1 /* 2 * Copyright (c) 2012-2017 Qualcomm Atheros, Inc. 3 * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. 4 * 5 * Permission to use, copy, modify, and/or distribute this software for any 6 * purpose with or without fee is hereby granted, provided that the above 7 * copyright notice and this permission notice appear in all copies. 8 * 9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 */ 17 18 #include <linux/etherdevice.h> 19 #include <net/ieee80211_radiotap.h> 20 #include <linux/if_arp.h> 21 #include <linux/moduleparam.h> 22 #include <linux/ip.h> 23 #include <linux/ipv6.h> 24 #include <net/ipv6.h> 25 #include <linux/prefetch.h> 26 27 #include "wil6210.h" 28 #include "wmi.h" 29 #include "txrx.h" 30 #include "trace.h" 31 #include "txrx_edma.h" 32 33 bool rx_align_2; 34 module_param(rx_align_2, bool, 0444); 35 MODULE_PARM_DESC(rx_align_2, " align Rx buffers on 4*n+2, default - no"); 36 37 bool rx_large_buf; 38 module_param(rx_large_buf, bool, 0444); 39 MODULE_PARM_DESC(rx_large_buf, " allocate 8KB RX buffers, default - no"); 40 41 /* Drop Tx packets in case Tx ring is full */ 42 bool drop_if_ring_full; 43 44 static inline uint wil_rx_snaplen(void) 45 { 46 return rx_align_2 ? 6 : 0; 47 } 48 49 /* wil_ring_wmark_low - low watermark for available descriptor space */ 50 static inline int wil_ring_wmark_low(struct wil_ring *ring) 51 { 52 return ring->size / 8; 53 } 54 55 /* wil_ring_wmark_high - high watermark for available descriptor space */ 56 static inline int wil_ring_wmark_high(struct wil_ring *ring) 57 { 58 return ring->size / 4; 59 } 60 61 /* returns true if num avail descriptors is lower than wmark_low */ 62 static inline int wil_ring_avail_low(struct wil_ring *ring) 63 { 64 return wil_ring_avail_tx(ring) < wil_ring_wmark_low(ring); 65 } 66 67 /* returns true if num avail descriptors is higher than wmark_high */ 68 static inline int wil_ring_avail_high(struct wil_ring *ring) 69 { 70 return wil_ring_avail_tx(ring) > wil_ring_wmark_high(ring); 71 } 72 73 /* returns true when all tx vrings are empty */ 74 bool wil_is_tx_idle(struct wil6210_priv *wil) 75 { 76 int i; 77 unsigned long data_comp_to; 78 int min_ring_id = wil_get_min_tx_ring_id(wil); 79 80 for (i = min_ring_id; i < WIL6210_MAX_TX_RINGS; i++) { 81 struct wil_ring *vring = &wil->ring_tx[i]; 82 int vring_index = vring - wil->ring_tx; 83 struct wil_ring_tx_data *txdata = 84 &wil->ring_tx_data[vring_index]; 85 86 spin_lock(&txdata->lock); 87 88 if (!vring->va || !txdata->enabled) { 89 spin_unlock(&txdata->lock); 90 continue; 91 } 92 93 data_comp_to = jiffies + msecs_to_jiffies( 94 WIL_DATA_COMPLETION_TO_MS); 95 if (test_bit(wil_status_napi_en, wil->status)) { 96 while (!wil_ring_is_empty(vring)) { 97 if (time_after(jiffies, data_comp_to)) { 98 wil_dbg_pm(wil, 99 "TO waiting for idle tx\n"); 100 spin_unlock(&txdata->lock); 101 return false; 102 } 103 wil_dbg_ratelimited(wil, 104 "tx vring is not empty -> NAPI\n"); 105 spin_unlock(&txdata->lock); 106 napi_synchronize(&wil->napi_tx); 107 msleep(20); 108 spin_lock(&txdata->lock); 109 if (!vring->va || !txdata->enabled) 110 break; 111 } 112 } 113 114 spin_unlock(&txdata->lock); 115 } 116 117 return true; 118 } 119 120 static int wil_vring_alloc(struct wil6210_priv *wil, struct wil_ring *vring) 121 { 122 struct device *dev = wil_to_dev(wil); 123 size_t sz = vring->size * sizeof(vring->va[0]); 124 uint i; 125 126 wil_dbg_misc(wil, "vring_alloc:\n"); 127 128 BUILD_BUG_ON(sizeof(vring->va[0]) != 32); 129 130 vring->swhead = 0; 131 vring->swtail = 0; 132 vring->ctx = kcalloc(vring->size, sizeof(vring->ctx[0]), GFP_KERNEL); 133 if (!vring->ctx) { 134 vring->va = NULL; 135 return -ENOMEM; 136 } 137 138 /* vring->va should be aligned on its size rounded up to power of 2 139 * This is granted by the dma_alloc_coherent. 140 * 141 * HW has limitation that all vrings addresses must share the same 142 * upper 16 msb bits part of 48 bits address. To workaround that, 143 * if we are using more than 32 bit addresses switch to 32 bit 144 * allocation before allocating vring memory. 145 * 146 * There's no check for the return value of dma_set_mask_and_coherent, 147 * since we assume if we were able to set the mask during 148 * initialization in this system it will not fail if we set it again 149 */ 150 if (wil->dma_addr_size > 32) 151 dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); 152 153 vring->va = dma_alloc_coherent(dev, sz, &vring->pa, GFP_KERNEL); 154 if (!vring->va) { 155 kfree(vring->ctx); 156 vring->ctx = NULL; 157 return -ENOMEM; 158 } 159 160 if (wil->dma_addr_size > 32) 161 dma_set_mask_and_coherent(dev, 162 DMA_BIT_MASK(wil->dma_addr_size)); 163 164 /* initially, all descriptors are SW owned 165 * For Tx and Rx, ownership bit is at the same location, thus 166 * we can use any 167 */ 168 for (i = 0; i < vring->size; i++) { 169 volatile struct vring_tx_desc *_d = 170 &vring->va[i].tx.legacy; 171 172 _d->dma.status = TX_DMA_STATUS_DU; 173 } 174 175 wil_dbg_misc(wil, "vring[%d] 0x%p:%pad 0x%p\n", vring->size, 176 vring->va, &vring->pa, vring->ctx); 177 178 return 0; 179 } 180 181 static void wil_txdesc_unmap(struct device *dev, union wil_tx_desc *desc, 182 struct wil_ctx *ctx) 183 { 184 struct vring_tx_desc *d = &desc->legacy; 185 dma_addr_t pa = wil_desc_addr(&d->dma.addr); 186 u16 dmalen = le16_to_cpu(d->dma.length); 187 188 switch (ctx->mapped_as) { 189 case wil_mapped_as_single: 190 dma_unmap_single(dev, pa, dmalen, DMA_TO_DEVICE); 191 break; 192 case wil_mapped_as_page: 193 dma_unmap_page(dev, pa, dmalen, DMA_TO_DEVICE); 194 break; 195 default: 196 break; 197 } 198 } 199 200 static void wil_vring_free(struct wil6210_priv *wil, struct wil_ring *vring) 201 { 202 struct device *dev = wil_to_dev(wil); 203 size_t sz = vring->size * sizeof(vring->va[0]); 204 205 lockdep_assert_held(&wil->mutex); 206 if (!vring->is_rx) { 207 int vring_index = vring - wil->ring_tx; 208 209 wil_dbg_misc(wil, "free Tx vring %d [%d] 0x%p:%pad 0x%p\n", 210 vring_index, vring->size, vring->va, 211 &vring->pa, vring->ctx); 212 } else { 213 wil_dbg_misc(wil, "free Rx vring [%d] 0x%p:%pad 0x%p\n", 214 vring->size, vring->va, 215 &vring->pa, vring->ctx); 216 } 217 218 while (!wil_ring_is_empty(vring)) { 219 dma_addr_t pa; 220 u16 dmalen; 221 struct wil_ctx *ctx; 222 223 if (!vring->is_rx) { 224 struct vring_tx_desc dd, *d = ⅆ 225 volatile struct vring_tx_desc *_d = 226 &vring->va[vring->swtail].tx.legacy; 227 228 ctx = &vring->ctx[vring->swtail]; 229 if (!ctx) { 230 wil_dbg_txrx(wil, 231 "ctx(%d) was already completed\n", 232 vring->swtail); 233 vring->swtail = wil_ring_next_tail(vring); 234 continue; 235 } 236 *d = *_d; 237 wil_txdesc_unmap(dev, (union wil_tx_desc *)d, ctx); 238 if (ctx->skb) 239 dev_kfree_skb_any(ctx->skb); 240 vring->swtail = wil_ring_next_tail(vring); 241 } else { /* rx */ 242 struct vring_rx_desc dd, *d = ⅆ 243 volatile struct vring_rx_desc *_d = 244 &vring->va[vring->swhead].rx.legacy; 245 246 ctx = &vring->ctx[vring->swhead]; 247 *d = *_d; 248 pa = wil_desc_addr(&d->dma.addr); 249 dmalen = le16_to_cpu(d->dma.length); 250 dma_unmap_single(dev, pa, dmalen, DMA_FROM_DEVICE); 251 kfree_skb(ctx->skb); 252 wil_ring_advance_head(vring, 1); 253 } 254 } 255 dma_free_coherent(dev, sz, (void *)vring->va, vring->pa); 256 kfree(vring->ctx); 257 vring->pa = 0; 258 vring->va = NULL; 259 vring->ctx = NULL; 260 } 261 262 /** 263 * Allocate one skb for Rx VRING 264 * 265 * Safe to call from IRQ 266 */ 267 static int wil_vring_alloc_skb(struct wil6210_priv *wil, struct wil_ring *vring, 268 u32 i, int headroom) 269 { 270 struct device *dev = wil_to_dev(wil); 271 unsigned int sz = wil->rx_buf_len + ETH_HLEN + wil_rx_snaplen(); 272 struct vring_rx_desc dd, *d = ⅆ 273 volatile struct vring_rx_desc *_d = &vring->va[i].rx.legacy; 274 dma_addr_t pa; 275 struct sk_buff *skb = dev_alloc_skb(sz + headroom); 276 277 if (unlikely(!skb)) 278 return -ENOMEM; 279 280 skb_reserve(skb, headroom); 281 skb_put(skb, sz); 282 283 /** 284 * Make sure that the network stack calculates checksum for packets 285 * which failed the HW checksum calculation 286 */ 287 skb->ip_summed = CHECKSUM_NONE; 288 289 pa = dma_map_single(dev, skb->data, skb->len, DMA_FROM_DEVICE); 290 if (unlikely(dma_mapping_error(dev, pa))) { 291 kfree_skb(skb); 292 return -ENOMEM; 293 } 294 295 d->dma.d0 = RX_DMA_D0_CMD_DMA_RT | RX_DMA_D0_CMD_DMA_IT; 296 wil_desc_addr_set(&d->dma.addr, pa); 297 /* ip_length don't care */ 298 /* b11 don't care */ 299 /* error don't care */ 300 d->dma.status = 0; /* BIT(0) should be 0 for HW_OWNED */ 301 d->dma.length = cpu_to_le16(sz); 302 *_d = *d; 303 vring->ctx[i].skb = skb; 304 305 return 0; 306 } 307 308 /** 309 * Adds radiotap header 310 * 311 * Any error indicated as "Bad FCS" 312 * 313 * Vendor data for 04:ce:14-1 (Wilocity-1) consists of: 314 * - Rx descriptor: 32 bytes 315 * - Phy info 316 */ 317 static void wil_rx_add_radiotap_header(struct wil6210_priv *wil, 318 struct sk_buff *skb) 319 { 320 struct wil6210_rtap { 321 struct ieee80211_radiotap_header rthdr; 322 /* fields should be in the order of bits in rthdr.it_present */ 323 /* flags */ 324 u8 flags; 325 /* channel */ 326 __le16 chnl_freq __aligned(2); 327 __le16 chnl_flags; 328 /* MCS */ 329 u8 mcs_present; 330 u8 mcs_flags; 331 u8 mcs_index; 332 } __packed; 333 struct vring_rx_desc *d = wil_skb_rxdesc(skb); 334 struct wil6210_rtap *rtap; 335 int rtap_len = sizeof(struct wil6210_rtap); 336 struct ieee80211_channel *ch = wil->monitor_chandef.chan; 337 338 if (skb_headroom(skb) < rtap_len && 339 pskb_expand_head(skb, rtap_len, 0, GFP_ATOMIC)) { 340 wil_err(wil, "Unable to expand headroom to %d\n", rtap_len); 341 return; 342 } 343 344 rtap = skb_push(skb, rtap_len); 345 memset(rtap, 0, rtap_len); 346 347 rtap->rthdr.it_version = PKTHDR_RADIOTAP_VERSION; 348 rtap->rthdr.it_len = cpu_to_le16(rtap_len); 349 rtap->rthdr.it_present = cpu_to_le32((1 << IEEE80211_RADIOTAP_FLAGS) | 350 (1 << IEEE80211_RADIOTAP_CHANNEL) | 351 (1 << IEEE80211_RADIOTAP_MCS)); 352 if (d->dma.status & RX_DMA_STATUS_ERROR) 353 rtap->flags |= IEEE80211_RADIOTAP_F_BADFCS; 354 355 rtap->chnl_freq = cpu_to_le16(ch ? ch->center_freq : 58320); 356 rtap->chnl_flags = cpu_to_le16(0); 357 358 rtap->mcs_present = IEEE80211_RADIOTAP_MCS_HAVE_MCS; 359 rtap->mcs_flags = 0; 360 rtap->mcs_index = wil_rxdesc_mcs(d); 361 } 362 363 static bool wil_is_rx_idle(struct wil6210_priv *wil) 364 { 365 struct vring_rx_desc *_d; 366 struct wil_ring *ring = &wil->ring_rx; 367 368 _d = (struct vring_rx_desc *)&ring->va[ring->swhead].rx.legacy; 369 if (_d->dma.status & RX_DMA_STATUS_DU) 370 return false; 371 372 return true; 373 } 374 375 static int wil_rx_get_cid_by_skb(struct wil6210_priv *wil, struct sk_buff *skb) 376 { 377 struct vring_rx_desc *d = wil_skb_rxdesc(skb); 378 int mid = wil_rxdesc_mid(d); 379 struct wil6210_vif *vif = wil->vifs[mid]; 380 /* cid from DMA descriptor is limited to 3 bits. 381 * In case of cid>=8, the value would be cid modulo 8 and we need to 382 * find real cid by locating the transmitter (ta) inside sta array 383 */ 384 int cid = wil_rxdesc_cid(d); 385 unsigned int snaplen = wil_rx_snaplen(); 386 struct ieee80211_hdr_3addr *hdr; 387 int i; 388 unsigned char *ta; 389 u8 ftype; 390 391 /* in monitor mode there are no connections */ 392 if (vif->wdev.iftype == NL80211_IFTYPE_MONITOR) 393 return cid; 394 395 ftype = wil_rxdesc_ftype(d) << 2; 396 if (likely(ftype == IEEE80211_FTYPE_DATA)) { 397 if (unlikely(skb->len < ETH_HLEN + snaplen)) { 398 wil_err_ratelimited(wil, 399 "Short data frame, len = %d\n", 400 skb->len); 401 return -ENOENT; 402 } 403 ta = wil_skb_get_sa(skb); 404 } else { 405 if (unlikely(skb->len < sizeof(struct ieee80211_hdr_3addr))) { 406 wil_err_ratelimited(wil, "Short frame, len = %d\n", 407 skb->len); 408 return -ENOENT; 409 } 410 hdr = (void *)skb->data; 411 ta = hdr->addr2; 412 } 413 414 if (wil->max_assoc_sta <= WIL6210_RX_DESC_MAX_CID) 415 return cid; 416 417 /* assuming no concurrency between AP interfaces and STA interfaces. 418 * multista is used only in P2P_GO or AP mode. In other modes return 419 * cid from the rx descriptor 420 */ 421 if (vif->wdev.iftype != NL80211_IFTYPE_P2P_GO && 422 vif->wdev.iftype != NL80211_IFTYPE_AP) 423 return cid; 424 425 /* For Rx packets cid from rx descriptor is limited to 3 bits (0..7), 426 * to find the real cid, compare transmitter address with the stored 427 * stations mac address in the driver sta array 428 */ 429 for (i = cid; i < wil->max_assoc_sta; i += WIL6210_RX_DESC_MAX_CID) { 430 if (wil->sta[i].status != wil_sta_unused && 431 ether_addr_equal(wil->sta[i].addr, ta)) { 432 cid = i; 433 break; 434 } 435 } 436 if (i >= wil->max_assoc_sta) { 437 wil_err_ratelimited(wil, "Could not find cid for frame with transmit addr = %pM, iftype = %d, frametype = %d, len = %d\n", 438 ta, vif->wdev.iftype, ftype, skb->len); 439 cid = -ENOENT; 440 } 441 442 return cid; 443 } 444 445 /** 446 * reap 1 frame from @swhead 447 * 448 * Rx descriptor copied to skb->cb 449 * 450 * Safe to call from IRQ 451 */ 452 static struct sk_buff *wil_vring_reap_rx(struct wil6210_priv *wil, 453 struct wil_ring *vring) 454 { 455 struct device *dev = wil_to_dev(wil); 456 struct wil6210_vif *vif; 457 struct net_device *ndev; 458 volatile struct vring_rx_desc *_d; 459 struct vring_rx_desc *d; 460 struct sk_buff *skb; 461 dma_addr_t pa; 462 unsigned int snaplen = wil_rx_snaplen(); 463 unsigned int sz = wil->rx_buf_len + ETH_HLEN + snaplen; 464 u16 dmalen; 465 u8 ftype; 466 int cid, mid; 467 int i; 468 struct wil_net_stats *stats; 469 470 BUILD_BUG_ON(sizeof(struct skb_rx_info) > sizeof(skb->cb)); 471 472 again: 473 if (unlikely(wil_ring_is_empty(vring))) 474 return NULL; 475 476 i = (int)vring->swhead; 477 _d = &vring->va[i].rx.legacy; 478 if (unlikely(!(_d->dma.status & RX_DMA_STATUS_DU))) { 479 /* it is not error, we just reached end of Rx done area */ 480 return NULL; 481 } 482 483 skb = vring->ctx[i].skb; 484 vring->ctx[i].skb = NULL; 485 wil_ring_advance_head(vring, 1); 486 if (!skb) { 487 wil_err(wil, "No Rx skb at [%d]\n", i); 488 goto again; 489 } 490 d = wil_skb_rxdesc(skb); 491 *d = *_d; 492 pa = wil_desc_addr(&d->dma.addr); 493 494 dma_unmap_single(dev, pa, sz, DMA_FROM_DEVICE); 495 dmalen = le16_to_cpu(d->dma.length); 496 497 trace_wil6210_rx(i, d); 498 wil_dbg_txrx(wil, "Rx[%3d] : %d bytes\n", i, dmalen); 499 wil_hex_dump_txrx("RxD ", DUMP_PREFIX_NONE, 32, 4, 500 (const void *)d, sizeof(*d), false); 501 502 mid = wil_rxdesc_mid(d); 503 vif = wil->vifs[mid]; 504 505 if (unlikely(!vif)) { 506 wil_dbg_txrx(wil, "skipped RX descriptor with invalid mid %d", 507 mid); 508 kfree_skb(skb); 509 goto again; 510 } 511 ndev = vif_to_ndev(vif); 512 if (unlikely(dmalen > sz)) { 513 wil_err_ratelimited(wil, "Rx size too large: %d bytes!\n", 514 dmalen); 515 kfree_skb(skb); 516 goto again; 517 } 518 skb_trim(skb, dmalen); 519 520 prefetch(skb->data); 521 522 wil_hex_dump_txrx("Rx ", DUMP_PREFIX_OFFSET, 16, 1, 523 skb->data, skb_headlen(skb), false); 524 525 cid = wil_rx_get_cid_by_skb(wil, skb); 526 if (cid == -ENOENT) { 527 kfree_skb(skb); 528 goto again; 529 } 530 wil_skb_set_cid(skb, (u8)cid); 531 stats = &wil->sta[cid].stats; 532 533 stats->last_mcs_rx = wil_rxdesc_mcs(d); 534 if (stats->last_mcs_rx < ARRAY_SIZE(stats->rx_per_mcs)) 535 stats->rx_per_mcs[stats->last_mcs_rx]++; 536 537 /* use radiotap header only if required */ 538 if (ndev->type == ARPHRD_IEEE80211_RADIOTAP) 539 wil_rx_add_radiotap_header(wil, skb); 540 541 /* no extra checks if in sniffer mode */ 542 if (ndev->type != ARPHRD_ETHER) 543 return skb; 544 /* Non-data frames may be delivered through Rx DMA channel (ex: BAR) 545 * Driver should recognize it by frame type, that is found 546 * in Rx descriptor. If type is not data, it is 802.11 frame as is 547 */ 548 ftype = wil_rxdesc_ftype(d) << 2; 549 if (unlikely(ftype != IEEE80211_FTYPE_DATA)) { 550 u8 fc1 = wil_rxdesc_fc1(d); 551 int tid = wil_rxdesc_tid(d); 552 u16 seq = wil_rxdesc_seq(d); 553 554 wil_dbg_txrx(wil, 555 "Non-data frame FC[7:0] 0x%02x MID %d CID %d TID %d Seq 0x%03x\n", 556 fc1, mid, cid, tid, seq); 557 stats->rx_non_data_frame++; 558 if (wil_is_back_req(fc1)) { 559 wil_dbg_txrx(wil, 560 "BAR: MID %d CID %d TID %d Seq 0x%03x\n", 561 mid, cid, tid, seq); 562 wil_rx_bar(wil, vif, cid, tid, seq); 563 } else { 564 /* print again all info. One can enable only this 565 * without overhead for printing every Rx frame 566 */ 567 wil_dbg_txrx(wil, 568 "Unhandled non-data frame FC[7:0] 0x%02x MID %d CID %d TID %d Seq 0x%03x\n", 569 fc1, mid, cid, tid, seq); 570 wil_hex_dump_txrx("RxD ", DUMP_PREFIX_NONE, 32, 4, 571 (const void *)d, sizeof(*d), false); 572 wil_hex_dump_txrx("Rx ", DUMP_PREFIX_OFFSET, 16, 1, 573 skb->data, skb_headlen(skb), false); 574 } 575 kfree_skb(skb); 576 goto again; 577 } 578 579 /* L4 IDENT is on when HW calculated checksum, check status 580 * and in case of error drop the packet 581 * higher stack layers will handle retransmission (if required) 582 */ 583 if (likely(d->dma.status & RX_DMA_STATUS_L4I)) { 584 /* L4 protocol identified, csum calculated */ 585 if (likely((d->dma.error & RX_DMA_ERROR_L4_ERR) == 0)) 586 skb->ip_summed = CHECKSUM_UNNECESSARY; 587 /* If HW reports bad checksum, let IP stack re-check it 588 * For example, HW don't understand Microsoft IP stack that 589 * mis-calculates TCP checksum - if it should be 0x0, 590 * it writes 0xffff in violation of RFC 1624 591 */ 592 else 593 stats->rx_csum_err++; 594 } 595 596 if (snaplen) { 597 /* Packet layout 598 * +-------+-------+---------+------------+------+ 599 * | SA(6) | DA(6) | SNAP(6) | ETHTYPE(2) | DATA | 600 * +-------+-------+---------+------------+------+ 601 * Need to remove SNAP, shifting SA and DA forward 602 */ 603 memmove(skb->data + snaplen, skb->data, 2 * ETH_ALEN); 604 skb_pull(skb, snaplen); 605 } 606 607 return skb; 608 } 609 610 /** 611 * allocate and fill up to @count buffers in rx ring 612 * buffers posted at @swtail 613 * Note: we have a single RX queue for servicing all VIFs, but we 614 * allocate skbs with headroom according to main interface only. This 615 * means it will not work with monitor interface together with other VIFs. 616 * Currently we only support monitor interface on its own without other VIFs, 617 * and we will need to fix this code once we add support. 618 */ 619 static int wil_rx_refill(struct wil6210_priv *wil, int count) 620 { 621 struct net_device *ndev = wil->main_ndev; 622 struct wil_ring *v = &wil->ring_rx; 623 u32 next_tail; 624 int rc = 0; 625 int headroom = ndev->type == ARPHRD_IEEE80211_RADIOTAP ? 626 WIL6210_RTAP_SIZE : 0; 627 628 for (; next_tail = wil_ring_next_tail(v), 629 (next_tail != v->swhead) && (count-- > 0); 630 v->swtail = next_tail) { 631 rc = wil_vring_alloc_skb(wil, v, v->swtail, headroom); 632 if (unlikely(rc)) { 633 wil_err_ratelimited(wil, "Error %d in rx refill[%d]\n", 634 rc, v->swtail); 635 break; 636 } 637 } 638 639 /* make sure all writes to descriptors (shared memory) are done before 640 * committing them to HW 641 */ 642 wmb(); 643 644 wil_w(wil, v->hwtail, v->swtail); 645 646 return rc; 647 } 648 649 /** 650 * reverse_memcmp - Compare two areas of memory, in reverse order 651 * @cs: One area of memory 652 * @ct: Another area of memory 653 * @count: The size of the area. 654 * 655 * Cut'n'paste from original memcmp (see lib/string.c) 656 * with minimal modifications 657 */ 658 int reverse_memcmp(const void *cs, const void *ct, size_t count) 659 { 660 const unsigned char *su1, *su2; 661 int res = 0; 662 663 for (su1 = cs + count - 1, su2 = ct + count - 1; count > 0; 664 --su1, --su2, count--) { 665 res = *su1 - *su2; 666 if (res) 667 break; 668 } 669 return res; 670 } 671 672 static int wil_rx_crypto_check(struct wil6210_priv *wil, struct sk_buff *skb) 673 { 674 struct vring_rx_desc *d = wil_skb_rxdesc(skb); 675 int cid = wil_skb_get_cid(skb); 676 int tid = wil_rxdesc_tid(d); 677 int key_id = wil_rxdesc_key_id(d); 678 int mc = wil_rxdesc_mcast(d); 679 struct wil_sta_info *s = &wil->sta[cid]; 680 struct wil_tid_crypto_rx *c = mc ? &s->group_crypto_rx : 681 &s->tid_crypto_rx[tid]; 682 struct wil_tid_crypto_rx_single *cc = &c->key_id[key_id]; 683 const u8 *pn = (u8 *)&d->mac.pn_15_0; 684 685 if (!cc->key_set) { 686 wil_err_ratelimited(wil, 687 "Key missing. CID %d TID %d MCast %d KEY_ID %d\n", 688 cid, tid, mc, key_id); 689 return -EINVAL; 690 } 691 692 if (reverse_memcmp(pn, cc->pn, IEEE80211_GCMP_PN_LEN) <= 0) { 693 wil_err_ratelimited(wil, 694 "Replay attack. CID %d TID %d MCast %d KEY_ID %d PN %6phN last %6phN\n", 695 cid, tid, mc, key_id, pn, cc->pn); 696 return -EINVAL; 697 } 698 memcpy(cc->pn, pn, IEEE80211_GCMP_PN_LEN); 699 700 return 0; 701 } 702 703 static int wil_rx_error_check(struct wil6210_priv *wil, struct sk_buff *skb, 704 struct wil_net_stats *stats) 705 { 706 struct vring_rx_desc *d = wil_skb_rxdesc(skb); 707 708 if ((d->dma.status & RX_DMA_STATUS_ERROR) && 709 (d->dma.error & RX_DMA_ERROR_MIC)) { 710 stats->rx_mic_error++; 711 wil_dbg_txrx(wil, "MIC error, dropping packet\n"); 712 return -EFAULT; 713 } 714 715 return 0; 716 } 717 718 static void wil_get_netif_rx_params(struct sk_buff *skb, int *cid, 719 int *security) 720 { 721 struct vring_rx_desc *d = wil_skb_rxdesc(skb); 722 723 *cid = wil_skb_get_cid(skb); 724 *security = wil_rxdesc_security(d); 725 } 726 727 /* 728 * Check if skb is ptk eapol key message 729 * 730 * returns a pointer to the start of the eapol key structure, NULL 731 * if frame is not PTK eapol key 732 */ 733 static struct wil_eapol_key *wil_is_ptk_eapol_key(struct wil6210_priv *wil, 734 struct sk_buff *skb) 735 { 736 u8 *buf; 737 const struct wil_1x_hdr *hdr; 738 struct wil_eapol_key *key; 739 u16 key_info; 740 int len = skb->len; 741 742 if (!skb_mac_header_was_set(skb)) { 743 wil_err(wil, "mac header was not set\n"); 744 return NULL; 745 } 746 747 len -= skb_mac_offset(skb); 748 749 if (len < sizeof(struct ethhdr) + sizeof(struct wil_1x_hdr) + 750 sizeof(struct wil_eapol_key)) 751 return NULL; 752 753 buf = skb_mac_header(skb) + sizeof(struct ethhdr); 754 755 hdr = (const struct wil_1x_hdr *)buf; 756 if (hdr->type != WIL_1X_TYPE_EAPOL_KEY) 757 return NULL; 758 759 key = (struct wil_eapol_key *)(buf + sizeof(struct wil_1x_hdr)); 760 if (key->type != WIL_EAPOL_KEY_TYPE_WPA && 761 key->type != WIL_EAPOL_KEY_TYPE_RSN) 762 return NULL; 763 764 key_info = be16_to_cpu(key->key_info); 765 if (!(key_info & WIL_KEY_INFO_KEY_TYPE)) /* check if pairwise */ 766 return NULL; 767 768 return key; 769 } 770 771 static bool wil_skb_is_eap_3(struct wil6210_priv *wil, struct sk_buff *skb) 772 { 773 struct wil_eapol_key *key; 774 u16 key_info; 775 776 key = wil_is_ptk_eapol_key(wil, skb); 777 if (!key) 778 return false; 779 780 key_info = be16_to_cpu(key->key_info); 781 if (key_info & (WIL_KEY_INFO_MIC | 782 WIL_KEY_INFO_ENCR_KEY_DATA)) { 783 /* 3/4 of 4-Way Handshake */ 784 wil_dbg_misc(wil, "EAPOL key message 3\n"); 785 return true; 786 } 787 /* 1/4 of 4-Way Handshake */ 788 wil_dbg_misc(wil, "EAPOL key message 1\n"); 789 790 return false; 791 } 792 793 static bool wil_skb_is_eap_4(struct wil6210_priv *wil, struct sk_buff *skb) 794 { 795 struct wil_eapol_key *key; 796 u32 *nonce, i; 797 798 key = wil_is_ptk_eapol_key(wil, skb); 799 if (!key) 800 return false; 801 802 nonce = (u32 *)key->key_nonce; 803 for (i = 0; i < WIL_EAP_NONCE_LEN / sizeof(u32); i++, nonce++) { 804 if (*nonce != 0) { 805 /* message 2/4 */ 806 wil_dbg_misc(wil, "EAPOL key message 2\n"); 807 return false; 808 } 809 } 810 wil_dbg_misc(wil, "EAPOL key message 4\n"); 811 812 return true; 813 } 814 815 void wil_enable_tx_key_worker(struct work_struct *work) 816 { 817 struct wil6210_vif *vif = container_of(work, 818 struct wil6210_vif, enable_tx_key_worker); 819 struct wil6210_priv *wil = vif_to_wil(vif); 820 int rc, cid; 821 822 rtnl_lock(); 823 if (vif->ptk_rekey_state != WIL_REKEY_WAIT_M4_SENT) { 824 wil_dbg_misc(wil, "Invalid rekey state = %d\n", 825 vif->ptk_rekey_state); 826 rtnl_unlock(); 827 return; 828 } 829 830 cid = wil_find_cid_by_idx(wil, vif->mid, 0); 831 if (!wil_cid_valid(wil, cid)) { 832 wil_err(wil, "Invalid cid = %d\n", cid); 833 rtnl_unlock(); 834 return; 835 } 836 837 wil_dbg_misc(wil, "Apply PTK key after eapol was sent out\n"); 838 rc = wmi_add_cipher_key(vif, 0, wil->sta[cid].addr, 0, NULL, 839 WMI_KEY_USE_APPLY_PTK); 840 841 vif->ptk_rekey_state = WIL_REKEY_IDLE; 842 rtnl_unlock(); 843 844 if (rc) 845 wil_err(wil, "Apply PTK key failed %d\n", rc); 846 } 847 848 void wil_tx_complete_handle_eapol(struct wil6210_vif *vif, struct sk_buff *skb) 849 { 850 struct wil6210_priv *wil = vif_to_wil(vif); 851 struct wireless_dev *wdev = vif_to_wdev(vif); 852 bool q = false; 853 854 if (wdev->iftype != NL80211_IFTYPE_STATION || 855 !test_bit(WMI_FW_CAPABILITY_SPLIT_REKEY, wil->fw_capabilities)) 856 return; 857 858 /* check if skb is an EAP message 4/4 */ 859 if (!wil_skb_is_eap_4(wil, skb)) 860 return; 861 862 spin_lock_bh(&wil->eap_lock); 863 switch (vif->ptk_rekey_state) { 864 case WIL_REKEY_IDLE: 865 /* ignore idle state, can happen due to M4 retransmission */ 866 break; 867 case WIL_REKEY_M3_RECEIVED: 868 vif->ptk_rekey_state = WIL_REKEY_IDLE; 869 break; 870 case WIL_REKEY_WAIT_M4_SENT: 871 q = true; 872 break; 873 default: 874 wil_err(wil, "Unknown rekey state = %d", 875 vif->ptk_rekey_state); 876 } 877 spin_unlock_bh(&wil->eap_lock); 878 879 if (q) { 880 q = queue_work(wil->wmi_wq, &vif->enable_tx_key_worker); 881 wil_dbg_misc(wil, "queue_work of enable_tx_key_worker -> %d\n", 882 q); 883 } 884 } 885 886 static void wil_rx_handle_eapol(struct wil6210_vif *vif, struct sk_buff *skb) 887 { 888 struct wil6210_priv *wil = vif_to_wil(vif); 889 struct wireless_dev *wdev = vif_to_wdev(vif); 890 891 if (wdev->iftype != NL80211_IFTYPE_STATION || 892 !test_bit(WMI_FW_CAPABILITY_SPLIT_REKEY, wil->fw_capabilities)) 893 return; 894 895 /* check if skb is a EAP message 3/4 */ 896 if (!wil_skb_is_eap_3(wil, skb)) 897 return; 898 899 if (vif->ptk_rekey_state == WIL_REKEY_IDLE) 900 vif->ptk_rekey_state = WIL_REKEY_M3_RECEIVED; 901 } 902 903 /* 904 * Pass Rx packet to the netif. Update statistics. 905 * Called in softirq context (NAPI poll). 906 */ 907 void wil_netif_rx(struct sk_buff *skb, struct net_device *ndev, int cid, 908 struct wil_net_stats *stats, bool gro) 909 { 910 gro_result_t rc = GRO_NORMAL; 911 struct wil6210_vif *vif = ndev_to_vif(ndev); 912 struct wil6210_priv *wil = ndev_to_wil(ndev); 913 struct wireless_dev *wdev = vif_to_wdev(vif); 914 unsigned int len = skb->len; 915 u8 *sa, *da = wil_skb_get_da(skb); 916 /* here looking for DA, not A1, thus Rxdesc's 'mcast' indication 917 * is not suitable, need to look at data 918 */ 919 int mcast = is_multicast_ether_addr(da); 920 struct sk_buff *xmit_skb = NULL; 921 static const char * const gro_res_str[] = { 922 [GRO_MERGED] = "GRO_MERGED", 923 [GRO_MERGED_FREE] = "GRO_MERGED_FREE", 924 [GRO_HELD] = "GRO_HELD", 925 [GRO_NORMAL] = "GRO_NORMAL", 926 [GRO_DROP] = "GRO_DROP", 927 [GRO_CONSUMED] = "GRO_CONSUMED", 928 }; 929 930 if (wdev->iftype == NL80211_IFTYPE_STATION) { 931 sa = wil_skb_get_sa(skb); 932 if (mcast && ether_addr_equal(sa, ndev->dev_addr)) { 933 /* mcast packet looped back to us */ 934 rc = GRO_DROP; 935 dev_kfree_skb(skb); 936 goto stats; 937 } 938 } else if (wdev->iftype == NL80211_IFTYPE_AP && !vif->ap_isolate) { 939 if (mcast) { 940 /* send multicast frames both to higher layers in 941 * local net stack and back to the wireless medium 942 */ 943 xmit_skb = skb_copy(skb, GFP_ATOMIC); 944 } else { 945 int xmit_cid = wil_find_cid(wil, vif->mid, da); 946 947 if (xmit_cid >= 0) { 948 /* The destination station is associated to 949 * this AP (in this VLAN), so send the frame 950 * directly to it and do not pass it to local 951 * net stack. 952 */ 953 xmit_skb = skb; 954 skb = NULL; 955 } 956 } 957 } 958 if (xmit_skb) { 959 /* Send to wireless media and increase priority by 256 to 960 * keep the received priority instead of reclassifying 961 * the frame (see cfg80211_classify8021d). 962 */ 963 xmit_skb->dev = ndev; 964 xmit_skb->priority += 256; 965 xmit_skb->protocol = htons(ETH_P_802_3); 966 skb_reset_network_header(xmit_skb); 967 skb_reset_mac_header(xmit_skb); 968 wil_dbg_txrx(wil, "Rx -> Tx %d bytes\n", len); 969 dev_queue_xmit(xmit_skb); 970 } 971 972 if (skb) { /* deliver to local stack */ 973 skb->protocol = eth_type_trans(skb, ndev); 974 skb->dev = ndev; 975 976 if (skb->protocol == cpu_to_be16(ETH_P_PAE)) 977 wil_rx_handle_eapol(vif, skb); 978 979 if (gro) 980 rc = napi_gro_receive(&wil->napi_rx, skb); 981 else 982 netif_rx_ni(skb); 983 wil_dbg_txrx(wil, "Rx complete %d bytes => %s\n", 984 len, gro_res_str[rc]); 985 } 986 stats: 987 /* statistics. rc set to GRO_NORMAL for AP bridging */ 988 if (unlikely(rc == GRO_DROP)) { 989 ndev->stats.rx_dropped++; 990 stats->rx_dropped++; 991 wil_dbg_txrx(wil, "Rx drop %d bytes\n", len); 992 } else { 993 ndev->stats.rx_packets++; 994 stats->rx_packets++; 995 ndev->stats.rx_bytes += len; 996 stats->rx_bytes += len; 997 if (mcast) 998 ndev->stats.multicast++; 999 } 1000 } 1001 1002 void wil_netif_rx_any(struct sk_buff *skb, struct net_device *ndev) 1003 { 1004 int cid, security; 1005 struct wil6210_priv *wil = ndev_to_wil(ndev); 1006 struct wil_net_stats *stats; 1007 1008 wil->txrx_ops.get_netif_rx_params(skb, &cid, &security); 1009 1010 stats = &wil->sta[cid].stats; 1011 1012 skb_orphan(skb); 1013 1014 if (security && (wil->txrx_ops.rx_crypto_check(wil, skb) != 0)) { 1015 wil_dbg_txrx(wil, "Rx drop %d bytes\n", skb->len); 1016 dev_kfree_skb(skb); 1017 ndev->stats.rx_dropped++; 1018 stats->rx_replay++; 1019 stats->rx_dropped++; 1020 return; 1021 } 1022 1023 /* check errors reported by HW and update statistics */ 1024 if (unlikely(wil->txrx_ops.rx_error_check(wil, skb, stats))) { 1025 dev_kfree_skb(skb); 1026 return; 1027 } 1028 1029 wil_netif_rx(skb, ndev, cid, stats, true); 1030 } 1031 1032 /** 1033 * Proceed all completed skb's from Rx VRING 1034 * 1035 * Safe to call from NAPI poll, i.e. softirq with interrupts enabled 1036 */ 1037 void wil_rx_handle(struct wil6210_priv *wil, int *quota) 1038 { 1039 struct net_device *ndev = wil->main_ndev; 1040 struct wireless_dev *wdev = ndev->ieee80211_ptr; 1041 struct wil_ring *v = &wil->ring_rx; 1042 struct sk_buff *skb; 1043 1044 if (unlikely(!v->va)) { 1045 wil_err(wil, "Rx IRQ while Rx not yet initialized\n"); 1046 return; 1047 } 1048 wil_dbg_txrx(wil, "rx_handle\n"); 1049 while ((*quota > 0) && (NULL != (skb = wil_vring_reap_rx(wil, v)))) { 1050 (*quota)--; 1051 1052 /* monitor is currently supported on main interface only */ 1053 if (wdev->iftype == NL80211_IFTYPE_MONITOR) { 1054 skb->dev = ndev; 1055 skb_reset_mac_header(skb); 1056 skb->ip_summed = CHECKSUM_UNNECESSARY; 1057 skb->pkt_type = PACKET_OTHERHOST; 1058 skb->protocol = htons(ETH_P_802_2); 1059 wil_netif_rx_any(skb, ndev); 1060 } else { 1061 wil_rx_reorder(wil, skb); 1062 } 1063 } 1064 wil_rx_refill(wil, v->size); 1065 } 1066 1067 static void wil_rx_buf_len_init(struct wil6210_priv *wil) 1068 { 1069 wil->rx_buf_len = rx_large_buf ? 1070 WIL_MAX_ETH_MTU : TXRX_BUF_LEN_DEFAULT - WIL_MAX_MPDU_OVERHEAD; 1071 if (mtu_max > wil->rx_buf_len) { 1072 /* do not allow RX buffers to be smaller than mtu_max, for 1073 * backward compatibility (mtu_max parameter was also used 1074 * to support receiving large packets) 1075 */ 1076 wil_info(wil, "Override RX buffer to mtu_max(%d)\n", mtu_max); 1077 wil->rx_buf_len = mtu_max; 1078 } 1079 } 1080 1081 static int wil_rx_init(struct wil6210_priv *wil, uint order) 1082 { 1083 struct wil_ring *vring = &wil->ring_rx; 1084 int rc; 1085 1086 wil_dbg_misc(wil, "rx_init\n"); 1087 1088 if (vring->va) { 1089 wil_err(wil, "Rx ring already allocated\n"); 1090 return -EINVAL; 1091 } 1092 1093 wil_rx_buf_len_init(wil); 1094 1095 vring->size = 1 << order; 1096 vring->is_rx = true; 1097 rc = wil_vring_alloc(wil, vring); 1098 if (rc) 1099 return rc; 1100 1101 rc = wmi_rx_chain_add(wil, vring); 1102 if (rc) 1103 goto err_free; 1104 1105 rc = wil_rx_refill(wil, vring->size); 1106 if (rc) 1107 goto err_free; 1108 1109 return 0; 1110 err_free: 1111 wil_vring_free(wil, vring); 1112 1113 return rc; 1114 } 1115 1116 static void wil_rx_fini(struct wil6210_priv *wil) 1117 { 1118 struct wil_ring *vring = &wil->ring_rx; 1119 1120 wil_dbg_misc(wil, "rx_fini\n"); 1121 1122 if (vring->va) 1123 wil_vring_free(wil, vring); 1124 } 1125 1126 static int wil_tx_desc_map(union wil_tx_desc *desc, dma_addr_t pa, 1127 u32 len, int vring_index) 1128 { 1129 struct vring_tx_desc *d = &desc->legacy; 1130 1131 wil_desc_addr_set(&d->dma.addr, pa); 1132 d->dma.ip_length = 0; 1133 /* 0..6: mac_length; 7:ip_version 0-IP6 1-IP4*/ 1134 d->dma.b11 = 0/*14 | BIT(7)*/; 1135 d->dma.error = 0; 1136 d->dma.status = 0; /* BIT(0) should be 0 for HW_OWNED */ 1137 d->dma.length = cpu_to_le16((u16)len); 1138 d->dma.d0 = (vring_index << DMA_CFG_DESC_TX_0_QID_POS); 1139 d->mac.d[0] = 0; 1140 d->mac.d[1] = 0; 1141 d->mac.d[2] = 0; 1142 d->mac.ucode_cmd = 0; 1143 /* translation type: 0 - bypass; 1 - 802.3; 2 - native wifi */ 1144 d->mac.d[2] = BIT(MAC_CFG_DESC_TX_2_SNAP_HDR_INSERTION_EN_POS) | 1145 (1 << MAC_CFG_DESC_TX_2_L2_TRANSLATION_TYPE_POS); 1146 1147 return 0; 1148 } 1149 1150 void wil_tx_data_init(struct wil_ring_tx_data *txdata) 1151 { 1152 spin_lock_bh(&txdata->lock); 1153 txdata->dot1x_open = 0; 1154 txdata->enabled = 0; 1155 txdata->idle = 0; 1156 txdata->last_idle = 0; 1157 txdata->begin = 0; 1158 txdata->agg_wsize = 0; 1159 txdata->agg_timeout = 0; 1160 txdata->agg_amsdu = 0; 1161 txdata->addba_in_progress = false; 1162 txdata->mid = U8_MAX; 1163 spin_unlock_bh(&txdata->lock); 1164 } 1165 1166 static int wil_vring_init_tx(struct wil6210_vif *vif, int id, int size, 1167 int cid, int tid) 1168 { 1169 struct wil6210_priv *wil = vif_to_wil(vif); 1170 int rc; 1171 struct wmi_vring_cfg_cmd cmd = { 1172 .action = cpu_to_le32(WMI_VRING_CMD_ADD), 1173 .vring_cfg = { 1174 .tx_sw_ring = { 1175 .max_mpdu_size = 1176 cpu_to_le16(wil_mtu2macbuf(mtu_max)), 1177 .ring_size = cpu_to_le16(size), 1178 }, 1179 .ringid = id, 1180 .encap_trans_type = WMI_VRING_ENC_TYPE_802_3, 1181 .mac_ctrl = 0, 1182 .to_resolution = 0, 1183 .agg_max_wsize = 0, 1184 .schd_params = { 1185 .priority = cpu_to_le16(0), 1186 .timeslot_us = cpu_to_le16(0xfff), 1187 }, 1188 }, 1189 }; 1190 struct { 1191 struct wmi_cmd_hdr wmi; 1192 struct wmi_vring_cfg_done_event cmd; 1193 } __packed reply = { 1194 .cmd = {.status = WMI_FW_STATUS_FAILURE}, 1195 }; 1196 struct wil_ring *vring = &wil->ring_tx[id]; 1197 struct wil_ring_tx_data *txdata = &wil->ring_tx_data[id]; 1198 1199 if (cid >= WIL6210_RX_DESC_MAX_CID) { 1200 cmd.vring_cfg.cidxtid = CIDXTID_EXTENDED_CID_TID; 1201 cmd.vring_cfg.cid = cid; 1202 cmd.vring_cfg.tid = tid; 1203 } else { 1204 cmd.vring_cfg.cidxtid = mk_cidxtid(cid, tid); 1205 } 1206 1207 wil_dbg_misc(wil, "vring_init_tx: max_mpdu_size %d\n", 1208 cmd.vring_cfg.tx_sw_ring.max_mpdu_size); 1209 lockdep_assert_held(&wil->mutex); 1210 1211 if (vring->va) { 1212 wil_err(wil, "Tx ring [%d] already allocated\n", id); 1213 rc = -EINVAL; 1214 goto out; 1215 } 1216 1217 wil_tx_data_init(txdata); 1218 vring->is_rx = false; 1219 vring->size = size; 1220 rc = wil_vring_alloc(wil, vring); 1221 if (rc) 1222 goto out; 1223 1224 wil->ring2cid_tid[id][0] = cid; 1225 wil->ring2cid_tid[id][1] = tid; 1226 1227 cmd.vring_cfg.tx_sw_ring.ring_mem_base = cpu_to_le64(vring->pa); 1228 1229 if (!vif->privacy) 1230 txdata->dot1x_open = true; 1231 rc = wmi_call(wil, WMI_VRING_CFG_CMDID, vif->mid, &cmd, sizeof(cmd), 1232 WMI_VRING_CFG_DONE_EVENTID, &reply, sizeof(reply), 1233 WIL_WMI_CALL_GENERAL_TO_MS); 1234 if (rc) 1235 goto out_free; 1236 1237 if (reply.cmd.status != WMI_FW_STATUS_SUCCESS) { 1238 wil_err(wil, "Tx config failed, status 0x%02x\n", 1239 reply.cmd.status); 1240 rc = -EINVAL; 1241 goto out_free; 1242 } 1243 1244 spin_lock_bh(&txdata->lock); 1245 vring->hwtail = le32_to_cpu(reply.cmd.tx_vring_tail_ptr); 1246 txdata->mid = vif->mid; 1247 txdata->enabled = 1; 1248 spin_unlock_bh(&txdata->lock); 1249 1250 if (txdata->dot1x_open && (agg_wsize >= 0)) 1251 wil_addba_tx_request(wil, id, agg_wsize); 1252 1253 return 0; 1254 out_free: 1255 spin_lock_bh(&txdata->lock); 1256 txdata->dot1x_open = false; 1257 txdata->enabled = 0; 1258 spin_unlock_bh(&txdata->lock); 1259 wil_vring_free(wil, vring); 1260 wil->ring2cid_tid[id][0] = wil->max_assoc_sta; 1261 wil->ring2cid_tid[id][1] = 0; 1262 1263 out: 1264 1265 return rc; 1266 } 1267 1268 static int wil_tx_vring_modify(struct wil6210_vif *vif, int ring_id, int cid, 1269 int tid) 1270 { 1271 struct wil6210_priv *wil = vif_to_wil(vif); 1272 int rc; 1273 struct wmi_vring_cfg_cmd cmd = { 1274 .action = cpu_to_le32(WMI_VRING_CMD_MODIFY), 1275 .vring_cfg = { 1276 .tx_sw_ring = { 1277 .max_mpdu_size = 1278 cpu_to_le16(wil_mtu2macbuf(mtu_max)), 1279 .ring_size = 0, 1280 }, 1281 .ringid = ring_id, 1282 .cidxtid = mk_cidxtid(cid, tid), 1283 .encap_trans_type = WMI_VRING_ENC_TYPE_802_3, 1284 .mac_ctrl = 0, 1285 .to_resolution = 0, 1286 .agg_max_wsize = 0, 1287 .schd_params = { 1288 .priority = cpu_to_le16(0), 1289 .timeslot_us = cpu_to_le16(0xfff), 1290 }, 1291 }, 1292 }; 1293 struct { 1294 struct wmi_cmd_hdr wmi; 1295 struct wmi_vring_cfg_done_event cmd; 1296 } __packed reply = { 1297 .cmd = {.status = WMI_FW_STATUS_FAILURE}, 1298 }; 1299 struct wil_ring *vring = &wil->ring_tx[ring_id]; 1300 struct wil_ring_tx_data *txdata = &wil->ring_tx_data[ring_id]; 1301 1302 wil_dbg_misc(wil, "vring_modify: ring %d cid %d tid %d\n", ring_id, 1303 cid, tid); 1304 lockdep_assert_held(&wil->mutex); 1305 1306 if (!vring->va) { 1307 wil_err(wil, "Tx ring [%d] not allocated\n", ring_id); 1308 return -EINVAL; 1309 } 1310 1311 if (wil->ring2cid_tid[ring_id][0] != cid || 1312 wil->ring2cid_tid[ring_id][1] != tid) { 1313 wil_err(wil, "ring info does not match cid=%u tid=%u\n", 1314 wil->ring2cid_tid[ring_id][0], 1315 wil->ring2cid_tid[ring_id][1]); 1316 } 1317 1318 cmd.vring_cfg.tx_sw_ring.ring_mem_base = cpu_to_le64(vring->pa); 1319 1320 rc = wmi_call(wil, WMI_VRING_CFG_CMDID, vif->mid, &cmd, sizeof(cmd), 1321 WMI_VRING_CFG_DONE_EVENTID, &reply, sizeof(reply), 1322 WIL_WMI_CALL_GENERAL_TO_MS); 1323 if (rc) 1324 goto fail; 1325 1326 if (reply.cmd.status != WMI_FW_STATUS_SUCCESS) { 1327 wil_err(wil, "Tx modify failed, status 0x%02x\n", 1328 reply.cmd.status); 1329 rc = -EINVAL; 1330 goto fail; 1331 } 1332 1333 /* set BA aggregation window size to 0 to force a new BA with the 1334 * new AP 1335 */ 1336 txdata->agg_wsize = 0; 1337 if (txdata->dot1x_open && agg_wsize >= 0) 1338 wil_addba_tx_request(wil, ring_id, agg_wsize); 1339 1340 return 0; 1341 fail: 1342 spin_lock_bh(&txdata->lock); 1343 txdata->dot1x_open = false; 1344 txdata->enabled = 0; 1345 spin_unlock_bh(&txdata->lock); 1346 wil->ring2cid_tid[ring_id][0] = wil->max_assoc_sta; 1347 wil->ring2cid_tid[ring_id][1] = 0; 1348 return rc; 1349 } 1350 1351 int wil_vring_init_bcast(struct wil6210_vif *vif, int id, int size) 1352 { 1353 struct wil6210_priv *wil = vif_to_wil(vif); 1354 int rc; 1355 struct wmi_bcast_vring_cfg_cmd cmd = { 1356 .action = cpu_to_le32(WMI_VRING_CMD_ADD), 1357 .vring_cfg = { 1358 .tx_sw_ring = { 1359 .max_mpdu_size = 1360 cpu_to_le16(wil_mtu2macbuf(mtu_max)), 1361 .ring_size = cpu_to_le16(size), 1362 }, 1363 .ringid = id, 1364 .encap_trans_type = WMI_VRING_ENC_TYPE_802_3, 1365 }, 1366 }; 1367 struct { 1368 struct wmi_cmd_hdr wmi; 1369 struct wmi_vring_cfg_done_event cmd; 1370 } __packed reply = { 1371 .cmd = {.status = WMI_FW_STATUS_FAILURE}, 1372 }; 1373 struct wil_ring *vring = &wil->ring_tx[id]; 1374 struct wil_ring_tx_data *txdata = &wil->ring_tx_data[id]; 1375 1376 wil_dbg_misc(wil, "vring_init_bcast: max_mpdu_size %d\n", 1377 cmd.vring_cfg.tx_sw_ring.max_mpdu_size); 1378 lockdep_assert_held(&wil->mutex); 1379 1380 if (vring->va) { 1381 wil_err(wil, "Tx ring [%d] already allocated\n", id); 1382 rc = -EINVAL; 1383 goto out; 1384 } 1385 1386 wil_tx_data_init(txdata); 1387 vring->is_rx = false; 1388 vring->size = size; 1389 rc = wil_vring_alloc(wil, vring); 1390 if (rc) 1391 goto out; 1392 1393 wil->ring2cid_tid[id][0] = wil->max_assoc_sta; /* CID */ 1394 wil->ring2cid_tid[id][1] = 0; /* TID */ 1395 1396 cmd.vring_cfg.tx_sw_ring.ring_mem_base = cpu_to_le64(vring->pa); 1397 1398 if (!vif->privacy) 1399 txdata->dot1x_open = true; 1400 rc = wmi_call(wil, WMI_BCAST_VRING_CFG_CMDID, vif->mid, 1401 &cmd, sizeof(cmd), 1402 WMI_VRING_CFG_DONE_EVENTID, &reply, sizeof(reply), 1403 WIL_WMI_CALL_GENERAL_TO_MS); 1404 if (rc) 1405 goto out_free; 1406 1407 if (reply.cmd.status != WMI_FW_STATUS_SUCCESS) { 1408 wil_err(wil, "Tx config failed, status 0x%02x\n", 1409 reply.cmd.status); 1410 rc = -EINVAL; 1411 goto out_free; 1412 } 1413 1414 spin_lock_bh(&txdata->lock); 1415 vring->hwtail = le32_to_cpu(reply.cmd.tx_vring_tail_ptr); 1416 txdata->mid = vif->mid; 1417 txdata->enabled = 1; 1418 spin_unlock_bh(&txdata->lock); 1419 1420 return 0; 1421 out_free: 1422 spin_lock_bh(&txdata->lock); 1423 txdata->enabled = 0; 1424 txdata->dot1x_open = false; 1425 spin_unlock_bh(&txdata->lock); 1426 wil_vring_free(wil, vring); 1427 out: 1428 1429 return rc; 1430 } 1431 1432 static struct wil_ring *wil_find_tx_ucast(struct wil6210_priv *wil, 1433 struct wil6210_vif *vif, 1434 struct sk_buff *skb) 1435 { 1436 int i, cid; 1437 const u8 *da = wil_skb_get_da(skb); 1438 int min_ring_id = wil_get_min_tx_ring_id(wil); 1439 1440 cid = wil_find_cid(wil, vif->mid, da); 1441 1442 if (cid < 0 || cid >= wil->max_assoc_sta) 1443 return NULL; 1444 1445 /* TODO: fix for multiple TID */ 1446 for (i = min_ring_id; i < ARRAY_SIZE(wil->ring2cid_tid); i++) { 1447 if (!wil->ring_tx_data[i].dot1x_open && 1448 skb->protocol != cpu_to_be16(ETH_P_PAE)) 1449 continue; 1450 if (wil->ring2cid_tid[i][0] == cid) { 1451 struct wil_ring *v = &wil->ring_tx[i]; 1452 struct wil_ring_tx_data *txdata = &wil->ring_tx_data[i]; 1453 1454 wil_dbg_txrx(wil, "find_tx_ucast: (%pM) -> [%d]\n", 1455 da, i); 1456 if (v->va && txdata->enabled) { 1457 return v; 1458 } else { 1459 wil_dbg_txrx(wil, 1460 "find_tx_ucast: vring[%d] not valid\n", 1461 i); 1462 return NULL; 1463 } 1464 } 1465 } 1466 1467 return NULL; 1468 } 1469 1470 static int wil_tx_ring(struct wil6210_priv *wil, struct wil6210_vif *vif, 1471 struct wil_ring *ring, struct sk_buff *skb); 1472 1473 static struct wil_ring *wil_find_tx_ring_sta(struct wil6210_priv *wil, 1474 struct wil6210_vif *vif, 1475 struct sk_buff *skb) 1476 { 1477 struct wil_ring *ring; 1478 int i; 1479 u8 cid; 1480 struct wil_ring_tx_data *txdata; 1481 int min_ring_id = wil_get_min_tx_ring_id(wil); 1482 1483 /* In the STA mode, it is expected to have only 1 VRING 1484 * for the AP we connected to. 1485 * find 1-st vring eligible for this skb and use it. 1486 */ 1487 for (i = min_ring_id; i < WIL6210_MAX_TX_RINGS; i++) { 1488 ring = &wil->ring_tx[i]; 1489 txdata = &wil->ring_tx_data[i]; 1490 if (!ring->va || !txdata->enabled || txdata->mid != vif->mid) 1491 continue; 1492 1493 cid = wil->ring2cid_tid[i][0]; 1494 if (cid >= wil->max_assoc_sta) /* skip BCAST */ 1495 continue; 1496 1497 if (!wil->ring_tx_data[i].dot1x_open && 1498 skb->protocol != cpu_to_be16(ETH_P_PAE)) 1499 continue; 1500 1501 wil_dbg_txrx(wil, "Tx -> ring %d\n", i); 1502 1503 return ring; 1504 } 1505 1506 wil_dbg_txrx(wil, "Tx while no rings active?\n"); 1507 1508 return NULL; 1509 } 1510 1511 /* Use one of 2 strategies: 1512 * 1513 * 1. New (real broadcast): 1514 * use dedicated broadcast vring 1515 * 2. Old (pseudo-DMS): 1516 * Find 1-st vring and return it; 1517 * duplicate skb and send it to other active vrings; 1518 * in all cases override dest address to unicast peer's address 1519 * Use old strategy when new is not supported yet: 1520 * - for PBSS 1521 */ 1522 static struct wil_ring *wil_find_tx_bcast_1(struct wil6210_priv *wil, 1523 struct wil6210_vif *vif, 1524 struct sk_buff *skb) 1525 { 1526 struct wil_ring *v; 1527 struct wil_ring_tx_data *txdata; 1528 int i = vif->bcast_ring; 1529 1530 if (i < 0) 1531 return NULL; 1532 v = &wil->ring_tx[i]; 1533 txdata = &wil->ring_tx_data[i]; 1534 if (!v->va || !txdata->enabled) 1535 return NULL; 1536 if (!wil->ring_tx_data[i].dot1x_open && 1537 skb->protocol != cpu_to_be16(ETH_P_PAE)) 1538 return NULL; 1539 1540 return v; 1541 } 1542 1543 static void wil_set_da_for_vring(struct wil6210_priv *wil, 1544 struct sk_buff *skb, int vring_index) 1545 { 1546 u8 *da = wil_skb_get_da(skb); 1547 int cid = wil->ring2cid_tid[vring_index][0]; 1548 1549 ether_addr_copy(da, wil->sta[cid].addr); 1550 } 1551 1552 static struct wil_ring *wil_find_tx_bcast_2(struct wil6210_priv *wil, 1553 struct wil6210_vif *vif, 1554 struct sk_buff *skb) 1555 { 1556 struct wil_ring *v, *v2; 1557 struct sk_buff *skb2; 1558 int i; 1559 u8 cid; 1560 const u8 *src = wil_skb_get_sa(skb); 1561 struct wil_ring_tx_data *txdata, *txdata2; 1562 int min_ring_id = wil_get_min_tx_ring_id(wil); 1563 1564 /* find 1-st vring eligible for data */ 1565 for (i = min_ring_id; i < WIL6210_MAX_TX_RINGS; i++) { 1566 v = &wil->ring_tx[i]; 1567 txdata = &wil->ring_tx_data[i]; 1568 if (!v->va || !txdata->enabled || txdata->mid != vif->mid) 1569 continue; 1570 1571 cid = wil->ring2cid_tid[i][0]; 1572 if (cid >= wil->max_assoc_sta) /* skip BCAST */ 1573 continue; 1574 if (!wil->ring_tx_data[i].dot1x_open && 1575 skb->protocol != cpu_to_be16(ETH_P_PAE)) 1576 continue; 1577 1578 /* don't Tx back to source when re-routing Rx->Tx at the AP */ 1579 if (0 == memcmp(wil->sta[cid].addr, src, ETH_ALEN)) 1580 continue; 1581 1582 goto found; 1583 } 1584 1585 wil_dbg_txrx(wil, "Tx while no vrings active?\n"); 1586 1587 return NULL; 1588 1589 found: 1590 wil_dbg_txrx(wil, "BCAST -> ring %d\n", i); 1591 wil_set_da_for_vring(wil, skb, i); 1592 1593 /* find other active vrings and duplicate skb for each */ 1594 for (i++; i < WIL6210_MAX_TX_RINGS; i++) { 1595 v2 = &wil->ring_tx[i]; 1596 txdata2 = &wil->ring_tx_data[i]; 1597 if (!v2->va || txdata2->mid != vif->mid) 1598 continue; 1599 cid = wil->ring2cid_tid[i][0]; 1600 if (cid >= wil->max_assoc_sta) /* skip BCAST */ 1601 continue; 1602 if (!wil->ring_tx_data[i].dot1x_open && 1603 skb->protocol != cpu_to_be16(ETH_P_PAE)) 1604 continue; 1605 1606 if (0 == memcmp(wil->sta[cid].addr, src, ETH_ALEN)) 1607 continue; 1608 1609 skb2 = skb_copy(skb, GFP_ATOMIC); 1610 if (skb2) { 1611 wil_dbg_txrx(wil, "BCAST DUP -> ring %d\n", i); 1612 wil_set_da_for_vring(wil, skb2, i); 1613 wil_tx_ring(wil, vif, v2, skb2); 1614 /* successful call to wil_tx_ring takes skb2 ref */ 1615 dev_kfree_skb_any(skb2); 1616 } else { 1617 wil_err(wil, "skb_copy failed\n"); 1618 } 1619 } 1620 1621 return v; 1622 } 1623 1624 static inline 1625 void wil_tx_desc_set_nr_frags(struct vring_tx_desc *d, int nr_frags) 1626 { 1627 d->mac.d[2] |= (nr_frags << MAC_CFG_DESC_TX_2_NUM_OF_DESCRIPTORS_POS); 1628 } 1629 1630 /** 1631 * Sets the descriptor @d up for csum and/or TSO offloading. The corresponding 1632 * @skb is used to obtain the protocol and headers length. 1633 * @tso_desc_type is a descriptor type for TSO: 0 - a header, 1 - first data, 1634 * 2 - middle, 3 - last descriptor. 1635 */ 1636 1637 static void wil_tx_desc_offload_setup_tso(struct vring_tx_desc *d, 1638 struct sk_buff *skb, 1639 int tso_desc_type, bool is_ipv4, 1640 int tcp_hdr_len, int skb_net_hdr_len) 1641 { 1642 d->dma.b11 = ETH_HLEN; /* MAC header length */ 1643 d->dma.b11 |= is_ipv4 << DMA_CFG_DESC_TX_OFFLOAD_CFG_L3T_IPV4_POS; 1644 1645 d->dma.d0 |= (2 << DMA_CFG_DESC_TX_0_L4_TYPE_POS); 1646 /* L4 header len: TCP header length */ 1647 d->dma.d0 |= (tcp_hdr_len & DMA_CFG_DESC_TX_0_L4_LENGTH_MSK); 1648 1649 /* Setup TSO: bit and desc type */ 1650 d->dma.d0 |= (BIT(DMA_CFG_DESC_TX_0_TCP_SEG_EN_POS)) | 1651 (tso_desc_type << DMA_CFG_DESC_TX_0_SEGMENT_BUF_DETAILS_POS); 1652 d->dma.d0 |= (is_ipv4 << DMA_CFG_DESC_TX_0_IPV4_CHECKSUM_EN_POS); 1653 1654 d->dma.ip_length = skb_net_hdr_len; 1655 /* Enable TCP/UDP checksum */ 1656 d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_TCP_UDP_CHECKSUM_EN_POS); 1657 /* Calculate pseudo-header */ 1658 d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_PSEUDO_HEADER_CALC_EN_POS); 1659 } 1660 1661 /** 1662 * Sets the descriptor @d up for csum. The corresponding 1663 * @skb is used to obtain the protocol and headers length. 1664 * Returns the protocol: 0 - not TCP, 1 - TCPv4, 2 - TCPv6. 1665 * Note, if d==NULL, the function only returns the protocol result. 1666 * 1667 * It is very similar to previous wil_tx_desc_offload_setup_tso. This 1668 * is "if unrolling" to optimize the critical path. 1669 */ 1670 1671 static int wil_tx_desc_offload_setup(struct vring_tx_desc *d, 1672 struct sk_buff *skb){ 1673 int protocol; 1674 1675 if (skb->ip_summed != CHECKSUM_PARTIAL) 1676 return 0; 1677 1678 d->dma.b11 = ETH_HLEN; /* MAC header length */ 1679 1680 switch (skb->protocol) { 1681 case cpu_to_be16(ETH_P_IP): 1682 protocol = ip_hdr(skb)->protocol; 1683 d->dma.b11 |= BIT(DMA_CFG_DESC_TX_OFFLOAD_CFG_L3T_IPV4_POS); 1684 break; 1685 case cpu_to_be16(ETH_P_IPV6): 1686 protocol = ipv6_hdr(skb)->nexthdr; 1687 break; 1688 default: 1689 return -EINVAL; 1690 } 1691 1692 switch (protocol) { 1693 case IPPROTO_TCP: 1694 d->dma.d0 |= (2 << DMA_CFG_DESC_TX_0_L4_TYPE_POS); 1695 /* L4 header len: TCP header length */ 1696 d->dma.d0 |= 1697 (tcp_hdrlen(skb) & DMA_CFG_DESC_TX_0_L4_LENGTH_MSK); 1698 break; 1699 case IPPROTO_UDP: 1700 /* L4 header len: UDP header length */ 1701 d->dma.d0 |= 1702 (sizeof(struct udphdr) & DMA_CFG_DESC_TX_0_L4_LENGTH_MSK); 1703 break; 1704 default: 1705 return -EINVAL; 1706 } 1707 1708 d->dma.ip_length = skb_network_header_len(skb); 1709 /* Enable TCP/UDP checksum */ 1710 d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_TCP_UDP_CHECKSUM_EN_POS); 1711 /* Calculate pseudo-header */ 1712 d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_PSEUDO_HEADER_CALC_EN_POS); 1713 1714 return 0; 1715 } 1716 1717 static inline void wil_tx_last_desc(struct vring_tx_desc *d) 1718 { 1719 d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_CMD_EOP_POS) | 1720 BIT(DMA_CFG_DESC_TX_0_CMD_MARK_WB_POS) | 1721 BIT(DMA_CFG_DESC_TX_0_CMD_DMA_IT_POS); 1722 } 1723 1724 static inline void wil_set_tx_desc_last_tso(volatile struct vring_tx_desc *d) 1725 { 1726 d->dma.d0 |= wil_tso_type_lst << 1727 DMA_CFG_DESC_TX_0_SEGMENT_BUF_DETAILS_POS; 1728 } 1729 1730 static int __wil_tx_vring_tso(struct wil6210_priv *wil, struct wil6210_vif *vif, 1731 struct wil_ring *vring, struct sk_buff *skb) 1732 { 1733 struct device *dev = wil_to_dev(wil); 1734 1735 /* point to descriptors in shared memory */ 1736 volatile struct vring_tx_desc *_desc = NULL, *_hdr_desc, 1737 *_first_desc = NULL; 1738 1739 /* pointers to shadow descriptors */ 1740 struct vring_tx_desc desc_mem, hdr_desc_mem, first_desc_mem, 1741 *d = &hdr_desc_mem, *hdr_desc = &hdr_desc_mem, 1742 *first_desc = &first_desc_mem; 1743 1744 /* pointer to shadow descriptors' context */ 1745 struct wil_ctx *hdr_ctx, *first_ctx = NULL; 1746 1747 int descs_used = 0; /* total number of used descriptors */ 1748 int sg_desc_cnt = 0; /* number of descriptors for current mss*/ 1749 1750 u32 swhead = vring->swhead; 1751 int used, avail = wil_ring_avail_tx(vring); 1752 int nr_frags = skb_shinfo(skb)->nr_frags; 1753 int min_desc_required = nr_frags + 1; 1754 int mss = skb_shinfo(skb)->gso_size; /* payload size w/o headers */ 1755 int f, len, hdrlen, headlen; 1756 int vring_index = vring - wil->ring_tx; 1757 struct wil_ring_tx_data *txdata = &wil->ring_tx_data[vring_index]; 1758 uint i = swhead; 1759 dma_addr_t pa; 1760 const skb_frag_t *frag = NULL; 1761 int rem_data = mss; 1762 int lenmss; 1763 int hdr_compensation_need = true; 1764 int desc_tso_type = wil_tso_type_first; 1765 bool is_ipv4; 1766 int tcp_hdr_len; 1767 int skb_net_hdr_len; 1768 int gso_type; 1769 int rc = -EINVAL; 1770 1771 wil_dbg_txrx(wil, "tx_vring_tso: %d bytes to vring %d\n", skb->len, 1772 vring_index); 1773 1774 if (unlikely(!txdata->enabled)) 1775 return -EINVAL; 1776 1777 /* A typical page 4K is 3-4 payloads, we assume each fragment 1778 * is a full payload, that's how min_desc_required has been 1779 * calculated. In real we might need more or less descriptors, 1780 * this is the initial check only. 1781 */ 1782 if (unlikely(avail < min_desc_required)) { 1783 wil_err_ratelimited(wil, 1784 "TSO: Tx ring[%2d] full. No space for %d fragments\n", 1785 vring_index, min_desc_required); 1786 return -ENOMEM; 1787 } 1788 1789 /* Header Length = MAC header len + IP header len + TCP header len*/ 1790 hdrlen = ETH_HLEN + 1791 (int)skb_network_header_len(skb) + 1792 tcp_hdrlen(skb); 1793 1794 gso_type = skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV6 | SKB_GSO_TCPV4); 1795 switch (gso_type) { 1796 case SKB_GSO_TCPV4: 1797 /* TCP v4, zero out the IP length and IPv4 checksum fields 1798 * as required by the offloading doc 1799 */ 1800 ip_hdr(skb)->tot_len = 0; 1801 ip_hdr(skb)->check = 0; 1802 is_ipv4 = true; 1803 break; 1804 case SKB_GSO_TCPV6: 1805 /* TCP v6, zero out the payload length */ 1806 ipv6_hdr(skb)->payload_len = 0; 1807 is_ipv4 = false; 1808 break; 1809 default: 1810 /* other than TCPv4 or TCPv6 types are not supported for TSO. 1811 * It is also illegal for both to be set simultaneously 1812 */ 1813 return -EINVAL; 1814 } 1815 1816 if (skb->ip_summed != CHECKSUM_PARTIAL) 1817 return -EINVAL; 1818 1819 /* tcp header length and skb network header length are fixed for all 1820 * packet's descriptors - read then once here 1821 */ 1822 tcp_hdr_len = tcp_hdrlen(skb); 1823 skb_net_hdr_len = skb_network_header_len(skb); 1824 1825 _hdr_desc = &vring->va[i].tx.legacy; 1826 1827 pa = dma_map_single(dev, skb->data, hdrlen, DMA_TO_DEVICE); 1828 if (unlikely(dma_mapping_error(dev, pa))) { 1829 wil_err(wil, "TSO: Skb head DMA map error\n"); 1830 goto err_exit; 1831 } 1832 1833 wil->txrx_ops.tx_desc_map((union wil_tx_desc *)hdr_desc, pa, 1834 hdrlen, vring_index); 1835 wil_tx_desc_offload_setup_tso(hdr_desc, skb, wil_tso_type_hdr, is_ipv4, 1836 tcp_hdr_len, skb_net_hdr_len); 1837 wil_tx_last_desc(hdr_desc); 1838 1839 vring->ctx[i].mapped_as = wil_mapped_as_single; 1840 hdr_ctx = &vring->ctx[i]; 1841 1842 descs_used++; 1843 headlen = skb_headlen(skb) - hdrlen; 1844 1845 for (f = headlen ? -1 : 0; f < nr_frags; f++) { 1846 if (headlen) { 1847 len = headlen; 1848 wil_dbg_txrx(wil, "TSO: process skb head, len %u\n", 1849 len); 1850 } else { 1851 frag = &skb_shinfo(skb)->frags[f]; 1852 len = skb_frag_size(frag); 1853 wil_dbg_txrx(wil, "TSO: frag[%d]: len %u\n", f, len); 1854 } 1855 1856 while (len) { 1857 wil_dbg_txrx(wil, 1858 "TSO: len %d, rem_data %d, descs_used %d\n", 1859 len, rem_data, descs_used); 1860 1861 if (descs_used == avail) { 1862 wil_err_ratelimited(wil, "TSO: ring overflow\n"); 1863 rc = -ENOMEM; 1864 goto mem_error; 1865 } 1866 1867 lenmss = min_t(int, rem_data, len); 1868 i = (swhead + descs_used) % vring->size; 1869 wil_dbg_txrx(wil, "TSO: lenmss %d, i %d\n", lenmss, i); 1870 1871 if (!headlen) { 1872 pa = skb_frag_dma_map(dev, frag, 1873 skb_frag_size(frag) - len, 1874 lenmss, DMA_TO_DEVICE); 1875 vring->ctx[i].mapped_as = wil_mapped_as_page; 1876 } else { 1877 pa = dma_map_single(dev, 1878 skb->data + 1879 skb_headlen(skb) - headlen, 1880 lenmss, 1881 DMA_TO_DEVICE); 1882 vring->ctx[i].mapped_as = wil_mapped_as_single; 1883 headlen -= lenmss; 1884 } 1885 1886 if (unlikely(dma_mapping_error(dev, pa))) { 1887 wil_err(wil, "TSO: DMA map page error\n"); 1888 goto mem_error; 1889 } 1890 1891 _desc = &vring->va[i].tx.legacy; 1892 1893 if (!_first_desc) { 1894 _first_desc = _desc; 1895 first_ctx = &vring->ctx[i]; 1896 d = first_desc; 1897 } else { 1898 d = &desc_mem; 1899 } 1900 1901 wil->txrx_ops.tx_desc_map((union wil_tx_desc *)d, 1902 pa, lenmss, vring_index); 1903 wil_tx_desc_offload_setup_tso(d, skb, desc_tso_type, 1904 is_ipv4, tcp_hdr_len, 1905 skb_net_hdr_len); 1906 1907 /* use tso_type_first only once */ 1908 desc_tso_type = wil_tso_type_mid; 1909 1910 descs_used++; /* desc used so far */ 1911 sg_desc_cnt++; /* desc used for this segment */ 1912 len -= lenmss; 1913 rem_data -= lenmss; 1914 1915 wil_dbg_txrx(wil, 1916 "TSO: len %d, rem_data %d, descs_used %d, sg_desc_cnt %d,\n", 1917 len, rem_data, descs_used, sg_desc_cnt); 1918 1919 /* Close the segment if reached mss size or last frag*/ 1920 if (rem_data == 0 || (f == nr_frags - 1 && len == 0)) { 1921 if (hdr_compensation_need) { 1922 /* first segment include hdr desc for 1923 * release 1924 */ 1925 hdr_ctx->nr_frags = sg_desc_cnt; 1926 wil_tx_desc_set_nr_frags(first_desc, 1927 sg_desc_cnt + 1928 1); 1929 hdr_compensation_need = false; 1930 } else { 1931 wil_tx_desc_set_nr_frags(first_desc, 1932 sg_desc_cnt); 1933 } 1934 first_ctx->nr_frags = sg_desc_cnt - 1; 1935 1936 wil_tx_last_desc(d); 1937 1938 /* first descriptor may also be the last 1939 * for this mss - make sure not to copy 1940 * it twice 1941 */ 1942 if (first_desc != d) 1943 *_first_desc = *first_desc; 1944 1945 /*last descriptor will be copied at the end 1946 * of this TS processing 1947 */ 1948 if (f < nr_frags - 1 || len > 0) 1949 *_desc = *d; 1950 1951 rem_data = mss; 1952 _first_desc = NULL; 1953 sg_desc_cnt = 0; 1954 } else if (first_desc != d) /* update mid descriptor */ 1955 *_desc = *d; 1956 } 1957 } 1958 1959 if (!_desc) 1960 goto mem_error; 1961 1962 /* first descriptor may also be the last. 1963 * in this case d pointer is invalid 1964 */ 1965 if (_first_desc == _desc) 1966 d = first_desc; 1967 1968 /* Last data descriptor */ 1969 wil_set_tx_desc_last_tso(d); 1970 *_desc = *d; 1971 1972 /* Fill the total number of descriptors in first desc (hdr)*/ 1973 wil_tx_desc_set_nr_frags(hdr_desc, descs_used); 1974 *_hdr_desc = *hdr_desc; 1975 1976 /* hold reference to skb 1977 * to prevent skb release before accounting 1978 * in case of immediate "tx done" 1979 */ 1980 vring->ctx[i].skb = skb_get(skb); 1981 1982 /* performance monitoring */ 1983 used = wil_ring_used_tx(vring); 1984 if (wil_val_in_range(wil->ring_idle_trsh, 1985 used, used + descs_used)) { 1986 txdata->idle += get_cycles() - txdata->last_idle; 1987 wil_dbg_txrx(wil, "Ring[%2d] not idle %d -> %d\n", 1988 vring_index, used, used + descs_used); 1989 } 1990 1991 /* Make sure to advance the head only after descriptor update is done. 1992 * This will prevent a race condition where the completion thread 1993 * will see the DU bit set from previous run and will handle the 1994 * skb before it was completed. 1995 */ 1996 wmb(); 1997 1998 /* advance swhead */ 1999 wil_ring_advance_head(vring, descs_used); 2000 wil_dbg_txrx(wil, "TSO: Tx swhead %d -> %d\n", swhead, vring->swhead); 2001 2002 /* make sure all writes to descriptors (shared memory) are done before 2003 * committing them to HW 2004 */ 2005 wmb(); 2006 2007 if (wil->tx_latency) 2008 *(ktime_t *)&skb->cb = ktime_get(); 2009 else 2010 memset(skb->cb, 0, sizeof(ktime_t)); 2011 2012 wil_w(wil, vring->hwtail, vring->swhead); 2013 return 0; 2014 2015 mem_error: 2016 while (descs_used > 0) { 2017 struct wil_ctx *ctx; 2018 2019 i = (swhead + descs_used - 1) % vring->size; 2020 d = (struct vring_tx_desc *)&vring->va[i].tx.legacy; 2021 _desc = &vring->va[i].tx.legacy; 2022 *d = *_desc; 2023 _desc->dma.status = TX_DMA_STATUS_DU; 2024 ctx = &vring->ctx[i]; 2025 wil_txdesc_unmap(dev, (union wil_tx_desc *)d, ctx); 2026 memset(ctx, 0, sizeof(*ctx)); 2027 descs_used--; 2028 } 2029 err_exit: 2030 return rc; 2031 } 2032 2033 static int __wil_tx_ring(struct wil6210_priv *wil, struct wil6210_vif *vif, 2034 struct wil_ring *ring, struct sk_buff *skb) 2035 { 2036 struct device *dev = wil_to_dev(wil); 2037 struct vring_tx_desc dd, *d = ⅆ 2038 volatile struct vring_tx_desc *_d; 2039 u32 swhead = ring->swhead; 2040 int avail = wil_ring_avail_tx(ring); 2041 int nr_frags = skb_shinfo(skb)->nr_frags; 2042 uint f = 0; 2043 int ring_index = ring - wil->ring_tx; 2044 struct wil_ring_tx_data *txdata = &wil->ring_tx_data[ring_index]; 2045 uint i = swhead; 2046 dma_addr_t pa; 2047 int used; 2048 bool mcast = (ring_index == vif->bcast_ring); 2049 uint len = skb_headlen(skb); 2050 2051 wil_dbg_txrx(wil, "tx_ring: %d bytes to ring %d, nr_frags %d\n", 2052 skb->len, ring_index, nr_frags); 2053 2054 if (unlikely(!txdata->enabled)) 2055 return -EINVAL; 2056 2057 if (unlikely(avail < 1 + nr_frags)) { 2058 wil_err_ratelimited(wil, 2059 "Tx ring[%2d] full. No space for %d fragments\n", 2060 ring_index, 1 + nr_frags); 2061 return -ENOMEM; 2062 } 2063 _d = &ring->va[i].tx.legacy; 2064 2065 pa = dma_map_single(dev, skb->data, skb_headlen(skb), DMA_TO_DEVICE); 2066 2067 wil_dbg_txrx(wil, "Tx[%2d] skb %d bytes 0x%p -> %pad\n", ring_index, 2068 skb_headlen(skb), skb->data, &pa); 2069 wil_hex_dump_txrx("Tx ", DUMP_PREFIX_OFFSET, 16, 1, 2070 skb->data, skb_headlen(skb), false); 2071 2072 if (unlikely(dma_mapping_error(dev, pa))) 2073 return -EINVAL; 2074 ring->ctx[i].mapped_as = wil_mapped_as_single; 2075 /* 1-st segment */ 2076 wil->txrx_ops.tx_desc_map((union wil_tx_desc *)d, pa, len, 2077 ring_index); 2078 if (unlikely(mcast)) { 2079 d->mac.d[0] |= BIT(MAC_CFG_DESC_TX_0_MCS_EN_POS); /* MCS 0 */ 2080 if (unlikely(len > WIL_BCAST_MCS0_LIMIT)) /* set MCS 1 */ 2081 d->mac.d[0] |= (1 << MAC_CFG_DESC_TX_0_MCS_INDEX_POS); 2082 } 2083 /* Process TCP/UDP checksum offloading */ 2084 if (unlikely(wil_tx_desc_offload_setup(d, skb))) { 2085 wil_err(wil, "Tx[%2d] Failed to set cksum, drop packet\n", 2086 ring_index); 2087 goto dma_error; 2088 } 2089 2090 ring->ctx[i].nr_frags = nr_frags; 2091 wil_tx_desc_set_nr_frags(d, nr_frags + 1); 2092 2093 /* middle segments */ 2094 for (; f < nr_frags; f++) { 2095 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 2096 int len = skb_frag_size(frag); 2097 2098 *_d = *d; 2099 wil_dbg_txrx(wil, "Tx[%2d] desc[%4d]\n", ring_index, i); 2100 wil_hex_dump_txrx("TxD ", DUMP_PREFIX_NONE, 32, 4, 2101 (const void *)d, sizeof(*d), false); 2102 i = (swhead + f + 1) % ring->size; 2103 _d = &ring->va[i].tx.legacy; 2104 pa = skb_frag_dma_map(dev, frag, 0, skb_frag_size(frag), 2105 DMA_TO_DEVICE); 2106 if (unlikely(dma_mapping_error(dev, pa))) { 2107 wil_err(wil, "Tx[%2d] failed to map fragment\n", 2108 ring_index); 2109 goto dma_error; 2110 } 2111 ring->ctx[i].mapped_as = wil_mapped_as_page; 2112 wil->txrx_ops.tx_desc_map((union wil_tx_desc *)d, 2113 pa, len, ring_index); 2114 /* no need to check return code - 2115 * if it succeeded for 1-st descriptor, 2116 * it will succeed here too 2117 */ 2118 wil_tx_desc_offload_setup(d, skb); 2119 } 2120 /* for the last seg only */ 2121 d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_CMD_EOP_POS); 2122 d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_CMD_MARK_WB_POS); 2123 d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_CMD_DMA_IT_POS); 2124 *_d = *d; 2125 wil_dbg_txrx(wil, "Tx[%2d] desc[%4d]\n", ring_index, i); 2126 wil_hex_dump_txrx("TxD ", DUMP_PREFIX_NONE, 32, 4, 2127 (const void *)d, sizeof(*d), false); 2128 2129 /* hold reference to skb 2130 * to prevent skb release before accounting 2131 * in case of immediate "tx done" 2132 */ 2133 ring->ctx[i].skb = skb_get(skb); 2134 2135 /* performance monitoring */ 2136 used = wil_ring_used_tx(ring); 2137 if (wil_val_in_range(wil->ring_idle_trsh, 2138 used, used + nr_frags + 1)) { 2139 txdata->idle += get_cycles() - txdata->last_idle; 2140 wil_dbg_txrx(wil, "Ring[%2d] not idle %d -> %d\n", 2141 ring_index, used, used + nr_frags + 1); 2142 } 2143 2144 /* Make sure to advance the head only after descriptor update is done. 2145 * This will prevent a race condition where the completion thread 2146 * will see the DU bit set from previous run and will handle the 2147 * skb before it was completed. 2148 */ 2149 wmb(); 2150 2151 /* advance swhead */ 2152 wil_ring_advance_head(ring, nr_frags + 1); 2153 wil_dbg_txrx(wil, "Tx[%2d] swhead %d -> %d\n", ring_index, swhead, 2154 ring->swhead); 2155 trace_wil6210_tx(ring_index, swhead, skb->len, nr_frags); 2156 2157 /* make sure all writes to descriptors (shared memory) are done before 2158 * committing them to HW 2159 */ 2160 wmb(); 2161 2162 if (wil->tx_latency) 2163 *(ktime_t *)&skb->cb = ktime_get(); 2164 else 2165 memset(skb->cb, 0, sizeof(ktime_t)); 2166 2167 wil_w(wil, ring->hwtail, ring->swhead); 2168 2169 return 0; 2170 dma_error: 2171 /* unmap what we have mapped */ 2172 nr_frags = f + 1; /* frags mapped + one for skb head */ 2173 for (f = 0; f < nr_frags; f++) { 2174 struct wil_ctx *ctx; 2175 2176 i = (swhead + f) % ring->size; 2177 ctx = &ring->ctx[i]; 2178 _d = &ring->va[i].tx.legacy; 2179 *d = *_d; 2180 _d->dma.status = TX_DMA_STATUS_DU; 2181 wil->txrx_ops.tx_desc_unmap(dev, 2182 (union wil_tx_desc *)d, 2183 ctx); 2184 2185 memset(ctx, 0, sizeof(*ctx)); 2186 } 2187 2188 return -EINVAL; 2189 } 2190 2191 static int wil_tx_ring(struct wil6210_priv *wil, struct wil6210_vif *vif, 2192 struct wil_ring *ring, struct sk_buff *skb) 2193 { 2194 int ring_index = ring - wil->ring_tx; 2195 struct wil_ring_tx_data *txdata = &wil->ring_tx_data[ring_index]; 2196 int rc; 2197 2198 spin_lock(&txdata->lock); 2199 2200 if (test_bit(wil_status_suspending, wil->status) || 2201 test_bit(wil_status_suspended, wil->status) || 2202 test_bit(wil_status_resuming, wil->status)) { 2203 wil_dbg_txrx(wil, 2204 "suspend/resume in progress. drop packet\n"); 2205 spin_unlock(&txdata->lock); 2206 return -EINVAL; 2207 } 2208 2209 rc = (skb_is_gso(skb) ? wil->txrx_ops.tx_ring_tso : __wil_tx_ring) 2210 (wil, vif, ring, skb); 2211 2212 spin_unlock(&txdata->lock); 2213 2214 return rc; 2215 } 2216 2217 /** 2218 * Check status of tx vrings and stop/wake net queues if needed 2219 * It will start/stop net queues of a specific VIF net_device. 2220 * 2221 * This function does one of two checks: 2222 * In case check_stop is true, will check if net queues need to be stopped. If 2223 * the conditions for stopping are met, netif_tx_stop_all_queues() is called. 2224 * In case check_stop is false, will check if net queues need to be waked. If 2225 * the conditions for waking are met, netif_tx_wake_all_queues() is called. 2226 * vring is the vring which is currently being modified by either adding 2227 * descriptors (tx) into it or removing descriptors (tx complete) from it. Can 2228 * be null when irrelevant (e.g. connect/disconnect events). 2229 * 2230 * The implementation is to stop net queues if modified vring has low 2231 * descriptor availability. Wake if all vrings are not in low descriptor 2232 * availability and modified vring has high descriptor availability. 2233 */ 2234 static inline void __wil_update_net_queues(struct wil6210_priv *wil, 2235 struct wil6210_vif *vif, 2236 struct wil_ring *ring, 2237 bool check_stop) 2238 { 2239 int i; 2240 int min_ring_id = wil_get_min_tx_ring_id(wil); 2241 2242 if (unlikely(!vif)) 2243 return; 2244 2245 if (ring) 2246 wil_dbg_txrx(wil, "vring %d, mid %d, check_stop=%d, stopped=%d", 2247 (int)(ring - wil->ring_tx), vif->mid, check_stop, 2248 vif->net_queue_stopped); 2249 else 2250 wil_dbg_txrx(wil, "check_stop=%d, mid=%d, stopped=%d", 2251 check_stop, vif->mid, vif->net_queue_stopped); 2252 2253 if (ring && drop_if_ring_full) 2254 /* no need to stop/wake net queues */ 2255 return; 2256 2257 if (check_stop == vif->net_queue_stopped) 2258 /* net queues already in desired state */ 2259 return; 2260 2261 if (check_stop) { 2262 if (!ring || unlikely(wil_ring_avail_low(ring))) { 2263 /* not enough room in the vring */ 2264 netif_tx_stop_all_queues(vif_to_ndev(vif)); 2265 vif->net_queue_stopped = true; 2266 wil_dbg_txrx(wil, "netif_tx_stop called\n"); 2267 } 2268 return; 2269 } 2270 2271 /* Do not wake the queues in suspend flow */ 2272 if (test_bit(wil_status_suspending, wil->status) || 2273 test_bit(wil_status_suspended, wil->status)) 2274 return; 2275 2276 /* check wake */ 2277 for (i = min_ring_id; i < WIL6210_MAX_TX_RINGS; i++) { 2278 struct wil_ring *cur_ring = &wil->ring_tx[i]; 2279 struct wil_ring_tx_data *txdata = &wil->ring_tx_data[i]; 2280 2281 if (txdata->mid != vif->mid || !cur_ring->va || 2282 !txdata->enabled || cur_ring == ring) 2283 continue; 2284 2285 if (wil_ring_avail_low(cur_ring)) { 2286 wil_dbg_txrx(wil, "ring %d full, can't wake\n", 2287 (int)(cur_ring - wil->ring_tx)); 2288 return; 2289 } 2290 } 2291 2292 if (!ring || wil_ring_avail_high(ring)) { 2293 /* enough room in the ring */ 2294 wil_dbg_txrx(wil, "calling netif_tx_wake\n"); 2295 netif_tx_wake_all_queues(vif_to_ndev(vif)); 2296 vif->net_queue_stopped = false; 2297 } 2298 } 2299 2300 void wil_update_net_queues(struct wil6210_priv *wil, struct wil6210_vif *vif, 2301 struct wil_ring *ring, bool check_stop) 2302 { 2303 spin_lock(&wil->net_queue_lock); 2304 __wil_update_net_queues(wil, vif, ring, check_stop); 2305 spin_unlock(&wil->net_queue_lock); 2306 } 2307 2308 void wil_update_net_queues_bh(struct wil6210_priv *wil, struct wil6210_vif *vif, 2309 struct wil_ring *ring, bool check_stop) 2310 { 2311 spin_lock_bh(&wil->net_queue_lock); 2312 __wil_update_net_queues(wil, vif, ring, check_stop); 2313 spin_unlock_bh(&wil->net_queue_lock); 2314 } 2315 2316 netdev_tx_t wil_start_xmit(struct sk_buff *skb, struct net_device *ndev) 2317 { 2318 struct wil6210_vif *vif = ndev_to_vif(ndev); 2319 struct wil6210_priv *wil = vif_to_wil(vif); 2320 const u8 *da = wil_skb_get_da(skb); 2321 bool bcast = is_multicast_ether_addr(da); 2322 struct wil_ring *ring; 2323 static bool pr_once_fw; 2324 int rc; 2325 2326 wil_dbg_txrx(wil, "start_xmit\n"); 2327 if (unlikely(!test_bit(wil_status_fwready, wil->status))) { 2328 if (!pr_once_fw) { 2329 wil_err(wil, "FW not ready\n"); 2330 pr_once_fw = true; 2331 } 2332 goto drop; 2333 } 2334 if (unlikely(!test_bit(wil_vif_fwconnected, vif->status))) { 2335 wil_dbg_ratelimited(wil, 2336 "VIF not connected, packet dropped\n"); 2337 goto drop; 2338 } 2339 if (unlikely(vif->wdev.iftype == NL80211_IFTYPE_MONITOR)) { 2340 wil_err(wil, "Xmit in monitor mode not supported\n"); 2341 goto drop; 2342 } 2343 pr_once_fw = false; 2344 2345 /* find vring */ 2346 if (vif->wdev.iftype == NL80211_IFTYPE_STATION && !vif->pbss) { 2347 /* in STA mode (ESS), all to same VRING (to AP) */ 2348 ring = wil_find_tx_ring_sta(wil, vif, skb); 2349 } else if (bcast) { 2350 if (vif->pbss) 2351 /* in pbss, no bcast VRING - duplicate skb in 2352 * all stations VRINGs 2353 */ 2354 ring = wil_find_tx_bcast_2(wil, vif, skb); 2355 else if (vif->wdev.iftype == NL80211_IFTYPE_AP) 2356 /* AP has a dedicated bcast VRING */ 2357 ring = wil_find_tx_bcast_1(wil, vif, skb); 2358 else 2359 /* unexpected combination, fallback to duplicating 2360 * the skb in all stations VRINGs 2361 */ 2362 ring = wil_find_tx_bcast_2(wil, vif, skb); 2363 } else { 2364 /* unicast, find specific VRING by dest. address */ 2365 ring = wil_find_tx_ucast(wil, vif, skb); 2366 } 2367 if (unlikely(!ring)) { 2368 wil_dbg_txrx(wil, "No Tx RING found for %pM\n", da); 2369 goto drop; 2370 } 2371 /* set up vring entry */ 2372 rc = wil_tx_ring(wil, vif, ring, skb); 2373 2374 switch (rc) { 2375 case 0: 2376 /* shall we stop net queues? */ 2377 wil_update_net_queues_bh(wil, vif, ring, true); 2378 /* statistics will be updated on the tx_complete */ 2379 dev_kfree_skb_any(skb); 2380 return NETDEV_TX_OK; 2381 case -ENOMEM: 2382 if (drop_if_ring_full) 2383 goto drop; 2384 return NETDEV_TX_BUSY; 2385 default: 2386 break; /* goto drop; */ 2387 } 2388 drop: 2389 ndev->stats.tx_dropped++; 2390 dev_kfree_skb_any(skb); 2391 2392 return NET_XMIT_DROP; 2393 } 2394 2395 void wil_tx_latency_calc(struct wil6210_priv *wil, struct sk_buff *skb, 2396 struct wil_sta_info *sta) 2397 { 2398 int skb_time_us; 2399 int bin; 2400 2401 if (!wil->tx_latency) 2402 return; 2403 2404 if (ktime_to_ms(*(ktime_t *)&skb->cb) == 0) 2405 return; 2406 2407 skb_time_us = ktime_us_delta(ktime_get(), *(ktime_t *)&skb->cb); 2408 bin = skb_time_us / wil->tx_latency_res; 2409 bin = min_t(int, bin, WIL_NUM_LATENCY_BINS - 1); 2410 2411 wil_dbg_txrx(wil, "skb time %dus => bin %d\n", skb_time_us, bin); 2412 sta->tx_latency_bins[bin]++; 2413 sta->stats.tx_latency_total_us += skb_time_us; 2414 if (skb_time_us < sta->stats.tx_latency_min_us) 2415 sta->stats.tx_latency_min_us = skb_time_us; 2416 if (skb_time_us > sta->stats.tx_latency_max_us) 2417 sta->stats.tx_latency_max_us = skb_time_us; 2418 } 2419 2420 /** 2421 * Clean up transmitted skb's from the Tx VRING 2422 * 2423 * Return number of descriptors cleared 2424 * 2425 * Safe to call from IRQ 2426 */ 2427 int wil_tx_complete(struct wil6210_vif *vif, int ringid) 2428 { 2429 struct wil6210_priv *wil = vif_to_wil(vif); 2430 struct net_device *ndev = vif_to_ndev(vif); 2431 struct device *dev = wil_to_dev(wil); 2432 struct wil_ring *vring = &wil->ring_tx[ringid]; 2433 struct wil_ring_tx_data *txdata = &wil->ring_tx_data[ringid]; 2434 int done = 0; 2435 int cid = wil->ring2cid_tid[ringid][0]; 2436 struct wil_net_stats *stats = NULL; 2437 volatile struct vring_tx_desc *_d; 2438 int used_before_complete; 2439 int used_new; 2440 2441 if (unlikely(!vring->va)) { 2442 wil_err(wil, "Tx irq[%d]: vring not initialized\n", ringid); 2443 return 0; 2444 } 2445 2446 if (unlikely(!txdata->enabled)) { 2447 wil_info(wil, "Tx irq[%d]: vring disabled\n", ringid); 2448 return 0; 2449 } 2450 2451 wil_dbg_txrx(wil, "tx_complete: (%d)\n", ringid); 2452 2453 used_before_complete = wil_ring_used_tx(vring); 2454 2455 if (cid < wil->max_assoc_sta) 2456 stats = &wil->sta[cid].stats; 2457 2458 while (!wil_ring_is_empty(vring)) { 2459 int new_swtail; 2460 struct wil_ctx *ctx = &vring->ctx[vring->swtail]; 2461 /** 2462 * For the fragmented skb, HW will set DU bit only for the 2463 * last fragment. look for it. 2464 * In TSO the first DU will include hdr desc 2465 */ 2466 int lf = (vring->swtail + ctx->nr_frags) % vring->size; 2467 /* TODO: check we are not past head */ 2468 2469 _d = &vring->va[lf].tx.legacy; 2470 if (unlikely(!(_d->dma.status & TX_DMA_STATUS_DU))) 2471 break; 2472 2473 new_swtail = (lf + 1) % vring->size; 2474 while (vring->swtail != new_swtail) { 2475 struct vring_tx_desc dd, *d = ⅆ 2476 u16 dmalen; 2477 struct sk_buff *skb; 2478 2479 ctx = &vring->ctx[vring->swtail]; 2480 skb = ctx->skb; 2481 _d = &vring->va[vring->swtail].tx.legacy; 2482 2483 *d = *_d; 2484 2485 dmalen = le16_to_cpu(d->dma.length); 2486 trace_wil6210_tx_done(ringid, vring->swtail, dmalen, 2487 d->dma.error); 2488 wil_dbg_txrx(wil, 2489 "TxC[%2d][%3d] : %d bytes, status 0x%02x err 0x%02x\n", 2490 ringid, vring->swtail, dmalen, 2491 d->dma.status, d->dma.error); 2492 wil_hex_dump_txrx("TxCD ", DUMP_PREFIX_NONE, 32, 4, 2493 (const void *)d, sizeof(*d), false); 2494 2495 wil->txrx_ops.tx_desc_unmap(dev, 2496 (union wil_tx_desc *)d, 2497 ctx); 2498 2499 if (skb) { 2500 if (likely(d->dma.error == 0)) { 2501 ndev->stats.tx_packets++; 2502 ndev->stats.tx_bytes += skb->len; 2503 if (stats) { 2504 stats->tx_packets++; 2505 stats->tx_bytes += skb->len; 2506 2507 wil_tx_latency_calc(wil, skb, 2508 &wil->sta[cid]); 2509 } 2510 } else { 2511 ndev->stats.tx_errors++; 2512 if (stats) 2513 stats->tx_errors++; 2514 } 2515 2516 if (skb->protocol == cpu_to_be16(ETH_P_PAE)) 2517 wil_tx_complete_handle_eapol(vif, skb); 2518 2519 wil_consume_skb(skb, d->dma.error == 0); 2520 } 2521 memset(ctx, 0, sizeof(*ctx)); 2522 /* Make sure the ctx is zeroed before updating the tail 2523 * to prevent a case where wil_tx_ring will see 2524 * this descriptor as used and handle it before ctx zero 2525 * is completed. 2526 */ 2527 wmb(); 2528 /* There is no need to touch HW descriptor: 2529 * - ststus bit TX_DMA_STATUS_DU is set by design, 2530 * so hardware will not try to process this desc., 2531 * - rest of descriptor will be initialized on Tx. 2532 */ 2533 vring->swtail = wil_ring_next_tail(vring); 2534 done++; 2535 } 2536 } 2537 2538 /* performance monitoring */ 2539 used_new = wil_ring_used_tx(vring); 2540 if (wil_val_in_range(wil->ring_idle_trsh, 2541 used_new, used_before_complete)) { 2542 wil_dbg_txrx(wil, "Ring[%2d] idle %d -> %d\n", 2543 ringid, used_before_complete, used_new); 2544 txdata->last_idle = get_cycles(); 2545 } 2546 2547 /* shall we wake net queues? */ 2548 if (done) 2549 wil_update_net_queues(wil, vif, vring, false); 2550 2551 return done; 2552 } 2553 2554 static inline int wil_tx_init(struct wil6210_priv *wil) 2555 { 2556 return 0; 2557 } 2558 2559 static inline void wil_tx_fini(struct wil6210_priv *wil) {} 2560 2561 static void wil_get_reorder_params(struct wil6210_priv *wil, 2562 struct sk_buff *skb, int *tid, int *cid, 2563 int *mid, u16 *seq, int *mcast, int *retry) 2564 { 2565 struct vring_rx_desc *d = wil_skb_rxdesc(skb); 2566 2567 *tid = wil_rxdesc_tid(d); 2568 *cid = wil_skb_get_cid(skb); 2569 *mid = wil_rxdesc_mid(d); 2570 *seq = wil_rxdesc_seq(d); 2571 *mcast = wil_rxdesc_mcast(d); 2572 *retry = wil_rxdesc_retry(d); 2573 } 2574 2575 void wil_init_txrx_ops_legacy_dma(struct wil6210_priv *wil) 2576 { 2577 wil->txrx_ops.configure_interrupt_moderation = 2578 wil_configure_interrupt_moderation; 2579 /* TX ops */ 2580 wil->txrx_ops.tx_desc_map = wil_tx_desc_map; 2581 wil->txrx_ops.tx_desc_unmap = wil_txdesc_unmap; 2582 wil->txrx_ops.tx_ring_tso = __wil_tx_vring_tso; 2583 wil->txrx_ops.ring_init_tx = wil_vring_init_tx; 2584 wil->txrx_ops.ring_fini_tx = wil_vring_free; 2585 wil->txrx_ops.ring_init_bcast = wil_vring_init_bcast; 2586 wil->txrx_ops.tx_init = wil_tx_init; 2587 wil->txrx_ops.tx_fini = wil_tx_fini; 2588 wil->txrx_ops.tx_ring_modify = wil_tx_vring_modify; 2589 /* RX ops */ 2590 wil->txrx_ops.rx_init = wil_rx_init; 2591 wil->txrx_ops.wmi_addba_rx_resp = wmi_addba_rx_resp; 2592 wil->txrx_ops.get_reorder_params = wil_get_reorder_params; 2593 wil->txrx_ops.get_netif_rx_params = 2594 wil_get_netif_rx_params; 2595 wil->txrx_ops.rx_crypto_check = wil_rx_crypto_check; 2596 wil->txrx_ops.rx_error_check = wil_rx_error_check; 2597 wil->txrx_ops.is_rx_idle = wil_is_rx_idle; 2598 wil->txrx_ops.rx_fini = wil_rx_fini; 2599 } 2600