1 /* 2 * Copyright (c) 2012-2015 Qualcomm Atheros, Inc. 3 * 4 * Permission to use, copy, modify, and/or distribute this software for any 5 * purpose with or without fee is hereby granted, provided that the above 6 * copyright notice and this permission notice appear in all copies. 7 * 8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 15 */ 16 17 #include <linux/etherdevice.h> 18 #include <net/ieee80211_radiotap.h> 19 #include <linux/if_arp.h> 20 #include <linux/moduleparam.h> 21 #include <linux/ip.h> 22 #include <linux/ipv6.h> 23 #include <net/ipv6.h> 24 #include <linux/prefetch.h> 25 26 #include "wil6210.h" 27 #include "wmi.h" 28 #include "txrx.h" 29 #include "trace.h" 30 31 static bool rtap_include_phy_info; 32 module_param(rtap_include_phy_info, bool, S_IRUGO); 33 MODULE_PARM_DESC(rtap_include_phy_info, 34 " Include PHY info in the radiotap header, default - no"); 35 36 bool rx_align_2; 37 module_param(rx_align_2, bool, S_IRUGO); 38 MODULE_PARM_DESC(rx_align_2, " align Rx buffers on 4*n+2, default - no"); 39 40 static inline uint wil_rx_snaplen(void) 41 { 42 return rx_align_2 ? 6 : 0; 43 } 44 45 static inline int wil_vring_is_empty(struct vring *vring) 46 { 47 return vring->swhead == vring->swtail; 48 } 49 50 static inline u32 wil_vring_next_tail(struct vring *vring) 51 { 52 return (vring->swtail + 1) % vring->size; 53 } 54 55 static inline void wil_vring_advance_head(struct vring *vring, int n) 56 { 57 vring->swhead = (vring->swhead + n) % vring->size; 58 } 59 60 static inline int wil_vring_is_full(struct vring *vring) 61 { 62 return wil_vring_next_tail(vring) == vring->swhead; 63 } 64 65 /* Used space in Tx Vring */ 66 static inline int wil_vring_used_tx(struct vring *vring) 67 { 68 u32 swhead = vring->swhead; 69 u32 swtail = vring->swtail; 70 return (vring->size + swhead - swtail) % vring->size; 71 } 72 73 /* Available space in Tx Vring */ 74 static inline int wil_vring_avail_tx(struct vring *vring) 75 { 76 return vring->size - wil_vring_used_tx(vring) - 1; 77 } 78 79 /* wil_vring_wmark_low - low watermark for available descriptor space */ 80 static inline int wil_vring_wmark_low(struct vring *vring) 81 { 82 return vring->size/8; 83 } 84 85 /* wil_vring_wmark_high - high watermark for available descriptor space */ 86 static inline int wil_vring_wmark_high(struct vring *vring) 87 { 88 return vring->size/4; 89 } 90 91 /* wil_val_in_range - check if value in [min,max) */ 92 static inline bool wil_val_in_range(int val, int min, int max) 93 { 94 return val >= min && val < max; 95 } 96 97 static int wil_vring_alloc(struct wil6210_priv *wil, struct vring *vring) 98 { 99 struct device *dev = wil_to_dev(wil); 100 size_t sz = vring->size * sizeof(vring->va[0]); 101 uint i; 102 103 wil_dbg_misc(wil, "%s()\n", __func__); 104 105 BUILD_BUG_ON(sizeof(vring->va[0]) != 32); 106 107 vring->swhead = 0; 108 vring->swtail = 0; 109 vring->ctx = kcalloc(vring->size, sizeof(vring->ctx[0]), GFP_KERNEL); 110 if (!vring->ctx) { 111 vring->va = NULL; 112 return -ENOMEM; 113 } 114 /* vring->va should be aligned on its size rounded up to power of 2 115 * This is granted by the dma_alloc_coherent 116 */ 117 vring->va = dma_alloc_coherent(dev, sz, &vring->pa, GFP_KERNEL); 118 if (!vring->va) { 119 kfree(vring->ctx); 120 vring->ctx = NULL; 121 return -ENOMEM; 122 } 123 /* initially, all descriptors are SW owned 124 * For Tx and Rx, ownership bit is at the same location, thus 125 * we can use any 126 */ 127 for (i = 0; i < vring->size; i++) { 128 volatile struct vring_tx_desc *_d = &vring->va[i].tx; 129 130 _d->dma.status = TX_DMA_STATUS_DU; 131 } 132 133 wil_dbg_misc(wil, "vring[%d] 0x%p:%pad 0x%p\n", vring->size, 134 vring->va, &vring->pa, vring->ctx); 135 136 return 0; 137 } 138 139 static void wil_txdesc_unmap(struct device *dev, struct vring_tx_desc *d, 140 struct wil_ctx *ctx) 141 { 142 dma_addr_t pa = wil_desc_addr(&d->dma.addr); 143 u16 dmalen = le16_to_cpu(d->dma.length); 144 145 switch (ctx->mapped_as) { 146 case wil_mapped_as_single: 147 dma_unmap_single(dev, pa, dmalen, DMA_TO_DEVICE); 148 break; 149 case wil_mapped_as_page: 150 dma_unmap_page(dev, pa, dmalen, DMA_TO_DEVICE); 151 break; 152 default: 153 break; 154 } 155 } 156 157 static void wil_vring_free(struct wil6210_priv *wil, struct vring *vring, 158 int tx) 159 { 160 struct device *dev = wil_to_dev(wil); 161 size_t sz = vring->size * sizeof(vring->va[0]); 162 163 lockdep_assert_held(&wil->mutex); 164 if (tx) { 165 int vring_index = vring - wil->vring_tx; 166 167 wil_dbg_misc(wil, "free Tx vring %d [%d] 0x%p:%pad 0x%p\n", 168 vring_index, vring->size, vring->va, 169 &vring->pa, vring->ctx); 170 } else { 171 wil_dbg_misc(wil, "free Rx vring [%d] 0x%p:%pad 0x%p\n", 172 vring->size, vring->va, 173 &vring->pa, vring->ctx); 174 } 175 176 while (!wil_vring_is_empty(vring)) { 177 dma_addr_t pa; 178 u16 dmalen; 179 struct wil_ctx *ctx; 180 181 if (tx) { 182 struct vring_tx_desc dd, *d = ⅆ 183 volatile struct vring_tx_desc *_d = 184 &vring->va[vring->swtail].tx; 185 186 ctx = &vring->ctx[vring->swtail]; 187 *d = *_d; 188 wil_txdesc_unmap(dev, d, ctx); 189 if (ctx->skb) 190 dev_kfree_skb_any(ctx->skb); 191 vring->swtail = wil_vring_next_tail(vring); 192 } else { /* rx */ 193 struct vring_rx_desc dd, *d = ⅆ 194 volatile struct vring_rx_desc *_d = 195 &vring->va[vring->swhead].rx; 196 197 ctx = &vring->ctx[vring->swhead]; 198 *d = *_d; 199 pa = wil_desc_addr(&d->dma.addr); 200 dmalen = le16_to_cpu(d->dma.length); 201 dma_unmap_single(dev, pa, dmalen, DMA_FROM_DEVICE); 202 kfree_skb(ctx->skb); 203 wil_vring_advance_head(vring, 1); 204 } 205 } 206 dma_free_coherent(dev, sz, (void *)vring->va, vring->pa); 207 kfree(vring->ctx); 208 vring->pa = 0; 209 vring->va = NULL; 210 vring->ctx = NULL; 211 } 212 213 /** 214 * Allocate one skb for Rx VRING 215 * 216 * Safe to call from IRQ 217 */ 218 static int wil_vring_alloc_skb(struct wil6210_priv *wil, struct vring *vring, 219 u32 i, int headroom) 220 { 221 struct device *dev = wil_to_dev(wil); 222 unsigned int sz = mtu_max + ETH_HLEN + wil_rx_snaplen(); 223 struct vring_rx_desc dd, *d = ⅆ 224 volatile struct vring_rx_desc *_d = &vring->va[i].rx; 225 dma_addr_t pa; 226 struct sk_buff *skb = dev_alloc_skb(sz + headroom); 227 228 if (unlikely(!skb)) 229 return -ENOMEM; 230 231 skb_reserve(skb, headroom); 232 skb_put(skb, sz); 233 234 pa = dma_map_single(dev, skb->data, skb->len, DMA_FROM_DEVICE); 235 if (unlikely(dma_mapping_error(dev, pa))) { 236 kfree_skb(skb); 237 return -ENOMEM; 238 } 239 240 d->dma.d0 = RX_DMA_D0_CMD_DMA_RT | RX_DMA_D0_CMD_DMA_IT; 241 wil_desc_addr_set(&d->dma.addr, pa); 242 /* ip_length don't care */ 243 /* b11 don't care */ 244 /* error don't care */ 245 d->dma.status = 0; /* BIT(0) should be 0 for HW_OWNED */ 246 d->dma.length = cpu_to_le16(sz); 247 *_d = *d; 248 vring->ctx[i].skb = skb; 249 250 return 0; 251 } 252 253 /** 254 * Adds radiotap header 255 * 256 * Any error indicated as "Bad FCS" 257 * 258 * Vendor data for 04:ce:14-1 (Wilocity-1) consists of: 259 * - Rx descriptor: 32 bytes 260 * - Phy info 261 */ 262 static void wil_rx_add_radiotap_header(struct wil6210_priv *wil, 263 struct sk_buff *skb) 264 { 265 struct wireless_dev *wdev = wil->wdev; 266 struct wil6210_rtap { 267 struct ieee80211_radiotap_header rthdr; 268 /* fields should be in the order of bits in rthdr.it_present */ 269 /* flags */ 270 u8 flags; 271 /* channel */ 272 __le16 chnl_freq __aligned(2); 273 __le16 chnl_flags; 274 /* MCS */ 275 u8 mcs_present; 276 u8 mcs_flags; 277 u8 mcs_index; 278 } __packed; 279 struct wil6210_rtap_vendor { 280 struct wil6210_rtap rtap; 281 /* vendor */ 282 u8 vendor_oui[3] __aligned(2); 283 u8 vendor_ns; 284 __le16 vendor_skip; 285 u8 vendor_data[0]; 286 } __packed; 287 struct vring_rx_desc *d = wil_skb_rxdesc(skb); 288 struct wil6210_rtap_vendor *rtap_vendor; 289 int rtap_len = sizeof(struct wil6210_rtap); 290 int phy_length = 0; /* phy info header size, bytes */ 291 static char phy_data[128]; 292 struct ieee80211_channel *ch = wdev->preset_chandef.chan; 293 294 if (rtap_include_phy_info) { 295 rtap_len = sizeof(*rtap_vendor) + sizeof(*d); 296 /* calculate additional length */ 297 if (d->dma.status & RX_DMA_STATUS_PHY_INFO) { 298 /** 299 * PHY info starts from 8-byte boundary 300 * there are 8-byte lines, last line may be partially 301 * written (HW bug), thus FW configures for last line 302 * to be excessive. Driver skips this last line. 303 */ 304 int len = min_t(int, 8 + sizeof(phy_data), 305 wil_rxdesc_phy_length(d)); 306 307 if (len > 8) { 308 void *p = skb_tail_pointer(skb); 309 void *pa = PTR_ALIGN(p, 8); 310 311 if (skb_tailroom(skb) >= len + (pa - p)) { 312 phy_length = len - 8; 313 memcpy(phy_data, pa, phy_length); 314 } 315 } 316 } 317 rtap_len += phy_length; 318 } 319 320 if (skb_headroom(skb) < rtap_len && 321 pskb_expand_head(skb, rtap_len, 0, GFP_ATOMIC)) { 322 wil_err(wil, "Unable to expand headrom to %d\n", rtap_len); 323 return; 324 } 325 326 rtap_vendor = (void *)skb_push(skb, rtap_len); 327 memset(rtap_vendor, 0, rtap_len); 328 329 rtap_vendor->rtap.rthdr.it_version = PKTHDR_RADIOTAP_VERSION; 330 rtap_vendor->rtap.rthdr.it_len = cpu_to_le16(rtap_len); 331 rtap_vendor->rtap.rthdr.it_present = cpu_to_le32( 332 (1 << IEEE80211_RADIOTAP_FLAGS) | 333 (1 << IEEE80211_RADIOTAP_CHANNEL) | 334 (1 << IEEE80211_RADIOTAP_MCS)); 335 if (d->dma.status & RX_DMA_STATUS_ERROR) 336 rtap_vendor->rtap.flags |= IEEE80211_RADIOTAP_F_BADFCS; 337 338 rtap_vendor->rtap.chnl_freq = cpu_to_le16(ch ? ch->center_freq : 58320); 339 rtap_vendor->rtap.chnl_flags = cpu_to_le16(0); 340 341 rtap_vendor->rtap.mcs_present = IEEE80211_RADIOTAP_MCS_HAVE_MCS; 342 rtap_vendor->rtap.mcs_flags = 0; 343 rtap_vendor->rtap.mcs_index = wil_rxdesc_mcs(d); 344 345 if (rtap_include_phy_info) { 346 rtap_vendor->rtap.rthdr.it_present |= cpu_to_le32(1 << 347 IEEE80211_RADIOTAP_VENDOR_NAMESPACE); 348 /* OUI for Wilocity 04:ce:14 */ 349 rtap_vendor->vendor_oui[0] = 0x04; 350 rtap_vendor->vendor_oui[1] = 0xce; 351 rtap_vendor->vendor_oui[2] = 0x14; 352 rtap_vendor->vendor_ns = 1; 353 /* Rx descriptor + PHY data */ 354 rtap_vendor->vendor_skip = cpu_to_le16(sizeof(*d) + 355 phy_length); 356 memcpy(rtap_vendor->vendor_data, (void *)d, sizeof(*d)); 357 memcpy(rtap_vendor->vendor_data + sizeof(*d), phy_data, 358 phy_length); 359 } 360 } 361 362 /* similar to ieee80211_ version, but FC contain only 1-st byte */ 363 static inline int wil_is_back_req(u8 fc) 364 { 365 return (fc & (IEEE80211_FCTL_FTYPE | IEEE80211_FCTL_STYPE)) == 366 (IEEE80211_FTYPE_CTL | IEEE80211_STYPE_BACK_REQ); 367 } 368 369 /** 370 * reap 1 frame from @swhead 371 * 372 * Rx descriptor copied to skb->cb 373 * 374 * Safe to call from IRQ 375 */ 376 static struct sk_buff *wil_vring_reap_rx(struct wil6210_priv *wil, 377 struct vring *vring) 378 { 379 struct device *dev = wil_to_dev(wil); 380 struct net_device *ndev = wil_to_ndev(wil); 381 volatile struct vring_rx_desc *_d; 382 struct vring_rx_desc *d; 383 struct sk_buff *skb; 384 dma_addr_t pa; 385 unsigned int snaplen = wil_rx_snaplen(); 386 unsigned int sz = mtu_max + ETH_HLEN + snaplen; 387 u16 dmalen; 388 u8 ftype; 389 int cid; 390 int i; 391 struct wil_net_stats *stats; 392 393 BUILD_BUG_ON(sizeof(struct vring_rx_desc) > sizeof(skb->cb)); 394 395 again: 396 if (unlikely(wil_vring_is_empty(vring))) 397 return NULL; 398 399 i = (int)vring->swhead; 400 _d = &vring->va[i].rx; 401 if (unlikely(!(_d->dma.status & RX_DMA_STATUS_DU))) { 402 /* it is not error, we just reached end of Rx done area */ 403 return NULL; 404 } 405 406 skb = vring->ctx[i].skb; 407 vring->ctx[i].skb = NULL; 408 wil_vring_advance_head(vring, 1); 409 if (!skb) { 410 wil_err(wil, "No Rx skb at [%d]\n", i); 411 goto again; 412 } 413 d = wil_skb_rxdesc(skb); 414 *d = *_d; 415 pa = wil_desc_addr(&d->dma.addr); 416 417 dma_unmap_single(dev, pa, sz, DMA_FROM_DEVICE); 418 dmalen = le16_to_cpu(d->dma.length); 419 420 trace_wil6210_rx(i, d); 421 wil_dbg_txrx(wil, "Rx[%3d] : %d bytes\n", i, dmalen); 422 wil_hex_dump_txrx("RxD ", DUMP_PREFIX_NONE, 32, 4, 423 (const void *)d, sizeof(*d), false); 424 425 cid = wil_rxdesc_cid(d); 426 stats = &wil->sta[cid].stats; 427 428 if (unlikely(dmalen > sz)) { 429 wil_err(wil, "Rx size too large: %d bytes!\n", dmalen); 430 stats->rx_large_frame++; 431 kfree_skb(skb); 432 goto again; 433 } 434 skb_trim(skb, dmalen); 435 436 prefetch(skb->data); 437 438 wil_hex_dump_txrx("Rx ", DUMP_PREFIX_OFFSET, 16, 1, 439 skb->data, skb_headlen(skb), false); 440 441 stats->last_mcs_rx = wil_rxdesc_mcs(d); 442 if (stats->last_mcs_rx < ARRAY_SIZE(stats->rx_per_mcs)) 443 stats->rx_per_mcs[stats->last_mcs_rx]++; 444 445 /* use radiotap header only if required */ 446 if (ndev->type == ARPHRD_IEEE80211_RADIOTAP) 447 wil_rx_add_radiotap_header(wil, skb); 448 449 /* no extra checks if in sniffer mode */ 450 if (ndev->type != ARPHRD_ETHER) 451 return skb; 452 /* Non-data frames may be delivered through Rx DMA channel (ex: BAR) 453 * Driver should recognize it by frame type, that is found 454 * in Rx descriptor. If type is not data, it is 802.11 frame as is 455 */ 456 ftype = wil_rxdesc_ftype(d) << 2; 457 if (unlikely(ftype != IEEE80211_FTYPE_DATA)) { 458 u8 fc1 = wil_rxdesc_fc1(d); 459 int mid = wil_rxdesc_mid(d); 460 int tid = wil_rxdesc_tid(d); 461 u16 seq = wil_rxdesc_seq(d); 462 463 wil_dbg_txrx(wil, 464 "Non-data frame FC[7:0] 0x%02x MID %d CID %d TID %d Seq 0x%03x\n", 465 fc1, mid, cid, tid, seq); 466 stats->rx_non_data_frame++; 467 if (wil_is_back_req(fc1)) { 468 wil_dbg_txrx(wil, 469 "BAR: MID %d CID %d TID %d Seq 0x%03x\n", 470 mid, cid, tid, seq); 471 wil_rx_bar(wil, cid, tid, seq); 472 } else { 473 /* print again all info. One can enable only this 474 * without overhead for printing every Rx frame 475 */ 476 wil_dbg_txrx(wil, 477 "Unhandled non-data frame FC[7:0] 0x%02x MID %d CID %d TID %d Seq 0x%03x\n", 478 fc1, mid, cid, tid, seq); 479 wil_hex_dump_txrx("RxD ", DUMP_PREFIX_NONE, 32, 4, 480 (const void *)d, sizeof(*d), false); 481 wil_hex_dump_txrx("Rx ", DUMP_PREFIX_OFFSET, 16, 1, 482 skb->data, skb_headlen(skb), false); 483 } 484 kfree_skb(skb); 485 goto again; 486 } 487 488 if (unlikely(skb->len < ETH_HLEN + snaplen)) { 489 wil_err(wil, "Short frame, len = %d\n", skb->len); 490 stats->rx_short_frame++; 491 kfree_skb(skb); 492 goto again; 493 } 494 495 /* L4 IDENT is on when HW calculated checksum, check status 496 * and in case of error drop the packet 497 * higher stack layers will handle retransmission (if required) 498 */ 499 if (likely(d->dma.status & RX_DMA_STATUS_L4I)) { 500 /* L4 protocol identified, csum calculated */ 501 if (likely((d->dma.error & RX_DMA_ERROR_L4_ERR) == 0)) 502 skb->ip_summed = CHECKSUM_UNNECESSARY; 503 /* If HW reports bad checksum, let IP stack re-check it 504 * For example, HW don't understand Microsoft IP stack that 505 * mis-calculates TCP checksum - if it should be 0x0, 506 * it writes 0xffff in violation of RFC 1624 507 */ 508 } 509 510 if (snaplen) { 511 /* Packet layout 512 * +-------+-------+---------+------------+------+ 513 * | SA(6) | DA(6) | SNAP(6) | ETHTYPE(2) | DATA | 514 * +-------+-------+---------+------------+------+ 515 * Need to remove SNAP, shifting SA and DA forward 516 */ 517 memmove(skb->data + snaplen, skb->data, 2 * ETH_ALEN); 518 skb_pull(skb, snaplen); 519 } 520 521 return skb; 522 } 523 524 /** 525 * allocate and fill up to @count buffers in rx ring 526 * buffers posted at @swtail 527 */ 528 static int wil_rx_refill(struct wil6210_priv *wil, int count) 529 { 530 struct net_device *ndev = wil_to_ndev(wil); 531 struct vring *v = &wil->vring_rx; 532 u32 next_tail; 533 int rc = 0; 534 int headroom = ndev->type == ARPHRD_IEEE80211_RADIOTAP ? 535 WIL6210_RTAP_SIZE : 0; 536 537 for (; next_tail = wil_vring_next_tail(v), 538 (next_tail != v->swhead) && (count-- > 0); 539 v->swtail = next_tail) { 540 rc = wil_vring_alloc_skb(wil, v, v->swtail, headroom); 541 if (unlikely(rc)) { 542 wil_err(wil, "Error %d in wil_rx_refill[%d]\n", 543 rc, v->swtail); 544 break; 545 } 546 } 547 wil_w(wil, v->hwtail, v->swtail); 548 549 return rc; 550 } 551 552 /* 553 * Pass Rx packet to the netif. Update statistics. 554 * Called in softirq context (NAPI poll). 555 */ 556 void wil_netif_rx_any(struct sk_buff *skb, struct net_device *ndev) 557 { 558 gro_result_t rc = GRO_NORMAL; 559 struct wil6210_priv *wil = ndev_to_wil(ndev); 560 struct wireless_dev *wdev = wil_to_wdev(wil); 561 unsigned int len = skb->len; 562 struct vring_rx_desc *d = wil_skb_rxdesc(skb); 563 int cid = wil_rxdesc_cid(d); /* always 0..7, no need to check */ 564 struct ethhdr *eth = (void *)skb->data; 565 /* here looking for DA, not A1, thus Rxdesc's 'mcast' indication 566 * is not suitable, need to look at data 567 */ 568 int mcast = is_multicast_ether_addr(eth->h_dest); 569 struct wil_net_stats *stats = &wil->sta[cid].stats; 570 struct sk_buff *xmit_skb = NULL; 571 static const char * const gro_res_str[] = { 572 [GRO_MERGED] = "GRO_MERGED", 573 [GRO_MERGED_FREE] = "GRO_MERGED_FREE", 574 [GRO_HELD] = "GRO_HELD", 575 [GRO_NORMAL] = "GRO_NORMAL", 576 [GRO_DROP] = "GRO_DROP", 577 }; 578 579 if (ndev->features & NETIF_F_RXHASH) 580 /* fake L4 to ensure it won't be re-calculated later 581 * set hash to any non-zero value to activate rps 582 * mechanism, core will be chosen according 583 * to user-level rps configuration. 584 */ 585 skb_set_hash(skb, 1, PKT_HASH_TYPE_L4); 586 587 skb_orphan(skb); 588 589 if (wdev->iftype == NL80211_IFTYPE_AP && !wil->ap_isolate) { 590 if (mcast) { 591 /* send multicast frames both to higher layers in 592 * local net stack and back to the wireless medium 593 */ 594 xmit_skb = skb_copy(skb, GFP_ATOMIC); 595 } else { 596 int xmit_cid = wil_find_cid(wil, eth->h_dest); 597 598 if (xmit_cid >= 0) { 599 /* The destination station is associated to 600 * this AP (in this VLAN), so send the frame 601 * directly to it and do not pass it to local 602 * net stack. 603 */ 604 xmit_skb = skb; 605 skb = NULL; 606 } 607 } 608 } 609 if (xmit_skb) { 610 /* Send to wireless media and increase priority by 256 to 611 * keep the received priority instead of reclassifying 612 * the frame (see cfg80211_classify8021d). 613 */ 614 xmit_skb->dev = ndev; 615 xmit_skb->priority += 256; 616 xmit_skb->protocol = htons(ETH_P_802_3); 617 skb_reset_network_header(xmit_skb); 618 skb_reset_mac_header(xmit_skb); 619 wil_dbg_txrx(wil, "Rx -> Tx %d bytes\n", len); 620 dev_queue_xmit(xmit_skb); 621 } 622 623 if (skb) { /* deliver to local stack */ 624 625 skb->protocol = eth_type_trans(skb, ndev); 626 rc = napi_gro_receive(&wil->napi_rx, skb); 627 wil_dbg_txrx(wil, "Rx complete %d bytes => %s\n", 628 len, gro_res_str[rc]); 629 } 630 /* statistics. rc set to GRO_NORMAL for AP bridging */ 631 if (unlikely(rc == GRO_DROP)) { 632 ndev->stats.rx_dropped++; 633 stats->rx_dropped++; 634 wil_dbg_txrx(wil, "Rx drop %d bytes\n", len); 635 } else { 636 ndev->stats.rx_packets++; 637 stats->rx_packets++; 638 ndev->stats.rx_bytes += len; 639 stats->rx_bytes += len; 640 if (mcast) 641 ndev->stats.multicast++; 642 } 643 } 644 645 /** 646 * Proceed all completed skb's from Rx VRING 647 * 648 * Safe to call from NAPI poll, i.e. softirq with interrupts enabled 649 */ 650 void wil_rx_handle(struct wil6210_priv *wil, int *quota) 651 { 652 struct net_device *ndev = wil_to_ndev(wil); 653 struct vring *v = &wil->vring_rx; 654 struct sk_buff *skb; 655 656 if (unlikely(!v->va)) { 657 wil_err(wil, "Rx IRQ while Rx not yet initialized\n"); 658 return; 659 } 660 wil_dbg_txrx(wil, "%s()\n", __func__); 661 while ((*quota > 0) && (NULL != (skb = wil_vring_reap_rx(wil, v)))) { 662 (*quota)--; 663 664 if (wil->wdev->iftype == NL80211_IFTYPE_MONITOR) { 665 skb->dev = ndev; 666 skb_reset_mac_header(skb); 667 skb->ip_summed = CHECKSUM_UNNECESSARY; 668 skb->pkt_type = PACKET_OTHERHOST; 669 skb->protocol = htons(ETH_P_802_2); 670 wil_netif_rx_any(skb, ndev); 671 } else { 672 wil_rx_reorder(wil, skb); 673 } 674 } 675 wil_rx_refill(wil, v->size); 676 } 677 678 int wil_rx_init(struct wil6210_priv *wil, u16 size) 679 { 680 struct vring *vring = &wil->vring_rx; 681 int rc; 682 683 wil_dbg_misc(wil, "%s()\n", __func__); 684 685 if (vring->va) { 686 wil_err(wil, "Rx ring already allocated\n"); 687 return -EINVAL; 688 } 689 690 vring->size = size; 691 rc = wil_vring_alloc(wil, vring); 692 if (rc) 693 return rc; 694 695 rc = wmi_rx_chain_add(wil, vring); 696 if (rc) 697 goto err_free; 698 699 rc = wil_rx_refill(wil, vring->size); 700 if (rc) 701 goto err_free; 702 703 return 0; 704 err_free: 705 wil_vring_free(wil, vring, 0); 706 707 return rc; 708 } 709 710 void wil_rx_fini(struct wil6210_priv *wil) 711 { 712 struct vring *vring = &wil->vring_rx; 713 714 wil_dbg_misc(wil, "%s()\n", __func__); 715 716 if (vring->va) 717 wil_vring_free(wil, vring, 0); 718 } 719 720 int wil_vring_init_tx(struct wil6210_priv *wil, int id, int size, 721 int cid, int tid) 722 { 723 int rc; 724 struct wmi_vring_cfg_cmd cmd = { 725 .action = cpu_to_le32(WMI_VRING_CMD_ADD), 726 .vring_cfg = { 727 .tx_sw_ring = { 728 .max_mpdu_size = 729 cpu_to_le16(wil_mtu2macbuf(mtu_max)), 730 .ring_size = cpu_to_le16(size), 731 }, 732 .ringid = id, 733 .cidxtid = mk_cidxtid(cid, tid), 734 .encap_trans_type = WMI_VRING_ENC_TYPE_802_3, 735 .mac_ctrl = 0, 736 .to_resolution = 0, 737 .agg_max_wsize = 0, 738 .schd_params = { 739 .priority = cpu_to_le16(0), 740 .timeslot_us = cpu_to_le16(0xfff), 741 }, 742 }, 743 }; 744 struct { 745 struct wil6210_mbox_hdr_wmi wmi; 746 struct wmi_vring_cfg_done_event cmd; 747 } __packed reply; 748 struct vring *vring = &wil->vring_tx[id]; 749 struct vring_tx_data *txdata = &wil->vring_tx_data[id]; 750 751 wil_dbg_misc(wil, "%s() max_mpdu_size %d\n", __func__, 752 cmd.vring_cfg.tx_sw_ring.max_mpdu_size); 753 lockdep_assert_held(&wil->mutex); 754 755 if (vring->va) { 756 wil_err(wil, "Tx ring [%d] already allocated\n", id); 757 rc = -EINVAL; 758 goto out; 759 } 760 761 memset(txdata, 0, sizeof(*txdata)); 762 spin_lock_init(&txdata->lock); 763 vring->size = size; 764 rc = wil_vring_alloc(wil, vring); 765 if (rc) 766 goto out; 767 768 wil->vring2cid_tid[id][0] = cid; 769 wil->vring2cid_tid[id][1] = tid; 770 771 cmd.vring_cfg.tx_sw_ring.ring_mem_base = cpu_to_le64(vring->pa); 772 773 if (!wil->privacy) 774 txdata->dot1x_open = true; 775 rc = wmi_call(wil, WMI_VRING_CFG_CMDID, &cmd, sizeof(cmd), 776 WMI_VRING_CFG_DONE_EVENTID, &reply, sizeof(reply), 100); 777 if (rc) 778 goto out_free; 779 780 if (reply.cmd.status != WMI_FW_STATUS_SUCCESS) { 781 wil_err(wil, "Tx config failed, status 0x%02x\n", 782 reply.cmd.status); 783 rc = -EINVAL; 784 goto out_free; 785 } 786 vring->hwtail = le32_to_cpu(reply.cmd.tx_vring_tail_ptr); 787 788 txdata->enabled = 1; 789 if (txdata->dot1x_open && (agg_wsize >= 0)) 790 wil_addba_tx_request(wil, id, agg_wsize); 791 792 return 0; 793 out_free: 794 txdata->dot1x_open = false; 795 txdata->enabled = 0; 796 wil_vring_free(wil, vring, 1); 797 out: 798 799 return rc; 800 } 801 802 int wil_vring_init_bcast(struct wil6210_priv *wil, int id, int size) 803 { 804 int rc; 805 struct wmi_bcast_vring_cfg_cmd cmd = { 806 .action = cpu_to_le32(WMI_VRING_CMD_ADD), 807 .vring_cfg = { 808 .tx_sw_ring = { 809 .max_mpdu_size = 810 cpu_to_le16(wil_mtu2macbuf(mtu_max)), 811 .ring_size = cpu_to_le16(size), 812 }, 813 .ringid = id, 814 .encap_trans_type = WMI_VRING_ENC_TYPE_802_3, 815 }, 816 }; 817 struct { 818 struct wil6210_mbox_hdr_wmi wmi; 819 struct wmi_vring_cfg_done_event cmd; 820 } __packed reply; 821 struct vring *vring = &wil->vring_tx[id]; 822 struct vring_tx_data *txdata = &wil->vring_tx_data[id]; 823 824 wil_dbg_misc(wil, "%s() max_mpdu_size %d\n", __func__, 825 cmd.vring_cfg.tx_sw_ring.max_mpdu_size); 826 lockdep_assert_held(&wil->mutex); 827 828 if (vring->va) { 829 wil_err(wil, "Tx ring [%d] already allocated\n", id); 830 rc = -EINVAL; 831 goto out; 832 } 833 834 memset(txdata, 0, sizeof(*txdata)); 835 spin_lock_init(&txdata->lock); 836 vring->size = size; 837 rc = wil_vring_alloc(wil, vring); 838 if (rc) 839 goto out; 840 841 wil->vring2cid_tid[id][0] = WIL6210_MAX_CID; /* CID */ 842 wil->vring2cid_tid[id][1] = 0; /* TID */ 843 844 cmd.vring_cfg.tx_sw_ring.ring_mem_base = cpu_to_le64(vring->pa); 845 846 if (!wil->privacy) 847 txdata->dot1x_open = true; 848 rc = wmi_call(wil, WMI_BCAST_VRING_CFG_CMDID, &cmd, sizeof(cmd), 849 WMI_VRING_CFG_DONE_EVENTID, &reply, sizeof(reply), 100); 850 if (rc) 851 goto out_free; 852 853 if (reply.cmd.status != WMI_FW_STATUS_SUCCESS) { 854 wil_err(wil, "Tx config failed, status 0x%02x\n", 855 reply.cmd.status); 856 rc = -EINVAL; 857 goto out_free; 858 } 859 vring->hwtail = le32_to_cpu(reply.cmd.tx_vring_tail_ptr); 860 861 txdata->enabled = 1; 862 863 return 0; 864 out_free: 865 txdata->enabled = 0; 866 txdata->dot1x_open = false; 867 wil_vring_free(wil, vring, 1); 868 out: 869 870 return rc; 871 } 872 873 void wil_vring_fini_tx(struct wil6210_priv *wil, int id) 874 { 875 struct vring *vring = &wil->vring_tx[id]; 876 struct vring_tx_data *txdata = &wil->vring_tx_data[id]; 877 878 lockdep_assert_held(&wil->mutex); 879 880 if (!vring->va) 881 return; 882 883 wil_dbg_misc(wil, "%s() id=%d\n", __func__, id); 884 885 spin_lock_bh(&txdata->lock); 886 txdata->dot1x_open = false; 887 txdata->enabled = 0; /* no Tx can be in progress or start anew */ 888 spin_unlock_bh(&txdata->lock); 889 /* make sure NAPI won't touch this vring */ 890 if (test_bit(wil_status_napi_en, wil->status)) 891 napi_synchronize(&wil->napi_tx); 892 893 wil_vring_free(wil, vring, 1); 894 memset(txdata, 0, sizeof(*txdata)); 895 } 896 897 static struct vring *wil_find_tx_ucast(struct wil6210_priv *wil, 898 struct sk_buff *skb) 899 { 900 int i; 901 struct ethhdr *eth = (void *)skb->data; 902 int cid = wil_find_cid(wil, eth->h_dest); 903 904 if (cid < 0) 905 return NULL; 906 907 /* TODO: fix for multiple TID */ 908 for (i = 0; i < ARRAY_SIZE(wil->vring2cid_tid); i++) { 909 if (!wil->vring_tx_data[i].dot1x_open && 910 (skb->protocol != cpu_to_be16(ETH_P_PAE))) 911 continue; 912 if (wil->vring2cid_tid[i][0] == cid) { 913 struct vring *v = &wil->vring_tx[i]; 914 915 wil_dbg_txrx(wil, "%s(%pM) -> [%d]\n", 916 __func__, eth->h_dest, i); 917 if (v->va) { 918 return v; 919 } else { 920 wil_dbg_txrx(wil, "vring[%d] not valid\n", i); 921 return NULL; 922 } 923 } 924 } 925 926 return NULL; 927 } 928 929 static int wil_tx_vring(struct wil6210_priv *wil, struct vring *vring, 930 struct sk_buff *skb); 931 932 static struct vring *wil_find_tx_vring_sta(struct wil6210_priv *wil, 933 struct sk_buff *skb) 934 { 935 struct vring *v; 936 int i; 937 u8 cid; 938 939 /* In the STA mode, it is expected to have only 1 VRING 940 * for the AP we connected to. 941 * find 1-st vring eligible for this skb and use it. 942 */ 943 for (i = 0; i < WIL6210_MAX_TX_RINGS; i++) { 944 v = &wil->vring_tx[i]; 945 if (!v->va) 946 continue; 947 948 cid = wil->vring2cid_tid[i][0]; 949 if (cid >= WIL6210_MAX_CID) /* skip BCAST */ 950 continue; 951 952 if (!wil->vring_tx_data[i].dot1x_open && 953 (skb->protocol != cpu_to_be16(ETH_P_PAE))) 954 continue; 955 956 wil_dbg_txrx(wil, "Tx -> ring %d\n", i); 957 958 return v; 959 } 960 961 wil_dbg_txrx(wil, "Tx while no vrings active?\n"); 962 963 return NULL; 964 } 965 966 /* Use one of 2 strategies: 967 * 968 * 1. New (real broadcast): 969 * use dedicated broadcast vring 970 * 2. Old (pseudo-DMS): 971 * Find 1-st vring and return it; 972 * duplicate skb and send it to other active vrings; 973 * in all cases override dest address to unicast peer's address 974 * Use old strategy when new is not supported yet: 975 * - for PBSS 976 */ 977 static struct vring *wil_find_tx_bcast_1(struct wil6210_priv *wil, 978 struct sk_buff *skb) 979 { 980 struct vring *v; 981 int i = wil->bcast_vring; 982 983 if (i < 0) 984 return NULL; 985 v = &wil->vring_tx[i]; 986 if (!v->va) 987 return NULL; 988 if (!wil->vring_tx_data[i].dot1x_open && 989 (skb->protocol != cpu_to_be16(ETH_P_PAE))) 990 return NULL; 991 992 return v; 993 } 994 995 static void wil_set_da_for_vring(struct wil6210_priv *wil, 996 struct sk_buff *skb, int vring_index) 997 { 998 struct ethhdr *eth = (void *)skb->data; 999 int cid = wil->vring2cid_tid[vring_index][0]; 1000 1001 ether_addr_copy(eth->h_dest, wil->sta[cid].addr); 1002 } 1003 1004 static struct vring *wil_find_tx_bcast_2(struct wil6210_priv *wil, 1005 struct sk_buff *skb) 1006 { 1007 struct vring *v, *v2; 1008 struct sk_buff *skb2; 1009 int i; 1010 u8 cid; 1011 struct ethhdr *eth = (void *)skb->data; 1012 char *src = eth->h_source; 1013 1014 /* find 1-st vring eligible for data */ 1015 for (i = 0; i < WIL6210_MAX_TX_RINGS; i++) { 1016 v = &wil->vring_tx[i]; 1017 if (!v->va) 1018 continue; 1019 1020 cid = wil->vring2cid_tid[i][0]; 1021 if (cid >= WIL6210_MAX_CID) /* skip BCAST */ 1022 continue; 1023 if (!wil->vring_tx_data[i].dot1x_open && 1024 (skb->protocol != cpu_to_be16(ETH_P_PAE))) 1025 continue; 1026 1027 /* don't Tx back to source when re-routing Rx->Tx at the AP */ 1028 if (0 == memcmp(wil->sta[cid].addr, src, ETH_ALEN)) 1029 continue; 1030 1031 goto found; 1032 } 1033 1034 wil_dbg_txrx(wil, "Tx while no vrings active?\n"); 1035 1036 return NULL; 1037 1038 found: 1039 wil_dbg_txrx(wil, "BCAST -> ring %d\n", i); 1040 wil_set_da_for_vring(wil, skb, i); 1041 1042 /* find other active vrings and duplicate skb for each */ 1043 for (i++; i < WIL6210_MAX_TX_RINGS; i++) { 1044 v2 = &wil->vring_tx[i]; 1045 if (!v2->va) 1046 continue; 1047 cid = wil->vring2cid_tid[i][0]; 1048 if (cid >= WIL6210_MAX_CID) /* skip BCAST */ 1049 continue; 1050 if (!wil->vring_tx_data[i].dot1x_open && 1051 (skb->protocol != cpu_to_be16(ETH_P_PAE))) 1052 continue; 1053 1054 if (0 == memcmp(wil->sta[cid].addr, src, ETH_ALEN)) 1055 continue; 1056 1057 skb2 = skb_copy(skb, GFP_ATOMIC); 1058 if (skb2) { 1059 wil_dbg_txrx(wil, "BCAST DUP -> ring %d\n", i); 1060 wil_set_da_for_vring(wil, skb2, i); 1061 wil_tx_vring(wil, v2, skb2); 1062 } else { 1063 wil_err(wil, "skb_copy failed\n"); 1064 } 1065 } 1066 1067 return v; 1068 } 1069 1070 static struct vring *wil_find_tx_bcast(struct wil6210_priv *wil, 1071 struct sk_buff *skb) 1072 { 1073 struct wireless_dev *wdev = wil->wdev; 1074 1075 if (wdev->iftype != NL80211_IFTYPE_AP) 1076 return wil_find_tx_bcast_2(wil, skb); 1077 1078 return wil_find_tx_bcast_1(wil, skb); 1079 } 1080 1081 static int wil_tx_desc_map(struct vring_tx_desc *d, dma_addr_t pa, u32 len, 1082 int vring_index) 1083 { 1084 wil_desc_addr_set(&d->dma.addr, pa); 1085 d->dma.ip_length = 0; 1086 /* 0..6: mac_length; 7:ip_version 0-IP6 1-IP4*/ 1087 d->dma.b11 = 0/*14 | BIT(7)*/; 1088 d->dma.error = 0; 1089 d->dma.status = 0; /* BIT(0) should be 0 for HW_OWNED */ 1090 d->dma.length = cpu_to_le16((u16)len); 1091 d->dma.d0 = (vring_index << DMA_CFG_DESC_TX_0_QID_POS); 1092 d->mac.d[0] = 0; 1093 d->mac.d[1] = 0; 1094 d->mac.d[2] = 0; 1095 d->mac.ucode_cmd = 0; 1096 /* translation type: 0 - bypass; 1 - 802.3; 2 - native wifi */ 1097 d->mac.d[2] = BIT(MAC_CFG_DESC_TX_2_SNAP_HDR_INSERTION_EN_POS) | 1098 (1 << MAC_CFG_DESC_TX_2_L2_TRANSLATION_TYPE_POS); 1099 1100 return 0; 1101 } 1102 1103 static inline 1104 void wil_tx_desc_set_nr_frags(struct vring_tx_desc *d, int nr_frags) 1105 { 1106 d->mac.d[2] |= (nr_frags << MAC_CFG_DESC_TX_2_NUM_OF_DESCRIPTORS_POS); 1107 } 1108 1109 /** 1110 * Sets the descriptor @d up for csum and/or TSO offloading. The corresponding 1111 * @skb is used to obtain the protocol and headers length. 1112 * @tso_desc_type is a descriptor type for TSO: 0 - a header, 1 - first data, 1113 * 2 - middle, 3 - last descriptor. 1114 */ 1115 1116 static void wil_tx_desc_offload_setup_tso(struct vring_tx_desc *d, 1117 struct sk_buff *skb, 1118 int tso_desc_type, bool is_ipv4, 1119 int tcp_hdr_len, int skb_net_hdr_len) 1120 { 1121 d->dma.b11 = ETH_HLEN; /* MAC header length */ 1122 d->dma.b11 |= is_ipv4 << DMA_CFG_DESC_TX_OFFLOAD_CFG_L3T_IPV4_POS; 1123 1124 d->dma.d0 |= (2 << DMA_CFG_DESC_TX_0_L4_TYPE_POS); 1125 /* L4 header len: TCP header length */ 1126 d->dma.d0 |= (tcp_hdr_len & DMA_CFG_DESC_TX_0_L4_LENGTH_MSK); 1127 1128 /* Setup TSO: bit and desc type */ 1129 d->dma.d0 |= (BIT(DMA_CFG_DESC_TX_0_TCP_SEG_EN_POS)) | 1130 (tso_desc_type << DMA_CFG_DESC_TX_0_SEGMENT_BUF_DETAILS_POS); 1131 d->dma.d0 |= (is_ipv4 << DMA_CFG_DESC_TX_0_IPV4_CHECKSUM_EN_POS); 1132 1133 d->dma.ip_length = skb_net_hdr_len; 1134 /* Enable TCP/UDP checksum */ 1135 d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_TCP_UDP_CHECKSUM_EN_POS); 1136 /* Calculate pseudo-header */ 1137 d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_PSEUDO_HEADER_CALC_EN_POS); 1138 } 1139 1140 /** 1141 * Sets the descriptor @d up for csum. The corresponding 1142 * @skb is used to obtain the protocol and headers length. 1143 * Returns the protocol: 0 - not TCP, 1 - TCPv4, 2 - TCPv6. 1144 * Note, if d==NULL, the function only returns the protocol result. 1145 * 1146 * It is very similar to previous wil_tx_desc_offload_setup_tso. This 1147 * is "if unrolling" to optimize the critical path. 1148 */ 1149 1150 static int wil_tx_desc_offload_setup(struct vring_tx_desc *d, 1151 struct sk_buff *skb){ 1152 int protocol; 1153 1154 if (skb->ip_summed != CHECKSUM_PARTIAL) 1155 return 0; 1156 1157 d->dma.b11 = ETH_HLEN; /* MAC header length */ 1158 1159 switch (skb->protocol) { 1160 case cpu_to_be16(ETH_P_IP): 1161 protocol = ip_hdr(skb)->protocol; 1162 d->dma.b11 |= BIT(DMA_CFG_DESC_TX_OFFLOAD_CFG_L3T_IPV4_POS); 1163 break; 1164 case cpu_to_be16(ETH_P_IPV6): 1165 protocol = ipv6_hdr(skb)->nexthdr; 1166 break; 1167 default: 1168 return -EINVAL; 1169 } 1170 1171 switch (protocol) { 1172 case IPPROTO_TCP: 1173 d->dma.d0 |= (2 << DMA_CFG_DESC_TX_0_L4_TYPE_POS); 1174 /* L4 header len: TCP header length */ 1175 d->dma.d0 |= 1176 (tcp_hdrlen(skb) & DMA_CFG_DESC_TX_0_L4_LENGTH_MSK); 1177 break; 1178 case IPPROTO_UDP: 1179 /* L4 header len: UDP header length */ 1180 d->dma.d0 |= 1181 (sizeof(struct udphdr) & DMA_CFG_DESC_TX_0_L4_LENGTH_MSK); 1182 break; 1183 default: 1184 return -EINVAL; 1185 } 1186 1187 d->dma.ip_length = skb_network_header_len(skb); 1188 /* Enable TCP/UDP checksum */ 1189 d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_TCP_UDP_CHECKSUM_EN_POS); 1190 /* Calculate pseudo-header */ 1191 d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_PSEUDO_HEADER_CALC_EN_POS); 1192 1193 return 0; 1194 } 1195 1196 static inline void wil_tx_last_desc(struct vring_tx_desc *d) 1197 { 1198 d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_CMD_EOP_POS) | 1199 BIT(DMA_CFG_DESC_TX_0_CMD_MARK_WB_POS) | 1200 BIT(DMA_CFG_DESC_TX_0_CMD_DMA_IT_POS); 1201 } 1202 1203 static inline void wil_set_tx_desc_last_tso(volatile struct vring_tx_desc *d) 1204 { 1205 d->dma.d0 |= wil_tso_type_lst << 1206 DMA_CFG_DESC_TX_0_SEGMENT_BUF_DETAILS_POS; 1207 } 1208 1209 static int __wil_tx_vring_tso(struct wil6210_priv *wil, struct vring *vring, 1210 struct sk_buff *skb) 1211 { 1212 struct device *dev = wil_to_dev(wil); 1213 1214 /* point to descriptors in shared memory */ 1215 volatile struct vring_tx_desc *_desc = NULL, *_hdr_desc, 1216 *_first_desc = NULL; 1217 1218 /* pointers to shadow descriptors */ 1219 struct vring_tx_desc desc_mem, hdr_desc_mem, first_desc_mem, 1220 *d = &hdr_desc_mem, *hdr_desc = &hdr_desc_mem, 1221 *first_desc = &first_desc_mem; 1222 1223 /* pointer to shadow descriptors' context */ 1224 struct wil_ctx *hdr_ctx, *first_ctx = NULL; 1225 1226 int descs_used = 0; /* total number of used descriptors */ 1227 int sg_desc_cnt = 0; /* number of descriptors for current mss*/ 1228 1229 u32 swhead = vring->swhead; 1230 int used, avail = wil_vring_avail_tx(vring); 1231 int nr_frags = skb_shinfo(skb)->nr_frags; 1232 int min_desc_required = nr_frags + 1; 1233 int mss = skb_shinfo(skb)->gso_size; /* payload size w/o headers */ 1234 int f, len, hdrlen, headlen; 1235 int vring_index = vring - wil->vring_tx; 1236 struct vring_tx_data *txdata = &wil->vring_tx_data[vring_index]; 1237 uint i = swhead; 1238 dma_addr_t pa; 1239 const skb_frag_t *frag = NULL; 1240 int rem_data = mss; 1241 int lenmss; 1242 int hdr_compensation_need = true; 1243 int desc_tso_type = wil_tso_type_first; 1244 bool is_ipv4; 1245 int tcp_hdr_len; 1246 int skb_net_hdr_len; 1247 int gso_type; 1248 int rc = -EINVAL; 1249 1250 wil_dbg_txrx(wil, "%s() %d bytes to vring %d\n", 1251 __func__, skb->len, vring_index); 1252 1253 if (unlikely(!txdata->enabled)) 1254 return -EINVAL; 1255 1256 /* A typical page 4K is 3-4 payloads, we assume each fragment 1257 * is a full payload, that's how min_desc_required has been 1258 * calculated. In real we might need more or less descriptors, 1259 * this is the initial check only. 1260 */ 1261 if (unlikely(avail < min_desc_required)) { 1262 wil_err_ratelimited(wil, 1263 "TSO: Tx ring[%2d] full. No space for %d fragments\n", 1264 vring_index, min_desc_required); 1265 return -ENOMEM; 1266 } 1267 1268 /* Header Length = MAC header len + IP header len + TCP header len*/ 1269 hdrlen = ETH_HLEN + 1270 (int)skb_network_header_len(skb) + 1271 tcp_hdrlen(skb); 1272 1273 gso_type = skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV6 | SKB_GSO_TCPV4); 1274 switch (gso_type) { 1275 case SKB_GSO_TCPV4: 1276 /* TCP v4, zero out the IP length and IPv4 checksum fields 1277 * as required by the offloading doc 1278 */ 1279 ip_hdr(skb)->tot_len = 0; 1280 ip_hdr(skb)->check = 0; 1281 is_ipv4 = true; 1282 break; 1283 case SKB_GSO_TCPV6: 1284 /* TCP v6, zero out the payload length */ 1285 ipv6_hdr(skb)->payload_len = 0; 1286 is_ipv4 = false; 1287 break; 1288 default: 1289 /* other than TCPv4 or TCPv6 types are not supported for TSO. 1290 * It is also illegal for both to be set simultaneously 1291 */ 1292 return -EINVAL; 1293 } 1294 1295 if (skb->ip_summed != CHECKSUM_PARTIAL) 1296 return -EINVAL; 1297 1298 /* tcp header length and skb network header length are fixed for all 1299 * packet's descriptors - read then once here 1300 */ 1301 tcp_hdr_len = tcp_hdrlen(skb); 1302 skb_net_hdr_len = skb_network_header_len(skb); 1303 1304 _hdr_desc = &vring->va[i].tx; 1305 1306 pa = dma_map_single(dev, skb->data, hdrlen, DMA_TO_DEVICE); 1307 if (unlikely(dma_mapping_error(dev, pa))) { 1308 wil_err(wil, "TSO: Skb head DMA map error\n"); 1309 goto err_exit; 1310 } 1311 1312 wil_tx_desc_map(hdr_desc, pa, hdrlen, vring_index); 1313 wil_tx_desc_offload_setup_tso(hdr_desc, skb, wil_tso_type_hdr, is_ipv4, 1314 tcp_hdr_len, skb_net_hdr_len); 1315 wil_tx_last_desc(hdr_desc); 1316 1317 vring->ctx[i].mapped_as = wil_mapped_as_single; 1318 hdr_ctx = &vring->ctx[i]; 1319 1320 descs_used++; 1321 headlen = skb_headlen(skb) - hdrlen; 1322 1323 for (f = headlen ? -1 : 0; f < nr_frags; f++) { 1324 if (headlen) { 1325 len = headlen; 1326 wil_dbg_txrx(wil, "TSO: process skb head, len %u\n", 1327 len); 1328 } else { 1329 frag = &skb_shinfo(skb)->frags[f]; 1330 len = frag->size; 1331 wil_dbg_txrx(wil, "TSO: frag[%d]: len %u\n", f, len); 1332 } 1333 1334 while (len) { 1335 wil_dbg_txrx(wil, 1336 "TSO: len %d, rem_data %d, descs_used %d\n", 1337 len, rem_data, descs_used); 1338 1339 if (descs_used == avail) { 1340 wil_err_ratelimited(wil, "TSO: ring overflow\n"); 1341 rc = -ENOMEM; 1342 goto mem_error; 1343 } 1344 1345 lenmss = min_t(int, rem_data, len); 1346 i = (swhead + descs_used) % vring->size; 1347 wil_dbg_txrx(wil, "TSO: lenmss %d, i %d\n", lenmss, i); 1348 1349 if (!headlen) { 1350 pa = skb_frag_dma_map(dev, frag, 1351 frag->size - len, lenmss, 1352 DMA_TO_DEVICE); 1353 vring->ctx[i].mapped_as = wil_mapped_as_page; 1354 } else { 1355 pa = dma_map_single(dev, 1356 skb->data + 1357 skb_headlen(skb) - headlen, 1358 lenmss, 1359 DMA_TO_DEVICE); 1360 vring->ctx[i].mapped_as = wil_mapped_as_single; 1361 headlen -= lenmss; 1362 } 1363 1364 if (unlikely(dma_mapping_error(dev, pa))) { 1365 wil_err(wil, "TSO: DMA map page error\n"); 1366 goto mem_error; 1367 } 1368 1369 _desc = &vring->va[i].tx; 1370 1371 if (!_first_desc) { 1372 _first_desc = _desc; 1373 first_ctx = &vring->ctx[i]; 1374 d = first_desc; 1375 } else { 1376 d = &desc_mem; 1377 } 1378 1379 wil_tx_desc_map(d, pa, lenmss, vring_index); 1380 wil_tx_desc_offload_setup_tso(d, skb, desc_tso_type, 1381 is_ipv4, tcp_hdr_len, 1382 skb_net_hdr_len); 1383 1384 /* use tso_type_first only once */ 1385 desc_tso_type = wil_tso_type_mid; 1386 1387 descs_used++; /* desc used so far */ 1388 sg_desc_cnt++; /* desc used for this segment */ 1389 len -= lenmss; 1390 rem_data -= lenmss; 1391 1392 wil_dbg_txrx(wil, 1393 "TSO: len %d, rem_data %d, descs_used %d, sg_desc_cnt %d,\n", 1394 len, rem_data, descs_used, sg_desc_cnt); 1395 1396 /* Close the segment if reached mss size or last frag*/ 1397 if (rem_data == 0 || (f == nr_frags - 1 && len == 0)) { 1398 if (hdr_compensation_need) { 1399 /* first segment include hdr desc for 1400 * release 1401 */ 1402 hdr_ctx->nr_frags = sg_desc_cnt; 1403 wil_tx_desc_set_nr_frags(first_desc, 1404 sg_desc_cnt + 1405 1); 1406 hdr_compensation_need = false; 1407 } else { 1408 wil_tx_desc_set_nr_frags(first_desc, 1409 sg_desc_cnt); 1410 } 1411 first_ctx->nr_frags = sg_desc_cnt - 1; 1412 1413 wil_tx_last_desc(d); 1414 1415 /* first descriptor may also be the last 1416 * for this mss - make sure not to copy 1417 * it twice 1418 */ 1419 if (first_desc != d) 1420 *_first_desc = *first_desc; 1421 1422 /*last descriptor will be copied at the end 1423 * of this TS processing 1424 */ 1425 if (f < nr_frags - 1 || len > 0) 1426 *_desc = *d; 1427 1428 rem_data = mss; 1429 _first_desc = NULL; 1430 sg_desc_cnt = 0; 1431 } else if (first_desc != d) /* update mid descriptor */ 1432 *_desc = *d; 1433 } 1434 } 1435 1436 /* first descriptor may also be the last. 1437 * in this case d pointer is invalid 1438 */ 1439 if (_first_desc == _desc) 1440 d = first_desc; 1441 1442 /* Last data descriptor */ 1443 wil_set_tx_desc_last_tso(d); 1444 *_desc = *d; 1445 1446 /* Fill the total number of descriptors in first desc (hdr)*/ 1447 wil_tx_desc_set_nr_frags(hdr_desc, descs_used); 1448 *_hdr_desc = *hdr_desc; 1449 1450 /* hold reference to skb 1451 * to prevent skb release before accounting 1452 * in case of immediate "tx done" 1453 */ 1454 vring->ctx[i].skb = skb_get(skb); 1455 1456 /* performance monitoring */ 1457 used = wil_vring_used_tx(vring); 1458 if (wil_val_in_range(vring_idle_trsh, 1459 used, used + descs_used)) { 1460 txdata->idle += get_cycles() - txdata->last_idle; 1461 wil_dbg_txrx(wil, "Ring[%2d] not idle %d -> %d\n", 1462 vring_index, used, used + descs_used); 1463 } 1464 1465 /* advance swhead */ 1466 wil_vring_advance_head(vring, descs_used); 1467 wil_dbg_txrx(wil, "TSO: Tx swhead %d -> %d\n", swhead, vring->swhead); 1468 1469 /* make sure all writes to descriptors (shared memory) are done before 1470 * committing them to HW 1471 */ 1472 wmb(); 1473 1474 wil_w(wil, vring->hwtail, vring->swhead); 1475 return 0; 1476 1477 mem_error: 1478 while (descs_used > 0) { 1479 struct wil_ctx *ctx; 1480 1481 i = (swhead + descs_used) % vring->size; 1482 d = (struct vring_tx_desc *)&vring->va[i].tx; 1483 _desc = &vring->va[i].tx; 1484 *d = *_desc; 1485 _desc->dma.status = TX_DMA_STATUS_DU; 1486 ctx = &vring->ctx[i]; 1487 wil_txdesc_unmap(dev, d, ctx); 1488 memset(ctx, 0, sizeof(*ctx)); 1489 descs_used--; 1490 } 1491 err_exit: 1492 return rc; 1493 } 1494 1495 static int __wil_tx_vring(struct wil6210_priv *wil, struct vring *vring, 1496 struct sk_buff *skb) 1497 { 1498 struct device *dev = wil_to_dev(wil); 1499 struct vring_tx_desc dd, *d = ⅆ 1500 volatile struct vring_tx_desc *_d; 1501 u32 swhead = vring->swhead; 1502 int avail = wil_vring_avail_tx(vring); 1503 int nr_frags = skb_shinfo(skb)->nr_frags; 1504 uint f = 0; 1505 int vring_index = vring - wil->vring_tx; 1506 struct vring_tx_data *txdata = &wil->vring_tx_data[vring_index]; 1507 uint i = swhead; 1508 dma_addr_t pa; 1509 int used; 1510 bool mcast = (vring_index == wil->bcast_vring); 1511 uint len = skb_headlen(skb); 1512 1513 wil_dbg_txrx(wil, "%s() %d bytes to vring %d\n", 1514 __func__, skb->len, vring_index); 1515 1516 if (unlikely(!txdata->enabled)) 1517 return -EINVAL; 1518 1519 if (unlikely(avail < 1 + nr_frags)) { 1520 wil_err_ratelimited(wil, 1521 "Tx ring[%2d] full. No space for %d fragments\n", 1522 vring_index, 1 + nr_frags); 1523 return -ENOMEM; 1524 } 1525 _d = &vring->va[i].tx; 1526 1527 pa = dma_map_single(dev, skb->data, skb_headlen(skb), DMA_TO_DEVICE); 1528 1529 wil_dbg_txrx(wil, "Tx[%2d] skb %d bytes 0x%p -> %pad\n", vring_index, 1530 skb_headlen(skb), skb->data, &pa); 1531 wil_hex_dump_txrx("Tx ", DUMP_PREFIX_OFFSET, 16, 1, 1532 skb->data, skb_headlen(skb), false); 1533 1534 if (unlikely(dma_mapping_error(dev, pa))) 1535 return -EINVAL; 1536 vring->ctx[i].mapped_as = wil_mapped_as_single; 1537 /* 1-st segment */ 1538 wil_tx_desc_map(d, pa, len, vring_index); 1539 if (unlikely(mcast)) { 1540 d->mac.d[0] |= BIT(MAC_CFG_DESC_TX_0_MCS_EN_POS); /* MCS 0 */ 1541 if (unlikely(len > WIL_BCAST_MCS0_LIMIT)) /* set MCS 1 */ 1542 d->mac.d[0] |= (1 << MAC_CFG_DESC_TX_0_MCS_INDEX_POS); 1543 } 1544 /* Process TCP/UDP checksum offloading */ 1545 if (unlikely(wil_tx_desc_offload_setup(d, skb))) { 1546 wil_err(wil, "Tx[%2d] Failed to set cksum, drop packet\n", 1547 vring_index); 1548 goto dma_error; 1549 } 1550 1551 vring->ctx[i].nr_frags = nr_frags; 1552 wil_tx_desc_set_nr_frags(d, nr_frags + 1); 1553 1554 /* middle segments */ 1555 for (; f < nr_frags; f++) { 1556 const struct skb_frag_struct *frag = 1557 &skb_shinfo(skb)->frags[f]; 1558 int len = skb_frag_size(frag); 1559 1560 *_d = *d; 1561 wil_dbg_txrx(wil, "Tx[%2d] desc[%4d]\n", vring_index, i); 1562 wil_hex_dump_txrx("TxD ", DUMP_PREFIX_NONE, 32, 4, 1563 (const void *)d, sizeof(*d), false); 1564 i = (swhead + f + 1) % vring->size; 1565 _d = &vring->va[i].tx; 1566 pa = skb_frag_dma_map(dev, frag, 0, skb_frag_size(frag), 1567 DMA_TO_DEVICE); 1568 if (unlikely(dma_mapping_error(dev, pa))) { 1569 wil_err(wil, "Tx[%2d] failed to map fragment\n", 1570 vring_index); 1571 goto dma_error; 1572 } 1573 vring->ctx[i].mapped_as = wil_mapped_as_page; 1574 wil_tx_desc_map(d, pa, len, vring_index); 1575 /* no need to check return code - 1576 * if it succeeded for 1-st descriptor, 1577 * it will succeed here too 1578 */ 1579 wil_tx_desc_offload_setup(d, skb); 1580 } 1581 /* for the last seg only */ 1582 d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_CMD_EOP_POS); 1583 d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_CMD_MARK_WB_POS); 1584 d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_CMD_DMA_IT_POS); 1585 *_d = *d; 1586 wil_dbg_txrx(wil, "Tx[%2d] desc[%4d]\n", vring_index, i); 1587 wil_hex_dump_txrx("TxD ", DUMP_PREFIX_NONE, 32, 4, 1588 (const void *)d, sizeof(*d), false); 1589 1590 /* hold reference to skb 1591 * to prevent skb release before accounting 1592 * in case of immediate "tx done" 1593 */ 1594 vring->ctx[i].skb = skb_get(skb); 1595 1596 /* performance monitoring */ 1597 used = wil_vring_used_tx(vring); 1598 if (wil_val_in_range(vring_idle_trsh, 1599 used, used + nr_frags + 1)) { 1600 txdata->idle += get_cycles() - txdata->last_idle; 1601 wil_dbg_txrx(wil, "Ring[%2d] not idle %d -> %d\n", 1602 vring_index, used, used + nr_frags + 1); 1603 } 1604 1605 /* advance swhead */ 1606 wil_vring_advance_head(vring, nr_frags + 1); 1607 wil_dbg_txrx(wil, "Tx[%2d] swhead %d -> %d\n", vring_index, swhead, 1608 vring->swhead); 1609 trace_wil6210_tx(vring_index, swhead, skb->len, nr_frags); 1610 1611 /* make sure all writes to descriptors (shared memory) are done before 1612 * committing them to HW 1613 */ 1614 wmb(); 1615 1616 wil_w(wil, vring->hwtail, vring->swhead); 1617 1618 return 0; 1619 dma_error: 1620 /* unmap what we have mapped */ 1621 nr_frags = f + 1; /* frags mapped + one for skb head */ 1622 for (f = 0; f < nr_frags; f++) { 1623 struct wil_ctx *ctx; 1624 1625 i = (swhead + f) % vring->size; 1626 ctx = &vring->ctx[i]; 1627 _d = &vring->va[i].tx; 1628 *d = *_d; 1629 _d->dma.status = TX_DMA_STATUS_DU; 1630 wil_txdesc_unmap(dev, d, ctx); 1631 1632 memset(ctx, 0, sizeof(*ctx)); 1633 } 1634 1635 return -EINVAL; 1636 } 1637 1638 static int wil_tx_vring(struct wil6210_priv *wil, struct vring *vring, 1639 struct sk_buff *skb) 1640 { 1641 int vring_index = vring - wil->vring_tx; 1642 struct vring_tx_data *txdata = &wil->vring_tx_data[vring_index]; 1643 int rc; 1644 1645 spin_lock(&txdata->lock); 1646 1647 rc = (skb_is_gso(skb) ? __wil_tx_vring_tso : __wil_tx_vring) 1648 (wil, vring, skb); 1649 1650 spin_unlock(&txdata->lock); 1651 1652 return rc; 1653 } 1654 1655 netdev_tx_t wil_start_xmit(struct sk_buff *skb, struct net_device *ndev) 1656 { 1657 struct wil6210_priv *wil = ndev_to_wil(ndev); 1658 struct ethhdr *eth = (void *)skb->data; 1659 bool bcast = is_multicast_ether_addr(eth->h_dest); 1660 struct vring *vring; 1661 static bool pr_once_fw; 1662 int rc; 1663 1664 wil_dbg_txrx(wil, "%s()\n", __func__); 1665 if (unlikely(!test_bit(wil_status_fwready, wil->status))) { 1666 if (!pr_once_fw) { 1667 wil_err(wil, "FW not ready\n"); 1668 pr_once_fw = true; 1669 } 1670 goto drop; 1671 } 1672 if (unlikely(!test_bit(wil_status_fwconnected, wil->status))) { 1673 wil_err_ratelimited(wil, "FW not connected\n"); 1674 goto drop; 1675 } 1676 if (unlikely(wil->wdev->iftype == NL80211_IFTYPE_MONITOR)) { 1677 wil_err(wil, "Xmit in monitor mode not supported\n"); 1678 goto drop; 1679 } 1680 pr_once_fw = false; 1681 1682 /* find vring */ 1683 if (wil->wdev->iftype == NL80211_IFTYPE_STATION) { 1684 /* in STA mode (ESS), all to same VRING */ 1685 vring = wil_find_tx_vring_sta(wil, skb); 1686 } else { /* direct communication, find matching VRING */ 1687 vring = bcast ? wil_find_tx_bcast(wil, skb) : 1688 wil_find_tx_ucast(wil, skb); 1689 } 1690 if (unlikely(!vring)) { 1691 wil_dbg_txrx(wil, "No Tx VRING found for %pM\n", eth->h_dest); 1692 goto drop; 1693 } 1694 /* set up vring entry */ 1695 rc = wil_tx_vring(wil, vring, skb); 1696 1697 /* do we still have enough room in the vring? */ 1698 if (unlikely(wil_vring_avail_tx(vring) < wil_vring_wmark_low(vring))) { 1699 netif_tx_stop_all_queues(wil_to_ndev(wil)); 1700 wil_dbg_txrx(wil, "netif_tx_stop : ring full\n"); 1701 } 1702 1703 switch (rc) { 1704 case 0: 1705 /* statistics will be updated on the tx_complete */ 1706 dev_kfree_skb_any(skb); 1707 return NETDEV_TX_OK; 1708 case -ENOMEM: 1709 return NETDEV_TX_BUSY; 1710 default: 1711 break; /* goto drop; */ 1712 } 1713 drop: 1714 ndev->stats.tx_dropped++; 1715 dev_kfree_skb_any(skb); 1716 1717 return NET_XMIT_DROP; 1718 } 1719 1720 static inline bool wil_need_txstat(struct sk_buff *skb) 1721 { 1722 struct ethhdr *eth = (void *)skb->data; 1723 1724 return is_unicast_ether_addr(eth->h_dest) && skb->sk && 1725 (skb_shinfo(skb)->tx_flags & SKBTX_WIFI_STATUS); 1726 } 1727 1728 static inline void wil_consume_skb(struct sk_buff *skb, bool acked) 1729 { 1730 if (unlikely(wil_need_txstat(skb))) 1731 skb_complete_wifi_ack(skb, acked); 1732 else 1733 acked ? dev_consume_skb_any(skb) : dev_kfree_skb_any(skb); 1734 } 1735 1736 /** 1737 * Clean up transmitted skb's from the Tx VRING 1738 * 1739 * Return number of descriptors cleared 1740 * 1741 * Safe to call from IRQ 1742 */ 1743 int wil_tx_complete(struct wil6210_priv *wil, int ringid) 1744 { 1745 struct net_device *ndev = wil_to_ndev(wil); 1746 struct device *dev = wil_to_dev(wil); 1747 struct vring *vring = &wil->vring_tx[ringid]; 1748 struct vring_tx_data *txdata = &wil->vring_tx_data[ringid]; 1749 int done = 0; 1750 int cid = wil->vring2cid_tid[ringid][0]; 1751 struct wil_net_stats *stats = NULL; 1752 volatile struct vring_tx_desc *_d; 1753 int used_before_complete; 1754 int used_new; 1755 1756 if (unlikely(!vring->va)) { 1757 wil_err(wil, "Tx irq[%d]: vring not initialized\n", ringid); 1758 return 0; 1759 } 1760 1761 if (unlikely(!txdata->enabled)) { 1762 wil_info(wil, "Tx irq[%d]: vring disabled\n", ringid); 1763 return 0; 1764 } 1765 1766 wil_dbg_txrx(wil, "%s(%d)\n", __func__, ringid); 1767 1768 used_before_complete = wil_vring_used_tx(vring); 1769 1770 if (cid < WIL6210_MAX_CID) 1771 stats = &wil->sta[cid].stats; 1772 1773 while (!wil_vring_is_empty(vring)) { 1774 int new_swtail; 1775 struct wil_ctx *ctx = &vring->ctx[vring->swtail]; 1776 /** 1777 * For the fragmented skb, HW will set DU bit only for the 1778 * last fragment. look for it. 1779 * In TSO the first DU will include hdr desc 1780 */ 1781 int lf = (vring->swtail + ctx->nr_frags) % vring->size; 1782 /* TODO: check we are not past head */ 1783 1784 _d = &vring->va[lf].tx; 1785 if (unlikely(!(_d->dma.status & TX_DMA_STATUS_DU))) 1786 break; 1787 1788 new_swtail = (lf + 1) % vring->size; 1789 while (vring->swtail != new_swtail) { 1790 struct vring_tx_desc dd, *d = ⅆ 1791 u16 dmalen; 1792 struct sk_buff *skb; 1793 1794 ctx = &vring->ctx[vring->swtail]; 1795 skb = ctx->skb; 1796 _d = &vring->va[vring->swtail].tx; 1797 1798 *d = *_d; 1799 1800 dmalen = le16_to_cpu(d->dma.length); 1801 trace_wil6210_tx_done(ringid, vring->swtail, dmalen, 1802 d->dma.error); 1803 wil_dbg_txrx(wil, 1804 "TxC[%2d][%3d] : %d bytes, status 0x%02x err 0x%02x\n", 1805 ringid, vring->swtail, dmalen, 1806 d->dma.status, d->dma.error); 1807 wil_hex_dump_txrx("TxCD ", DUMP_PREFIX_NONE, 32, 4, 1808 (const void *)d, sizeof(*d), false); 1809 1810 wil_txdesc_unmap(dev, d, ctx); 1811 1812 if (skb) { 1813 if (likely(d->dma.error == 0)) { 1814 ndev->stats.tx_packets++; 1815 ndev->stats.tx_bytes += skb->len; 1816 if (stats) { 1817 stats->tx_packets++; 1818 stats->tx_bytes += skb->len; 1819 } 1820 } else { 1821 ndev->stats.tx_errors++; 1822 if (stats) 1823 stats->tx_errors++; 1824 } 1825 wil_consume_skb(skb, d->dma.error == 0); 1826 } 1827 memset(ctx, 0, sizeof(*ctx)); 1828 /* There is no need to touch HW descriptor: 1829 * - ststus bit TX_DMA_STATUS_DU is set by design, 1830 * so hardware will not try to process this desc., 1831 * - rest of descriptor will be initialized on Tx. 1832 */ 1833 vring->swtail = wil_vring_next_tail(vring); 1834 done++; 1835 } 1836 } 1837 1838 /* performance monitoring */ 1839 used_new = wil_vring_used_tx(vring); 1840 if (wil_val_in_range(vring_idle_trsh, 1841 used_new, used_before_complete)) { 1842 wil_dbg_txrx(wil, "Ring[%2d] idle %d -> %d\n", 1843 ringid, used_before_complete, used_new); 1844 txdata->last_idle = get_cycles(); 1845 } 1846 1847 if (wil_vring_avail_tx(vring) > wil_vring_wmark_high(vring)) { 1848 wil_dbg_txrx(wil, "netif_tx_wake : ring not full\n"); 1849 netif_tx_wake_all_queues(wil_to_ndev(wil)); 1850 } 1851 1852 return done; 1853 } 1854