1 // SPDX-License-Identifier: ISC
2 /*
3  * Copyright (c) 2012-2017 Qualcomm Atheros, Inc.
4  * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
5  */
6 
7 #include <linux/etherdevice.h>
8 #include <net/ieee80211_radiotap.h>
9 #include <linux/if_arp.h>
10 #include <linux/moduleparam.h>
11 #include <linux/ip.h>
12 #include <linux/ipv6.h>
13 #include <linux/if_vlan.h>
14 #include <net/ipv6.h>
15 #include <linux/prefetch.h>
16 
17 #include "wil6210.h"
18 #include "wmi.h"
19 #include "txrx.h"
20 #include "trace.h"
21 #include "txrx_edma.h"
22 
23 bool rx_align_2;
24 module_param(rx_align_2, bool, 0444);
25 MODULE_PARM_DESC(rx_align_2, " align Rx buffers on 4*n+2, default - no");
26 
27 bool rx_large_buf;
28 module_param(rx_large_buf, bool, 0444);
29 MODULE_PARM_DESC(rx_large_buf, " allocate 8KB RX buffers, default - no");
30 
31 /* Drop Tx packets in case Tx ring is full */
32 bool drop_if_ring_full;
33 
34 static inline uint wil_rx_snaplen(void)
35 {
36 	return rx_align_2 ? 6 : 0;
37 }
38 
39 /* wil_ring_wmark_low - low watermark for available descriptor space */
40 static inline int wil_ring_wmark_low(struct wil_ring *ring)
41 {
42 	return ring->size / 8;
43 }
44 
45 /* wil_ring_wmark_high - high watermark for available descriptor space */
46 static inline int wil_ring_wmark_high(struct wil_ring *ring)
47 {
48 	return ring->size / 4;
49 }
50 
51 /* returns true if num avail descriptors is lower than wmark_low */
52 static inline int wil_ring_avail_low(struct wil_ring *ring)
53 {
54 	return wil_ring_avail_tx(ring) < wil_ring_wmark_low(ring);
55 }
56 
57 /* returns true if num avail descriptors is higher than wmark_high */
58 static inline int wil_ring_avail_high(struct wil_ring *ring)
59 {
60 	return wil_ring_avail_tx(ring) > wil_ring_wmark_high(ring);
61 }
62 
63 /* returns true when all tx vrings are empty */
64 bool wil_is_tx_idle(struct wil6210_priv *wil)
65 {
66 	int i;
67 	unsigned long data_comp_to;
68 	int min_ring_id = wil_get_min_tx_ring_id(wil);
69 
70 	for (i = min_ring_id; i < WIL6210_MAX_TX_RINGS; i++) {
71 		struct wil_ring *vring = &wil->ring_tx[i];
72 		int vring_index = vring - wil->ring_tx;
73 		struct wil_ring_tx_data *txdata =
74 			&wil->ring_tx_data[vring_index];
75 
76 		spin_lock(&txdata->lock);
77 
78 		if (!vring->va || !txdata->enabled) {
79 			spin_unlock(&txdata->lock);
80 			continue;
81 		}
82 
83 		data_comp_to = jiffies + msecs_to_jiffies(
84 					WIL_DATA_COMPLETION_TO_MS);
85 		if (test_bit(wil_status_napi_en, wil->status)) {
86 			while (!wil_ring_is_empty(vring)) {
87 				if (time_after(jiffies, data_comp_to)) {
88 					wil_dbg_pm(wil,
89 						   "TO waiting for idle tx\n");
90 					spin_unlock(&txdata->lock);
91 					return false;
92 				}
93 				wil_dbg_ratelimited(wil,
94 						    "tx vring is not empty -> NAPI\n");
95 				spin_unlock(&txdata->lock);
96 				napi_synchronize(&wil->napi_tx);
97 				msleep(20);
98 				spin_lock(&txdata->lock);
99 				if (!vring->va || !txdata->enabled)
100 					break;
101 			}
102 		}
103 
104 		spin_unlock(&txdata->lock);
105 	}
106 
107 	return true;
108 }
109 
110 static int wil_vring_alloc(struct wil6210_priv *wil, struct wil_ring *vring)
111 {
112 	struct device *dev = wil_to_dev(wil);
113 	size_t sz = vring->size * sizeof(vring->va[0]);
114 	uint i;
115 
116 	wil_dbg_misc(wil, "vring_alloc:\n");
117 
118 	BUILD_BUG_ON(sizeof(vring->va[0]) != 32);
119 
120 	vring->swhead = 0;
121 	vring->swtail = 0;
122 	vring->ctx = kcalloc(vring->size, sizeof(vring->ctx[0]), GFP_KERNEL);
123 	if (!vring->ctx) {
124 		vring->va = NULL;
125 		return -ENOMEM;
126 	}
127 
128 	/* vring->va should be aligned on its size rounded up to power of 2
129 	 * This is granted by the dma_alloc_coherent.
130 	 *
131 	 * HW has limitation that all vrings addresses must share the same
132 	 * upper 16 msb bits part of 48 bits address. To workaround that,
133 	 * if we are using more than 32 bit addresses switch to 32 bit
134 	 * allocation before allocating vring memory.
135 	 *
136 	 * There's no check for the return value of dma_set_mask_and_coherent,
137 	 * since we assume if we were able to set the mask during
138 	 * initialization in this system it will not fail if we set it again
139 	 */
140 	if (wil->dma_addr_size > 32)
141 		dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
142 
143 	vring->va = dma_alloc_coherent(dev, sz, &vring->pa, GFP_KERNEL);
144 	if (!vring->va) {
145 		kfree(vring->ctx);
146 		vring->ctx = NULL;
147 		return -ENOMEM;
148 	}
149 
150 	if (wil->dma_addr_size > 32)
151 		dma_set_mask_and_coherent(dev,
152 					  DMA_BIT_MASK(wil->dma_addr_size));
153 
154 	/* initially, all descriptors are SW owned
155 	 * For Tx and Rx, ownership bit is at the same location, thus
156 	 * we can use any
157 	 */
158 	for (i = 0; i < vring->size; i++) {
159 		volatile struct vring_tx_desc *_d =
160 			&vring->va[i].tx.legacy;
161 
162 		_d->dma.status = TX_DMA_STATUS_DU;
163 	}
164 
165 	wil_dbg_misc(wil, "vring[%d] 0x%p:%pad 0x%p\n", vring->size,
166 		     vring->va, &vring->pa, vring->ctx);
167 
168 	return 0;
169 }
170 
171 static void wil_txdesc_unmap(struct device *dev, union wil_tx_desc *desc,
172 			     struct wil_ctx *ctx)
173 {
174 	struct vring_tx_desc *d = &desc->legacy;
175 	dma_addr_t pa = wil_desc_addr(&d->dma.addr);
176 	u16 dmalen = le16_to_cpu(d->dma.length);
177 
178 	switch (ctx->mapped_as) {
179 	case wil_mapped_as_single:
180 		dma_unmap_single(dev, pa, dmalen, DMA_TO_DEVICE);
181 		break;
182 	case wil_mapped_as_page:
183 		dma_unmap_page(dev, pa, dmalen, DMA_TO_DEVICE);
184 		break;
185 	default:
186 		break;
187 	}
188 }
189 
190 static void wil_vring_free(struct wil6210_priv *wil, struct wil_ring *vring)
191 {
192 	struct device *dev = wil_to_dev(wil);
193 	size_t sz = vring->size * sizeof(vring->va[0]);
194 
195 	lockdep_assert_held(&wil->mutex);
196 	if (!vring->is_rx) {
197 		int vring_index = vring - wil->ring_tx;
198 
199 		wil_dbg_misc(wil, "free Tx vring %d [%d] 0x%p:%pad 0x%p\n",
200 			     vring_index, vring->size, vring->va,
201 			     &vring->pa, vring->ctx);
202 	} else {
203 		wil_dbg_misc(wil, "free Rx vring [%d] 0x%p:%pad 0x%p\n",
204 			     vring->size, vring->va,
205 			     &vring->pa, vring->ctx);
206 	}
207 
208 	while (!wil_ring_is_empty(vring)) {
209 		dma_addr_t pa;
210 		u16 dmalen;
211 		struct wil_ctx *ctx;
212 
213 		if (!vring->is_rx) {
214 			struct vring_tx_desc dd, *d = &dd;
215 			volatile struct vring_tx_desc *_d =
216 					&vring->va[vring->swtail].tx.legacy;
217 
218 			ctx = &vring->ctx[vring->swtail];
219 			if (!ctx) {
220 				wil_dbg_txrx(wil,
221 					     "ctx(%d) was already completed\n",
222 					     vring->swtail);
223 				vring->swtail = wil_ring_next_tail(vring);
224 				continue;
225 			}
226 			*d = *_d;
227 			wil_txdesc_unmap(dev, (union wil_tx_desc *)d, ctx);
228 			if (ctx->skb)
229 				dev_kfree_skb_any(ctx->skb);
230 			vring->swtail = wil_ring_next_tail(vring);
231 		} else { /* rx */
232 			struct vring_rx_desc dd, *d = &dd;
233 			volatile struct vring_rx_desc *_d =
234 				&vring->va[vring->swhead].rx.legacy;
235 
236 			ctx = &vring->ctx[vring->swhead];
237 			*d = *_d;
238 			pa = wil_desc_addr(&d->dma.addr);
239 			dmalen = le16_to_cpu(d->dma.length);
240 			dma_unmap_single(dev, pa, dmalen, DMA_FROM_DEVICE);
241 			kfree_skb(ctx->skb);
242 			wil_ring_advance_head(vring, 1);
243 		}
244 	}
245 	dma_free_coherent(dev, sz, (void *)vring->va, vring->pa);
246 	kfree(vring->ctx);
247 	vring->pa = 0;
248 	vring->va = NULL;
249 	vring->ctx = NULL;
250 }
251 
252 /* Allocate one skb for Rx VRING
253  *
254  * Safe to call from IRQ
255  */
256 static int wil_vring_alloc_skb(struct wil6210_priv *wil, struct wil_ring *vring,
257 			       u32 i, int headroom)
258 {
259 	struct device *dev = wil_to_dev(wil);
260 	unsigned int sz = wil->rx_buf_len + ETH_HLEN + wil_rx_snaplen();
261 	struct vring_rx_desc dd, *d = &dd;
262 	volatile struct vring_rx_desc *_d = &vring->va[i].rx.legacy;
263 	dma_addr_t pa;
264 	struct sk_buff *skb = dev_alloc_skb(sz + headroom);
265 
266 	if (unlikely(!skb))
267 		return -ENOMEM;
268 
269 	skb_reserve(skb, headroom);
270 	skb_put(skb, sz);
271 
272 	/**
273 	 * Make sure that the network stack calculates checksum for packets
274 	 * which failed the HW checksum calculation
275 	 */
276 	skb->ip_summed = CHECKSUM_NONE;
277 
278 	pa = dma_map_single(dev, skb->data, skb->len, DMA_FROM_DEVICE);
279 	if (unlikely(dma_mapping_error(dev, pa))) {
280 		kfree_skb(skb);
281 		return -ENOMEM;
282 	}
283 
284 	d->dma.d0 = RX_DMA_D0_CMD_DMA_RT | RX_DMA_D0_CMD_DMA_IT;
285 	wil_desc_addr_set(&d->dma.addr, pa);
286 	/* ip_length don't care */
287 	/* b11 don't care */
288 	/* error don't care */
289 	d->dma.status = 0; /* BIT(0) should be 0 for HW_OWNED */
290 	d->dma.length = cpu_to_le16(sz);
291 	*_d = *d;
292 	vring->ctx[i].skb = skb;
293 
294 	return 0;
295 }
296 
297 /* Adds radiotap header
298  *
299  * Any error indicated as "Bad FCS"
300  *
301  * Vendor data for 04:ce:14-1 (Wilocity-1) consists of:
302  *  - Rx descriptor: 32 bytes
303  *  - Phy info
304  */
305 static void wil_rx_add_radiotap_header(struct wil6210_priv *wil,
306 				       struct sk_buff *skb)
307 {
308 	struct wil6210_rtap {
309 		struct ieee80211_radiotap_header rthdr;
310 		/* fields should be in the order of bits in rthdr.it_present */
311 		/* flags */
312 		u8 flags;
313 		/* channel */
314 		__le16 chnl_freq __aligned(2);
315 		__le16 chnl_flags;
316 		/* MCS */
317 		u8 mcs_present;
318 		u8 mcs_flags;
319 		u8 mcs_index;
320 	} __packed;
321 	struct vring_rx_desc *d = wil_skb_rxdesc(skb);
322 	struct wil6210_rtap *rtap;
323 	int rtap_len = sizeof(struct wil6210_rtap);
324 	struct ieee80211_channel *ch = wil->monitor_chandef.chan;
325 
326 	if (skb_headroom(skb) < rtap_len &&
327 	    pskb_expand_head(skb, rtap_len, 0, GFP_ATOMIC)) {
328 		wil_err(wil, "Unable to expand headroom to %d\n", rtap_len);
329 		return;
330 	}
331 
332 	rtap = skb_push(skb, rtap_len);
333 	memset(rtap, 0, rtap_len);
334 
335 	rtap->rthdr.it_version = PKTHDR_RADIOTAP_VERSION;
336 	rtap->rthdr.it_len = cpu_to_le16(rtap_len);
337 	rtap->rthdr.it_present = cpu_to_le32((1 << IEEE80211_RADIOTAP_FLAGS) |
338 			(1 << IEEE80211_RADIOTAP_CHANNEL) |
339 			(1 << IEEE80211_RADIOTAP_MCS));
340 	if (d->dma.status & RX_DMA_STATUS_ERROR)
341 		rtap->flags |= IEEE80211_RADIOTAP_F_BADFCS;
342 
343 	rtap->chnl_freq = cpu_to_le16(ch ? ch->center_freq : 58320);
344 	rtap->chnl_flags = cpu_to_le16(0);
345 
346 	rtap->mcs_present = IEEE80211_RADIOTAP_MCS_HAVE_MCS;
347 	rtap->mcs_flags = 0;
348 	rtap->mcs_index = wil_rxdesc_mcs(d);
349 }
350 
351 static bool wil_is_rx_idle(struct wil6210_priv *wil)
352 {
353 	struct vring_rx_desc *_d;
354 	struct wil_ring *ring = &wil->ring_rx;
355 
356 	_d = (struct vring_rx_desc *)&ring->va[ring->swhead].rx.legacy;
357 	if (_d->dma.status & RX_DMA_STATUS_DU)
358 		return false;
359 
360 	return true;
361 }
362 
363 static int wil_rx_get_cid_by_skb(struct wil6210_priv *wil, struct sk_buff *skb)
364 {
365 	struct vring_rx_desc *d = wil_skb_rxdesc(skb);
366 	int mid = wil_rxdesc_mid(d);
367 	struct wil6210_vif *vif = wil->vifs[mid];
368 	/* cid from DMA descriptor is limited to 3 bits.
369 	 * In case of cid>=8, the value would be cid modulo 8 and we need to
370 	 * find real cid by locating the transmitter (ta) inside sta array
371 	 */
372 	int cid = wil_rxdesc_cid(d);
373 	unsigned int snaplen = wil_rx_snaplen();
374 	struct ieee80211_hdr_3addr *hdr;
375 	int i;
376 	unsigned char *ta;
377 	u8 ftype;
378 
379 	/* in monitor mode there are no connections */
380 	if (vif->wdev.iftype == NL80211_IFTYPE_MONITOR)
381 		return cid;
382 
383 	ftype = wil_rxdesc_ftype(d) << 2;
384 	if (likely(ftype == IEEE80211_FTYPE_DATA)) {
385 		if (unlikely(skb->len < ETH_HLEN + snaplen)) {
386 			wil_err_ratelimited(wil,
387 					    "Short data frame, len = %d\n",
388 					    skb->len);
389 			return -ENOENT;
390 		}
391 		ta = wil_skb_get_sa(skb);
392 	} else {
393 		if (unlikely(skb->len < sizeof(struct ieee80211_hdr_3addr))) {
394 			wil_err_ratelimited(wil, "Short frame, len = %d\n",
395 					    skb->len);
396 			return -ENOENT;
397 		}
398 		hdr = (void *)skb->data;
399 		ta = hdr->addr2;
400 	}
401 
402 	if (wil->max_assoc_sta <= WIL6210_RX_DESC_MAX_CID)
403 		return cid;
404 
405 	/* assuming no concurrency between AP interfaces and STA interfaces.
406 	 * multista is used only in P2P_GO or AP mode. In other modes return
407 	 * cid from the rx descriptor
408 	 */
409 	if (vif->wdev.iftype != NL80211_IFTYPE_P2P_GO &&
410 	    vif->wdev.iftype != NL80211_IFTYPE_AP)
411 		return cid;
412 
413 	/* For Rx packets cid from rx descriptor is limited to 3 bits (0..7),
414 	 * to find the real cid, compare transmitter address with the stored
415 	 * stations mac address in the driver sta array
416 	 */
417 	for (i = cid; i < wil->max_assoc_sta; i += WIL6210_RX_DESC_MAX_CID) {
418 		if (wil->sta[i].status != wil_sta_unused &&
419 		    ether_addr_equal(wil->sta[i].addr, ta)) {
420 			cid = i;
421 			break;
422 		}
423 	}
424 	if (i >= wil->max_assoc_sta) {
425 		wil_err_ratelimited(wil, "Could not find cid for frame with transmit addr = %pM, iftype = %d, frametype = %d, len = %d\n",
426 				    ta, vif->wdev.iftype, ftype, skb->len);
427 		cid = -ENOENT;
428 	}
429 
430 	return cid;
431 }
432 
433 /* reap 1 frame from @swhead
434  *
435  * Rx descriptor copied to skb->cb
436  *
437  * Safe to call from IRQ
438  */
439 static struct sk_buff *wil_vring_reap_rx(struct wil6210_priv *wil,
440 					 struct wil_ring *vring)
441 {
442 	struct device *dev = wil_to_dev(wil);
443 	struct wil6210_vif *vif;
444 	struct net_device *ndev;
445 	volatile struct vring_rx_desc *_d;
446 	struct vring_rx_desc *d;
447 	struct sk_buff *skb;
448 	dma_addr_t pa;
449 	unsigned int snaplen = wil_rx_snaplen();
450 	unsigned int sz = wil->rx_buf_len + ETH_HLEN + snaplen;
451 	u16 dmalen;
452 	u8 ftype;
453 	int cid, mid;
454 	int i;
455 	struct wil_net_stats *stats;
456 
457 	BUILD_BUG_ON(sizeof(struct skb_rx_info) > sizeof(skb->cb));
458 
459 again:
460 	if (unlikely(wil_ring_is_empty(vring)))
461 		return NULL;
462 
463 	i = (int)vring->swhead;
464 	_d = &vring->va[i].rx.legacy;
465 	if (unlikely(!(_d->dma.status & RX_DMA_STATUS_DU))) {
466 		/* it is not error, we just reached end of Rx done area */
467 		return NULL;
468 	}
469 
470 	skb = vring->ctx[i].skb;
471 	vring->ctx[i].skb = NULL;
472 	wil_ring_advance_head(vring, 1);
473 	if (!skb) {
474 		wil_err(wil, "No Rx skb at [%d]\n", i);
475 		goto again;
476 	}
477 	d = wil_skb_rxdesc(skb);
478 	*d = *_d;
479 	pa = wil_desc_addr(&d->dma.addr);
480 
481 	dma_unmap_single(dev, pa, sz, DMA_FROM_DEVICE);
482 	dmalen = le16_to_cpu(d->dma.length);
483 
484 	trace_wil6210_rx(i, d);
485 	wil_dbg_txrx(wil, "Rx[%3d] : %d bytes\n", i, dmalen);
486 	wil_hex_dump_txrx("RxD ", DUMP_PREFIX_NONE, 32, 4,
487 			  (const void *)d, sizeof(*d), false);
488 
489 	mid = wil_rxdesc_mid(d);
490 	vif = wil->vifs[mid];
491 
492 	if (unlikely(!vif)) {
493 		wil_dbg_txrx(wil, "skipped RX descriptor with invalid mid %d",
494 			     mid);
495 		kfree_skb(skb);
496 		goto again;
497 	}
498 	ndev = vif_to_ndev(vif);
499 	if (unlikely(dmalen > sz)) {
500 		wil_err_ratelimited(wil, "Rx size too large: %d bytes!\n",
501 				    dmalen);
502 		kfree_skb(skb);
503 		goto again;
504 	}
505 	skb_trim(skb, dmalen);
506 
507 	prefetch(skb->data);
508 
509 	wil_hex_dump_txrx("Rx ", DUMP_PREFIX_OFFSET, 16, 1,
510 			  skb->data, skb_headlen(skb), false);
511 
512 	cid = wil_rx_get_cid_by_skb(wil, skb);
513 	if (cid == -ENOENT) {
514 		kfree_skb(skb);
515 		goto again;
516 	}
517 	wil_skb_set_cid(skb, (u8)cid);
518 	stats = &wil->sta[cid].stats;
519 
520 	stats->last_mcs_rx = wil_rxdesc_mcs(d);
521 	if (stats->last_mcs_rx < ARRAY_SIZE(stats->rx_per_mcs))
522 		stats->rx_per_mcs[stats->last_mcs_rx]++;
523 
524 	/* use radiotap header only if required */
525 	if (ndev->type == ARPHRD_IEEE80211_RADIOTAP)
526 		wil_rx_add_radiotap_header(wil, skb);
527 
528 	/* no extra checks if in sniffer mode */
529 	if (ndev->type != ARPHRD_ETHER)
530 		return skb;
531 	/* Non-data frames may be delivered through Rx DMA channel (ex: BAR)
532 	 * Driver should recognize it by frame type, that is found
533 	 * in Rx descriptor. If type is not data, it is 802.11 frame as is
534 	 */
535 	ftype = wil_rxdesc_ftype(d) << 2;
536 	if (unlikely(ftype != IEEE80211_FTYPE_DATA)) {
537 		u8 fc1 = wil_rxdesc_fc1(d);
538 		int tid = wil_rxdesc_tid(d);
539 		u16 seq = wil_rxdesc_seq(d);
540 
541 		wil_dbg_txrx(wil,
542 			     "Non-data frame FC[7:0] 0x%02x MID %d CID %d TID %d Seq 0x%03x\n",
543 			     fc1, mid, cid, tid, seq);
544 		stats->rx_non_data_frame++;
545 		if (wil_is_back_req(fc1)) {
546 			wil_dbg_txrx(wil,
547 				     "BAR: MID %d CID %d TID %d Seq 0x%03x\n",
548 				     mid, cid, tid, seq);
549 			wil_rx_bar(wil, vif, cid, tid, seq);
550 		} else {
551 			/* print again all info. One can enable only this
552 			 * without overhead for printing every Rx frame
553 			 */
554 			wil_dbg_txrx(wil,
555 				     "Unhandled non-data frame FC[7:0] 0x%02x MID %d CID %d TID %d Seq 0x%03x\n",
556 				     fc1, mid, cid, tid, seq);
557 			wil_hex_dump_txrx("RxD ", DUMP_PREFIX_NONE, 32, 4,
558 					  (const void *)d, sizeof(*d), false);
559 			wil_hex_dump_txrx("Rx ", DUMP_PREFIX_OFFSET, 16, 1,
560 					  skb->data, skb_headlen(skb), false);
561 		}
562 		kfree_skb(skb);
563 		goto again;
564 	}
565 
566 	/* L4 IDENT is on when HW calculated checksum, check status
567 	 * and in case of error drop the packet
568 	 * higher stack layers will handle retransmission (if required)
569 	 */
570 	if (likely(d->dma.status & RX_DMA_STATUS_L4I)) {
571 		/* L4 protocol identified, csum calculated */
572 		if (likely((d->dma.error & RX_DMA_ERROR_L4_ERR) == 0))
573 			skb->ip_summed = CHECKSUM_UNNECESSARY;
574 		/* If HW reports bad checksum, let IP stack re-check it
575 		 * For example, HW don't understand Microsoft IP stack that
576 		 * mis-calculates TCP checksum - if it should be 0x0,
577 		 * it writes 0xffff in violation of RFC 1624
578 		 */
579 		else
580 			stats->rx_csum_err++;
581 	}
582 
583 	if (snaplen) {
584 		/* Packet layout
585 		 * +-------+-------+---------+------------+------+
586 		 * | SA(6) | DA(6) | SNAP(6) | ETHTYPE(2) | DATA |
587 		 * +-------+-------+---------+------------+------+
588 		 * Need to remove SNAP, shifting SA and DA forward
589 		 */
590 		memmove(skb->data + snaplen, skb->data, 2 * ETH_ALEN);
591 		skb_pull(skb, snaplen);
592 	}
593 
594 	return skb;
595 }
596 
597 /* allocate and fill up to @count buffers in rx ring
598  * buffers posted at @swtail
599  * Note: we have a single RX queue for servicing all VIFs, but we
600  * allocate skbs with headroom according to main interface only. This
601  * means it will not work with monitor interface together with other VIFs.
602  * Currently we only support monitor interface on its own without other VIFs,
603  * and we will need to fix this code once we add support.
604  */
605 static int wil_rx_refill(struct wil6210_priv *wil, int count)
606 {
607 	struct net_device *ndev = wil->main_ndev;
608 	struct wil_ring *v = &wil->ring_rx;
609 	u32 next_tail;
610 	int rc = 0;
611 	int headroom = ndev->type == ARPHRD_IEEE80211_RADIOTAP ?
612 			WIL6210_RTAP_SIZE : 0;
613 
614 	for (; next_tail = wil_ring_next_tail(v),
615 	     (next_tail != v->swhead) && (count-- > 0);
616 	     v->swtail = next_tail) {
617 		rc = wil_vring_alloc_skb(wil, v, v->swtail, headroom);
618 		if (unlikely(rc)) {
619 			wil_err_ratelimited(wil, "Error %d in rx refill[%d]\n",
620 					    rc, v->swtail);
621 			break;
622 		}
623 	}
624 
625 	/* make sure all writes to descriptors (shared memory) are done before
626 	 * committing them to HW
627 	 */
628 	wmb();
629 
630 	wil_w(wil, v->hwtail, v->swtail);
631 
632 	return rc;
633 }
634 
635 /**
636  * reverse_memcmp - Compare two areas of memory, in reverse order
637  * @cs: One area of memory
638  * @ct: Another area of memory
639  * @count: The size of the area.
640  *
641  * Cut'n'paste from original memcmp (see lib/string.c)
642  * with minimal modifications
643  */
644 int reverse_memcmp(const void *cs, const void *ct, size_t count)
645 {
646 	const unsigned char *su1, *su2;
647 	int res = 0;
648 
649 	for (su1 = cs + count - 1, su2 = ct + count - 1; count > 0;
650 	     --su1, --su2, count--) {
651 		res = *su1 - *su2;
652 		if (res)
653 			break;
654 	}
655 	return res;
656 }
657 
658 static int wil_rx_crypto_check(struct wil6210_priv *wil, struct sk_buff *skb)
659 {
660 	struct vring_rx_desc *d = wil_skb_rxdesc(skb);
661 	int cid = wil_skb_get_cid(skb);
662 	int tid = wil_rxdesc_tid(d);
663 	int key_id = wil_rxdesc_key_id(d);
664 	int mc = wil_rxdesc_mcast(d);
665 	struct wil_sta_info *s = &wil->sta[cid];
666 	struct wil_tid_crypto_rx *c = mc ? &s->group_crypto_rx :
667 				      &s->tid_crypto_rx[tid];
668 	struct wil_tid_crypto_rx_single *cc = &c->key_id[key_id];
669 	const u8 *pn = (u8 *)&d->mac.pn_15_0;
670 
671 	if (!cc->key_set) {
672 		wil_err_ratelimited(wil,
673 				    "Key missing. CID %d TID %d MCast %d KEY_ID %d\n",
674 				    cid, tid, mc, key_id);
675 		return -EINVAL;
676 	}
677 
678 	if (reverse_memcmp(pn, cc->pn, IEEE80211_GCMP_PN_LEN) <= 0) {
679 		wil_err_ratelimited(wil,
680 				    "Replay attack. CID %d TID %d MCast %d KEY_ID %d PN %6phN last %6phN\n",
681 				    cid, tid, mc, key_id, pn, cc->pn);
682 		return -EINVAL;
683 	}
684 	memcpy(cc->pn, pn, IEEE80211_GCMP_PN_LEN);
685 
686 	return 0;
687 }
688 
689 static int wil_rx_error_check(struct wil6210_priv *wil, struct sk_buff *skb,
690 			      struct wil_net_stats *stats)
691 {
692 	struct vring_rx_desc *d = wil_skb_rxdesc(skb);
693 
694 	if ((d->dma.status & RX_DMA_STATUS_ERROR) &&
695 	    (d->dma.error & RX_DMA_ERROR_MIC)) {
696 		stats->rx_mic_error++;
697 		wil_dbg_txrx(wil, "MIC error, dropping packet\n");
698 		return -EFAULT;
699 	}
700 
701 	return 0;
702 }
703 
704 static void wil_get_netif_rx_params(struct sk_buff *skb, int *cid,
705 				    int *security)
706 {
707 	struct vring_rx_desc *d = wil_skb_rxdesc(skb);
708 
709 	*cid = wil_skb_get_cid(skb);
710 	*security = wil_rxdesc_security(d);
711 }
712 
713 /*
714  * Check if skb is ptk eapol key message
715  *
716  * returns a pointer to the start of the eapol key structure, NULL
717  * if frame is not PTK eapol key
718  */
719 static struct wil_eapol_key *wil_is_ptk_eapol_key(struct wil6210_priv *wil,
720 						  struct sk_buff *skb)
721 {
722 	u8 *buf;
723 	const struct wil_1x_hdr *hdr;
724 	struct wil_eapol_key *key;
725 	u16 key_info;
726 	int len = skb->len;
727 
728 	if (!skb_mac_header_was_set(skb)) {
729 		wil_err(wil, "mac header was not set\n");
730 		return NULL;
731 	}
732 
733 	len -= skb_mac_offset(skb);
734 
735 	if (len < sizeof(struct ethhdr) + sizeof(struct wil_1x_hdr) +
736 	    sizeof(struct wil_eapol_key))
737 		return NULL;
738 
739 	buf = skb_mac_header(skb) + sizeof(struct ethhdr);
740 
741 	hdr = (const struct wil_1x_hdr *)buf;
742 	if (hdr->type != WIL_1X_TYPE_EAPOL_KEY)
743 		return NULL;
744 
745 	key = (struct wil_eapol_key *)(buf + sizeof(struct wil_1x_hdr));
746 	if (key->type != WIL_EAPOL_KEY_TYPE_WPA &&
747 	    key->type != WIL_EAPOL_KEY_TYPE_RSN)
748 		return NULL;
749 
750 	key_info = be16_to_cpu(key->key_info);
751 	if (!(key_info & WIL_KEY_INFO_KEY_TYPE)) /* check if pairwise */
752 		return NULL;
753 
754 	return key;
755 }
756 
757 static bool wil_skb_is_eap_3(struct wil6210_priv *wil, struct sk_buff *skb)
758 {
759 	struct wil_eapol_key *key;
760 	u16 key_info;
761 
762 	key = wil_is_ptk_eapol_key(wil, skb);
763 	if (!key)
764 		return false;
765 
766 	key_info = be16_to_cpu(key->key_info);
767 	if (key_info & (WIL_KEY_INFO_MIC |
768 			WIL_KEY_INFO_ENCR_KEY_DATA)) {
769 		/* 3/4 of 4-Way Handshake */
770 		wil_dbg_misc(wil, "EAPOL key message 3\n");
771 		return true;
772 	}
773 	/* 1/4 of 4-Way Handshake */
774 	wil_dbg_misc(wil, "EAPOL key message 1\n");
775 
776 	return false;
777 }
778 
779 static bool wil_skb_is_eap_4(struct wil6210_priv *wil, struct sk_buff *skb)
780 {
781 	struct wil_eapol_key *key;
782 	u32 *nonce, i;
783 
784 	key = wil_is_ptk_eapol_key(wil, skb);
785 	if (!key)
786 		return false;
787 
788 	nonce = (u32 *)key->key_nonce;
789 	for (i = 0; i < WIL_EAP_NONCE_LEN / sizeof(u32); i++, nonce++) {
790 		if (*nonce != 0) {
791 			/* message 2/4 */
792 			wil_dbg_misc(wil, "EAPOL key message 2\n");
793 			return false;
794 		}
795 	}
796 	wil_dbg_misc(wil, "EAPOL key message 4\n");
797 
798 	return true;
799 }
800 
801 void wil_enable_tx_key_worker(struct work_struct *work)
802 {
803 	struct wil6210_vif *vif = container_of(work,
804 			struct wil6210_vif, enable_tx_key_worker);
805 	struct wil6210_priv *wil = vif_to_wil(vif);
806 	int rc, cid;
807 
808 	rtnl_lock();
809 	if (vif->ptk_rekey_state != WIL_REKEY_WAIT_M4_SENT) {
810 		wil_dbg_misc(wil, "Invalid rekey state = %d\n",
811 			     vif->ptk_rekey_state);
812 		rtnl_unlock();
813 		return;
814 	}
815 
816 	cid =  wil_find_cid_by_idx(wil, vif->mid, 0);
817 	if (!wil_cid_valid(wil, cid)) {
818 		wil_err(wil, "Invalid cid = %d\n", cid);
819 		rtnl_unlock();
820 		return;
821 	}
822 
823 	wil_dbg_misc(wil, "Apply PTK key after eapol was sent out\n");
824 	rc = wmi_add_cipher_key(vif, 0, wil->sta[cid].addr, 0, NULL,
825 				WMI_KEY_USE_APPLY_PTK);
826 
827 	vif->ptk_rekey_state = WIL_REKEY_IDLE;
828 	rtnl_unlock();
829 
830 	if (rc)
831 		wil_err(wil, "Apply PTK key failed %d\n", rc);
832 }
833 
834 void wil_tx_complete_handle_eapol(struct wil6210_vif *vif, struct sk_buff *skb)
835 {
836 	struct wil6210_priv *wil = vif_to_wil(vif);
837 	struct wireless_dev *wdev = vif_to_wdev(vif);
838 	bool q = false;
839 
840 	if (wdev->iftype != NL80211_IFTYPE_STATION ||
841 	    !test_bit(WMI_FW_CAPABILITY_SPLIT_REKEY, wil->fw_capabilities))
842 		return;
843 
844 	/* check if skb is an EAP message 4/4 */
845 	if (!wil_skb_is_eap_4(wil, skb))
846 		return;
847 
848 	spin_lock_bh(&wil->eap_lock);
849 	switch (vif->ptk_rekey_state) {
850 	case WIL_REKEY_IDLE:
851 		/* ignore idle state, can happen due to M4 retransmission */
852 		break;
853 	case WIL_REKEY_M3_RECEIVED:
854 		vif->ptk_rekey_state = WIL_REKEY_IDLE;
855 		break;
856 	case WIL_REKEY_WAIT_M4_SENT:
857 		q = true;
858 		break;
859 	default:
860 		wil_err(wil, "Unknown rekey state = %d",
861 			vif->ptk_rekey_state);
862 	}
863 	spin_unlock_bh(&wil->eap_lock);
864 
865 	if (q) {
866 		q = queue_work(wil->wmi_wq, &vif->enable_tx_key_worker);
867 		wil_dbg_misc(wil, "queue_work of enable_tx_key_worker -> %d\n",
868 			     q);
869 	}
870 }
871 
872 static void wil_rx_handle_eapol(struct wil6210_vif *vif, struct sk_buff *skb)
873 {
874 	struct wil6210_priv *wil = vif_to_wil(vif);
875 	struct wireless_dev *wdev = vif_to_wdev(vif);
876 
877 	if (wdev->iftype != NL80211_IFTYPE_STATION ||
878 	    !test_bit(WMI_FW_CAPABILITY_SPLIT_REKEY, wil->fw_capabilities))
879 		return;
880 
881 	/* check if skb is a EAP message 3/4 */
882 	if (!wil_skb_is_eap_3(wil, skb))
883 		return;
884 
885 	if (vif->ptk_rekey_state == WIL_REKEY_IDLE)
886 		vif->ptk_rekey_state = WIL_REKEY_M3_RECEIVED;
887 }
888 
889 /*
890  * Pass Rx packet to the netif. Update statistics.
891  * Called in softirq context (NAPI poll).
892  */
893 void wil_netif_rx(struct sk_buff *skb, struct net_device *ndev, int cid,
894 		  struct wil_net_stats *stats, bool gro)
895 {
896 	struct wil6210_vif *vif = ndev_to_vif(ndev);
897 	struct wil6210_priv *wil = ndev_to_wil(ndev);
898 	struct wireless_dev *wdev = vif_to_wdev(vif);
899 	unsigned int len = skb->len;
900 	u8 *sa, *da = wil_skb_get_da(skb);
901 	/* here looking for DA, not A1, thus Rxdesc's 'mcast' indication
902 	 * is not suitable, need to look at data
903 	 */
904 	int mcast = is_multicast_ether_addr(da);
905 	struct sk_buff *xmit_skb = NULL;
906 
907 	if (wdev->iftype == NL80211_IFTYPE_STATION) {
908 		sa = wil_skb_get_sa(skb);
909 		if (mcast && ether_addr_equal(sa, ndev->dev_addr)) {
910 			/* mcast packet looped back to us */
911 			dev_kfree_skb(skb);
912 			ndev->stats.rx_dropped++;
913 			stats->rx_dropped++;
914 			wil_dbg_txrx(wil, "Rx drop %d bytes\n", len);
915 			return;
916 		}
917 	} else if (wdev->iftype == NL80211_IFTYPE_AP && !vif->ap_isolate) {
918 		if (mcast) {
919 			/* send multicast frames both to higher layers in
920 			 * local net stack and back to the wireless medium
921 			 */
922 			xmit_skb = skb_copy(skb, GFP_ATOMIC);
923 		} else {
924 			int xmit_cid = wil_find_cid(wil, vif->mid, da);
925 
926 			if (xmit_cid >= 0) {
927 				/* The destination station is associated to
928 				 * this AP (in this VLAN), so send the frame
929 				 * directly to it and do not pass it to local
930 				 * net stack.
931 				 */
932 				xmit_skb = skb;
933 				skb = NULL;
934 			}
935 		}
936 	}
937 	if (xmit_skb) {
938 		/* Send to wireless media and increase priority by 256 to
939 		 * keep the received priority instead of reclassifying
940 		 * the frame (see cfg80211_classify8021d).
941 		 */
942 		xmit_skb->dev = ndev;
943 		xmit_skb->priority += 256;
944 		xmit_skb->protocol = htons(ETH_P_802_3);
945 		skb_reset_network_header(xmit_skb);
946 		skb_reset_mac_header(xmit_skb);
947 		wil_dbg_txrx(wil, "Rx -> Tx %d bytes\n", len);
948 		dev_queue_xmit(xmit_skb);
949 	}
950 
951 	if (skb) { /* deliver to local stack */
952 		skb->protocol = eth_type_trans(skb, ndev);
953 		skb->dev = ndev;
954 
955 		if (skb->protocol == cpu_to_be16(ETH_P_PAE))
956 			wil_rx_handle_eapol(vif, skb);
957 
958 		if (gro)
959 			napi_gro_receive(&wil->napi_rx, skb);
960 		else
961 			netif_rx(skb);
962 	}
963 	ndev->stats.rx_packets++;
964 	stats->rx_packets++;
965 	ndev->stats.rx_bytes += len;
966 	stats->rx_bytes += len;
967 	if (mcast)
968 		ndev->stats.multicast++;
969 }
970 
971 void wil_netif_rx_any(struct sk_buff *skb, struct net_device *ndev)
972 {
973 	int cid, security;
974 	struct wil6210_priv *wil = ndev_to_wil(ndev);
975 	struct wil_net_stats *stats;
976 
977 	wil->txrx_ops.get_netif_rx_params(skb, &cid, &security);
978 
979 	stats = &wil->sta[cid].stats;
980 
981 	skb_orphan(skb);
982 
983 	if (security && (wil->txrx_ops.rx_crypto_check(wil, skb) != 0)) {
984 		wil_dbg_txrx(wil, "Rx drop %d bytes\n", skb->len);
985 		dev_kfree_skb(skb);
986 		ndev->stats.rx_dropped++;
987 		stats->rx_replay++;
988 		stats->rx_dropped++;
989 		return;
990 	}
991 
992 	/* check errors reported by HW and update statistics */
993 	if (unlikely(wil->txrx_ops.rx_error_check(wil, skb, stats))) {
994 		dev_kfree_skb(skb);
995 		return;
996 	}
997 
998 	wil_netif_rx(skb, ndev, cid, stats, true);
999 }
1000 
1001 /* Proceed all completed skb's from Rx VRING
1002  *
1003  * Safe to call from NAPI poll, i.e. softirq with interrupts enabled
1004  */
1005 void wil_rx_handle(struct wil6210_priv *wil, int *quota)
1006 {
1007 	struct net_device *ndev = wil->main_ndev;
1008 	struct wireless_dev *wdev = ndev->ieee80211_ptr;
1009 	struct wil_ring *v = &wil->ring_rx;
1010 	struct sk_buff *skb;
1011 
1012 	if (unlikely(!v->va)) {
1013 		wil_err(wil, "Rx IRQ while Rx not yet initialized\n");
1014 		return;
1015 	}
1016 	wil_dbg_txrx(wil, "rx_handle\n");
1017 	while ((*quota > 0) && (NULL != (skb = wil_vring_reap_rx(wil, v)))) {
1018 		(*quota)--;
1019 
1020 		/* monitor is currently supported on main interface only */
1021 		if (wdev->iftype == NL80211_IFTYPE_MONITOR) {
1022 			skb->dev = ndev;
1023 			skb_reset_mac_header(skb);
1024 			skb->ip_summed = CHECKSUM_UNNECESSARY;
1025 			skb->pkt_type = PACKET_OTHERHOST;
1026 			skb->protocol = htons(ETH_P_802_2);
1027 			wil_netif_rx_any(skb, ndev);
1028 		} else {
1029 			wil_rx_reorder(wil, skb);
1030 		}
1031 	}
1032 	wil_rx_refill(wil, v->size);
1033 }
1034 
1035 static void wil_rx_buf_len_init(struct wil6210_priv *wil)
1036 {
1037 	wil->rx_buf_len = rx_large_buf ?
1038 		WIL_MAX_ETH_MTU : TXRX_BUF_LEN_DEFAULT - WIL_MAX_MPDU_OVERHEAD;
1039 	if (mtu_max > wil->rx_buf_len) {
1040 		/* do not allow RX buffers to be smaller than mtu_max, for
1041 		 * backward compatibility (mtu_max parameter was also used
1042 		 * to support receiving large packets)
1043 		 */
1044 		wil_info(wil, "Override RX buffer to mtu_max(%d)\n", mtu_max);
1045 		wil->rx_buf_len = mtu_max;
1046 	}
1047 }
1048 
1049 static int wil_rx_init(struct wil6210_priv *wil, uint order)
1050 {
1051 	struct wil_ring *vring = &wil->ring_rx;
1052 	int rc;
1053 
1054 	wil_dbg_misc(wil, "rx_init\n");
1055 
1056 	if (vring->va) {
1057 		wil_err(wil, "Rx ring already allocated\n");
1058 		return -EINVAL;
1059 	}
1060 
1061 	wil_rx_buf_len_init(wil);
1062 
1063 	vring->size = 1 << order;
1064 	vring->is_rx = true;
1065 	rc = wil_vring_alloc(wil, vring);
1066 	if (rc)
1067 		return rc;
1068 
1069 	rc = wmi_rx_chain_add(wil, vring);
1070 	if (rc)
1071 		goto err_free;
1072 
1073 	rc = wil_rx_refill(wil, vring->size);
1074 	if (rc)
1075 		goto err_free;
1076 
1077 	return 0;
1078  err_free:
1079 	wil_vring_free(wil, vring);
1080 
1081 	return rc;
1082 }
1083 
1084 static void wil_rx_fini(struct wil6210_priv *wil)
1085 {
1086 	struct wil_ring *vring = &wil->ring_rx;
1087 
1088 	wil_dbg_misc(wil, "rx_fini\n");
1089 
1090 	if (vring->va)
1091 		wil_vring_free(wil, vring);
1092 }
1093 
1094 static int wil_tx_desc_map(union wil_tx_desc *desc, dma_addr_t pa,
1095 			   u32 len, int vring_index)
1096 {
1097 	struct vring_tx_desc *d = &desc->legacy;
1098 
1099 	wil_desc_addr_set(&d->dma.addr, pa);
1100 	d->dma.ip_length = 0;
1101 	/* 0..6: mac_length; 7:ip_version 0-IP6 1-IP4*/
1102 	d->dma.b11 = 0/*14 | BIT(7)*/;
1103 	d->dma.error = 0;
1104 	d->dma.status = 0; /* BIT(0) should be 0 for HW_OWNED */
1105 	d->dma.length = cpu_to_le16((u16)len);
1106 	d->dma.d0 = (vring_index << DMA_CFG_DESC_TX_0_QID_POS);
1107 	d->mac.d[0] = 0;
1108 	d->mac.d[1] = 0;
1109 	d->mac.d[2] = 0;
1110 	d->mac.ucode_cmd = 0;
1111 	/* translation type:  0 - bypass; 1 - 802.3; 2 - native wifi */
1112 	d->mac.d[2] = BIT(MAC_CFG_DESC_TX_2_SNAP_HDR_INSERTION_EN_POS) |
1113 		      (1 << MAC_CFG_DESC_TX_2_L2_TRANSLATION_TYPE_POS);
1114 
1115 	return 0;
1116 }
1117 
1118 void wil_tx_data_init(struct wil_ring_tx_data *txdata)
1119 {
1120 	spin_lock_bh(&txdata->lock);
1121 	txdata->dot1x_open = false;
1122 	txdata->enabled = 0;
1123 	txdata->idle = 0;
1124 	txdata->last_idle = 0;
1125 	txdata->begin = 0;
1126 	txdata->agg_wsize = 0;
1127 	txdata->agg_timeout = 0;
1128 	txdata->agg_amsdu = 0;
1129 	txdata->addba_in_progress = false;
1130 	txdata->mid = U8_MAX;
1131 	spin_unlock_bh(&txdata->lock);
1132 }
1133 
1134 static int wil_vring_init_tx(struct wil6210_vif *vif, int id, int size,
1135 			     int cid, int tid)
1136 {
1137 	struct wil6210_priv *wil = vif_to_wil(vif);
1138 	int rc;
1139 	struct wmi_vring_cfg_cmd cmd = {
1140 		.action = cpu_to_le32(WMI_VRING_CMD_ADD),
1141 		.vring_cfg = {
1142 			.tx_sw_ring = {
1143 				.max_mpdu_size =
1144 					cpu_to_le16(wil_mtu2macbuf(mtu_max)),
1145 				.ring_size = cpu_to_le16(size),
1146 			},
1147 			.ringid = id,
1148 			.encap_trans_type = WMI_VRING_ENC_TYPE_802_3,
1149 			.mac_ctrl = 0,
1150 			.to_resolution = 0,
1151 			.agg_max_wsize = 0,
1152 			.schd_params = {
1153 				.priority = cpu_to_le16(0),
1154 				.timeslot_us = cpu_to_le16(0xfff),
1155 			},
1156 		},
1157 	};
1158 	struct {
1159 		struct wmi_cmd_hdr wmi;
1160 		struct wmi_vring_cfg_done_event cmd;
1161 	} __packed reply = {
1162 		.cmd = {.status = WMI_FW_STATUS_FAILURE},
1163 	};
1164 	struct wil_ring *vring = &wil->ring_tx[id];
1165 	struct wil_ring_tx_data *txdata = &wil->ring_tx_data[id];
1166 
1167 	if (cid >= WIL6210_RX_DESC_MAX_CID) {
1168 		cmd.vring_cfg.cidxtid = CIDXTID_EXTENDED_CID_TID;
1169 		cmd.vring_cfg.cid = cid;
1170 		cmd.vring_cfg.tid = tid;
1171 	} else {
1172 		cmd.vring_cfg.cidxtid = mk_cidxtid(cid, tid);
1173 	}
1174 
1175 	wil_dbg_misc(wil, "vring_init_tx: max_mpdu_size %d\n",
1176 		     cmd.vring_cfg.tx_sw_ring.max_mpdu_size);
1177 	lockdep_assert_held(&wil->mutex);
1178 
1179 	if (vring->va) {
1180 		wil_err(wil, "Tx ring [%d] already allocated\n", id);
1181 		rc = -EINVAL;
1182 		goto out;
1183 	}
1184 
1185 	wil_tx_data_init(txdata);
1186 	vring->is_rx = false;
1187 	vring->size = size;
1188 	rc = wil_vring_alloc(wil, vring);
1189 	if (rc)
1190 		goto out;
1191 
1192 	wil->ring2cid_tid[id][0] = cid;
1193 	wil->ring2cid_tid[id][1] = tid;
1194 
1195 	cmd.vring_cfg.tx_sw_ring.ring_mem_base = cpu_to_le64(vring->pa);
1196 
1197 	if (!vif->privacy)
1198 		txdata->dot1x_open = true;
1199 	rc = wmi_call(wil, WMI_VRING_CFG_CMDID, vif->mid, &cmd, sizeof(cmd),
1200 		      WMI_VRING_CFG_DONE_EVENTID, &reply, sizeof(reply),
1201 		      WIL_WMI_CALL_GENERAL_TO_MS);
1202 	if (rc)
1203 		goto out_free;
1204 
1205 	if (reply.cmd.status != WMI_FW_STATUS_SUCCESS) {
1206 		wil_err(wil, "Tx config failed, status 0x%02x\n",
1207 			reply.cmd.status);
1208 		rc = -EINVAL;
1209 		goto out_free;
1210 	}
1211 
1212 	spin_lock_bh(&txdata->lock);
1213 	vring->hwtail = le32_to_cpu(reply.cmd.tx_vring_tail_ptr);
1214 	txdata->mid = vif->mid;
1215 	txdata->enabled = 1;
1216 	spin_unlock_bh(&txdata->lock);
1217 
1218 	if (txdata->dot1x_open && (agg_wsize >= 0))
1219 		wil_addba_tx_request(wil, id, agg_wsize);
1220 
1221 	return 0;
1222  out_free:
1223 	spin_lock_bh(&txdata->lock);
1224 	txdata->dot1x_open = false;
1225 	txdata->enabled = 0;
1226 	spin_unlock_bh(&txdata->lock);
1227 	wil_vring_free(wil, vring);
1228 	wil->ring2cid_tid[id][0] = wil->max_assoc_sta;
1229 	wil->ring2cid_tid[id][1] = 0;
1230 
1231  out:
1232 
1233 	return rc;
1234 }
1235 
1236 static int wil_tx_vring_modify(struct wil6210_vif *vif, int ring_id, int cid,
1237 			       int tid)
1238 {
1239 	struct wil6210_priv *wil = vif_to_wil(vif);
1240 	int rc;
1241 	struct wmi_vring_cfg_cmd cmd = {
1242 		.action = cpu_to_le32(WMI_VRING_CMD_MODIFY),
1243 		.vring_cfg = {
1244 			.tx_sw_ring = {
1245 				.max_mpdu_size =
1246 					cpu_to_le16(wil_mtu2macbuf(mtu_max)),
1247 				.ring_size = 0,
1248 			},
1249 			.ringid = ring_id,
1250 			.cidxtid = mk_cidxtid(cid, tid),
1251 			.encap_trans_type = WMI_VRING_ENC_TYPE_802_3,
1252 			.mac_ctrl = 0,
1253 			.to_resolution = 0,
1254 			.agg_max_wsize = 0,
1255 			.schd_params = {
1256 				.priority = cpu_to_le16(0),
1257 				.timeslot_us = cpu_to_le16(0xfff),
1258 			},
1259 		},
1260 	};
1261 	struct {
1262 		struct wmi_cmd_hdr wmi;
1263 		struct wmi_vring_cfg_done_event cmd;
1264 	} __packed reply = {
1265 		.cmd = {.status = WMI_FW_STATUS_FAILURE},
1266 	};
1267 	struct wil_ring *vring = &wil->ring_tx[ring_id];
1268 	struct wil_ring_tx_data *txdata = &wil->ring_tx_data[ring_id];
1269 
1270 	wil_dbg_misc(wil, "vring_modify: ring %d cid %d tid %d\n", ring_id,
1271 		     cid, tid);
1272 	lockdep_assert_held(&wil->mutex);
1273 
1274 	if (!vring->va) {
1275 		wil_err(wil, "Tx ring [%d] not allocated\n", ring_id);
1276 		return -EINVAL;
1277 	}
1278 
1279 	if (wil->ring2cid_tid[ring_id][0] != cid ||
1280 	    wil->ring2cid_tid[ring_id][1] != tid) {
1281 		wil_err(wil, "ring info does not match cid=%u tid=%u\n",
1282 			wil->ring2cid_tid[ring_id][0],
1283 			wil->ring2cid_tid[ring_id][1]);
1284 	}
1285 
1286 	cmd.vring_cfg.tx_sw_ring.ring_mem_base = cpu_to_le64(vring->pa);
1287 
1288 	rc = wmi_call(wil, WMI_VRING_CFG_CMDID, vif->mid, &cmd, sizeof(cmd),
1289 		      WMI_VRING_CFG_DONE_EVENTID, &reply, sizeof(reply),
1290 		      WIL_WMI_CALL_GENERAL_TO_MS);
1291 	if (rc)
1292 		goto fail;
1293 
1294 	if (reply.cmd.status != WMI_FW_STATUS_SUCCESS) {
1295 		wil_err(wil, "Tx modify failed, status 0x%02x\n",
1296 			reply.cmd.status);
1297 		rc = -EINVAL;
1298 		goto fail;
1299 	}
1300 
1301 	/* set BA aggregation window size to 0 to force a new BA with the
1302 	 * new AP
1303 	 */
1304 	txdata->agg_wsize = 0;
1305 	if (txdata->dot1x_open && agg_wsize >= 0)
1306 		wil_addba_tx_request(wil, ring_id, agg_wsize);
1307 
1308 	return 0;
1309 fail:
1310 	spin_lock_bh(&txdata->lock);
1311 	txdata->dot1x_open = false;
1312 	txdata->enabled = 0;
1313 	spin_unlock_bh(&txdata->lock);
1314 	wil->ring2cid_tid[ring_id][0] = wil->max_assoc_sta;
1315 	wil->ring2cid_tid[ring_id][1] = 0;
1316 	return rc;
1317 }
1318 
1319 int wil_vring_init_bcast(struct wil6210_vif *vif, int id, int size)
1320 {
1321 	struct wil6210_priv *wil = vif_to_wil(vif);
1322 	int rc;
1323 	struct wmi_bcast_vring_cfg_cmd cmd = {
1324 		.action = cpu_to_le32(WMI_VRING_CMD_ADD),
1325 		.vring_cfg = {
1326 			.tx_sw_ring = {
1327 				.max_mpdu_size =
1328 					cpu_to_le16(wil_mtu2macbuf(mtu_max)),
1329 				.ring_size = cpu_to_le16(size),
1330 			},
1331 			.ringid = id,
1332 			.encap_trans_type = WMI_VRING_ENC_TYPE_802_3,
1333 		},
1334 	};
1335 	struct {
1336 		struct wmi_cmd_hdr wmi;
1337 		struct wmi_vring_cfg_done_event cmd;
1338 	} __packed reply = {
1339 		.cmd = {.status = WMI_FW_STATUS_FAILURE},
1340 	};
1341 	struct wil_ring *vring = &wil->ring_tx[id];
1342 	struct wil_ring_tx_data *txdata = &wil->ring_tx_data[id];
1343 
1344 	wil_dbg_misc(wil, "vring_init_bcast: max_mpdu_size %d\n",
1345 		     cmd.vring_cfg.tx_sw_ring.max_mpdu_size);
1346 	lockdep_assert_held(&wil->mutex);
1347 
1348 	if (vring->va) {
1349 		wil_err(wil, "Tx ring [%d] already allocated\n", id);
1350 		rc = -EINVAL;
1351 		goto out;
1352 	}
1353 
1354 	wil_tx_data_init(txdata);
1355 	vring->is_rx = false;
1356 	vring->size = size;
1357 	rc = wil_vring_alloc(wil, vring);
1358 	if (rc)
1359 		goto out;
1360 
1361 	wil->ring2cid_tid[id][0] = wil->max_assoc_sta; /* CID */
1362 	wil->ring2cid_tid[id][1] = 0; /* TID */
1363 
1364 	cmd.vring_cfg.tx_sw_ring.ring_mem_base = cpu_to_le64(vring->pa);
1365 
1366 	if (!vif->privacy)
1367 		txdata->dot1x_open = true;
1368 	rc = wmi_call(wil, WMI_BCAST_VRING_CFG_CMDID, vif->mid,
1369 		      &cmd, sizeof(cmd),
1370 		      WMI_VRING_CFG_DONE_EVENTID, &reply, sizeof(reply),
1371 		      WIL_WMI_CALL_GENERAL_TO_MS);
1372 	if (rc)
1373 		goto out_free;
1374 
1375 	if (reply.cmd.status != WMI_FW_STATUS_SUCCESS) {
1376 		wil_err(wil, "Tx config failed, status 0x%02x\n",
1377 			reply.cmd.status);
1378 		rc = -EINVAL;
1379 		goto out_free;
1380 	}
1381 
1382 	spin_lock_bh(&txdata->lock);
1383 	vring->hwtail = le32_to_cpu(reply.cmd.tx_vring_tail_ptr);
1384 	txdata->mid = vif->mid;
1385 	txdata->enabled = 1;
1386 	spin_unlock_bh(&txdata->lock);
1387 
1388 	return 0;
1389  out_free:
1390 	spin_lock_bh(&txdata->lock);
1391 	txdata->enabled = 0;
1392 	txdata->dot1x_open = false;
1393 	spin_unlock_bh(&txdata->lock);
1394 	wil_vring_free(wil, vring);
1395  out:
1396 
1397 	return rc;
1398 }
1399 
1400 static struct wil_ring *wil_find_tx_ucast(struct wil6210_priv *wil,
1401 					  struct wil6210_vif *vif,
1402 					  struct sk_buff *skb)
1403 {
1404 	int i, cid;
1405 	const u8 *da = wil_skb_get_da(skb);
1406 	int min_ring_id = wil_get_min_tx_ring_id(wil);
1407 
1408 	cid = wil_find_cid(wil, vif->mid, da);
1409 
1410 	if (cid < 0 || cid >= wil->max_assoc_sta)
1411 		return NULL;
1412 
1413 	/* TODO: fix for multiple TID */
1414 	for (i = min_ring_id; i < ARRAY_SIZE(wil->ring2cid_tid); i++) {
1415 		if (!wil->ring_tx_data[i].dot1x_open &&
1416 		    skb->protocol != cpu_to_be16(ETH_P_PAE))
1417 			continue;
1418 		if (wil->ring2cid_tid[i][0] == cid) {
1419 			struct wil_ring *v = &wil->ring_tx[i];
1420 			struct wil_ring_tx_data *txdata = &wil->ring_tx_data[i];
1421 
1422 			wil_dbg_txrx(wil, "find_tx_ucast: (%pM) -> [%d]\n",
1423 				     da, i);
1424 			if (v->va && txdata->enabled) {
1425 				return v;
1426 			} else {
1427 				wil_dbg_txrx(wil,
1428 					     "find_tx_ucast: vring[%d] not valid\n",
1429 					     i);
1430 				return NULL;
1431 			}
1432 		}
1433 	}
1434 
1435 	return NULL;
1436 }
1437 
1438 static int wil_tx_ring(struct wil6210_priv *wil, struct wil6210_vif *vif,
1439 		       struct wil_ring *ring, struct sk_buff *skb);
1440 
1441 static struct wil_ring *wil_find_tx_ring_sta(struct wil6210_priv *wil,
1442 					     struct wil6210_vif *vif,
1443 					     struct sk_buff *skb)
1444 {
1445 	struct wil_ring *ring;
1446 	int i;
1447 	u8 cid;
1448 	struct wil_ring_tx_data  *txdata;
1449 	int min_ring_id = wil_get_min_tx_ring_id(wil);
1450 
1451 	/* In the STA mode, it is expected to have only 1 VRING
1452 	 * for the AP we connected to.
1453 	 * find 1-st vring eligible for this skb and use it.
1454 	 */
1455 	for (i = min_ring_id; i < WIL6210_MAX_TX_RINGS; i++) {
1456 		ring = &wil->ring_tx[i];
1457 		txdata = &wil->ring_tx_data[i];
1458 		if (!ring->va || !txdata->enabled || txdata->mid != vif->mid)
1459 			continue;
1460 
1461 		cid = wil->ring2cid_tid[i][0];
1462 		if (cid >= wil->max_assoc_sta) /* skip BCAST */
1463 			continue;
1464 
1465 		if (!wil->ring_tx_data[i].dot1x_open &&
1466 		    skb->protocol != cpu_to_be16(ETH_P_PAE))
1467 			continue;
1468 
1469 		wil_dbg_txrx(wil, "Tx -> ring %d\n", i);
1470 
1471 		return ring;
1472 	}
1473 
1474 	wil_dbg_txrx(wil, "Tx while no rings active?\n");
1475 
1476 	return NULL;
1477 }
1478 
1479 /* Use one of 2 strategies:
1480  *
1481  * 1. New (real broadcast):
1482  *    use dedicated broadcast vring
1483  * 2. Old (pseudo-DMS):
1484  *    Find 1-st vring and return it;
1485  *    duplicate skb and send it to other active vrings;
1486  *    in all cases override dest address to unicast peer's address
1487  * Use old strategy when new is not supported yet:
1488  *  - for PBSS
1489  */
1490 static struct wil_ring *wil_find_tx_bcast_1(struct wil6210_priv *wil,
1491 					    struct wil6210_vif *vif,
1492 					    struct sk_buff *skb)
1493 {
1494 	struct wil_ring *v;
1495 	struct wil_ring_tx_data *txdata;
1496 	int i = vif->bcast_ring;
1497 
1498 	if (i < 0)
1499 		return NULL;
1500 	v = &wil->ring_tx[i];
1501 	txdata = &wil->ring_tx_data[i];
1502 	if (!v->va || !txdata->enabled)
1503 		return NULL;
1504 	if (!wil->ring_tx_data[i].dot1x_open &&
1505 	    skb->protocol != cpu_to_be16(ETH_P_PAE))
1506 		return NULL;
1507 
1508 	return v;
1509 }
1510 
1511 /* apply multicast to unicast only for ARP and IP packets
1512  * (see NL80211_CMD_SET_MULTICAST_TO_UNICAST for more info)
1513  */
1514 static bool wil_check_multicast_to_unicast(struct wil6210_priv *wil,
1515 					   struct sk_buff *skb)
1516 {
1517 	const struct ethhdr *eth = (void *)skb->data;
1518 	const struct vlan_ethhdr *ethvlan = (void *)skb->data;
1519 	__be16 ethertype;
1520 
1521 	if (!wil->multicast_to_unicast)
1522 		return false;
1523 
1524 	/* multicast to unicast conversion only for some payload */
1525 	ethertype = eth->h_proto;
1526 	if (ethertype == htons(ETH_P_8021Q) && skb->len >= VLAN_ETH_HLEN)
1527 		ethertype = ethvlan->h_vlan_encapsulated_proto;
1528 	switch (ethertype) {
1529 	case htons(ETH_P_ARP):
1530 	case htons(ETH_P_IP):
1531 	case htons(ETH_P_IPV6):
1532 		break;
1533 	default:
1534 		return false;
1535 	}
1536 
1537 	return true;
1538 }
1539 
1540 static void wil_set_da_for_vring(struct wil6210_priv *wil,
1541 				 struct sk_buff *skb, int vring_index)
1542 {
1543 	u8 *da = wil_skb_get_da(skb);
1544 	int cid = wil->ring2cid_tid[vring_index][0];
1545 
1546 	ether_addr_copy(da, wil->sta[cid].addr);
1547 }
1548 
1549 static struct wil_ring *wil_find_tx_bcast_2(struct wil6210_priv *wil,
1550 					    struct wil6210_vif *vif,
1551 					    struct sk_buff *skb)
1552 {
1553 	struct wil_ring *v, *v2;
1554 	struct sk_buff *skb2;
1555 	int i;
1556 	u8 cid;
1557 	const u8 *src = wil_skb_get_sa(skb);
1558 	struct wil_ring_tx_data *txdata, *txdata2;
1559 	int min_ring_id = wil_get_min_tx_ring_id(wil);
1560 
1561 	/* find 1-st vring eligible for data */
1562 	for (i = min_ring_id; i < WIL6210_MAX_TX_RINGS; i++) {
1563 		v = &wil->ring_tx[i];
1564 		txdata = &wil->ring_tx_data[i];
1565 		if (!v->va || !txdata->enabled || txdata->mid != vif->mid)
1566 			continue;
1567 
1568 		cid = wil->ring2cid_tid[i][0];
1569 		if (cid >= wil->max_assoc_sta) /* skip BCAST */
1570 			continue;
1571 		if (!wil->ring_tx_data[i].dot1x_open &&
1572 		    skb->protocol != cpu_to_be16(ETH_P_PAE))
1573 			continue;
1574 
1575 		/* don't Tx back to source when re-routing Rx->Tx at the AP */
1576 		if (0 == memcmp(wil->sta[cid].addr, src, ETH_ALEN))
1577 			continue;
1578 
1579 		goto found;
1580 	}
1581 
1582 	wil_dbg_txrx(wil, "Tx while no vrings active?\n");
1583 
1584 	return NULL;
1585 
1586 found:
1587 	wil_dbg_txrx(wil, "BCAST -> ring %d\n", i);
1588 	wil_set_da_for_vring(wil, skb, i);
1589 
1590 	/* find other active vrings and duplicate skb for each */
1591 	for (i++; i < WIL6210_MAX_TX_RINGS; i++) {
1592 		v2 = &wil->ring_tx[i];
1593 		txdata2 = &wil->ring_tx_data[i];
1594 		if (!v2->va || txdata2->mid != vif->mid)
1595 			continue;
1596 		cid = wil->ring2cid_tid[i][0];
1597 		if (cid >= wil->max_assoc_sta) /* skip BCAST */
1598 			continue;
1599 		if (!wil->ring_tx_data[i].dot1x_open &&
1600 		    skb->protocol != cpu_to_be16(ETH_P_PAE))
1601 			continue;
1602 
1603 		if (0 == memcmp(wil->sta[cid].addr, src, ETH_ALEN))
1604 			continue;
1605 
1606 		skb2 = skb_copy(skb, GFP_ATOMIC);
1607 		if (skb2) {
1608 			wil_dbg_txrx(wil, "BCAST DUP -> ring %d\n", i);
1609 			wil_set_da_for_vring(wil, skb2, i);
1610 			wil_tx_ring(wil, vif, v2, skb2);
1611 			/* successful call to wil_tx_ring takes skb2 ref */
1612 			dev_kfree_skb_any(skb2);
1613 		} else {
1614 			wil_err(wil, "skb_copy failed\n");
1615 		}
1616 	}
1617 
1618 	return v;
1619 }
1620 
1621 static inline
1622 void wil_tx_desc_set_nr_frags(struct vring_tx_desc *d, int nr_frags)
1623 {
1624 	d->mac.d[2] |= (nr_frags << MAC_CFG_DESC_TX_2_NUM_OF_DESCRIPTORS_POS);
1625 }
1626 
1627 /* Sets the descriptor @d up for csum and/or TSO offloading. The corresponding
1628  * @skb is used to obtain the protocol and headers length.
1629  * @tso_desc_type is a descriptor type for TSO: 0 - a header, 1 - first data,
1630  * 2 - middle, 3 - last descriptor.
1631  */
1632 
1633 static void wil_tx_desc_offload_setup_tso(struct vring_tx_desc *d,
1634 					  struct sk_buff *skb,
1635 					  int tso_desc_type, bool is_ipv4,
1636 					  int tcp_hdr_len, int skb_net_hdr_len)
1637 {
1638 	d->dma.b11 = ETH_HLEN; /* MAC header length */
1639 	d->dma.b11 |= is_ipv4 << DMA_CFG_DESC_TX_OFFLOAD_CFG_L3T_IPV4_POS;
1640 
1641 	d->dma.d0 |= (2 << DMA_CFG_DESC_TX_0_L4_TYPE_POS);
1642 	/* L4 header len: TCP header length */
1643 	d->dma.d0 |= (tcp_hdr_len & DMA_CFG_DESC_TX_0_L4_LENGTH_MSK);
1644 
1645 	/* Setup TSO: bit and desc type */
1646 	d->dma.d0 |= (BIT(DMA_CFG_DESC_TX_0_TCP_SEG_EN_POS)) |
1647 		(tso_desc_type << DMA_CFG_DESC_TX_0_SEGMENT_BUF_DETAILS_POS);
1648 	d->dma.d0 |= (is_ipv4 << DMA_CFG_DESC_TX_0_IPV4_CHECKSUM_EN_POS);
1649 
1650 	d->dma.ip_length = skb_net_hdr_len;
1651 	/* Enable TCP/UDP checksum */
1652 	d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_TCP_UDP_CHECKSUM_EN_POS);
1653 	/* Calculate pseudo-header */
1654 	d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_PSEUDO_HEADER_CALC_EN_POS);
1655 }
1656 
1657 /* Sets the descriptor @d up for csum. The corresponding
1658  * @skb is used to obtain the protocol and headers length.
1659  * Returns the protocol: 0 - not TCP, 1 - TCPv4, 2 - TCPv6.
1660  * Note, if d==NULL, the function only returns the protocol result.
1661  *
1662  * It is very similar to previous wil_tx_desc_offload_setup_tso. This
1663  * is "if unrolling" to optimize the critical path.
1664  */
1665 
1666 static int wil_tx_desc_offload_setup(struct vring_tx_desc *d,
1667 				     struct sk_buff *skb){
1668 	int protocol;
1669 
1670 	if (skb->ip_summed != CHECKSUM_PARTIAL)
1671 		return 0;
1672 
1673 	d->dma.b11 = ETH_HLEN; /* MAC header length */
1674 
1675 	switch (skb->protocol) {
1676 	case cpu_to_be16(ETH_P_IP):
1677 		protocol = ip_hdr(skb)->protocol;
1678 		d->dma.b11 |= BIT(DMA_CFG_DESC_TX_OFFLOAD_CFG_L3T_IPV4_POS);
1679 		break;
1680 	case cpu_to_be16(ETH_P_IPV6):
1681 		protocol = ipv6_hdr(skb)->nexthdr;
1682 		break;
1683 	default:
1684 		return -EINVAL;
1685 	}
1686 
1687 	switch (protocol) {
1688 	case IPPROTO_TCP:
1689 		d->dma.d0 |= (2 << DMA_CFG_DESC_TX_0_L4_TYPE_POS);
1690 		/* L4 header len: TCP header length */
1691 		d->dma.d0 |=
1692 		(tcp_hdrlen(skb) & DMA_CFG_DESC_TX_0_L4_LENGTH_MSK);
1693 		break;
1694 	case IPPROTO_UDP:
1695 		/* L4 header len: UDP header length */
1696 		d->dma.d0 |=
1697 		(sizeof(struct udphdr) & DMA_CFG_DESC_TX_0_L4_LENGTH_MSK);
1698 		break;
1699 	default:
1700 		return -EINVAL;
1701 	}
1702 
1703 	d->dma.ip_length = skb_network_header_len(skb);
1704 	/* Enable TCP/UDP checksum */
1705 	d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_TCP_UDP_CHECKSUM_EN_POS);
1706 	/* Calculate pseudo-header */
1707 	d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_PSEUDO_HEADER_CALC_EN_POS);
1708 
1709 	return 0;
1710 }
1711 
1712 static inline void wil_tx_last_desc(struct vring_tx_desc *d)
1713 {
1714 	d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_CMD_EOP_POS) |
1715 	      BIT(DMA_CFG_DESC_TX_0_CMD_MARK_WB_POS) |
1716 	      BIT(DMA_CFG_DESC_TX_0_CMD_DMA_IT_POS);
1717 }
1718 
1719 static inline void wil_set_tx_desc_last_tso(volatile struct vring_tx_desc *d)
1720 {
1721 	d->dma.d0 |= wil_tso_type_lst <<
1722 		  DMA_CFG_DESC_TX_0_SEGMENT_BUF_DETAILS_POS;
1723 }
1724 
1725 static int __wil_tx_vring_tso(struct wil6210_priv *wil, struct wil6210_vif *vif,
1726 			      struct wil_ring *vring, struct sk_buff *skb)
1727 {
1728 	struct device *dev = wil_to_dev(wil);
1729 
1730 	/* point to descriptors in shared memory */
1731 	volatile struct vring_tx_desc *_desc = NULL, *_hdr_desc,
1732 				      *_first_desc = NULL;
1733 
1734 	/* pointers to shadow descriptors */
1735 	struct vring_tx_desc desc_mem, hdr_desc_mem, first_desc_mem,
1736 			     *d = &hdr_desc_mem, *hdr_desc = &hdr_desc_mem,
1737 			     *first_desc = &first_desc_mem;
1738 
1739 	/* pointer to shadow descriptors' context */
1740 	struct wil_ctx *hdr_ctx, *first_ctx = NULL;
1741 
1742 	int descs_used = 0; /* total number of used descriptors */
1743 	int sg_desc_cnt = 0; /* number of descriptors for current mss*/
1744 
1745 	u32 swhead = vring->swhead;
1746 	int used, avail = wil_ring_avail_tx(vring);
1747 	int nr_frags = skb_shinfo(skb)->nr_frags;
1748 	int min_desc_required = nr_frags + 1;
1749 	int mss = skb_shinfo(skb)->gso_size;	/* payload size w/o headers */
1750 	int f, len, hdrlen, headlen;
1751 	int vring_index = vring - wil->ring_tx;
1752 	struct wil_ring_tx_data *txdata = &wil->ring_tx_data[vring_index];
1753 	uint i = swhead;
1754 	dma_addr_t pa;
1755 	const skb_frag_t *frag = NULL;
1756 	int rem_data = mss;
1757 	int lenmss;
1758 	int hdr_compensation_need = true;
1759 	int desc_tso_type = wil_tso_type_first;
1760 	bool is_ipv4;
1761 	int tcp_hdr_len;
1762 	int skb_net_hdr_len;
1763 	int gso_type;
1764 	int rc = -EINVAL;
1765 
1766 	wil_dbg_txrx(wil, "tx_vring_tso: %d bytes to vring %d\n", skb->len,
1767 		     vring_index);
1768 
1769 	if (unlikely(!txdata->enabled))
1770 		return -EINVAL;
1771 
1772 	/* A typical page 4K is 3-4 payloads, we assume each fragment
1773 	 * is a full payload, that's how min_desc_required has been
1774 	 * calculated. In real we might need more or less descriptors,
1775 	 * this is the initial check only.
1776 	 */
1777 	if (unlikely(avail < min_desc_required)) {
1778 		wil_err_ratelimited(wil,
1779 				    "TSO: Tx ring[%2d] full. No space for %d fragments\n",
1780 				    vring_index, min_desc_required);
1781 		return -ENOMEM;
1782 	}
1783 
1784 	/* Header Length = MAC header len + IP header len + TCP header len*/
1785 	hdrlen = skb_tcp_all_headers(skb);
1786 
1787 	gso_type = skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV6 | SKB_GSO_TCPV4);
1788 	switch (gso_type) {
1789 	case SKB_GSO_TCPV4:
1790 		/* TCP v4, zero out the IP length and IPv4 checksum fields
1791 		 * as required by the offloading doc
1792 		 */
1793 		ip_hdr(skb)->tot_len = 0;
1794 		ip_hdr(skb)->check = 0;
1795 		is_ipv4 = true;
1796 		break;
1797 	case SKB_GSO_TCPV6:
1798 		/* TCP v6, zero out the payload length */
1799 		ipv6_hdr(skb)->payload_len = 0;
1800 		is_ipv4 = false;
1801 		break;
1802 	default:
1803 		/* other than TCPv4 or TCPv6 types are not supported for TSO.
1804 		 * It is also illegal for both to be set simultaneously
1805 		 */
1806 		return -EINVAL;
1807 	}
1808 
1809 	if (skb->ip_summed != CHECKSUM_PARTIAL)
1810 		return -EINVAL;
1811 
1812 	/* tcp header length and skb network header length are fixed for all
1813 	 * packet's descriptors - read then once here
1814 	 */
1815 	tcp_hdr_len = tcp_hdrlen(skb);
1816 	skb_net_hdr_len = skb_network_header_len(skb);
1817 
1818 	_hdr_desc = &vring->va[i].tx.legacy;
1819 
1820 	pa = dma_map_single(dev, skb->data, hdrlen, DMA_TO_DEVICE);
1821 	if (unlikely(dma_mapping_error(dev, pa))) {
1822 		wil_err(wil, "TSO: Skb head DMA map error\n");
1823 		goto err_exit;
1824 	}
1825 
1826 	wil->txrx_ops.tx_desc_map((union wil_tx_desc *)hdr_desc, pa,
1827 				  hdrlen, vring_index);
1828 	wil_tx_desc_offload_setup_tso(hdr_desc, skb, wil_tso_type_hdr, is_ipv4,
1829 				      tcp_hdr_len, skb_net_hdr_len);
1830 	wil_tx_last_desc(hdr_desc);
1831 
1832 	vring->ctx[i].mapped_as = wil_mapped_as_single;
1833 	hdr_ctx = &vring->ctx[i];
1834 
1835 	descs_used++;
1836 	headlen = skb_headlen(skb) - hdrlen;
1837 
1838 	for (f = headlen ? -1 : 0; f < nr_frags; f++)  {
1839 		if (headlen) {
1840 			len = headlen;
1841 			wil_dbg_txrx(wil, "TSO: process skb head, len %u\n",
1842 				     len);
1843 		} else {
1844 			frag = &skb_shinfo(skb)->frags[f];
1845 			len = skb_frag_size(frag);
1846 			wil_dbg_txrx(wil, "TSO: frag[%d]: len %u\n", f, len);
1847 		}
1848 
1849 		while (len) {
1850 			wil_dbg_txrx(wil,
1851 				     "TSO: len %d, rem_data %d, descs_used %d\n",
1852 				     len, rem_data, descs_used);
1853 
1854 			if (descs_used == avail)  {
1855 				wil_err_ratelimited(wil, "TSO: ring overflow\n");
1856 				rc = -ENOMEM;
1857 				goto mem_error;
1858 			}
1859 
1860 			lenmss = min_t(int, rem_data, len);
1861 			i = (swhead + descs_used) % vring->size;
1862 			wil_dbg_txrx(wil, "TSO: lenmss %d, i %d\n", lenmss, i);
1863 
1864 			if (!headlen) {
1865 				pa = skb_frag_dma_map(dev, frag,
1866 						      skb_frag_size(frag) - len,
1867 						      lenmss, DMA_TO_DEVICE);
1868 				vring->ctx[i].mapped_as = wil_mapped_as_page;
1869 			} else {
1870 				pa = dma_map_single(dev,
1871 						    skb->data +
1872 						    skb_headlen(skb) - headlen,
1873 						    lenmss,
1874 						    DMA_TO_DEVICE);
1875 				vring->ctx[i].mapped_as = wil_mapped_as_single;
1876 				headlen -= lenmss;
1877 			}
1878 
1879 			if (unlikely(dma_mapping_error(dev, pa))) {
1880 				wil_err(wil, "TSO: DMA map page error\n");
1881 				goto mem_error;
1882 			}
1883 
1884 			_desc = &vring->va[i].tx.legacy;
1885 
1886 			if (!_first_desc) {
1887 				_first_desc = _desc;
1888 				first_ctx = &vring->ctx[i];
1889 				d = first_desc;
1890 			} else {
1891 				d = &desc_mem;
1892 			}
1893 
1894 			wil->txrx_ops.tx_desc_map((union wil_tx_desc *)d,
1895 						  pa, lenmss, vring_index);
1896 			wil_tx_desc_offload_setup_tso(d, skb, desc_tso_type,
1897 						      is_ipv4, tcp_hdr_len,
1898 						      skb_net_hdr_len);
1899 
1900 			/* use tso_type_first only once */
1901 			desc_tso_type = wil_tso_type_mid;
1902 
1903 			descs_used++;  /* desc used so far */
1904 			sg_desc_cnt++; /* desc used for this segment */
1905 			len -= lenmss;
1906 			rem_data -= lenmss;
1907 
1908 			wil_dbg_txrx(wil,
1909 				     "TSO: len %d, rem_data %d, descs_used %d, sg_desc_cnt %d,\n",
1910 				     len, rem_data, descs_used, sg_desc_cnt);
1911 
1912 			/* Close the segment if reached mss size or last frag*/
1913 			if (rem_data == 0 || (f == nr_frags - 1 && len == 0)) {
1914 				if (hdr_compensation_need) {
1915 					/* first segment include hdr desc for
1916 					 * release
1917 					 */
1918 					hdr_ctx->nr_frags = sg_desc_cnt;
1919 					wil_tx_desc_set_nr_frags(first_desc,
1920 								 sg_desc_cnt +
1921 								 1);
1922 					hdr_compensation_need = false;
1923 				} else {
1924 					wil_tx_desc_set_nr_frags(first_desc,
1925 								 sg_desc_cnt);
1926 				}
1927 				first_ctx->nr_frags = sg_desc_cnt - 1;
1928 
1929 				wil_tx_last_desc(d);
1930 
1931 				/* first descriptor may also be the last
1932 				 * for this mss - make sure not to copy
1933 				 * it twice
1934 				 */
1935 				if (first_desc != d)
1936 					*_first_desc = *first_desc;
1937 
1938 				/*last descriptor will be copied at the end
1939 				 * of this TS processing
1940 				 */
1941 				if (f < nr_frags - 1 || len > 0)
1942 					*_desc = *d;
1943 
1944 				rem_data = mss;
1945 				_first_desc = NULL;
1946 				sg_desc_cnt = 0;
1947 			} else if (first_desc != d) /* update mid descriptor */
1948 					*_desc = *d;
1949 		}
1950 	}
1951 
1952 	if (!_desc)
1953 		goto mem_error;
1954 
1955 	/* first descriptor may also be the last.
1956 	 * in this case d pointer is invalid
1957 	 */
1958 	if (_first_desc == _desc)
1959 		d = first_desc;
1960 
1961 	/* Last data descriptor */
1962 	wil_set_tx_desc_last_tso(d);
1963 	*_desc = *d;
1964 
1965 	/* Fill the total number of descriptors in first desc (hdr)*/
1966 	wil_tx_desc_set_nr_frags(hdr_desc, descs_used);
1967 	*_hdr_desc = *hdr_desc;
1968 
1969 	/* hold reference to skb
1970 	 * to prevent skb release before accounting
1971 	 * in case of immediate "tx done"
1972 	 */
1973 	vring->ctx[i].skb = skb_get(skb);
1974 
1975 	/* performance monitoring */
1976 	used = wil_ring_used_tx(vring);
1977 	if (wil_val_in_range(wil->ring_idle_trsh,
1978 			     used, used + descs_used)) {
1979 		txdata->idle += get_cycles() - txdata->last_idle;
1980 		wil_dbg_txrx(wil,  "Ring[%2d] not idle %d -> %d\n",
1981 			     vring_index, used, used + descs_used);
1982 	}
1983 
1984 	/* Make sure to advance the head only after descriptor update is done.
1985 	 * This will prevent a race condition where the completion thread
1986 	 * will see the DU bit set from previous run and will handle the
1987 	 * skb before it was completed.
1988 	 */
1989 	wmb();
1990 
1991 	/* advance swhead */
1992 	wil_ring_advance_head(vring, descs_used);
1993 	wil_dbg_txrx(wil, "TSO: Tx swhead %d -> %d\n", swhead, vring->swhead);
1994 
1995 	/* make sure all writes to descriptors (shared memory) are done before
1996 	 * committing them to HW
1997 	 */
1998 	wmb();
1999 
2000 	if (wil->tx_latency)
2001 		*(ktime_t *)&skb->cb = ktime_get();
2002 	else
2003 		memset(skb->cb, 0, sizeof(ktime_t));
2004 
2005 	wil_w(wil, vring->hwtail, vring->swhead);
2006 	return 0;
2007 
2008 mem_error:
2009 	while (descs_used > 0) {
2010 		struct wil_ctx *ctx;
2011 
2012 		i = (swhead + descs_used - 1) % vring->size;
2013 		d = (struct vring_tx_desc *)&vring->va[i].tx.legacy;
2014 		_desc = &vring->va[i].tx.legacy;
2015 		*d = *_desc;
2016 		_desc->dma.status = TX_DMA_STATUS_DU;
2017 		ctx = &vring->ctx[i];
2018 		wil_txdesc_unmap(dev, (union wil_tx_desc *)d, ctx);
2019 		memset(ctx, 0, sizeof(*ctx));
2020 		descs_used--;
2021 	}
2022 err_exit:
2023 	return rc;
2024 }
2025 
2026 static int __wil_tx_ring(struct wil6210_priv *wil, struct wil6210_vif *vif,
2027 			 struct wil_ring *ring, struct sk_buff *skb)
2028 {
2029 	struct device *dev = wil_to_dev(wil);
2030 	struct vring_tx_desc dd, *d = &dd;
2031 	volatile struct vring_tx_desc *_d;
2032 	u32 swhead = ring->swhead;
2033 	int avail = wil_ring_avail_tx(ring);
2034 	int nr_frags = skb_shinfo(skb)->nr_frags;
2035 	uint f = 0;
2036 	int ring_index = ring - wil->ring_tx;
2037 	struct wil_ring_tx_data  *txdata = &wil->ring_tx_data[ring_index];
2038 	uint i = swhead;
2039 	dma_addr_t pa;
2040 	int used;
2041 	bool mcast = (ring_index == vif->bcast_ring);
2042 	uint len = skb_headlen(skb);
2043 
2044 	wil_dbg_txrx(wil, "tx_ring: %d bytes to ring %d, nr_frags %d\n",
2045 		     skb->len, ring_index, nr_frags);
2046 
2047 	if (unlikely(!txdata->enabled))
2048 		return -EINVAL;
2049 
2050 	if (unlikely(avail < 1 + nr_frags)) {
2051 		wil_err_ratelimited(wil,
2052 				    "Tx ring[%2d] full. No space for %d fragments\n",
2053 				    ring_index, 1 + nr_frags);
2054 		return -ENOMEM;
2055 	}
2056 	_d = &ring->va[i].tx.legacy;
2057 
2058 	pa = dma_map_single(dev, skb->data, skb_headlen(skb), DMA_TO_DEVICE);
2059 
2060 	wil_dbg_txrx(wil, "Tx[%2d] skb %d bytes 0x%p -> %pad\n", ring_index,
2061 		     skb_headlen(skb), skb->data, &pa);
2062 	wil_hex_dump_txrx("Tx ", DUMP_PREFIX_OFFSET, 16, 1,
2063 			  skb->data, skb_headlen(skb), false);
2064 
2065 	if (unlikely(dma_mapping_error(dev, pa)))
2066 		return -EINVAL;
2067 	ring->ctx[i].mapped_as = wil_mapped_as_single;
2068 	/* 1-st segment */
2069 	wil->txrx_ops.tx_desc_map((union wil_tx_desc *)d, pa, len,
2070 				   ring_index);
2071 	if (unlikely(mcast)) {
2072 		d->mac.d[0] |= BIT(MAC_CFG_DESC_TX_0_MCS_EN_POS); /* MCS 0 */
2073 		if (unlikely(len > WIL_BCAST_MCS0_LIMIT)) /* set MCS 1 */
2074 			d->mac.d[0] |= (1 << MAC_CFG_DESC_TX_0_MCS_INDEX_POS);
2075 	}
2076 	/* Process TCP/UDP checksum offloading */
2077 	if (unlikely(wil_tx_desc_offload_setup(d, skb))) {
2078 		wil_err(wil, "Tx[%2d] Failed to set cksum, drop packet\n",
2079 			ring_index);
2080 		goto dma_error;
2081 	}
2082 
2083 	ring->ctx[i].nr_frags = nr_frags;
2084 	wil_tx_desc_set_nr_frags(d, nr_frags + 1);
2085 
2086 	/* middle segments */
2087 	for (; f < nr_frags; f++) {
2088 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[f];
2089 		int len = skb_frag_size(frag);
2090 
2091 		*_d = *d;
2092 		wil_dbg_txrx(wil, "Tx[%2d] desc[%4d]\n", ring_index, i);
2093 		wil_hex_dump_txrx("TxD ", DUMP_PREFIX_NONE, 32, 4,
2094 				  (const void *)d, sizeof(*d), false);
2095 		i = (swhead + f + 1) % ring->size;
2096 		_d = &ring->va[i].tx.legacy;
2097 		pa = skb_frag_dma_map(dev, frag, 0, skb_frag_size(frag),
2098 				      DMA_TO_DEVICE);
2099 		if (unlikely(dma_mapping_error(dev, pa))) {
2100 			wil_err(wil, "Tx[%2d] failed to map fragment\n",
2101 				ring_index);
2102 			goto dma_error;
2103 		}
2104 		ring->ctx[i].mapped_as = wil_mapped_as_page;
2105 		wil->txrx_ops.tx_desc_map((union wil_tx_desc *)d,
2106 					   pa, len, ring_index);
2107 		/* no need to check return code -
2108 		 * if it succeeded for 1-st descriptor,
2109 		 * it will succeed here too
2110 		 */
2111 		wil_tx_desc_offload_setup(d, skb);
2112 	}
2113 	/* for the last seg only */
2114 	d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_CMD_EOP_POS);
2115 	d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_CMD_MARK_WB_POS);
2116 	d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_CMD_DMA_IT_POS);
2117 	*_d = *d;
2118 	wil_dbg_txrx(wil, "Tx[%2d] desc[%4d]\n", ring_index, i);
2119 	wil_hex_dump_txrx("TxD ", DUMP_PREFIX_NONE, 32, 4,
2120 			  (const void *)d, sizeof(*d), false);
2121 
2122 	/* hold reference to skb
2123 	 * to prevent skb release before accounting
2124 	 * in case of immediate "tx done"
2125 	 */
2126 	ring->ctx[i].skb = skb_get(skb);
2127 
2128 	/* performance monitoring */
2129 	used = wil_ring_used_tx(ring);
2130 	if (wil_val_in_range(wil->ring_idle_trsh,
2131 			     used, used + nr_frags + 1)) {
2132 		txdata->idle += get_cycles() - txdata->last_idle;
2133 		wil_dbg_txrx(wil,  "Ring[%2d] not idle %d -> %d\n",
2134 			     ring_index, used, used + nr_frags + 1);
2135 	}
2136 
2137 	/* Make sure to advance the head only after descriptor update is done.
2138 	 * This will prevent a race condition where the completion thread
2139 	 * will see the DU bit set from previous run and will handle the
2140 	 * skb before it was completed.
2141 	 */
2142 	wmb();
2143 
2144 	/* advance swhead */
2145 	wil_ring_advance_head(ring, nr_frags + 1);
2146 	wil_dbg_txrx(wil, "Tx[%2d] swhead %d -> %d\n", ring_index, swhead,
2147 		     ring->swhead);
2148 	trace_wil6210_tx(ring_index, swhead, skb->len, nr_frags);
2149 
2150 	/* make sure all writes to descriptors (shared memory) are done before
2151 	 * committing them to HW
2152 	 */
2153 	wmb();
2154 
2155 	if (wil->tx_latency)
2156 		*(ktime_t *)&skb->cb = ktime_get();
2157 	else
2158 		memset(skb->cb, 0, sizeof(ktime_t));
2159 
2160 	wil_w(wil, ring->hwtail, ring->swhead);
2161 
2162 	return 0;
2163  dma_error:
2164 	/* unmap what we have mapped */
2165 	nr_frags = f + 1; /* frags mapped + one for skb head */
2166 	for (f = 0; f < nr_frags; f++) {
2167 		struct wil_ctx *ctx;
2168 
2169 		i = (swhead + f) % ring->size;
2170 		ctx = &ring->ctx[i];
2171 		_d = &ring->va[i].tx.legacy;
2172 		*d = *_d;
2173 		_d->dma.status = TX_DMA_STATUS_DU;
2174 		wil->txrx_ops.tx_desc_unmap(dev,
2175 					    (union wil_tx_desc *)d,
2176 					    ctx);
2177 
2178 		memset(ctx, 0, sizeof(*ctx));
2179 	}
2180 
2181 	return -EINVAL;
2182 }
2183 
2184 static int wil_tx_ring(struct wil6210_priv *wil, struct wil6210_vif *vif,
2185 		       struct wil_ring *ring, struct sk_buff *skb)
2186 {
2187 	int ring_index = ring - wil->ring_tx;
2188 	struct wil_ring_tx_data *txdata = &wil->ring_tx_data[ring_index];
2189 	int rc;
2190 
2191 	spin_lock(&txdata->lock);
2192 
2193 	if (test_bit(wil_status_suspending, wil->status) ||
2194 	    test_bit(wil_status_suspended, wil->status) ||
2195 	    test_bit(wil_status_resuming, wil->status)) {
2196 		wil_dbg_txrx(wil,
2197 			     "suspend/resume in progress. drop packet\n");
2198 		spin_unlock(&txdata->lock);
2199 		return -EINVAL;
2200 	}
2201 
2202 	rc = (skb_is_gso(skb) ? wil->txrx_ops.tx_ring_tso : __wil_tx_ring)
2203 	     (wil, vif, ring, skb);
2204 
2205 	spin_unlock(&txdata->lock);
2206 
2207 	return rc;
2208 }
2209 
2210 /* Check status of tx vrings and stop/wake net queues if needed
2211  * It will start/stop net queues of a specific VIF net_device.
2212  *
2213  * This function does one of two checks:
2214  * In case check_stop is true, will check if net queues need to be stopped. If
2215  * the conditions for stopping are met, netif_tx_stop_all_queues() is called.
2216  * In case check_stop is false, will check if net queues need to be waked. If
2217  * the conditions for waking are met, netif_tx_wake_all_queues() is called.
2218  * vring is the vring which is currently being modified by either adding
2219  * descriptors (tx) into it or removing descriptors (tx complete) from it. Can
2220  * be null when irrelevant (e.g. connect/disconnect events).
2221  *
2222  * The implementation is to stop net queues if modified vring has low
2223  * descriptor availability. Wake if all vrings are not in low descriptor
2224  * availability and modified vring has high descriptor availability.
2225  */
2226 static inline void __wil_update_net_queues(struct wil6210_priv *wil,
2227 					   struct wil6210_vif *vif,
2228 					   struct wil_ring *ring,
2229 					   bool check_stop)
2230 {
2231 	int i;
2232 	int min_ring_id = wil_get_min_tx_ring_id(wil);
2233 
2234 	if (unlikely(!vif))
2235 		return;
2236 
2237 	if (ring)
2238 		wil_dbg_txrx(wil, "vring %d, mid %d, check_stop=%d, stopped=%d",
2239 			     (int)(ring - wil->ring_tx), vif->mid, check_stop,
2240 			     vif->net_queue_stopped);
2241 	else
2242 		wil_dbg_txrx(wil, "check_stop=%d, mid=%d, stopped=%d",
2243 			     check_stop, vif->mid, vif->net_queue_stopped);
2244 
2245 	if (ring && drop_if_ring_full)
2246 		/* no need to stop/wake net queues */
2247 		return;
2248 
2249 	if (check_stop == vif->net_queue_stopped)
2250 		/* net queues already in desired state */
2251 		return;
2252 
2253 	if (check_stop) {
2254 		if (!ring || unlikely(wil_ring_avail_low(ring))) {
2255 			/* not enough room in the vring */
2256 			netif_tx_stop_all_queues(vif_to_ndev(vif));
2257 			vif->net_queue_stopped = true;
2258 			wil_dbg_txrx(wil, "netif_tx_stop called\n");
2259 		}
2260 		return;
2261 	}
2262 
2263 	/* Do not wake the queues in suspend flow */
2264 	if (test_bit(wil_status_suspending, wil->status) ||
2265 	    test_bit(wil_status_suspended, wil->status))
2266 		return;
2267 
2268 	/* check wake */
2269 	for (i = min_ring_id; i < WIL6210_MAX_TX_RINGS; i++) {
2270 		struct wil_ring *cur_ring = &wil->ring_tx[i];
2271 		struct wil_ring_tx_data  *txdata = &wil->ring_tx_data[i];
2272 
2273 		if (txdata->mid != vif->mid || !cur_ring->va ||
2274 		    !txdata->enabled || cur_ring == ring)
2275 			continue;
2276 
2277 		if (wil_ring_avail_low(cur_ring)) {
2278 			wil_dbg_txrx(wil, "ring %d full, can't wake\n",
2279 				     (int)(cur_ring - wil->ring_tx));
2280 			return;
2281 		}
2282 	}
2283 
2284 	if (!ring || wil_ring_avail_high(ring)) {
2285 		/* enough room in the ring */
2286 		wil_dbg_txrx(wil, "calling netif_tx_wake\n");
2287 		netif_tx_wake_all_queues(vif_to_ndev(vif));
2288 		vif->net_queue_stopped = false;
2289 	}
2290 }
2291 
2292 void wil_update_net_queues(struct wil6210_priv *wil, struct wil6210_vif *vif,
2293 			   struct wil_ring *ring, bool check_stop)
2294 {
2295 	spin_lock(&wil->net_queue_lock);
2296 	__wil_update_net_queues(wil, vif, ring, check_stop);
2297 	spin_unlock(&wil->net_queue_lock);
2298 }
2299 
2300 void wil_update_net_queues_bh(struct wil6210_priv *wil, struct wil6210_vif *vif,
2301 			      struct wil_ring *ring, bool check_stop)
2302 {
2303 	spin_lock_bh(&wil->net_queue_lock);
2304 	__wil_update_net_queues(wil, vif, ring, check_stop);
2305 	spin_unlock_bh(&wil->net_queue_lock);
2306 }
2307 
2308 netdev_tx_t wil_start_xmit(struct sk_buff *skb, struct net_device *ndev)
2309 {
2310 	struct wil6210_vif *vif = ndev_to_vif(ndev);
2311 	struct wil6210_priv *wil = vif_to_wil(vif);
2312 	const u8 *da = wil_skb_get_da(skb);
2313 	bool bcast = is_multicast_ether_addr(da);
2314 	struct wil_ring *ring;
2315 	static bool pr_once_fw;
2316 	int rc;
2317 
2318 	wil_dbg_txrx(wil, "start_xmit\n");
2319 	if (unlikely(!test_bit(wil_status_fwready, wil->status))) {
2320 		if (!pr_once_fw) {
2321 			wil_err(wil, "FW not ready\n");
2322 			pr_once_fw = true;
2323 		}
2324 		goto drop;
2325 	}
2326 	if (unlikely(!test_bit(wil_vif_fwconnected, vif->status))) {
2327 		wil_dbg_ratelimited(wil,
2328 				    "VIF not connected, packet dropped\n");
2329 		goto drop;
2330 	}
2331 	if (unlikely(vif->wdev.iftype == NL80211_IFTYPE_MONITOR)) {
2332 		wil_err(wil, "Xmit in monitor mode not supported\n");
2333 		goto drop;
2334 	}
2335 	pr_once_fw = false;
2336 
2337 	/* find vring */
2338 	if (vif->wdev.iftype == NL80211_IFTYPE_STATION && !vif->pbss) {
2339 		/* in STA mode (ESS), all to same VRING (to AP) */
2340 		ring = wil_find_tx_ring_sta(wil, vif, skb);
2341 	} else if (bcast) {
2342 		if (vif->pbss || wil_check_multicast_to_unicast(wil, skb))
2343 			/* in pbss, no bcast VRING - duplicate skb in
2344 			 * all stations VRINGs
2345 			 */
2346 			ring = wil_find_tx_bcast_2(wil, vif, skb);
2347 		else if (vif->wdev.iftype == NL80211_IFTYPE_AP)
2348 			/* AP has a dedicated bcast VRING */
2349 			ring = wil_find_tx_bcast_1(wil, vif, skb);
2350 		else
2351 			/* unexpected combination, fallback to duplicating
2352 			 * the skb in all stations VRINGs
2353 			 */
2354 			ring = wil_find_tx_bcast_2(wil, vif, skb);
2355 	} else {
2356 		/* unicast, find specific VRING by dest. address */
2357 		ring = wil_find_tx_ucast(wil, vif, skb);
2358 	}
2359 	if (unlikely(!ring)) {
2360 		wil_dbg_txrx(wil, "No Tx RING found for %pM\n", da);
2361 		goto drop;
2362 	}
2363 	/* set up vring entry */
2364 	rc = wil_tx_ring(wil, vif, ring, skb);
2365 
2366 	switch (rc) {
2367 	case 0:
2368 		/* shall we stop net queues? */
2369 		wil_update_net_queues_bh(wil, vif, ring, true);
2370 		/* statistics will be updated on the tx_complete */
2371 		dev_kfree_skb_any(skb);
2372 		return NETDEV_TX_OK;
2373 	case -ENOMEM:
2374 		if (drop_if_ring_full)
2375 			goto drop;
2376 		return NETDEV_TX_BUSY;
2377 	default:
2378 		break; /* goto drop; */
2379 	}
2380  drop:
2381 	ndev->stats.tx_dropped++;
2382 	dev_kfree_skb_any(skb);
2383 
2384 	return NET_XMIT_DROP;
2385 }
2386 
2387 void wil_tx_latency_calc(struct wil6210_priv *wil, struct sk_buff *skb,
2388 			 struct wil_sta_info *sta)
2389 {
2390 	int skb_time_us;
2391 	int bin;
2392 
2393 	if (!wil->tx_latency)
2394 		return;
2395 
2396 	if (ktime_to_ms(*(ktime_t *)&skb->cb) == 0)
2397 		return;
2398 
2399 	skb_time_us = ktime_us_delta(ktime_get(), *(ktime_t *)&skb->cb);
2400 	bin = skb_time_us / wil->tx_latency_res;
2401 	bin = min_t(int, bin, WIL_NUM_LATENCY_BINS - 1);
2402 
2403 	wil_dbg_txrx(wil, "skb time %dus => bin %d\n", skb_time_us, bin);
2404 	sta->tx_latency_bins[bin]++;
2405 	sta->stats.tx_latency_total_us += skb_time_us;
2406 	if (skb_time_us < sta->stats.tx_latency_min_us)
2407 		sta->stats.tx_latency_min_us = skb_time_us;
2408 	if (skb_time_us > sta->stats.tx_latency_max_us)
2409 		sta->stats.tx_latency_max_us = skb_time_us;
2410 }
2411 
2412 /* Clean up transmitted skb's from the Tx VRING
2413  *
2414  * Return number of descriptors cleared
2415  *
2416  * Safe to call from IRQ
2417  */
2418 int wil_tx_complete(struct wil6210_vif *vif, int ringid)
2419 {
2420 	struct wil6210_priv *wil = vif_to_wil(vif);
2421 	struct net_device *ndev = vif_to_ndev(vif);
2422 	struct device *dev = wil_to_dev(wil);
2423 	struct wil_ring *vring = &wil->ring_tx[ringid];
2424 	struct wil_ring_tx_data *txdata = &wil->ring_tx_data[ringid];
2425 	int done = 0;
2426 	int cid = wil->ring2cid_tid[ringid][0];
2427 	struct wil_net_stats *stats = NULL;
2428 	volatile struct vring_tx_desc *_d;
2429 	int used_before_complete;
2430 	int used_new;
2431 
2432 	if (unlikely(!vring->va)) {
2433 		wil_err(wil, "Tx irq[%d]: vring not initialized\n", ringid);
2434 		return 0;
2435 	}
2436 
2437 	if (unlikely(!txdata->enabled)) {
2438 		wil_info(wil, "Tx irq[%d]: vring disabled\n", ringid);
2439 		return 0;
2440 	}
2441 
2442 	wil_dbg_txrx(wil, "tx_complete: (%d)\n", ringid);
2443 
2444 	used_before_complete = wil_ring_used_tx(vring);
2445 
2446 	if (cid < wil->max_assoc_sta)
2447 		stats = &wil->sta[cid].stats;
2448 
2449 	while (!wil_ring_is_empty(vring)) {
2450 		int new_swtail;
2451 		struct wil_ctx *ctx = &vring->ctx[vring->swtail];
2452 		/* For the fragmented skb, HW will set DU bit only for the
2453 		 * last fragment. look for it.
2454 		 * In TSO the first DU will include hdr desc
2455 		 */
2456 		int lf = (vring->swtail + ctx->nr_frags) % vring->size;
2457 		/* TODO: check we are not past head */
2458 
2459 		_d = &vring->va[lf].tx.legacy;
2460 		if (unlikely(!(_d->dma.status & TX_DMA_STATUS_DU)))
2461 			break;
2462 
2463 		new_swtail = (lf + 1) % vring->size;
2464 		while (vring->swtail != new_swtail) {
2465 			struct vring_tx_desc dd, *d = &dd;
2466 			u16 dmalen;
2467 			struct sk_buff *skb;
2468 
2469 			ctx = &vring->ctx[vring->swtail];
2470 			skb = ctx->skb;
2471 			_d = &vring->va[vring->swtail].tx.legacy;
2472 
2473 			*d = *_d;
2474 
2475 			dmalen = le16_to_cpu(d->dma.length);
2476 			trace_wil6210_tx_done(ringid, vring->swtail, dmalen,
2477 					      d->dma.error);
2478 			wil_dbg_txrx(wil,
2479 				     "TxC[%2d][%3d] : %d bytes, status 0x%02x err 0x%02x\n",
2480 				     ringid, vring->swtail, dmalen,
2481 				     d->dma.status, d->dma.error);
2482 			wil_hex_dump_txrx("TxCD ", DUMP_PREFIX_NONE, 32, 4,
2483 					  (const void *)d, sizeof(*d), false);
2484 
2485 			wil->txrx_ops.tx_desc_unmap(dev,
2486 						    (union wil_tx_desc *)d,
2487 						    ctx);
2488 
2489 			if (skb) {
2490 				if (likely(d->dma.error == 0)) {
2491 					ndev->stats.tx_packets++;
2492 					ndev->stats.tx_bytes += skb->len;
2493 					if (stats) {
2494 						stats->tx_packets++;
2495 						stats->tx_bytes += skb->len;
2496 
2497 						wil_tx_latency_calc(wil, skb,
2498 							&wil->sta[cid]);
2499 					}
2500 				} else {
2501 					ndev->stats.tx_errors++;
2502 					if (stats)
2503 						stats->tx_errors++;
2504 				}
2505 
2506 				if (skb->protocol == cpu_to_be16(ETH_P_PAE))
2507 					wil_tx_complete_handle_eapol(vif, skb);
2508 
2509 				wil_consume_skb(skb, d->dma.error == 0);
2510 			}
2511 			memset(ctx, 0, sizeof(*ctx));
2512 			/* Make sure the ctx is zeroed before updating the tail
2513 			 * to prevent a case where wil_tx_ring will see
2514 			 * this descriptor as used and handle it before ctx zero
2515 			 * is completed.
2516 			 */
2517 			wmb();
2518 			/* There is no need to touch HW descriptor:
2519 			 * - ststus bit TX_DMA_STATUS_DU is set by design,
2520 			 *   so hardware will not try to process this desc.,
2521 			 * - rest of descriptor will be initialized on Tx.
2522 			 */
2523 			vring->swtail = wil_ring_next_tail(vring);
2524 			done++;
2525 		}
2526 	}
2527 
2528 	/* performance monitoring */
2529 	used_new = wil_ring_used_tx(vring);
2530 	if (wil_val_in_range(wil->ring_idle_trsh,
2531 			     used_new, used_before_complete)) {
2532 		wil_dbg_txrx(wil, "Ring[%2d] idle %d -> %d\n",
2533 			     ringid, used_before_complete, used_new);
2534 		txdata->last_idle = get_cycles();
2535 	}
2536 
2537 	/* shall we wake net queues? */
2538 	if (done)
2539 		wil_update_net_queues(wil, vif, vring, false);
2540 
2541 	return done;
2542 }
2543 
2544 static inline int wil_tx_init(struct wil6210_priv *wil)
2545 {
2546 	return 0;
2547 }
2548 
2549 static inline void wil_tx_fini(struct wil6210_priv *wil) {}
2550 
2551 static void wil_get_reorder_params(struct wil6210_priv *wil,
2552 				   struct sk_buff *skb, int *tid, int *cid,
2553 				   int *mid, u16 *seq, int *mcast, int *retry)
2554 {
2555 	struct vring_rx_desc *d = wil_skb_rxdesc(skb);
2556 
2557 	*tid = wil_rxdesc_tid(d);
2558 	*cid = wil_skb_get_cid(skb);
2559 	*mid = wil_rxdesc_mid(d);
2560 	*seq = wil_rxdesc_seq(d);
2561 	*mcast = wil_rxdesc_mcast(d);
2562 	*retry = wil_rxdesc_retry(d);
2563 }
2564 
2565 void wil_init_txrx_ops_legacy_dma(struct wil6210_priv *wil)
2566 {
2567 	wil->txrx_ops.configure_interrupt_moderation =
2568 		wil_configure_interrupt_moderation;
2569 	/* TX ops */
2570 	wil->txrx_ops.tx_desc_map = wil_tx_desc_map;
2571 	wil->txrx_ops.tx_desc_unmap = wil_txdesc_unmap;
2572 	wil->txrx_ops.tx_ring_tso =  __wil_tx_vring_tso;
2573 	wil->txrx_ops.ring_init_tx = wil_vring_init_tx;
2574 	wil->txrx_ops.ring_fini_tx = wil_vring_free;
2575 	wil->txrx_ops.ring_init_bcast = wil_vring_init_bcast;
2576 	wil->txrx_ops.tx_init = wil_tx_init;
2577 	wil->txrx_ops.tx_fini = wil_tx_fini;
2578 	wil->txrx_ops.tx_ring_modify = wil_tx_vring_modify;
2579 	/* RX ops */
2580 	wil->txrx_ops.rx_init = wil_rx_init;
2581 	wil->txrx_ops.wmi_addba_rx_resp = wmi_addba_rx_resp;
2582 	wil->txrx_ops.get_reorder_params = wil_get_reorder_params;
2583 	wil->txrx_ops.get_netif_rx_params =
2584 		wil_get_netif_rx_params;
2585 	wil->txrx_ops.rx_crypto_check = wil_rx_crypto_check;
2586 	wil->txrx_ops.rx_error_check = wil_rx_error_check;
2587 	wil->txrx_ops.is_rx_idle = wil_is_rx_idle;
2588 	wil->txrx_ops.rx_fini = wil_rx_fini;
2589 }
2590