1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2019, Intel Corporation. */
3 
4 #include <linux/bpf_trace.h>
5 #include <net/xdp_sock_drv.h>
6 #include <net/xdp.h>
7 #include "ice.h"
8 #include "ice_base.h"
9 #include "ice_type.h"
10 #include "ice_xsk.h"
11 #include "ice_txrx.h"
12 #include "ice_txrx_lib.h"
13 #include "ice_lib.h"
14 
ice_xdp_buf(struct ice_rx_ring * rx_ring,u32 idx)15 static struct xdp_buff **ice_xdp_buf(struct ice_rx_ring *rx_ring, u32 idx)
16 {
17 	return &rx_ring->xdp_buf[idx];
18 }
19 
20 /**
21  * ice_qp_reset_stats - Resets all stats for rings of given index
22  * @vsi: VSI that contains rings of interest
23  * @q_idx: ring index in array
24  */
ice_qp_reset_stats(struct ice_vsi * vsi,u16 q_idx)25 static void ice_qp_reset_stats(struct ice_vsi *vsi, u16 q_idx)
26 {
27 	struct ice_vsi_stats *vsi_stat;
28 	struct ice_pf *pf;
29 
30 	pf = vsi->back;
31 	if (!pf->vsi_stats)
32 		return;
33 
34 	vsi_stat = pf->vsi_stats[vsi->idx];
35 	if (!vsi_stat)
36 		return;
37 
38 	memset(&vsi_stat->rx_ring_stats[q_idx]->rx_stats, 0,
39 	       sizeof(vsi_stat->rx_ring_stats[q_idx]->rx_stats));
40 	memset(&vsi_stat->tx_ring_stats[q_idx]->stats, 0,
41 	       sizeof(vsi_stat->tx_ring_stats[q_idx]->stats));
42 	if (ice_is_xdp_ena_vsi(vsi))
43 		memset(&vsi->xdp_rings[q_idx]->ring_stats->stats, 0,
44 		       sizeof(vsi->xdp_rings[q_idx]->ring_stats->stats));
45 }
46 
47 /**
48  * ice_qp_clean_rings - Cleans all the rings of a given index
49  * @vsi: VSI that contains rings of interest
50  * @q_idx: ring index in array
51  */
ice_qp_clean_rings(struct ice_vsi * vsi,u16 q_idx)52 static void ice_qp_clean_rings(struct ice_vsi *vsi, u16 q_idx)
53 {
54 	ice_clean_tx_ring(vsi->tx_rings[q_idx]);
55 	if (ice_is_xdp_ena_vsi(vsi))
56 		ice_clean_tx_ring(vsi->xdp_rings[q_idx]);
57 	ice_clean_rx_ring(vsi->rx_rings[q_idx]);
58 }
59 
60 /**
61  * ice_qvec_toggle_napi - Enables/disables NAPI for a given q_vector
62  * @vsi: VSI that has netdev
63  * @q_vector: q_vector that has NAPI context
64  * @enable: true for enable, false for disable
65  */
66 static void
ice_qvec_toggle_napi(struct ice_vsi * vsi,struct ice_q_vector * q_vector,bool enable)67 ice_qvec_toggle_napi(struct ice_vsi *vsi, struct ice_q_vector *q_vector,
68 		     bool enable)
69 {
70 	if (!vsi->netdev || !q_vector)
71 		return;
72 
73 	if (enable)
74 		napi_enable(&q_vector->napi);
75 	else
76 		napi_disable(&q_vector->napi);
77 }
78 
79 /**
80  * ice_qvec_dis_irq - Mask off queue interrupt generation on given ring
81  * @vsi: the VSI that contains queue vector being un-configured
82  * @rx_ring: Rx ring that will have its IRQ disabled
83  * @q_vector: queue vector
84  */
85 static void
ice_qvec_dis_irq(struct ice_vsi * vsi,struct ice_rx_ring * rx_ring,struct ice_q_vector * q_vector)86 ice_qvec_dis_irq(struct ice_vsi *vsi, struct ice_rx_ring *rx_ring,
87 		 struct ice_q_vector *q_vector)
88 {
89 	struct ice_pf *pf = vsi->back;
90 	struct ice_hw *hw = &pf->hw;
91 	u16 reg;
92 	u32 val;
93 
94 	/* QINT_TQCTL is being cleared in ice_vsi_stop_tx_ring, so handle
95 	 * here only QINT_RQCTL
96 	 */
97 	reg = rx_ring->reg_idx;
98 	val = rd32(hw, QINT_RQCTL(reg));
99 	val &= ~QINT_RQCTL_CAUSE_ENA_M;
100 	wr32(hw, QINT_RQCTL(reg), val);
101 
102 	if (q_vector) {
103 		wr32(hw, GLINT_DYN_CTL(q_vector->reg_idx), 0);
104 		ice_flush(hw);
105 		synchronize_irq(q_vector->irq.virq);
106 	}
107 }
108 
109 /**
110  * ice_qvec_cfg_msix - Enable IRQ for given queue vector
111  * @vsi: the VSI that contains queue vector
112  * @q_vector: queue vector
113  */
114 static void
ice_qvec_cfg_msix(struct ice_vsi * vsi,struct ice_q_vector * q_vector)115 ice_qvec_cfg_msix(struct ice_vsi *vsi, struct ice_q_vector *q_vector)
116 {
117 	u16 reg_idx = q_vector->reg_idx;
118 	struct ice_pf *pf = vsi->back;
119 	struct ice_hw *hw = &pf->hw;
120 	struct ice_tx_ring *tx_ring;
121 	struct ice_rx_ring *rx_ring;
122 
123 	ice_cfg_itr(hw, q_vector);
124 
125 	ice_for_each_tx_ring(tx_ring, q_vector->tx)
126 		ice_cfg_txq_interrupt(vsi, tx_ring->reg_idx, reg_idx,
127 				      q_vector->tx.itr_idx);
128 
129 	ice_for_each_rx_ring(rx_ring, q_vector->rx)
130 		ice_cfg_rxq_interrupt(vsi, rx_ring->reg_idx, reg_idx,
131 				      q_vector->rx.itr_idx);
132 
133 	ice_flush(hw);
134 }
135 
136 /**
137  * ice_qvec_ena_irq - Enable IRQ for given queue vector
138  * @vsi: the VSI that contains queue vector
139  * @q_vector: queue vector
140  */
ice_qvec_ena_irq(struct ice_vsi * vsi,struct ice_q_vector * q_vector)141 static void ice_qvec_ena_irq(struct ice_vsi *vsi, struct ice_q_vector *q_vector)
142 {
143 	struct ice_pf *pf = vsi->back;
144 	struct ice_hw *hw = &pf->hw;
145 
146 	ice_irq_dynamic_ena(hw, vsi, q_vector);
147 
148 	ice_flush(hw);
149 }
150 
151 /**
152  * ice_qp_dis - Disables a queue pair
153  * @vsi: VSI of interest
154  * @q_idx: ring index in array
155  *
156  * Returns 0 on success, negative on failure.
157  */
ice_qp_dis(struct ice_vsi * vsi,u16 q_idx)158 static int ice_qp_dis(struct ice_vsi *vsi, u16 q_idx)
159 {
160 	struct ice_txq_meta txq_meta = { };
161 	struct ice_q_vector *q_vector;
162 	struct ice_tx_ring *tx_ring;
163 	struct ice_rx_ring *rx_ring;
164 	int timeout = 50;
165 	int err;
166 
167 	if (q_idx >= vsi->num_rxq || q_idx >= vsi->num_txq)
168 		return -EINVAL;
169 
170 	tx_ring = vsi->tx_rings[q_idx];
171 	rx_ring = vsi->rx_rings[q_idx];
172 	q_vector = rx_ring->q_vector;
173 
174 	while (test_and_set_bit(ICE_CFG_BUSY, vsi->state)) {
175 		timeout--;
176 		if (!timeout)
177 			return -EBUSY;
178 		usleep_range(1000, 2000);
179 	}
180 
181 	synchronize_net();
182 	netif_tx_stop_queue(netdev_get_tx_queue(vsi->netdev, q_idx));
183 
184 	ice_qvec_dis_irq(vsi, rx_ring, q_vector);
185 	ice_qvec_toggle_napi(vsi, q_vector, false);
186 
187 	ice_fill_txq_meta(vsi, tx_ring, &txq_meta);
188 	err = ice_vsi_stop_tx_ring(vsi, ICE_NO_RESET, 0, tx_ring, &txq_meta);
189 	if (err)
190 		return err;
191 	if (ice_is_xdp_ena_vsi(vsi)) {
192 		struct ice_tx_ring *xdp_ring = vsi->xdp_rings[q_idx];
193 
194 		memset(&txq_meta, 0, sizeof(txq_meta));
195 		ice_fill_txq_meta(vsi, xdp_ring, &txq_meta);
196 		err = ice_vsi_stop_tx_ring(vsi, ICE_NO_RESET, 0, xdp_ring,
197 					   &txq_meta);
198 		if (err)
199 			return err;
200 	}
201 
202 	ice_vsi_ctrl_one_rx_ring(vsi, false, q_idx, false);
203 	ice_qp_clean_rings(vsi, q_idx);
204 	ice_qp_reset_stats(vsi, q_idx);
205 
206 	return 0;
207 }
208 
209 /**
210  * ice_qp_ena - Enables a queue pair
211  * @vsi: VSI of interest
212  * @q_idx: ring index in array
213  *
214  * Returns 0 on success, negative on failure.
215  */
ice_qp_ena(struct ice_vsi * vsi,u16 q_idx)216 static int ice_qp_ena(struct ice_vsi *vsi, u16 q_idx)
217 {
218 	struct ice_aqc_add_tx_qgrp *qg_buf;
219 	struct ice_q_vector *q_vector;
220 	struct ice_tx_ring *tx_ring;
221 	struct ice_rx_ring *rx_ring;
222 	u16 size;
223 	int err;
224 
225 	if (q_idx >= vsi->num_rxq || q_idx >= vsi->num_txq)
226 		return -EINVAL;
227 
228 	size = struct_size(qg_buf, txqs, 1);
229 	qg_buf = kzalloc(size, GFP_KERNEL);
230 	if (!qg_buf)
231 		return -ENOMEM;
232 
233 	qg_buf->num_txqs = 1;
234 
235 	tx_ring = vsi->tx_rings[q_idx];
236 	rx_ring = vsi->rx_rings[q_idx];
237 	q_vector = rx_ring->q_vector;
238 
239 	err = ice_vsi_cfg_txq(vsi, tx_ring, qg_buf);
240 	if (err)
241 		goto free_buf;
242 
243 	if (ice_is_xdp_ena_vsi(vsi)) {
244 		struct ice_tx_ring *xdp_ring = vsi->xdp_rings[q_idx];
245 
246 		memset(qg_buf, 0, size);
247 		qg_buf->num_txqs = 1;
248 		err = ice_vsi_cfg_txq(vsi, xdp_ring, qg_buf);
249 		if (err)
250 			goto free_buf;
251 		ice_set_ring_xdp(xdp_ring);
252 		ice_tx_xsk_pool(vsi, q_idx);
253 	}
254 
255 	err = ice_vsi_cfg_rxq(rx_ring);
256 	if (err)
257 		goto free_buf;
258 
259 	ice_qvec_cfg_msix(vsi, q_vector);
260 
261 	err = ice_vsi_ctrl_one_rx_ring(vsi, true, q_idx, true);
262 	if (err)
263 		goto free_buf;
264 
265 	ice_qvec_toggle_napi(vsi, q_vector, true);
266 	ice_qvec_ena_irq(vsi, q_vector);
267 
268 	netif_tx_start_queue(netdev_get_tx_queue(vsi->netdev, q_idx));
269 	clear_bit(ICE_CFG_BUSY, vsi->state);
270 free_buf:
271 	kfree(qg_buf);
272 	return err;
273 }
274 
275 /**
276  * ice_xsk_pool_disable - disable a buffer pool region
277  * @vsi: Current VSI
278  * @qid: queue ID
279  *
280  * Returns 0 on success, negative on failure
281  */
ice_xsk_pool_disable(struct ice_vsi * vsi,u16 qid)282 static int ice_xsk_pool_disable(struct ice_vsi *vsi, u16 qid)
283 {
284 	struct xsk_buff_pool *pool = xsk_get_pool_from_qid(vsi->netdev, qid);
285 
286 	if (!pool)
287 		return -EINVAL;
288 
289 	xsk_pool_dma_unmap(pool, ICE_RX_DMA_ATTR);
290 
291 	return 0;
292 }
293 
294 /**
295  * ice_xsk_pool_enable - enable a buffer pool region
296  * @vsi: Current VSI
297  * @pool: pointer to a requested buffer pool region
298  * @qid: queue ID
299  *
300  * Returns 0 on success, negative on failure
301  */
302 static int
ice_xsk_pool_enable(struct ice_vsi * vsi,struct xsk_buff_pool * pool,u16 qid)303 ice_xsk_pool_enable(struct ice_vsi *vsi, struct xsk_buff_pool *pool, u16 qid)
304 {
305 	int err;
306 
307 	if (vsi->type != ICE_VSI_PF)
308 		return -EINVAL;
309 
310 	if (qid >= vsi->netdev->real_num_rx_queues ||
311 	    qid >= vsi->netdev->real_num_tx_queues)
312 		return -EINVAL;
313 
314 	err = xsk_pool_dma_map(pool, ice_pf_to_dev(vsi->back),
315 			       ICE_RX_DMA_ATTR);
316 	if (err)
317 		return err;
318 
319 	return 0;
320 }
321 
322 /**
323  * ice_realloc_rx_xdp_bufs - reallocate for either XSK or normal buffer
324  * @rx_ring: Rx ring
325  * @pool_present: is pool for XSK present
326  *
327  * Try allocating memory and return ENOMEM, if failed to allocate.
328  * If allocation was successful, substitute buffer with allocated one.
329  * Returns 0 on success, negative on failure
330  */
331 static int
ice_realloc_rx_xdp_bufs(struct ice_rx_ring * rx_ring,bool pool_present)332 ice_realloc_rx_xdp_bufs(struct ice_rx_ring *rx_ring, bool pool_present)
333 {
334 	size_t elem_size = pool_present ? sizeof(*rx_ring->xdp_buf) :
335 					  sizeof(*rx_ring->rx_buf);
336 	void *sw_ring = kcalloc(rx_ring->count, elem_size, GFP_KERNEL);
337 
338 	if (!sw_ring)
339 		return -ENOMEM;
340 
341 	if (pool_present) {
342 		kfree(rx_ring->rx_buf);
343 		rx_ring->rx_buf = NULL;
344 		rx_ring->xdp_buf = sw_ring;
345 	} else {
346 		kfree(rx_ring->xdp_buf);
347 		rx_ring->xdp_buf = NULL;
348 		rx_ring->rx_buf = sw_ring;
349 	}
350 
351 	return 0;
352 }
353 
354 /**
355  * ice_realloc_zc_buf - reallocate XDP ZC queue pairs
356  * @vsi: Current VSI
357  * @zc: is zero copy set
358  *
359  * Reallocate buffer for rx_rings that might be used by XSK.
360  * XDP requires more memory, than rx_buf provides.
361  * Returns 0 on success, negative on failure
362  */
ice_realloc_zc_buf(struct ice_vsi * vsi,bool zc)363 int ice_realloc_zc_buf(struct ice_vsi *vsi, bool zc)
364 {
365 	struct ice_rx_ring *rx_ring;
366 	uint i;
367 
368 	ice_for_each_rxq(vsi, i) {
369 		rx_ring = vsi->rx_rings[i];
370 		if (!rx_ring->xsk_pool)
371 			continue;
372 
373 		if (ice_realloc_rx_xdp_bufs(rx_ring, zc))
374 			return -ENOMEM;
375 	}
376 
377 	return 0;
378 }
379 
380 /**
381  * ice_xsk_pool_setup - enable/disable a buffer pool region depending on its state
382  * @vsi: Current VSI
383  * @pool: buffer pool to enable/associate to a ring, NULL to disable
384  * @qid: queue ID
385  *
386  * Returns 0 on success, negative on failure
387  */
ice_xsk_pool_setup(struct ice_vsi * vsi,struct xsk_buff_pool * pool,u16 qid)388 int ice_xsk_pool_setup(struct ice_vsi *vsi, struct xsk_buff_pool *pool, u16 qid)
389 {
390 	bool if_running, pool_present = !!pool;
391 	int ret = 0, pool_failure = 0;
392 
393 	if (qid >= vsi->num_rxq || qid >= vsi->num_txq) {
394 		netdev_err(vsi->netdev, "Please use queue id in scope of combined queues count\n");
395 		pool_failure = -EINVAL;
396 		goto failure;
397 	}
398 
399 	if_running = !test_bit(ICE_VSI_DOWN, vsi->state) &&
400 		     ice_is_xdp_ena_vsi(vsi);
401 
402 	if (if_running) {
403 		struct ice_rx_ring *rx_ring = vsi->rx_rings[qid];
404 
405 		ret = ice_qp_dis(vsi, qid);
406 		if (ret) {
407 			netdev_err(vsi->netdev, "ice_qp_dis error = %d\n", ret);
408 			goto xsk_pool_if_up;
409 		}
410 
411 		ret = ice_realloc_rx_xdp_bufs(rx_ring, pool_present);
412 		if (ret)
413 			goto xsk_pool_if_up;
414 	}
415 
416 	pool_failure = pool_present ? ice_xsk_pool_enable(vsi, pool, qid) :
417 				      ice_xsk_pool_disable(vsi, qid);
418 
419 xsk_pool_if_up:
420 	if (if_running) {
421 		ret = ice_qp_ena(vsi, qid);
422 		if (!ret && pool_present)
423 			napi_schedule(&vsi->rx_rings[qid]->xdp_ring->q_vector->napi);
424 		else if (ret)
425 			netdev_err(vsi->netdev, "ice_qp_ena error = %d\n", ret);
426 	}
427 
428 failure:
429 	if (pool_failure) {
430 		netdev_err(vsi->netdev, "Could not %sable buffer pool, error = %d\n",
431 			   pool_present ? "en" : "dis", pool_failure);
432 		return pool_failure;
433 	}
434 
435 	return ret;
436 }
437 
438 /**
439  * ice_fill_rx_descs - pick buffers from XSK buffer pool and use it
440  * @pool: XSK Buffer pool to pull the buffers from
441  * @xdp: SW ring of xdp_buff that will hold the buffers
442  * @rx_desc: Pointer to Rx descriptors that will be filled
443  * @count: The number of buffers to allocate
444  *
445  * This function allocates a number of Rx buffers from the fill ring
446  * or the internal recycle mechanism and places them on the Rx ring.
447  *
448  * Note that ring wrap should be handled by caller of this function.
449  *
450  * Returns the amount of allocated Rx descriptors
451  */
ice_fill_rx_descs(struct xsk_buff_pool * pool,struct xdp_buff ** xdp,union ice_32b_rx_flex_desc * rx_desc,u16 count)452 static u16 ice_fill_rx_descs(struct xsk_buff_pool *pool, struct xdp_buff **xdp,
453 			     union ice_32b_rx_flex_desc *rx_desc, u16 count)
454 {
455 	dma_addr_t dma;
456 	u16 buffs;
457 	int i;
458 
459 	buffs = xsk_buff_alloc_batch(pool, xdp, count);
460 	for (i = 0; i < buffs; i++) {
461 		dma = xsk_buff_xdp_get_dma(*xdp);
462 		rx_desc->read.pkt_addr = cpu_to_le64(dma);
463 		rx_desc->wb.status_error0 = 0;
464 
465 		rx_desc++;
466 		xdp++;
467 	}
468 
469 	return buffs;
470 }
471 
472 /**
473  * __ice_alloc_rx_bufs_zc - allocate a number of Rx buffers
474  * @rx_ring: Rx ring
475  * @count: The number of buffers to allocate
476  *
477  * Place the @count of descriptors onto Rx ring. Handle the ring wrap
478  * for case where space from next_to_use up to the end of ring is less
479  * than @count. Finally do a tail bump.
480  *
481  * Returns true if all allocations were successful, false if any fail.
482  */
__ice_alloc_rx_bufs_zc(struct ice_rx_ring * rx_ring,u16 count)483 static bool __ice_alloc_rx_bufs_zc(struct ice_rx_ring *rx_ring, u16 count)
484 {
485 	u32 nb_buffs_extra = 0, nb_buffs = 0;
486 	union ice_32b_rx_flex_desc *rx_desc;
487 	u16 ntu = rx_ring->next_to_use;
488 	u16 total_count = count;
489 	struct xdp_buff **xdp;
490 
491 	rx_desc = ICE_RX_DESC(rx_ring, ntu);
492 	xdp = ice_xdp_buf(rx_ring, ntu);
493 
494 	if (ntu + count >= rx_ring->count) {
495 		nb_buffs_extra = ice_fill_rx_descs(rx_ring->xsk_pool, xdp,
496 						   rx_desc,
497 						   rx_ring->count - ntu);
498 		if (nb_buffs_extra != rx_ring->count - ntu) {
499 			ntu += nb_buffs_extra;
500 			goto exit;
501 		}
502 		rx_desc = ICE_RX_DESC(rx_ring, 0);
503 		xdp = ice_xdp_buf(rx_ring, 0);
504 		ntu = 0;
505 		count -= nb_buffs_extra;
506 		ice_release_rx_desc(rx_ring, 0);
507 	}
508 
509 	nb_buffs = ice_fill_rx_descs(rx_ring->xsk_pool, xdp, rx_desc, count);
510 
511 	ntu += nb_buffs;
512 	if (ntu == rx_ring->count)
513 		ntu = 0;
514 
515 exit:
516 	if (rx_ring->next_to_use != ntu)
517 		ice_release_rx_desc(rx_ring, ntu);
518 
519 	return total_count == (nb_buffs_extra + nb_buffs);
520 }
521 
522 /**
523  * ice_alloc_rx_bufs_zc - allocate a number of Rx buffers
524  * @rx_ring: Rx ring
525  * @count: The number of buffers to allocate
526  *
527  * Wrapper for internal allocation routine; figure out how many tail
528  * bumps should take place based on the given threshold
529  *
530  * Returns true if all calls to internal alloc routine succeeded
531  */
ice_alloc_rx_bufs_zc(struct ice_rx_ring * rx_ring,u16 count)532 bool ice_alloc_rx_bufs_zc(struct ice_rx_ring *rx_ring, u16 count)
533 {
534 	u16 rx_thresh = ICE_RING_QUARTER(rx_ring);
535 	u16 leftover, i, tail_bumps;
536 
537 	tail_bumps = count / rx_thresh;
538 	leftover = count - (tail_bumps * rx_thresh);
539 
540 	for (i = 0; i < tail_bumps; i++)
541 		if (!__ice_alloc_rx_bufs_zc(rx_ring, rx_thresh))
542 			return false;
543 	return __ice_alloc_rx_bufs_zc(rx_ring, leftover);
544 }
545 
546 /**
547  * ice_construct_skb_zc - Create an sk_buff from zero-copy buffer
548  * @rx_ring: Rx ring
549  * @xdp: Pointer to XDP buffer
550  *
551  * This function allocates a new skb from a zero-copy Rx buffer.
552  *
553  * Returns the skb on success, NULL on failure.
554  */
555 static struct sk_buff *
ice_construct_skb_zc(struct ice_rx_ring * rx_ring,struct xdp_buff * xdp)556 ice_construct_skb_zc(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp)
557 {
558 	unsigned int totalsize = xdp->data_end - xdp->data_meta;
559 	unsigned int metasize = xdp->data - xdp->data_meta;
560 	struct skb_shared_info *sinfo = NULL;
561 	struct sk_buff *skb;
562 	u32 nr_frags = 0;
563 
564 	if (unlikely(xdp_buff_has_frags(xdp))) {
565 		sinfo = xdp_get_shared_info_from_buff(xdp);
566 		nr_frags = sinfo->nr_frags;
567 	}
568 	net_prefetch(xdp->data_meta);
569 
570 	skb = __napi_alloc_skb(&rx_ring->q_vector->napi, totalsize,
571 			       GFP_ATOMIC | __GFP_NOWARN);
572 	if (unlikely(!skb))
573 		return NULL;
574 
575 	memcpy(__skb_put(skb, totalsize), xdp->data_meta,
576 	       ALIGN(totalsize, sizeof(long)));
577 
578 	if (metasize) {
579 		skb_metadata_set(skb, metasize);
580 		__skb_pull(skb, metasize);
581 	}
582 
583 	if (likely(!xdp_buff_has_frags(xdp)))
584 		goto out;
585 
586 	for (int i = 0; i < nr_frags; i++) {
587 		struct skb_shared_info *skinfo = skb_shinfo(skb);
588 		skb_frag_t *frag = &sinfo->frags[i];
589 		struct page *page;
590 		void *addr;
591 
592 		page = dev_alloc_page();
593 		if (!page) {
594 			dev_kfree_skb(skb);
595 			return NULL;
596 		}
597 		addr = page_to_virt(page);
598 
599 		memcpy(addr, skb_frag_page(frag), skb_frag_size(frag));
600 
601 		__skb_fill_page_desc_noacc(skinfo, skinfo->nr_frags++,
602 					   addr, 0, skb_frag_size(frag));
603 	}
604 
605 out:
606 	xsk_buff_free(xdp);
607 	return skb;
608 }
609 
610 /**
611  * ice_clean_xdp_irq_zc - produce AF_XDP descriptors to CQ
612  * @xdp_ring: XDP Tx ring
613  */
ice_clean_xdp_irq_zc(struct ice_tx_ring * xdp_ring)614 static u32 ice_clean_xdp_irq_zc(struct ice_tx_ring *xdp_ring)
615 {
616 	u16 ntc = xdp_ring->next_to_clean;
617 	struct ice_tx_desc *tx_desc;
618 	u16 cnt = xdp_ring->count;
619 	struct ice_tx_buf *tx_buf;
620 	u16 completed_frames = 0;
621 	u16 xsk_frames = 0;
622 	u16 last_rs;
623 	int i;
624 
625 	last_rs = xdp_ring->next_to_use ? xdp_ring->next_to_use - 1 : cnt - 1;
626 	tx_desc = ICE_TX_DESC(xdp_ring, last_rs);
627 	if (tx_desc->cmd_type_offset_bsz &
628 	    cpu_to_le64(ICE_TX_DESC_DTYPE_DESC_DONE)) {
629 		if (last_rs >= ntc)
630 			completed_frames = last_rs - ntc + 1;
631 		else
632 			completed_frames = last_rs + cnt - ntc + 1;
633 	}
634 
635 	if (!completed_frames)
636 		return 0;
637 
638 	if (likely(!xdp_ring->xdp_tx_active)) {
639 		xsk_frames = completed_frames;
640 		goto skip;
641 	}
642 
643 	ntc = xdp_ring->next_to_clean;
644 	for (i = 0; i < completed_frames; i++) {
645 		tx_buf = &xdp_ring->tx_buf[ntc];
646 
647 		if (tx_buf->type == ICE_TX_BUF_XSK_TX) {
648 			tx_buf->type = ICE_TX_BUF_EMPTY;
649 			xsk_buff_free(tx_buf->xdp);
650 			xdp_ring->xdp_tx_active--;
651 		} else {
652 			xsk_frames++;
653 		}
654 
655 		ntc++;
656 		if (ntc >= xdp_ring->count)
657 			ntc = 0;
658 	}
659 skip:
660 	tx_desc->cmd_type_offset_bsz = 0;
661 	xdp_ring->next_to_clean += completed_frames;
662 	if (xdp_ring->next_to_clean >= cnt)
663 		xdp_ring->next_to_clean -= cnt;
664 	if (xsk_frames)
665 		xsk_tx_completed(xdp_ring->xsk_pool, xsk_frames);
666 
667 	return completed_frames;
668 }
669 
670 /**
671  * ice_xmit_xdp_tx_zc - AF_XDP ZC handler for XDP_TX
672  * @xdp: XDP buffer to xmit
673  * @xdp_ring: XDP ring to produce descriptor onto
674  *
675  * note that this function works directly on xdp_buff, no need to convert
676  * it to xdp_frame. xdp_buff pointer is stored to ice_tx_buf so that cleaning
677  * side will be able to xsk_buff_free() it.
678  *
679  * Returns ICE_XDP_TX for successfully produced desc, ICE_XDP_CONSUMED if there
680  * was not enough space on XDP ring
681  */
ice_xmit_xdp_tx_zc(struct xdp_buff * xdp,struct ice_tx_ring * xdp_ring)682 static int ice_xmit_xdp_tx_zc(struct xdp_buff *xdp,
683 			      struct ice_tx_ring *xdp_ring)
684 {
685 	struct skb_shared_info *sinfo = NULL;
686 	u32 size = xdp->data_end - xdp->data;
687 	u32 ntu = xdp_ring->next_to_use;
688 	struct ice_tx_desc *tx_desc;
689 	struct ice_tx_buf *tx_buf;
690 	struct xdp_buff *head;
691 	u32 nr_frags = 0;
692 	u32 free_space;
693 	u32 frag = 0;
694 
695 	free_space = ICE_DESC_UNUSED(xdp_ring);
696 	if (free_space < ICE_RING_QUARTER(xdp_ring))
697 		free_space += ice_clean_xdp_irq_zc(xdp_ring);
698 
699 	if (unlikely(!free_space))
700 		goto busy;
701 
702 	if (unlikely(xdp_buff_has_frags(xdp))) {
703 		sinfo = xdp_get_shared_info_from_buff(xdp);
704 		nr_frags = sinfo->nr_frags;
705 		if (free_space < nr_frags + 1)
706 			goto busy;
707 	}
708 
709 	tx_desc = ICE_TX_DESC(xdp_ring, ntu);
710 	tx_buf = &xdp_ring->tx_buf[ntu];
711 	head = xdp;
712 
713 	for (;;) {
714 		dma_addr_t dma;
715 
716 		dma = xsk_buff_xdp_get_dma(xdp);
717 		xsk_buff_raw_dma_sync_for_device(xdp_ring->xsk_pool, dma, size);
718 
719 		tx_buf->xdp = xdp;
720 		tx_buf->type = ICE_TX_BUF_XSK_TX;
721 		tx_desc->buf_addr = cpu_to_le64(dma);
722 		tx_desc->cmd_type_offset_bsz = ice_build_ctob(0, 0, size, 0);
723 		/* account for each xdp_buff from xsk_buff_pool */
724 		xdp_ring->xdp_tx_active++;
725 
726 		if (++ntu == xdp_ring->count)
727 			ntu = 0;
728 
729 		if (frag == nr_frags)
730 			break;
731 
732 		tx_desc = ICE_TX_DESC(xdp_ring, ntu);
733 		tx_buf = &xdp_ring->tx_buf[ntu];
734 
735 		xdp = xsk_buff_get_frag(head);
736 		size = skb_frag_size(&sinfo->frags[frag]);
737 		frag++;
738 	}
739 
740 	xdp_ring->next_to_use = ntu;
741 	/* update last descriptor from a frame with EOP */
742 	tx_desc->cmd_type_offset_bsz |=
743 		cpu_to_le64(ICE_TX_DESC_CMD_EOP << ICE_TXD_QW1_CMD_S);
744 
745 	return ICE_XDP_TX;
746 
747 busy:
748 	xdp_ring->ring_stats->tx_stats.tx_busy++;
749 
750 	return ICE_XDP_CONSUMED;
751 }
752 
753 /**
754  * ice_run_xdp_zc - Executes an XDP program in zero-copy path
755  * @rx_ring: Rx ring
756  * @xdp: xdp_buff used as input to the XDP program
757  * @xdp_prog: XDP program to run
758  * @xdp_ring: ring to be used for XDP_TX action
759  *
760  * Returns any of ICE_XDP_{PASS, CONSUMED, TX, REDIR}
761  */
762 static int
ice_run_xdp_zc(struct ice_rx_ring * rx_ring,struct xdp_buff * xdp,struct bpf_prog * xdp_prog,struct ice_tx_ring * xdp_ring)763 ice_run_xdp_zc(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp,
764 	       struct bpf_prog *xdp_prog, struct ice_tx_ring *xdp_ring)
765 {
766 	int err, result = ICE_XDP_PASS;
767 	u32 act;
768 
769 	act = bpf_prog_run_xdp(xdp_prog, xdp);
770 
771 	if (likely(act == XDP_REDIRECT)) {
772 		err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog);
773 		if (!err)
774 			return ICE_XDP_REDIR;
775 		if (xsk_uses_need_wakeup(rx_ring->xsk_pool) && err == -ENOBUFS)
776 			result = ICE_XDP_EXIT;
777 		else
778 			result = ICE_XDP_CONSUMED;
779 		goto out_failure;
780 	}
781 
782 	switch (act) {
783 	case XDP_PASS:
784 		break;
785 	case XDP_TX:
786 		result = ice_xmit_xdp_tx_zc(xdp, xdp_ring);
787 		if (result == ICE_XDP_CONSUMED)
788 			goto out_failure;
789 		break;
790 	case XDP_DROP:
791 		result = ICE_XDP_CONSUMED;
792 		break;
793 	default:
794 		bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act);
795 		fallthrough;
796 	case XDP_ABORTED:
797 		result = ICE_XDP_CONSUMED;
798 out_failure:
799 		trace_xdp_exception(rx_ring->netdev, xdp_prog, act);
800 		break;
801 	}
802 
803 	return result;
804 }
805 
806 static int
ice_add_xsk_frag(struct ice_rx_ring * rx_ring,struct xdp_buff * first,struct xdp_buff * xdp,const unsigned int size)807 ice_add_xsk_frag(struct ice_rx_ring *rx_ring, struct xdp_buff *first,
808 		 struct xdp_buff *xdp, const unsigned int size)
809 {
810 	struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(first);
811 
812 	if (!size)
813 		return 0;
814 
815 	if (!xdp_buff_has_frags(first)) {
816 		sinfo->nr_frags = 0;
817 		sinfo->xdp_frags_size = 0;
818 		xdp_buff_set_frags_flag(first);
819 	}
820 
821 	if (unlikely(sinfo->nr_frags == MAX_SKB_FRAGS)) {
822 		xsk_buff_free(first);
823 		return -ENOMEM;
824 	}
825 
826 	__skb_fill_page_desc_noacc(sinfo, sinfo->nr_frags++,
827 				   virt_to_page(xdp->data_hard_start),
828 				   XDP_PACKET_HEADROOM, size);
829 	sinfo->xdp_frags_size += size;
830 	xsk_buff_add_frag(xdp);
831 
832 	return 0;
833 }
834 
835 /**
836  * ice_clean_rx_irq_zc - consumes packets from the hardware ring
837  * @rx_ring: AF_XDP Rx ring
838  * @budget: NAPI budget
839  *
840  * Returns number of processed packets on success, remaining budget on failure.
841  */
ice_clean_rx_irq_zc(struct ice_rx_ring * rx_ring,int budget)842 int ice_clean_rx_irq_zc(struct ice_rx_ring *rx_ring, int budget)
843 {
844 	unsigned int total_rx_bytes = 0, total_rx_packets = 0;
845 	struct xsk_buff_pool *xsk_pool = rx_ring->xsk_pool;
846 	u32 ntc = rx_ring->next_to_clean;
847 	u32 ntu = rx_ring->next_to_use;
848 	struct xdp_buff *first = NULL;
849 	struct ice_tx_ring *xdp_ring;
850 	unsigned int xdp_xmit = 0;
851 	struct bpf_prog *xdp_prog;
852 	u32 cnt = rx_ring->count;
853 	bool failure = false;
854 	int entries_to_alloc;
855 
856 	/* ZC patch is enabled only when XDP program is set,
857 	 * so here it can not be NULL
858 	 */
859 	xdp_prog = READ_ONCE(rx_ring->xdp_prog);
860 	xdp_ring = rx_ring->xdp_ring;
861 
862 	if (ntc != rx_ring->first_desc)
863 		first = *ice_xdp_buf(rx_ring, rx_ring->first_desc);
864 
865 	while (likely(total_rx_packets < (unsigned int)budget)) {
866 		union ice_32b_rx_flex_desc *rx_desc;
867 		unsigned int size, xdp_res = 0;
868 		struct xdp_buff *xdp;
869 		struct sk_buff *skb;
870 		u16 stat_err_bits;
871 		u16 vlan_tag = 0;
872 		u16 rx_ptype;
873 
874 		rx_desc = ICE_RX_DESC(rx_ring, ntc);
875 
876 		stat_err_bits = BIT(ICE_RX_FLEX_DESC_STATUS0_DD_S);
877 		if (!ice_test_staterr(rx_desc->wb.status_error0, stat_err_bits))
878 			break;
879 
880 		/* This memory barrier is needed to keep us from reading
881 		 * any other fields out of the rx_desc until we have
882 		 * verified the descriptor has been written back.
883 		 */
884 		dma_rmb();
885 
886 		if (unlikely(ntc == ntu))
887 			break;
888 
889 		xdp = *ice_xdp_buf(rx_ring, ntc);
890 
891 		size = le16_to_cpu(rx_desc->wb.pkt_len) &
892 				   ICE_RX_FLX_DESC_PKT_LEN_M;
893 
894 		xsk_buff_set_size(xdp, size);
895 		xsk_buff_dma_sync_for_cpu(xdp, xsk_pool);
896 
897 		if (!first) {
898 			first = xdp;
899 		} else if (ice_add_xsk_frag(rx_ring, first, xdp, size)) {
900 			break;
901 		}
902 
903 		if (++ntc == cnt)
904 			ntc = 0;
905 
906 		if (ice_is_non_eop(rx_ring, rx_desc))
907 			continue;
908 
909 		xdp_res = ice_run_xdp_zc(rx_ring, first, xdp_prog, xdp_ring);
910 		if (likely(xdp_res & (ICE_XDP_TX | ICE_XDP_REDIR))) {
911 			xdp_xmit |= xdp_res;
912 		} else if (xdp_res == ICE_XDP_EXIT) {
913 			failure = true;
914 			first = NULL;
915 			rx_ring->first_desc = ntc;
916 			break;
917 		} else if (xdp_res == ICE_XDP_CONSUMED) {
918 			xsk_buff_free(first);
919 		} else if (xdp_res == ICE_XDP_PASS) {
920 			goto construct_skb;
921 		}
922 
923 		total_rx_bytes += xdp_get_buff_len(first);
924 		total_rx_packets++;
925 
926 		first = NULL;
927 		rx_ring->first_desc = ntc;
928 		continue;
929 
930 construct_skb:
931 		/* XDP_PASS path */
932 		skb = ice_construct_skb_zc(rx_ring, first);
933 		if (!skb) {
934 			rx_ring->ring_stats->rx_stats.alloc_buf_failed++;
935 			break;
936 		}
937 
938 		first = NULL;
939 		rx_ring->first_desc = ntc;
940 
941 		if (eth_skb_pad(skb)) {
942 			skb = NULL;
943 			continue;
944 		}
945 
946 		total_rx_bytes += skb->len;
947 		total_rx_packets++;
948 
949 		vlan_tag = ice_get_vlan_tag_from_rx_desc(rx_desc);
950 
951 		rx_ptype = le16_to_cpu(rx_desc->wb.ptype_flex_flags0) &
952 				       ICE_RX_FLEX_DESC_PTYPE_M;
953 
954 		ice_process_skb_fields(rx_ring, rx_desc, skb, rx_ptype);
955 		ice_receive_skb(rx_ring, skb, vlan_tag);
956 	}
957 
958 	rx_ring->next_to_clean = ntc;
959 	entries_to_alloc = ICE_RX_DESC_UNUSED(rx_ring);
960 	if (entries_to_alloc > ICE_RING_QUARTER(rx_ring))
961 		failure |= !ice_alloc_rx_bufs_zc(rx_ring, entries_to_alloc);
962 
963 	ice_finalize_xdp_rx(xdp_ring, xdp_xmit, 0);
964 	ice_update_rx_ring_stats(rx_ring, total_rx_packets, total_rx_bytes);
965 
966 	if (xsk_uses_need_wakeup(xsk_pool)) {
967 		/* ntu could have changed when allocating entries above, so
968 		 * use rx_ring value instead of stack based one
969 		 */
970 		if (failure || ntc == rx_ring->next_to_use)
971 			xsk_set_rx_need_wakeup(xsk_pool);
972 		else
973 			xsk_clear_rx_need_wakeup(xsk_pool);
974 
975 		return (int)total_rx_packets;
976 	}
977 
978 	return failure ? budget : (int)total_rx_packets;
979 }
980 
981 /**
982  * ice_xmit_pkt - produce a single HW Tx descriptor out of AF_XDP descriptor
983  * @xdp_ring: XDP ring to produce the HW Tx descriptor on
984  * @desc: AF_XDP descriptor to pull the DMA address and length from
985  * @total_bytes: bytes accumulator that will be used for stats update
986  */
ice_xmit_pkt(struct ice_tx_ring * xdp_ring,struct xdp_desc * desc,unsigned int * total_bytes)987 static void ice_xmit_pkt(struct ice_tx_ring *xdp_ring, struct xdp_desc *desc,
988 			 unsigned int *total_bytes)
989 {
990 	struct ice_tx_desc *tx_desc;
991 	dma_addr_t dma;
992 
993 	dma = xsk_buff_raw_get_dma(xdp_ring->xsk_pool, desc->addr);
994 	xsk_buff_raw_dma_sync_for_device(xdp_ring->xsk_pool, dma, desc->len);
995 
996 	tx_desc = ICE_TX_DESC(xdp_ring, xdp_ring->next_to_use++);
997 	tx_desc->buf_addr = cpu_to_le64(dma);
998 	tx_desc->cmd_type_offset_bsz = ice_build_ctob(xsk_is_eop_desc(desc),
999 						      0, desc->len, 0);
1000 
1001 	*total_bytes += desc->len;
1002 }
1003 
1004 /**
1005  * ice_xmit_pkt_batch - produce a batch of HW Tx descriptors out of AF_XDP descriptors
1006  * @xdp_ring: XDP ring to produce the HW Tx descriptors on
1007  * @descs: AF_XDP descriptors to pull the DMA addresses and lengths from
1008  * @total_bytes: bytes accumulator that will be used for stats update
1009  */
ice_xmit_pkt_batch(struct ice_tx_ring * xdp_ring,struct xdp_desc * descs,unsigned int * total_bytes)1010 static void ice_xmit_pkt_batch(struct ice_tx_ring *xdp_ring, struct xdp_desc *descs,
1011 			       unsigned int *total_bytes)
1012 {
1013 	u16 ntu = xdp_ring->next_to_use;
1014 	struct ice_tx_desc *tx_desc;
1015 	u32 i;
1016 
1017 	loop_unrolled_for(i = 0; i < PKTS_PER_BATCH; i++) {
1018 		dma_addr_t dma;
1019 
1020 		dma = xsk_buff_raw_get_dma(xdp_ring->xsk_pool, descs[i].addr);
1021 		xsk_buff_raw_dma_sync_for_device(xdp_ring->xsk_pool, dma, descs[i].len);
1022 
1023 		tx_desc = ICE_TX_DESC(xdp_ring, ntu++);
1024 		tx_desc->buf_addr = cpu_to_le64(dma);
1025 		tx_desc->cmd_type_offset_bsz = ice_build_ctob(xsk_is_eop_desc(&descs[i]),
1026 							      0, descs[i].len, 0);
1027 
1028 		*total_bytes += descs[i].len;
1029 	}
1030 
1031 	xdp_ring->next_to_use = ntu;
1032 }
1033 
1034 /**
1035  * ice_fill_tx_hw_ring - produce the number of Tx descriptors onto ring
1036  * @xdp_ring: XDP ring to produce the HW Tx descriptors on
1037  * @descs: AF_XDP descriptors to pull the DMA addresses and lengths from
1038  * @nb_pkts: count of packets to be send
1039  * @total_bytes: bytes accumulator that will be used for stats update
1040  */
ice_fill_tx_hw_ring(struct ice_tx_ring * xdp_ring,struct xdp_desc * descs,u32 nb_pkts,unsigned int * total_bytes)1041 static void ice_fill_tx_hw_ring(struct ice_tx_ring *xdp_ring, struct xdp_desc *descs,
1042 				u32 nb_pkts, unsigned int *total_bytes)
1043 {
1044 	u32 batched, leftover, i;
1045 
1046 	batched = ALIGN_DOWN(nb_pkts, PKTS_PER_BATCH);
1047 	leftover = nb_pkts & (PKTS_PER_BATCH - 1);
1048 	for (i = 0; i < batched; i += PKTS_PER_BATCH)
1049 		ice_xmit_pkt_batch(xdp_ring, &descs[i], total_bytes);
1050 	for (; i < batched + leftover; i++)
1051 		ice_xmit_pkt(xdp_ring, &descs[i], total_bytes);
1052 }
1053 
1054 /**
1055  * ice_xmit_zc - take entries from XSK Tx ring and place them onto HW Tx ring
1056  * @xdp_ring: XDP ring to produce the HW Tx descriptors on
1057  *
1058  * Returns true if there is no more work that needs to be done, false otherwise
1059  */
ice_xmit_zc(struct ice_tx_ring * xdp_ring)1060 bool ice_xmit_zc(struct ice_tx_ring *xdp_ring)
1061 {
1062 	struct xdp_desc *descs = xdp_ring->xsk_pool->tx_descs;
1063 	u32 nb_pkts, nb_processed = 0;
1064 	unsigned int total_bytes = 0;
1065 	int budget;
1066 
1067 	ice_clean_xdp_irq_zc(xdp_ring);
1068 
1069 	if (!netif_carrier_ok(xdp_ring->vsi->netdev) ||
1070 	    !netif_running(xdp_ring->vsi->netdev))
1071 		return true;
1072 
1073 	budget = ICE_DESC_UNUSED(xdp_ring);
1074 	budget = min_t(u16, budget, ICE_RING_QUARTER(xdp_ring));
1075 
1076 	nb_pkts = xsk_tx_peek_release_desc_batch(xdp_ring->xsk_pool, budget);
1077 	if (!nb_pkts)
1078 		return true;
1079 
1080 	if (xdp_ring->next_to_use + nb_pkts >= xdp_ring->count) {
1081 		nb_processed = xdp_ring->count - xdp_ring->next_to_use;
1082 		ice_fill_tx_hw_ring(xdp_ring, descs, nb_processed, &total_bytes);
1083 		xdp_ring->next_to_use = 0;
1084 	}
1085 
1086 	ice_fill_tx_hw_ring(xdp_ring, &descs[nb_processed], nb_pkts - nb_processed,
1087 			    &total_bytes);
1088 
1089 	ice_set_rs_bit(xdp_ring);
1090 	ice_xdp_ring_update_tail(xdp_ring);
1091 	ice_update_tx_ring_stats(xdp_ring, nb_pkts, total_bytes);
1092 
1093 	if (xsk_uses_need_wakeup(xdp_ring->xsk_pool))
1094 		xsk_set_tx_need_wakeup(xdp_ring->xsk_pool);
1095 
1096 	return nb_pkts < budget;
1097 }
1098 
1099 /**
1100  * ice_xsk_wakeup - Implements ndo_xsk_wakeup
1101  * @netdev: net_device
1102  * @queue_id: queue to wake up
1103  * @flags: ignored in our case, since we have Rx and Tx in the same NAPI
1104  *
1105  * Returns negative on error, zero otherwise.
1106  */
1107 int
ice_xsk_wakeup(struct net_device * netdev,u32 queue_id,u32 __always_unused flags)1108 ice_xsk_wakeup(struct net_device *netdev, u32 queue_id,
1109 	       u32 __always_unused flags)
1110 {
1111 	struct ice_netdev_priv *np = netdev_priv(netdev);
1112 	struct ice_q_vector *q_vector;
1113 	struct ice_vsi *vsi = np->vsi;
1114 	struct ice_tx_ring *ring;
1115 
1116 	if (test_bit(ICE_VSI_DOWN, vsi->state) || !netif_carrier_ok(netdev))
1117 		return -ENETDOWN;
1118 
1119 	if (!ice_is_xdp_ena_vsi(vsi))
1120 		return -EINVAL;
1121 
1122 	if (queue_id >= vsi->num_txq || queue_id >= vsi->num_rxq)
1123 		return -EINVAL;
1124 
1125 	ring = vsi->rx_rings[queue_id]->xdp_ring;
1126 
1127 	if (!ring->xsk_pool)
1128 		return -EINVAL;
1129 
1130 	/* The idea here is that if NAPI is running, mark a miss, so
1131 	 * it will run again. If not, trigger an interrupt and
1132 	 * schedule the NAPI from interrupt context. If NAPI would be
1133 	 * scheduled here, the interrupt affinity would not be
1134 	 * honored.
1135 	 */
1136 	q_vector = ring->q_vector;
1137 	if (!napi_if_scheduled_mark_missed(&q_vector->napi))
1138 		ice_trigger_sw_intr(&vsi->back->hw, q_vector);
1139 
1140 	return 0;
1141 }
1142 
1143 /**
1144  * ice_xsk_any_rx_ring_ena - Checks if Rx rings have AF_XDP buff pool attached
1145  * @vsi: VSI to be checked
1146  *
1147  * Returns true if any of the Rx rings has an AF_XDP buff pool attached
1148  */
ice_xsk_any_rx_ring_ena(struct ice_vsi * vsi)1149 bool ice_xsk_any_rx_ring_ena(struct ice_vsi *vsi)
1150 {
1151 	int i;
1152 
1153 	ice_for_each_rxq(vsi, i) {
1154 		if (xsk_get_pool_from_qid(vsi->netdev, i))
1155 			return true;
1156 	}
1157 
1158 	return false;
1159 }
1160 
1161 /**
1162  * ice_xsk_clean_rx_ring - clean buffer pool queues connected to a given Rx ring
1163  * @rx_ring: ring to be cleaned
1164  */
ice_xsk_clean_rx_ring(struct ice_rx_ring * rx_ring)1165 void ice_xsk_clean_rx_ring(struct ice_rx_ring *rx_ring)
1166 {
1167 	u16 ntc = rx_ring->next_to_clean;
1168 	u16 ntu = rx_ring->next_to_use;
1169 
1170 	while (ntc != ntu) {
1171 		struct xdp_buff *xdp = *ice_xdp_buf(rx_ring, ntc);
1172 
1173 		xsk_buff_free(xdp);
1174 		ntc++;
1175 		if (ntc >= rx_ring->count)
1176 			ntc = 0;
1177 	}
1178 }
1179 
1180 /**
1181  * ice_xsk_clean_xdp_ring - Clean the XDP Tx ring and its buffer pool queues
1182  * @xdp_ring: XDP_Tx ring
1183  */
ice_xsk_clean_xdp_ring(struct ice_tx_ring * xdp_ring)1184 void ice_xsk_clean_xdp_ring(struct ice_tx_ring *xdp_ring)
1185 {
1186 	u16 ntc = xdp_ring->next_to_clean, ntu = xdp_ring->next_to_use;
1187 	u32 xsk_frames = 0;
1188 
1189 	while (ntc != ntu) {
1190 		struct ice_tx_buf *tx_buf = &xdp_ring->tx_buf[ntc];
1191 
1192 		if (tx_buf->type == ICE_TX_BUF_XSK_TX) {
1193 			tx_buf->type = ICE_TX_BUF_EMPTY;
1194 			xsk_buff_free(tx_buf->xdp);
1195 		} else {
1196 			xsk_frames++;
1197 		}
1198 
1199 		ntc++;
1200 		if (ntc >= xdp_ring->count)
1201 			ntc = 0;
1202 	}
1203 
1204 	if (xsk_frames)
1205 		xsk_tx_completed(xdp_ring->xsk_pool, xsk_frames);
1206 }
1207