6 #include <net/xdp.h>
355  * ice_realloc_zc_buf - reallocate XDP ZC queue pairs
360  * XDP requires more memory, than rx_buf provides.
441  * @xdp: SW ring of xdp_buff that will hold the buffers
452 static u16 ice_fill_rx_descs(struct xsk_buff_pool *pool, struct xdp_buff **xdp,  in ice_fill_rx_descs()  argument
459 	buffs = xsk_buff_alloc_batch(pool, xdp, count);  in ice_fill_rx_descs()
461 		dma = xsk_buff_xdp_get_dma(*xdp);  in ice_fill_rx_descs()
466 		xdp++;  in ice_fill_rx_descs()
489 	struct xdp_buff **xdp;  in __ice_alloc_rx_bufs_zc()  local
492 	xdp = ice_xdp_buf(rx_ring, ntu);  in __ice_alloc_rx_bufs_zc()
495 		nb_buffs_extra = ice_fill_rx_descs(rx_ring->xsk_pool, xdp,  in __ice_alloc_rx_bufs_zc()
503 		xdp = ice_xdp_buf(rx_ring, 0);  in __ice_alloc_rx_bufs_zc()
509 	nb_buffs = ice_fill_rx_descs(rx_ring->xsk_pool, xdp, rx_desc, count);  in __ice_alloc_rx_bufs_zc()
549  * @xdp: Pointer to XDP buffer
556 ice_construct_skb_zc(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp)  in ice_construct_skb_zc()  argument
558 	unsigned int totalsize = xdp->data_end - xdp->data_meta;  in ice_construct_skb_zc()
559 	unsigned int metasize = xdp->data - xdp->data_meta;  in ice_construct_skb_zc()
564 	if (unlikely(xdp_buff_has_frags(xdp))) {  in ice_construct_skb_zc()
565 		sinfo = xdp_get_shared_info_from_buff(xdp);  in ice_construct_skb_zc()
568 	net_prefetch(xdp->data_meta);  in ice_construct_skb_zc()
575 	memcpy(__skb_put(skb, totalsize), xdp->data_meta,  in ice_construct_skb_zc()
583 	if (likely(!xdp_buff_has_frags(xdp)))  in ice_construct_skb_zc()
606 	xsk_buff_free(xdp);  in ice_construct_skb_zc()
612  * @xdp_ring: XDP Tx ring
649 			xsk_buff_free(tx_buf->xdp);  in ice_clean_xdp_irq_zc()
672  * @xdp: XDP buffer to xmit
673  * @xdp_ring: XDP ring to produce descriptor onto
680  * was not enough space on XDP ring
682 static int ice_xmit_xdp_tx_zc(struct xdp_buff *xdp,  in ice_xmit_xdp_tx_zc()  argument
686 	u32 size = xdp->data_end - xdp->data;  in ice_xmit_xdp_tx_zc()
702 	if (unlikely(xdp_buff_has_frags(xdp))) {  in ice_xmit_xdp_tx_zc()
703 		sinfo = xdp_get_shared_info_from_buff(xdp);  in ice_xmit_xdp_tx_zc()
711 	head = xdp;  in ice_xmit_xdp_tx_zc()
716 		dma = xsk_buff_xdp_get_dma(xdp);  in ice_xmit_xdp_tx_zc()
719 		tx_buf->xdp = xdp;  in ice_xmit_xdp_tx_zc()
735 		xdp = xsk_buff_get_frag(head);  in ice_xmit_xdp_tx_zc()
754  * ice_run_xdp_zc - Executes an XDP program in zero-copy path
756  * @xdp: xdp_buff used as input to the XDP program
757  * @xdp_prog: XDP program to run
763 ice_run_xdp_zc(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp,  in ice_run_xdp_zc()  argument
769 	act = bpf_prog_run_xdp(xdp_prog, xdp);  in ice_run_xdp_zc()
772 		err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog);  in ice_run_xdp_zc()
786 		result = ice_xmit_xdp_tx_zc(xdp, xdp_ring);  in ice_run_xdp_zc()
808 		 struct xdp_buff *xdp, const unsigned int size)  in ice_add_xsk_frag()  argument
827 				   virt_to_page(xdp->data_hard_start),  in ice_add_xsk_frag()
830 	xsk_buff_add_frag(xdp);  in ice_add_xsk_frag()
856 	/* ZC patch is enabled only when XDP program is set,  in ice_clean_rx_irq_zc()
868 		struct xdp_buff *xdp;  in ice_clean_rx_irq_zc()  local
889 		xdp = *ice_xdp_buf(rx_ring, ntc);  in ice_clean_rx_irq_zc()
894 		xsk_buff_set_size(xdp, size);  in ice_clean_rx_irq_zc()
895 		xsk_buff_dma_sync_for_cpu(xdp, xsk_pool);  in ice_clean_rx_irq_zc()
898 			first = xdp;  in ice_clean_rx_irq_zc()
899 		} else if (ice_add_xsk_frag(rx_ring, first, xdp, size)) {  in ice_clean_rx_irq_zc()
983  * @xdp_ring: XDP ring to produce the HW Tx descriptor on
1006  * @xdp_ring: XDP ring to produce the HW Tx descriptors on
1036  * @xdp_ring: XDP ring to produce the HW Tx descriptors on
1056  * @xdp_ring: XDP ring to produce the HW Tx descriptors on
1171 		struct xdp_buff *xdp = *ice_xdp_buf(rx_ring, ntc);  in ice_xsk_clean_rx_ring()  local
1173 		xsk_buff_free(xdp);  in ice_xsk_clean_rx_ring()
1181  * ice_xsk_clean_xdp_ring - Clean the XDP Tx ring and its buffer pool queues
1194 			xsk_buff_free(tx_buf->xdp);  in ice_xsk_clean_xdp_ring()