56 static struct sk_buff *enetc_tx_swbd_get_skb(struct enetc_tx_swbd *tx_swbd)  in enetc_tx_swbd_get_skb()  argument
58 	if (tx_swbd->is_xdp_tx || tx_swbd->is_xdp_redirect)  in enetc_tx_swbd_get_skb()
61 	return tx_swbd->skb;  in enetc_tx_swbd_get_skb()
65 enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd *tx_swbd)  in enetc_tx_swbd_get_xdp_frame()  argument
67 	if (tx_swbd->is_xdp_redirect)  in enetc_tx_swbd_get_xdp_frame()
68 		return tx_swbd->xdp_frame;  in enetc_tx_swbd_get_xdp_frame()
74 				struct enetc_tx_swbd *tx_swbd)  in enetc_unmap_tx_buff()  argument
80 	if (tx_swbd->is_dma_page)  in enetc_unmap_tx_buff()
81 		dma_unmap_page(tx_ring->dev, tx_swbd->dma,  in enetc_unmap_tx_buff()
82 			       tx_swbd->is_xdp_tx ? PAGE_SIZE : tx_swbd->len,  in enetc_unmap_tx_buff()
83 			       tx_swbd->dir);  in enetc_unmap_tx_buff()
85 		dma_unmap_single(tx_ring->dev, tx_swbd->dma,  in enetc_unmap_tx_buff()
86 				 tx_swbd->len, tx_swbd->dir);  in enetc_unmap_tx_buff()
87 	tx_swbd->dma = 0;  in enetc_unmap_tx_buff()
91 				struct enetc_tx_swbd *tx_swbd)  in enetc_free_tx_frame()  argument
93 	struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);  in enetc_free_tx_frame()
94 	struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);  in enetc_free_tx_frame()
96 	if (tx_swbd->dma)  in enetc_free_tx_frame()
97 		enetc_unmap_tx_buff(tx_ring, tx_swbd);  in enetc_free_tx_frame()
100 		xdp_return_frame(tx_swbd->xdp_frame);  in enetc_free_tx_frame()
101 		tx_swbd->xdp_frame = NULL;  in enetc_free_tx_frame()
104 		tx_swbd->skb = NULL;  in enetc_free_tx_frame()
153 	struct enetc_tx_swbd *tx_swbd;  in enetc_map_tx_buffs()  local
177 	tx_swbd = &tx_ring->tx_swbd[i];  in enetc_map_tx_buffs()
178 	tx_swbd->dma = dma;  in enetc_map_tx_buffs()
179 	tx_swbd->len = len;  in enetc_map_tx_buffs()
180 	tx_swbd->is_dma_page = 0;  in enetc_map_tx_buffs()
181 	tx_swbd->dir = DMA_TO_DEVICE;  in enetc_map_tx_buffs()
196 	tx_swbd->do_twostep_tstamp = do_twostep_tstamp;  in enetc_map_tx_buffs()
197 	tx_swbd->qbv_en = !!(priv->active_offloads & ENETC_F_QBV);  in enetc_map_tx_buffs()
198 	tx_swbd->check_wb = tx_swbd->do_twostep_tstamp || tx_swbd->qbv_en;  in enetc_map_tx_buffs()
221 		tx_swbd++;  in enetc_map_tx_buffs()
226 			tx_swbd = tx_ring->tx_swbd;  in enetc_map_tx_buffs()
291 		tx_swbd++;  in enetc_map_tx_buffs()
296 			tx_swbd = tx_ring->tx_swbd;  in enetc_map_tx_buffs()
304 		tx_swbd->dma = dma;  in enetc_map_tx_buffs()
305 		tx_swbd->len = len;  in enetc_map_tx_buffs()
306 		tx_swbd->is_dma_page = 1;  in enetc_map_tx_buffs()
307 		tx_swbd->dir = DMA_TO_DEVICE;  in enetc_map_tx_buffs()
316 	tx_ring->tx_swbd[i].is_eof = true;  in enetc_map_tx_buffs()
317 	tx_ring->tx_swbd[i].skb = skb;  in enetc_map_tx_buffs()
332 		tx_swbd = &tx_ring->tx_swbd[i];  in enetc_map_tx_buffs()
333 		enetc_free_tx_frame(tx_ring, tx_swbd);  in enetc_map_tx_buffs()
343 				 struct enetc_tx_swbd *tx_swbd,  in enetc_map_tx_tso_hdr()  argument
368 	tx_swbd->len = hdr_len;  in enetc_map_tx_tso_hdr()
369 	tx_swbd->do_twostep_tstamp = false;  in enetc_map_tx_tso_hdr()
370 	tx_swbd->check_wb = false;  in enetc_map_tx_tso_hdr()
380 		tx_swbd = &tx_ring->tx_swbd[*i];  in enetc_map_tx_tso_hdr()
396 				 struct enetc_tx_swbd *tx_swbd,  in enetc_map_tx_tso_data()  argument
414 		tx_swbd->is_eof = 1;  in enetc_map_tx_tso_data()
421 	tx_swbd->dma = addr;  in enetc_map_tx_tso_data()
422 	tx_swbd->len = size;  in enetc_map_tx_tso_data()
423 	tx_swbd->dir = DMA_TO_DEVICE;  in enetc_map_tx_tso_data()
494 	struct enetc_tx_swbd *tx_swbd;  in enetc_map_tx_tso_buffs()  local
511 		tx_swbd = &tx_ring->tx_swbd[i];  in enetc_map_tx_tso_buffs()
524 		enetc_map_tx_tso_hdr(tx_ring, skb, tx_swbd, txbd, &i, hdr_len, data_len);  in enetc_map_tx_tso_buffs()
536 			tx_swbd = &tx_ring->tx_swbd[i];  in enetc_map_tx_tso_buffs()
547 			err = enetc_map_tx_tso_data(tx_ring, skb, tx_swbd, txbd,  in enetc_map_tx_tso_buffs()
565 			tx_swbd->skb = skb;  in enetc_map_tx_tso_buffs()
581 		tx_swbd = &tx_ring->tx_swbd[i];  in enetc_map_tx_tso_buffs()
582 		enetc_free_tx_frame(tx_ring, tx_swbd);  in enetc_map_tx_tso_buffs()
779 				      struct enetc_tx_swbd *tx_swbd)  in enetc_recycle_xdp_tx_buff()  argument
783 		.dma = tx_swbd->dma,  in enetc_recycle_xdp_tx_buff()
784 		.page = tx_swbd->page,  in enetc_recycle_xdp_tx_buff()
785 		.page_offset = tx_swbd->page_offset,  in enetc_recycle_xdp_tx_buff()
786 		.dir = tx_swbd->dir,  in enetc_recycle_xdp_tx_buff()
787 		.len = tx_swbd->len,  in enetc_recycle_xdp_tx_buff()
822 	struct enetc_tx_swbd *tx_swbd;  in enetc_clean_tx_ring()  local
828 	tx_swbd = &tx_ring->tx_swbd[i];  in enetc_clean_tx_ring()
835 		struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);  in enetc_clean_tx_ring()
836 		struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);  in enetc_clean_tx_ring()
837 		bool is_eof = tx_swbd->is_eof;  in enetc_clean_tx_ring()
839 		if (unlikely(tx_swbd->check_wb)) {  in enetc_clean_tx_ring()
843 			    tx_swbd->do_twostep_tstamp) {  in enetc_clean_tx_ring()
849 			if (tx_swbd->qbv_en &&  in enetc_clean_tx_ring()
854 		if (tx_swbd->is_xdp_tx)  in enetc_clean_tx_ring()
855 			enetc_recycle_xdp_tx_buff(tx_ring, tx_swbd);  in enetc_clean_tx_ring()
856 		else if (likely(tx_swbd->dma))  in enetc_clean_tx_ring()
857 			enetc_unmap_tx_buff(tx_ring, tx_swbd);  in enetc_clean_tx_ring()
875 		tx_byte_cnt += tx_swbd->len;  in enetc_clean_tx_ring()
879 		memset(tx_swbd, 0, sizeof(*tx_swbd));  in enetc_clean_tx_ring()
882 		tx_swbd++;  in enetc_clean_tx_ring()
886 			tx_swbd = tx_ring->tx_swbd;  in enetc_clean_tx_ring()
1257 				  struct enetc_tx_swbd *tx_swbd,  in enetc_xdp_map_tx_buff()  argument
1265 	txbd->addr = cpu_to_le64(tx_swbd->dma + tx_swbd->page_offset);  in enetc_xdp_map_tx_buff()
1266 	txbd->buf_len = cpu_to_le16(tx_swbd->len);  in enetc_xdp_map_tx_buff()
1269 	memcpy(&tx_ring->tx_swbd[i], tx_swbd, sizeof(*tx_swbd));  in enetc_xdp_map_tx_buff()
1507 		struct enetc_tx_swbd *tx_swbd = &xdp_tx_arr[n];  in enetc_rx_swbd_to_xdp_tx_swbd()  local
1510 		tx_swbd->dma = rx_swbd->dma;  in enetc_rx_swbd_to_xdp_tx_swbd()
1511 		tx_swbd->dir = rx_swbd->dir;  in enetc_rx_swbd_to_xdp_tx_swbd()
1512 		tx_swbd->page = rx_swbd->page;  in enetc_rx_swbd_to_xdp_tx_swbd()
1513 		tx_swbd->page_offset = rx_swbd->page_offset;  in enetc_rx_swbd_to_xdp_tx_swbd()
1514 		tx_swbd->len = rx_swbd->len;  in enetc_rx_swbd_to_xdp_tx_swbd()
1515 		tx_swbd->is_dma_page = true;  in enetc_rx_swbd_to_xdp_tx_swbd()
1516 		tx_swbd->is_xdp_tx = true;  in enetc_rx_swbd_to_xdp_tx_swbd()
1517 		tx_swbd->is_eof = false;  in enetc_rx_swbd_to_xdp_tx_swbd()
1805 	res->tx_swbd = vcalloc(bd_count, sizeof(*res->tx_swbd));  in enetc_alloc_tx_resource()
1806 	if (!res->tx_swbd)  in enetc_alloc_tx_resource()
1826 	vfree(res->tx_swbd);  in enetc_alloc_tx_resource()
1827 	res->tx_swbd = NULL;  in enetc_alloc_tx_resource()
1837 	vfree(res->tx_swbd);  in enetc_free_tx_resource()
1958 	tx_ring->tx_swbd = res ? res->tx_swbd : NULL;  in enetc_assign_tx_resource()
2008 		struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i];  in enetc_free_tx_ring()  local
2010 		enetc_free_tx_frame(tx_ring, tx_swbd);  in enetc_free_tx_ring()