1939 static bool ixgbe_can_reuse_rx_page(struct ixgbe_rx_buffer *rx_buffer,  in ixgbe_can_reuse_rx_page()  argument
1942 	unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;  in ixgbe_can_reuse_rx_page()
1943 	struct page *page = rx_buffer->page;  in ixgbe_can_reuse_rx_page()
1961 	if (rx_buffer->page_offset > IXGBE_LAST_OFFSET)  in ixgbe_can_reuse_rx_page()
1971 		rx_buffer->pagecnt_bias = USHRT_MAX;  in ixgbe_can_reuse_rx_page()
1993 			      struct ixgbe_rx_buffer *rx_buffer,  in ixgbe_add_rx_frag()  argument
2004 	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,  in ixgbe_add_rx_frag()
2005 			rx_buffer->page_offset, size, truesize);  in ixgbe_add_rx_frag()
2007 	rx_buffer->page_offset ^= truesize;  in ixgbe_add_rx_frag()
2009 	rx_buffer->page_offset += truesize;  in ixgbe_add_rx_frag()
2019 	struct ixgbe_rx_buffer *rx_buffer;  in ixgbe_get_rx_buffer()  local
2021 	rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];  in ixgbe_get_rx_buffer()
2024 		page_count(rx_buffer->page);  in ixgbe_get_rx_buffer()
2028 	prefetchw(rx_buffer->page);  in ixgbe_get_rx_buffer()
2029 	*skb = rx_buffer->skb;  in ixgbe_get_rx_buffer()
2045 				      rx_buffer->dma,  in ixgbe_get_rx_buffer()
2046 				      rx_buffer->page_offset,  in ixgbe_get_rx_buffer()
2050 	rx_buffer->pagecnt_bias--;  in ixgbe_get_rx_buffer()
2052 	return rx_buffer;  in ixgbe_get_rx_buffer()
2056 				struct ixgbe_rx_buffer *rx_buffer,  in ixgbe_put_rx_buffer()  argument
2060 	if (ixgbe_can_reuse_rx_page(rx_buffer, rx_buffer_pgcnt)) {  in ixgbe_put_rx_buffer()
2062 		ixgbe_reuse_rx_page(rx_ring, rx_buffer);  in ixgbe_put_rx_buffer()
2064 		if (!IS_ERR(skb) && IXGBE_CB(skb)->dma == rx_buffer->dma) {  in ixgbe_put_rx_buffer()
2069 			dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,  in ixgbe_put_rx_buffer()
2074 		__page_frag_cache_drain(rx_buffer->page,  in ixgbe_put_rx_buffer()
2075 					rx_buffer->pagecnt_bias);  in ixgbe_put_rx_buffer()
2079 	rx_buffer->page = NULL;  in ixgbe_put_rx_buffer()
2080 	rx_buffer->skb = NULL;  in ixgbe_put_rx_buffer()
2084 					   struct ixgbe_rx_buffer *rx_buffer,  in ixgbe_construct_skb()  argument
2123 			IXGBE_CB(skb)->dma = rx_buffer->dma;  in ixgbe_construct_skb()
2125 		skb_add_rx_frag(skb, 0, rx_buffer->page,  in ixgbe_construct_skb()
2126 				xdp->data - page_address(rx_buffer->page),  in ixgbe_construct_skb()
2129 		rx_buffer->page_offset ^= truesize;  in ixgbe_construct_skb()
2131 		rx_buffer->page_offset += truesize;  in ixgbe_construct_skb()
2136 		rx_buffer->pagecnt_bias++;  in ixgbe_construct_skb()
2143 				       struct ixgbe_rx_buffer *rx_buffer,  in ixgbe_build_skb()  argument
2177 		IXGBE_CB(skb)->dma = rx_buffer->dma;  in ixgbe_build_skb()
2181 	rx_buffer->page_offset ^= truesize;  in ixgbe_build_skb()
2183 	rx_buffer->page_offset += truesize;  in ixgbe_build_skb()
2261 				 struct ixgbe_rx_buffer *rx_buffer,  in ixgbe_rx_buffer_flip()  argument
2266 	rx_buffer->page_offset ^= truesize;  in ixgbe_rx_buffer_flip()
2268 	rx_buffer->page_offset += truesize;  in ixgbe_rx_buffer_flip()
2308 		struct ixgbe_rx_buffer *rx_buffer;  in ixgbe_clean_rx_irq()  local
2330 		rx_buffer = ixgbe_get_rx_buffer(rx_ring, rx_desc, &skb, size, &rx_buffer_pgcnt);  in ixgbe_clean_rx_irq()
2336 			hard_start = page_address(rx_buffer->page) +  in ixgbe_clean_rx_irq()
2337 				     rx_buffer->page_offset - offset;  in ixgbe_clean_rx_irq()
2352 				ixgbe_rx_buffer_flip(rx_ring, rx_buffer, size);  in ixgbe_clean_rx_irq()
2354 				rx_buffer->pagecnt_bias++;  in ixgbe_clean_rx_irq()
2359 			ixgbe_add_rx_frag(rx_ring, rx_buffer, skb, size);  in ixgbe_clean_rx_irq()
2361 			skb = ixgbe_build_skb(rx_ring, rx_buffer,  in ixgbe_clean_rx_irq()
2364 			skb = ixgbe_construct_skb(rx_ring, rx_buffer,  in ixgbe_clean_rx_irq()
2371 			rx_buffer->pagecnt_bias++;  in ixgbe_clean_rx_irq()
2375 		ixgbe_put_rx_buffer(rx_ring, rx_buffer, skb, rx_buffer_pgcnt);  in ixgbe_clean_rx_irq()
5289 	struct ixgbe_rx_buffer *rx_buffer = &rx_ring->rx_buffer_info[i];  in ixgbe_clean_rx_ring()  local
5298 		if (rx_buffer->skb) {  in ixgbe_clean_rx_ring()
5299 			struct sk_buff *skb = rx_buffer->skb;  in ixgbe_clean_rx_ring()
5313 					      rx_buffer->dma,  in ixgbe_clean_rx_ring()
5314 					      rx_buffer->page_offset,  in ixgbe_clean_rx_ring()
5319 		dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,  in ixgbe_clean_rx_ring()
5323 		__page_frag_cache_drain(rx_buffer->page,  in ixgbe_clean_rx_ring()
5324 					rx_buffer->pagecnt_bias);  in ixgbe_clean_rx_ring()
5327 		rx_buffer++;  in ixgbe_clean_rx_ring()
5330 			rx_buffer = rx_ring->rx_buffer_info;  in ixgbe_clean_rx_ring()