1969 static bool i40e_can_reuse_rx_page(struct i40e_rx_buffer *rx_buffer,  in i40e_can_reuse_rx_page()  argument
1972 	unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;  in i40e_can_reuse_rx_page()
1973 	struct page *page = rx_buffer->page;  in i40e_can_reuse_rx_page()
1983 	if (unlikely((rx_buffer->page_count - pagecnt_bias) > 1)) {  in i40e_can_reuse_rx_page()
1990 	if (rx_buffer->page_offset > I40E_LAST_OFFSET) {  in i40e_can_reuse_rx_page()
2002 		rx_buffer->pagecnt_bias = USHRT_MAX;  in i40e_can_reuse_rx_page()
2013 static void i40e_rx_buffer_flip(struct i40e_rx_buffer *rx_buffer,  in i40e_rx_buffer_flip()  argument
2017 	rx_buffer->page_offset ^= truesize;  in i40e_rx_buffer_flip()
2019 	rx_buffer->page_offset += truesize;  in i40e_rx_buffer_flip()
2034 	struct i40e_rx_buffer *rx_buffer;  in i40e_get_rx_buffer()  local
2036 	rx_buffer = i40e_rx_bi(rx_ring, rx_ring->next_to_process);  in i40e_get_rx_buffer()
2037 	rx_buffer->page_count =  in i40e_get_rx_buffer()
2039 		page_count(rx_buffer->page);  in i40e_get_rx_buffer()
2043 	prefetch_page_address(rx_buffer->page);  in i40e_get_rx_buffer()
2047 				      rx_buffer->dma,  in i40e_get_rx_buffer()
2048 				      rx_buffer->page_offset,  in i40e_get_rx_buffer()
2053 	rx_buffer->pagecnt_bias--;  in i40e_get_rx_buffer()
2055 	return rx_buffer;  in i40e_get_rx_buffer()
2067 			       struct i40e_rx_buffer *rx_buffer)  in i40e_put_rx_buffer()  argument
2069 	if (i40e_can_reuse_rx_page(rx_buffer, &rx_ring->rx_stats)) {  in i40e_put_rx_buffer()
2071 		i40e_reuse_rx_page(rx_ring, rx_buffer);  in i40e_put_rx_buffer()
2074 		dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,  in i40e_put_rx_buffer()
2077 		__page_frag_cache_drain(rx_buffer->page,  in i40e_put_rx_buffer()
2078 					rx_buffer->pagecnt_bias);  in i40e_put_rx_buffer()
2080 		rx_buffer->page = NULL;  in i40e_put_rx_buffer()
2095 	struct i40e_rx_buffer *rx_buffer;  in i40e_process_rx_buffs()  local
2100 		rx_buffer = i40e_rx_bi(rx_ring, next);  in i40e_process_rx_buffs()
2104 		if (!rx_buffer->page)  in i40e_process_rx_buffs()
2108 			i40e_rx_buffer_flip(rx_buffer, xdp->frame_sz);  in i40e_process_rx_buffs()
2110 			rx_buffer->pagecnt_bias++;  in i40e_process_rx_buffs()
2116 		i40e_put_rx_buffer(rx_ring, rx_buffer);  in i40e_process_rx_buffs()
2133 	struct i40e_rx_buffer *rx_buffer;  in i40e_construct_skb()  local
2179 	rx_buffer = i40e_rx_bi(rx_ring, rx_ring->next_to_clean);  in i40e_construct_skb()
2187 		skb_add_rx_frag(skb, 0, rx_buffer->page,  in i40e_construct_skb()
2188 				rx_buffer->page_offset + headlen,  in i40e_construct_skb()
2191 		i40e_rx_buffer_flip(rx_buffer, xdp->frame_sz);  in i40e_construct_skb()
2194 		rx_buffer->pagecnt_bias++;  in i40e_construct_skb()
2265 		struct i40e_rx_buffer *rx_buffer;  in i40e_build_skb()  local
2267 		rx_buffer = i40e_rx_bi(rx_ring, rx_ring->next_to_clean);  in i40e_build_skb()
2269 		i40e_rx_buffer_flip(rx_buffer, xdp->frame_sz);  in i40e_build_skb()
2435 			     struct i40e_rx_buffer *rx_buffer, u32 size)  in i40e_add_xdp_frag()  argument
2448 	__skb_fill_page_desc_noacc(sinfo, sinfo->nr_frags++, rx_buffer->page,  in i40e_add_xdp_frag()
2449 				   rx_buffer->page_offset, size);  in i40e_add_xdp_frag()
2453 	if (page_is_pfmemalloc(rx_buffer->page))  in i40e_add_xdp_frag()
2468 				  struct i40e_rx_buffer *rx_buffer)  in i40e_consume_xdp_buff()  argument
2471 	i40e_put_rx_buffer(rx_ring, rx_buffer);  in i40e_consume_xdp_buff()
2506 		struct i40e_rx_buffer *rx_buffer;  in i40e_clean_rx_irq()  local
2540 			rx_buffer = i40e_rx_bi(rx_ring, ntp);  in i40e_clean_rx_irq()
2542 			i40e_reuse_rx_page(rx_ring, rx_buffer);  in i40e_clean_rx_irq()
2561 		rx_buffer = i40e_get_rx_buffer(rx_ring, size);  in i40e_clean_rx_irq()
2569 			hard_start = page_address(rx_buffer->page) +  in i40e_clean_rx_irq()
2570 				     rx_buffer->page_offset - offset;  in i40e_clean_rx_irq()
2576 		} else if (i40e_add_xdp_frag(xdp, &nfrags, rx_buffer, size) &&  in i40e_clean_rx_irq()
2579 			i40e_consume_xdp_buff(rx_ring, xdp, rx_buffer);  in i40e_clean_rx_irq()
2595 				i40e_rx_buffer_flip(rx_buffer, xdp->frame_sz);  in i40e_clean_rx_irq()
2597 				rx_buffer->pagecnt_bias++;  in i40e_clean_rx_irq()
2609 				i40e_consume_xdp_buff(rx_ring, xdp, rx_buffer);  in i40e_clean_rx_irq()
2630 		i40e_put_rx_buffer(rx_ring, rx_buffer);  in i40e_clean_rx_irq()