60 	struct xdp_buff_xsk *xskb;  in xp_create_and_assign_umem()  local
99 		xskb = &pool->heads[i];  in xp_create_and_assign_umem()
100 		xskb->pool = pool;  in xp_create_and_assign_umem()
101 		xskb->xdp.frame_sz = umem->chunk_size - umem->headroom;  in xp_create_and_assign_umem()
102 		INIT_LIST_HEAD(&xskb->free_list_node);  in xp_create_and_assign_umem()
103 		INIT_LIST_HEAD(&xskb->xskb_list_node);  in xp_create_and_assign_umem()
105 			pool->free_heads[i] = xskb;  in xp_create_and_assign_umem()
107 			xp_init_xskb_addr(xskb, pool, i * pool->chunk_size);  in xp_create_and_assign_umem()
401 			struct xdp_buff_xsk *xskb = &pool->heads[i];  in xp_init_dma_info()  local
403 			xp_init_xskb_dma(xskb, pool, dma_map->dma_pages, xskb->orig_addr);  in xp_init_dma_info()
491 	struct xdp_buff_xsk *xskb;  in __xp_alloc()  local
515 		xskb = pool->free_heads[--pool->free_heads_cnt];  in __xp_alloc()
516 		xp_init_xskb_addr(xskb, pool, addr);  in __xp_alloc()
518 			xp_init_xskb_dma(xskb, pool, pool->dma_pages, addr);  in __xp_alloc()
520 		xskb = &pool->heads[xp_aligned_extract_idx(pool, addr)];  in __xp_alloc()
524 	return xskb;  in __xp_alloc()
529 	struct xdp_buff_xsk *xskb;  in xp_alloc()  local
532 		xskb = __xp_alloc(pool);  in xp_alloc()
533 		if (!xskb)  in xp_alloc()
537 		xskb = list_first_entry(&pool->free_list, struct xdp_buff_xsk,  in xp_alloc()
539 		list_del_init(&xskb->free_list_node);  in xp_alloc()
542 	xskb->xdp.data = xskb->xdp.data_hard_start + XDP_PACKET_HEADROOM;  in xp_alloc()
543 	xskb->xdp.data_meta = xskb->xdp.data;  in xp_alloc()
544 	xskb->xdp.flags = 0;  in xp_alloc()
547 		dma_sync_single_range_for_device(pool->dev, xskb->dma, 0,  in xp_alloc()
551 	return &xskb->xdp;  in xp_alloc()
567 		struct xdp_buff_xsk *xskb;  in xp_alloc_new_from_fq()  local
582 			xskb = pool->free_heads[--pool->free_heads_cnt];  in xp_alloc_new_from_fq()
583 			xp_init_xskb_addr(xskb, pool, addr);  in xp_alloc_new_from_fq()
585 				xp_init_xskb_dma(xskb, pool, pool->dma_pages, addr);  in xp_alloc_new_from_fq()
587 			xskb = &pool->heads[xp_aligned_extract_idx(pool, addr)];  in xp_alloc_new_from_fq()
590 		*xdp = &xskb->xdp;  in xp_alloc_new_from_fq()
600 	struct xdp_buff_xsk *xskb;  in xp_alloc_reused()  local
607 		xskb = list_first_entry(&pool->free_list, struct xdp_buff_xsk, free_list_node);  in xp_alloc_reused()
608 		list_del_init(&xskb->free_list_node);  in xp_alloc_reused()
610 		*xdp = &xskb->xdp;  in xp_alloc_reused()
657 void xp_free(struct xdp_buff_xsk *xskb)  in xp_free()  argument
659 	if (!list_empty(&xskb->free_list_node))  in xp_free()
662 	xskb->pool->free_list_cnt++;  in xp_free()
663 	list_add(&xskb->free_list_node, &xskb->pool->free_list);  in xp_free()
683 void xp_dma_sync_for_cpu_slow(struct xdp_buff_xsk *xskb)  in xp_dma_sync_for_cpu_slow()  argument
685 	dma_sync_single_range_for_cpu(xskb->pool->dev, xskb->dma, 0,  in xp_dma_sync_for_cpu_slow()
686 				      xskb->pool->frame_len, DMA_BIDIRECTIONAL);  in xp_dma_sync_for_cpu_slow()