Lines Matching full:xdp
2 /* include/net/xdp.h
15 * DOC: XDP RX-queue information
17 * The XDP RX-queue info (xdp_rxq_info) is associated with the driver
22 * reference to this xdp_rxq_info structure. This provides the XDP
32 * The struct is not directly tied to the XDP prog. A new XDP prog
43 MEM_TYPE_PAGE_ORDER0, /* Orig XDP full page model */
49 /* XDP flags for ndo_xdp_xmit */
74 XDP_FLAGS_HAS_FRAGS = BIT(0), /* non-linear xdp buff */
75 XDP_FLAGS_FRAGS_PF_MEMALLOC = BIT(1), /* xdp paged memory is under
91 static __always_inline bool xdp_buff_has_frags(struct xdp_buff *xdp) in xdp_buff_has_frags() argument
93 return !!(xdp->flags & XDP_FLAGS_HAS_FRAGS); in xdp_buff_has_frags()
96 static __always_inline void xdp_buff_set_frags_flag(struct xdp_buff *xdp) in xdp_buff_set_frags_flag() argument
98 xdp->flags |= XDP_FLAGS_HAS_FRAGS; in xdp_buff_set_frags_flag()
101 static __always_inline void xdp_buff_clear_frags_flag(struct xdp_buff *xdp) in xdp_buff_clear_frags_flag() argument
103 xdp->flags &= ~XDP_FLAGS_HAS_FRAGS; in xdp_buff_clear_frags_flag()
106 static __always_inline bool xdp_buff_is_frag_pfmemalloc(struct xdp_buff *xdp) in xdp_buff_is_frag_pfmemalloc() argument
108 return !!(xdp->flags & XDP_FLAGS_FRAGS_PF_MEMALLOC); in xdp_buff_is_frag_pfmemalloc()
111 static __always_inline void xdp_buff_set_frag_pfmemalloc(struct xdp_buff *xdp) in xdp_buff_set_frag_pfmemalloc() argument
113 xdp->flags |= XDP_FLAGS_FRAGS_PF_MEMALLOC; in xdp_buff_set_frag_pfmemalloc()
117 xdp_init_buff(struct xdp_buff *xdp, u32 frame_sz, struct xdp_rxq_info *rxq) in xdp_init_buff() argument
119 xdp->frame_sz = frame_sz; in xdp_init_buff()
120 xdp->rxq = rxq; in xdp_init_buff()
121 xdp->flags = 0; in xdp_init_buff()
125 xdp_prepare_buff(struct xdp_buff *xdp, unsigned char *hard_start, in xdp_prepare_buff() argument
130 xdp->data_hard_start = hard_start; in xdp_prepare_buff()
131 xdp->data = data; in xdp_prepare_buff()
132 xdp->data_end = data + data_len; in xdp_prepare_buff()
133 xdp->data_meta = meta_valid ? data : data + 1; in xdp_prepare_buff()
138 * This macro reserves tailroom in the XDP buffer by limiting the
139 * XDP/BPF data access to data_hard_end. Notice same area (and size)
142 #define xdp_data_hard_end(xdp) \ argument
143 ((xdp)->data_hard_start + (xdp)->frame_sz - \
147 xdp_get_shared_info_from_buff(struct xdp_buff *xdp) in xdp_get_shared_info_from_buff() argument
149 return (struct skb_shared_info *)xdp_data_hard_end(xdp); in xdp_get_shared_info_from_buff()
152 static __always_inline unsigned int xdp_get_buff_len(struct xdp_buff *xdp) in xdp_get_buff_len() argument
154 unsigned int len = xdp->data_end - xdp->data; in xdp_get_buff_len()
157 if (likely(!xdp_buff_has_frags(xdp))) in xdp_get_buff_len()
160 sinfo = xdp_get_shared_info_from_buff(xdp); in xdp_get_buff_len()
242 struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp);
252 void xdp_convert_frame_to_buff(struct xdp_frame *frame, struct xdp_buff *xdp) in xdp_convert_frame_to_buff() argument
254 xdp->data_hard_start = frame->data - frame->headroom - sizeof(*frame); in xdp_convert_frame_to_buff()
255 xdp->data = frame->data; in xdp_convert_frame_to_buff()
256 xdp->data_end = frame->data + frame->len; in xdp_convert_frame_to_buff()
257 xdp->data_meta = frame->data - frame->metasize; in xdp_convert_frame_to_buff()
258 xdp->frame_sz = frame->frame_sz; in xdp_convert_frame_to_buff()
259 xdp->flags = frame->flags; in xdp_convert_frame_to_buff()
263 int xdp_update_frame_from_buff(struct xdp_buff *xdp, in xdp_update_frame_from_buff() argument
269 headroom = xdp->data - xdp->data_hard_start; in xdp_update_frame_from_buff()
270 metasize = xdp->data - xdp->data_meta; in xdp_update_frame_from_buff()
276 if (unlikely(xdp->data_end > xdp_data_hard_end(xdp))) { in xdp_update_frame_from_buff()
281 xdp_frame->data = xdp->data; in xdp_update_frame_from_buff()
282 xdp_frame->len = xdp->data_end - xdp->data; in xdp_update_frame_from_buff()
285 xdp_frame->frame_sz = xdp->frame_sz; in xdp_update_frame_from_buff()
286 xdp_frame->flags = xdp->flags; in xdp_update_frame_from_buff()
293 struct xdp_frame *xdp_convert_buff_to_frame(struct xdp_buff *xdp) in xdp_convert_buff_to_frame() argument
297 if (xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL) in xdp_convert_buff_to_frame()
298 return xdp_convert_zc_to_xdp_frame(xdp); in xdp_convert_buff_to_frame()
301 xdp_frame = xdp->data_hard_start; in xdp_convert_buff_to_frame()
302 if (unlikely(xdp_update_frame_from_buff(xdp, xdp_frame) < 0)) in xdp_convert_buff_to_frame()
306 xdp_frame->mem = xdp->rxq->mem; in xdp_convert_buff_to_frame()
312 struct xdp_buff *xdp);
315 void xdp_return_buff(struct xdp_buff *xdp);
355 /* Drivers not supporting XDP metadata can use this helper, which
359 xdp_set_data_meta_invalid(struct xdp_buff *xdp) in xdp_set_data_meta_invalid() argument
361 xdp->data_meta = xdp->data + 1; in xdp_set_data_meta_invalid()
365 xdp_data_meta_unsupported(const struct xdp_buff *xdp) in xdp_data_meta_unsupported() argument
367 return unlikely(xdp->data_meta > xdp->data); in xdp_data_meta_unsupported()
483 struct xdp_buff *xdp) in bpf_prog_run_xdp() argument
485 /* Driver XDP hooks are invoked within a single NAPI poll cycle and thus in bpf_prog_run_xdp()
489 u32 act = __bpf_prog_run(prog, xdp, BPF_DISPATCHER_FUNC(xdp)); in bpf_prog_run_xdp()
492 if (act == XDP_TX && netif_is_bond_slave(xdp->rxq->dev)) in bpf_prog_run_xdp()
493 act = xdp_master_redirect(xdp); in bpf_prog_run_xdp()