1ddc64d0aSThomas Gleixner /* SPDX-License-Identifier: GPL-2.0-only */
2aecd67b6SJesper Dangaard Brouer /* include/net/xdp.h
3aecd67b6SJesper Dangaard Brouer *
4aecd67b6SJesper Dangaard Brouer * Copyright (c) 2017 Jesper Dangaard Brouer, Red Hat Inc.
5aecd67b6SJesper Dangaard Brouer */
6aecd67b6SJesper Dangaard Brouer #ifndef __LINUX_NET_XDP_H__
7aecd67b6SJesper Dangaard Brouer #define __LINUX_NET_XDP_H__
8aecd67b6SJesper Dangaard Brouer
90cd917a4SJesper Dangaard Brouer #include <linux/bitfield.h>
10*680ee045SJakub Kicinski #include <linux/filter.h>
11*680ee045SJakub Kicinski #include <linux/netdevice.h>
12*680ee045SJakub Kicinski #include <linux/skbuff.h> /* skb_shared_info */
13f95f0f95SJesper Dangaard Brouer
14aecd67b6SJesper Dangaard Brouer /**
15aecd67b6SJesper Dangaard Brouer * DOC: XDP RX-queue information
16aecd67b6SJesper Dangaard Brouer *
17aecd67b6SJesper Dangaard Brouer * The XDP RX-queue info (xdp_rxq_info) is associated with the driver
18aecd67b6SJesper Dangaard Brouer * level RX-ring queues. It is information that is specific to how
19aecd67b6SJesper Dangaard Brouer * the driver have configured a given RX-ring queue.
20aecd67b6SJesper Dangaard Brouer *
216bbc7103SKev Jackson * Each xdp_buff frame received in the driver carries a (pointer)
22aecd67b6SJesper Dangaard Brouer * reference to this xdp_rxq_info structure. This provides the XDP
23aecd67b6SJesper Dangaard Brouer * data-path read-access to RX-info for both kernel and bpf-side
24aecd67b6SJesper Dangaard Brouer * (limited subset).
25aecd67b6SJesper Dangaard Brouer *
26aecd67b6SJesper Dangaard Brouer * For now, direct access is only safe while running in NAPI/softirq
276bbc7103SKev Jackson * context. Contents are read-mostly and must not be updated during
28aecd67b6SJesper Dangaard Brouer * driver NAPI/softirq poll.
29aecd67b6SJesper Dangaard Brouer *
30aecd67b6SJesper Dangaard Brouer * The driver usage API is a register and unregister API.
31aecd67b6SJesper Dangaard Brouer *
32aecd67b6SJesper Dangaard Brouer * The struct is not directly tied to the XDP prog. A new XDP prog
33aecd67b6SJesper Dangaard Brouer * can be attached as long as it doesn't change the underlying
34aecd67b6SJesper Dangaard Brouer * RX-ring. If the RX-ring does change significantly, the NIC driver
35aecd67b6SJesper Dangaard Brouer * naturally need to stop the RX-ring before purging and reallocating
366bbc7103SKev Jackson * memory. In that process the driver MUST call unregister (which
376bbc7103SKev Jackson * also applies for driver shutdown and unload). The register API is
38aecd67b6SJesper Dangaard Brouer * also mandatory during RX-ring setup.
39aecd67b6SJesper Dangaard Brouer */
40aecd67b6SJesper Dangaard Brouer
415ab073ffSJesper Dangaard Brouer enum xdp_mem_type {
425ab073ffSJesper Dangaard Brouer MEM_TYPE_PAGE_SHARED = 0, /* Split-page refcnt based model */
435ab073ffSJesper Dangaard Brouer MEM_TYPE_PAGE_ORDER0, /* Orig XDP full page model */
4457d0a1c1SJesper Dangaard Brouer MEM_TYPE_PAGE_POOL,
452b43470aSBjörn Töpel MEM_TYPE_XSK_BUFF_POOL,
465ab073ffSJesper Dangaard Brouer MEM_TYPE_MAX,
475ab073ffSJesper Dangaard Brouer };
485ab073ffSJesper Dangaard Brouer
4942b33468SJesper Dangaard Brouer /* XDP flags for ndo_xdp_xmit */
5042b33468SJesper Dangaard Brouer #define XDP_XMIT_FLUSH (1U << 0) /* doorbell signal consumer */
5142b33468SJesper Dangaard Brouer #define XDP_XMIT_FLAGS_MASK XDP_XMIT_FLUSH
5242b33468SJesper Dangaard Brouer
535ab073ffSJesper Dangaard Brouer struct xdp_mem_info {
545ab073ffSJesper Dangaard Brouer u32 type; /* enum xdp_mem_type, but known size type */
558d5d8852SJesper Dangaard Brouer u32 id;
565ab073ffSJesper Dangaard Brouer };
575ab073ffSJesper Dangaard Brouer
5857d0a1c1SJesper Dangaard Brouer struct page_pool;
5957d0a1c1SJesper Dangaard Brouer
60aecd67b6SJesper Dangaard Brouer struct xdp_rxq_info {
61aecd67b6SJesper Dangaard Brouer struct net_device *dev;
62aecd67b6SJesper Dangaard Brouer u32 queue_index;
63aecd67b6SJesper Dangaard Brouer u32 reg_state;
645ab073ffSJesper Dangaard Brouer struct xdp_mem_info mem;
65b02e5a0eSBjörn Töpel unsigned int napi_id;
66bf25146aSEelco Chaudron u32 frag_size;
67aecd67b6SJesper Dangaard Brouer } ____cacheline_aligned; /* perf critical, avoid false-sharing */
68aecd67b6SJesper Dangaard Brouer
6964b59025SDavid Ahern struct xdp_txq_info {
7064b59025SDavid Ahern struct net_device *dev;
7164b59025SDavid Ahern };
7264b59025SDavid Ahern
732e88d4ffSLorenzo Bianconi enum xdp_buff_flags {
742e88d4ffSLorenzo Bianconi XDP_FLAGS_HAS_FRAGS = BIT(0), /* non-linear xdp buff */
75d65a1906SLorenzo Bianconi XDP_FLAGS_FRAGS_PF_MEMALLOC = BIT(1), /* xdp paged memory is under
76d65a1906SLorenzo Bianconi * pressure
77d65a1906SLorenzo Bianconi */
782e88d4ffSLorenzo Bianconi };
792e88d4ffSLorenzo Bianconi
80106ca27fSJesper Dangaard Brouer struct xdp_buff {
81106ca27fSJesper Dangaard Brouer void *data;
82106ca27fSJesper Dangaard Brouer void *data_end;
83106ca27fSJesper Dangaard Brouer void *data_meta;
84106ca27fSJesper Dangaard Brouer void *data_hard_start;
85106ca27fSJesper Dangaard Brouer struct xdp_rxq_info *rxq;
8664b59025SDavid Ahern struct xdp_txq_info *txq;
87f95f0f95SJesper Dangaard Brouer u32 frame_sz; /* frame size to deduce data_hard_end/reserved tailroom*/
882e88d4ffSLorenzo Bianconi u32 flags; /* supported values defined in xdp_buff_flags */
89106ca27fSJesper Dangaard Brouer };
905ab073ffSJesper Dangaard Brouer
xdp_buff_has_frags(struct xdp_buff * xdp)912e88d4ffSLorenzo Bianconi static __always_inline bool xdp_buff_has_frags(struct xdp_buff *xdp)
922e88d4ffSLorenzo Bianconi {
932e88d4ffSLorenzo Bianconi return !!(xdp->flags & XDP_FLAGS_HAS_FRAGS);
942e88d4ffSLorenzo Bianconi }
952e88d4ffSLorenzo Bianconi
xdp_buff_set_frags_flag(struct xdp_buff * xdp)962e88d4ffSLorenzo Bianconi static __always_inline void xdp_buff_set_frags_flag(struct xdp_buff *xdp)
972e88d4ffSLorenzo Bianconi {
982e88d4ffSLorenzo Bianconi xdp->flags |= XDP_FLAGS_HAS_FRAGS;
992e88d4ffSLorenzo Bianconi }
1002e88d4ffSLorenzo Bianconi
xdp_buff_clear_frags_flag(struct xdp_buff * xdp)1012e88d4ffSLorenzo Bianconi static __always_inline void xdp_buff_clear_frags_flag(struct xdp_buff *xdp)
1022e88d4ffSLorenzo Bianconi {
1032e88d4ffSLorenzo Bianconi xdp->flags &= ~XDP_FLAGS_HAS_FRAGS;
1042e88d4ffSLorenzo Bianconi }
1052e88d4ffSLorenzo Bianconi
xdp_buff_is_frag_pfmemalloc(struct xdp_buff * xdp)106d65a1906SLorenzo Bianconi static __always_inline bool xdp_buff_is_frag_pfmemalloc(struct xdp_buff *xdp)
107d65a1906SLorenzo Bianconi {
108d65a1906SLorenzo Bianconi return !!(xdp->flags & XDP_FLAGS_FRAGS_PF_MEMALLOC);
109d65a1906SLorenzo Bianconi }
110d65a1906SLorenzo Bianconi
xdp_buff_set_frag_pfmemalloc(struct xdp_buff * xdp)111d65a1906SLorenzo Bianconi static __always_inline void xdp_buff_set_frag_pfmemalloc(struct xdp_buff *xdp)
112d65a1906SLorenzo Bianconi {
113d65a1906SLorenzo Bianconi xdp->flags |= XDP_FLAGS_FRAGS_PF_MEMALLOC;
114d65a1906SLorenzo Bianconi }
115d65a1906SLorenzo Bianconi
11643b5169dSLorenzo Bianconi static __always_inline void
xdp_init_buff(struct xdp_buff * xdp,u32 frame_sz,struct xdp_rxq_info * rxq)11743b5169dSLorenzo Bianconi xdp_init_buff(struct xdp_buff *xdp, u32 frame_sz, struct xdp_rxq_info *rxq)
11843b5169dSLorenzo Bianconi {
11943b5169dSLorenzo Bianconi xdp->frame_sz = frame_sz;
12043b5169dSLorenzo Bianconi xdp->rxq = rxq;
1212e88d4ffSLorenzo Bianconi xdp->flags = 0;
12243b5169dSLorenzo Bianconi }
12343b5169dSLorenzo Bianconi
124be9df4afSLorenzo Bianconi static __always_inline void
xdp_prepare_buff(struct xdp_buff * xdp,unsigned char * hard_start,int headroom,int data_len,const bool meta_valid)125be9df4afSLorenzo Bianconi xdp_prepare_buff(struct xdp_buff *xdp, unsigned char *hard_start,
126be9df4afSLorenzo Bianconi int headroom, int data_len, const bool meta_valid)
127be9df4afSLorenzo Bianconi {
128be9df4afSLorenzo Bianconi unsigned char *data = hard_start + headroom;
129be9df4afSLorenzo Bianconi
130be9df4afSLorenzo Bianconi xdp->data_hard_start = hard_start;
131be9df4afSLorenzo Bianconi xdp->data = data;
132be9df4afSLorenzo Bianconi xdp->data_end = data + data_len;
133be9df4afSLorenzo Bianconi xdp->data_meta = meta_valid ? data : data + 1;
134be9df4afSLorenzo Bianconi }
135be9df4afSLorenzo Bianconi
136f95f0f95SJesper Dangaard Brouer /* Reserve memory area at end-of data area.
137f95f0f95SJesper Dangaard Brouer *
138f95f0f95SJesper Dangaard Brouer * This macro reserves tailroom in the XDP buffer by limiting the
139f95f0f95SJesper Dangaard Brouer * XDP/BPF data access to data_hard_end. Notice same area (and size)
140f95f0f95SJesper Dangaard Brouer * is used for XDP_PASS, when constructing the SKB via build_skb().
141f95f0f95SJesper Dangaard Brouer */
142f95f0f95SJesper Dangaard Brouer #define xdp_data_hard_end(xdp) \
143f95f0f95SJesper Dangaard Brouer ((xdp)->data_hard_start + (xdp)->frame_sz - \
144f95f0f95SJesper Dangaard Brouer SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
145f95f0f95SJesper Dangaard Brouer
1462f0bc54bSLorenzo Bianconi static inline struct skb_shared_info *
xdp_get_shared_info_from_buff(struct xdp_buff * xdp)1472f0bc54bSLorenzo Bianconi xdp_get_shared_info_from_buff(struct xdp_buff *xdp)
1482f0bc54bSLorenzo Bianconi {
1492f0bc54bSLorenzo Bianconi return (struct skb_shared_info *)xdp_data_hard_end(xdp);
1502f0bc54bSLorenzo Bianconi }
1512f0bc54bSLorenzo Bianconi
xdp_get_buff_len(struct xdp_buff * xdp)1520165cc81SLorenzo Bianconi static __always_inline unsigned int xdp_get_buff_len(struct xdp_buff *xdp)
1530165cc81SLorenzo Bianconi {
1540165cc81SLorenzo Bianconi unsigned int len = xdp->data_end - xdp->data;
1550165cc81SLorenzo Bianconi struct skb_shared_info *sinfo;
1560165cc81SLorenzo Bianconi
1570165cc81SLorenzo Bianconi if (likely(!xdp_buff_has_frags(xdp)))
1580165cc81SLorenzo Bianconi goto out;
1590165cc81SLorenzo Bianconi
1600165cc81SLorenzo Bianconi sinfo = xdp_get_shared_info_from_buff(xdp);
1610165cc81SLorenzo Bianconi len += sinfo->xdp_frags_size;
1620165cc81SLorenzo Bianconi out:
1630165cc81SLorenzo Bianconi return len;
1640165cc81SLorenzo Bianconi }
1650165cc81SLorenzo Bianconi
166c0048cffSJesper Dangaard Brouer struct xdp_frame {
167c0048cffSJesper Dangaard Brouer void *data;
168c0048cffSJesper Dangaard Brouer u16 len;
169c0048cffSJesper Dangaard Brouer u16 headroom;
170b860a1b9SJesper Dangaard Brouer u32 metasize; /* uses lower 8-bits */
171c0048cffSJesper Dangaard Brouer /* Lifetime of xdp_rxq_info is limited to NAPI/enqueue time,
172c0048cffSJesper Dangaard Brouer * while mem info is valid on remote CPU.
173c0048cffSJesper Dangaard Brouer */
174c0048cffSJesper Dangaard Brouer struct xdp_mem_info mem;
17570280ed9SJesper Dangaard Brouer struct net_device *dev_rx; /* used by cpumap */
176b860a1b9SJesper Dangaard Brouer u32 frame_sz;
1772e88d4ffSLorenzo Bianconi u32 flags; /* supported values defined in xdp_buff_flags */
178c0048cffSJesper Dangaard Brouer };
179c0048cffSJesper Dangaard Brouer
xdp_frame_has_frags(struct xdp_frame * frame)1802e88d4ffSLorenzo Bianconi static __always_inline bool xdp_frame_has_frags(struct xdp_frame *frame)
1812e88d4ffSLorenzo Bianconi {
1822e88d4ffSLorenzo Bianconi return !!(frame->flags & XDP_FLAGS_HAS_FRAGS);
1832e88d4ffSLorenzo Bianconi }
1842e88d4ffSLorenzo Bianconi
xdp_frame_is_frag_pfmemalloc(struct xdp_frame * frame)185d65a1906SLorenzo Bianconi static __always_inline bool xdp_frame_is_frag_pfmemalloc(struct xdp_frame *frame)
186d65a1906SLorenzo Bianconi {
187d65a1906SLorenzo Bianconi return !!(frame->flags & XDP_FLAGS_FRAGS_PF_MEMALLOC);
188d65a1906SLorenzo Bianconi }
189d65a1906SLorenzo Bianconi
19089653987SLorenzo Bianconi #define XDP_BULK_QUEUE_SIZE 16
19189653987SLorenzo Bianconi struct xdp_frame_bulk {
19289653987SLorenzo Bianconi int count;
19389653987SLorenzo Bianconi void *xa;
19489653987SLorenzo Bianconi void *q[XDP_BULK_QUEUE_SIZE];
19589653987SLorenzo Bianconi };
19689653987SLorenzo Bianconi
xdp_frame_bulk_init(struct xdp_frame_bulk * bq)19789653987SLorenzo Bianconi static __always_inline void xdp_frame_bulk_init(struct xdp_frame_bulk *bq)
19889653987SLorenzo Bianconi {
19989653987SLorenzo Bianconi /* bq->count will be zero'ed when bq->xa gets updated */
20089653987SLorenzo Bianconi bq->xa = NULL;
20189653987SLorenzo Bianconi }
202dee72f8aSDavid S. Miller
2032f0bc54bSLorenzo Bianconi static inline struct skb_shared_info *
xdp_get_shared_info_from_frame(struct xdp_frame * frame)2042f0bc54bSLorenzo Bianconi xdp_get_shared_info_from_frame(struct xdp_frame *frame)
2052f0bc54bSLorenzo Bianconi {
2062f0bc54bSLorenzo Bianconi void *data_hard_start = frame->data - frame->headroom - sizeof(*frame);
2072f0bc54bSLorenzo Bianconi
2082f0bc54bSLorenzo Bianconi return (struct skb_shared_info *)(data_hard_start + frame->frame_sz -
2092f0bc54bSLorenzo Bianconi SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));
2102f0bc54bSLorenzo Bianconi }
2112f0bc54bSLorenzo Bianconi
21292164774SLorenzo Bianconi struct xdp_cpumap_stats {
21328b1520eSLorenzo Bianconi unsigned int redirect;
21492164774SLorenzo Bianconi unsigned int pass;
21592164774SLorenzo Bianconi unsigned int drop;
21692164774SLorenzo Bianconi };
21792164774SLorenzo Bianconi
218a8d5b4abSToshiaki Makita /* Clear kernel pointers in xdp_frame */
xdp_scrub_frame(struct xdp_frame * frame)219a8d5b4abSToshiaki Makita static inline void xdp_scrub_frame(struct xdp_frame *frame)
220a8d5b4abSToshiaki Makita {
221a8d5b4abSToshiaki Makita frame->data = NULL;
222a8d5b4abSToshiaki Makita frame->dev_rx = NULL;
223a8d5b4abSToshiaki Makita }
224a8d5b4abSToshiaki Makita
225d65a1906SLorenzo Bianconi static inline void
xdp_update_skb_shared_info(struct sk_buff * skb,u8 nr_frags,unsigned int size,unsigned int truesize,bool pfmemalloc)226d65a1906SLorenzo Bianconi xdp_update_skb_shared_info(struct sk_buff *skb, u8 nr_frags,
227d65a1906SLorenzo Bianconi unsigned int size, unsigned int truesize,
228d65a1906SLorenzo Bianconi bool pfmemalloc)
229d65a1906SLorenzo Bianconi {
230d65a1906SLorenzo Bianconi skb_shinfo(skb)->nr_frags = nr_frags;
231d65a1906SLorenzo Bianconi
232d65a1906SLorenzo Bianconi skb->len += size;
233d65a1906SLorenzo Bianconi skb->data_len += size;
234d65a1906SLorenzo Bianconi skb->truesize += truesize;
235d65a1906SLorenzo Bianconi skb->pfmemalloc |= pfmemalloc;
236d65a1906SLorenzo Bianconi }
237d65a1906SLorenzo Bianconi
23834cc0b33SJesper Dangaard Brouer /* Avoids inlining WARN macro in fast-path */
23934cc0b33SJesper Dangaard Brouer void xdp_warn(const char *msg, const char *func, const int line);
24034cc0b33SJesper Dangaard Brouer #define XDP_WARN(msg) xdp_warn(msg, __func__, __LINE__)
24134cc0b33SJesper Dangaard Brouer
242b0d1beefSBjörn Töpel struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp);
24397a0e1eaSLorenzo Bianconi struct sk_buff *__xdp_build_skb_from_frame(struct xdp_frame *xdpf,
24497a0e1eaSLorenzo Bianconi struct sk_buff *skb,
24597a0e1eaSLorenzo Bianconi struct net_device *dev);
24689f479f0SLorenzo Bianconi struct sk_buff *xdp_build_skb_from_frame(struct xdp_frame *xdpf,
24789f479f0SLorenzo Bianconi struct net_device *dev);
24865e6dcf7SLorenzo Bianconi int xdp_alloc_skb_bulk(void **skbs, int n_skb, gfp_t gfp);
249e624d4edSHangbin Liu struct xdp_frame *xdpf_clone(struct xdp_frame *xdpf);
250b0d1beefSBjörn Töpel
251fc379872SLorenzo Bianconi static inline
xdp_convert_frame_to_buff(struct xdp_frame * frame,struct xdp_buff * xdp)252fc379872SLorenzo Bianconi void xdp_convert_frame_to_buff(struct xdp_frame *frame, struct xdp_buff *xdp)
253fc379872SLorenzo Bianconi {
254fc379872SLorenzo Bianconi xdp->data_hard_start = frame->data - frame->headroom - sizeof(*frame);
255fc379872SLorenzo Bianconi xdp->data = frame->data;
256fc379872SLorenzo Bianconi xdp->data_end = frame->data + frame->len;
257fc379872SLorenzo Bianconi xdp->data_meta = frame->data - frame->metasize;
258fc379872SLorenzo Bianconi xdp->frame_sz = frame->frame_sz;
2592e88d4ffSLorenzo Bianconi xdp->flags = frame->flags;
260fc379872SLorenzo Bianconi }
261fc379872SLorenzo Bianconi
262c0048cffSJesper Dangaard Brouer static inline
xdp_update_frame_from_buff(struct xdp_buff * xdp,struct xdp_frame * xdp_frame)263daa5cdc3SDavid Ahern int xdp_update_frame_from_buff(struct xdp_buff *xdp,
264daa5cdc3SDavid Ahern struct xdp_frame *xdp_frame)
265c0048cffSJesper Dangaard Brouer {
266daa5cdc3SDavid Ahern int metasize, headroom;
26702b55e56SBjörn Töpel
268c0048cffSJesper Dangaard Brouer /* Assure headroom is available for storing info */
269c0048cffSJesper Dangaard Brouer headroom = xdp->data - xdp->data_hard_start;
270c0048cffSJesper Dangaard Brouer metasize = xdp->data - xdp->data_meta;
271c0048cffSJesper Dangaard Brouer metasize = metasize > 0 ? metasize : 0;
272c0048cffSJesper Dangaard Brouer if (unlikely((headroom - metasize) < sizeof(*xdp_frame)))
273daa5cdc3SDavid Ahern return -ENOSPC;
274c0048cffSJesper Dangaard Brouer
27534cc0b33SJesper Dangaard Brouer /* Catch if driver didn't reserve tailroom for skb_shared_info */
27634cc0b33SJesper Dangaard Brouer if (unlikely(xdp->data_end > xdp_data_hard_end(xdp))) {
27734cc0b33SJesper Dangaard Brouer XDP_WARN("Driver BUG: missing reserved tailroom");
278daa5cdc3SDavid Ahern return -ENOSPC;
27934cc0b33SJesper Dangaard Brouer }
28034cc0b33SJesper Dangaard Brouer
281c0048cffSJesper Dangaard Brouer xdp_frame->data = xdp->data;
282c0048cffSJesper Dangaard Brouer xdp_frame->len = xdp->data_end - xdp->data;
283c0048cffSJesper Dangaard Brouer xdp_frame->headroom = headroom - sizeof(*xdp_frame);
284c0048cffSJesper Dangaard Brouer xdp_frame->metasize = metasize;
28534cc0b33SJesper Dangaard Brouer xdp_frame->frame_sz = xdp->frame_sz;
2862e88d4ffSLorenzo Bianconi xdp_frame->flags = xdp->flags;
287c0048cffSJesper Dangaard Brouer
288daa5cdc3SDavid Ahern return 0;
289daa5cdc3SDavid Ahern }
290daa5cdc3SDavid Ahern
291daa5cdc3SDavid Ahern /* Convert xdp_buff to xdp_frame */
292daa5cdc3SDavid Ahern static inline
xdp_convert_buff_to_frame(struct xdp_buff * xdp)293daa5cdc3SDavid Ahern struct xdp_frame *xdp_convert_buff_to_frame(struct xdp_buff *xdp)
294daa5cdc3SDavid Ahern {
295daa5cdc3SDavid Ahern struct xdp_frame *xdp_frame;
296daa5cdc3SDavid Ahern
297daa5cdc3SDavid Ahern if (xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL)
298daa5cdc3SDavid Ahern return xdp_convert_zc_to_xdp_frame(xdp);
299daa5cdc3SDavid Ahern
300daa5cdc3SDavid Ahern /* Store info in top of packet */
301daa5cdc3SDavid Ahern xdp_frame = xdp->data_hard_start;
302daa5cdc3SDavid Ahern if (unlikely(xdp_update_frame_from_buff(xdp, xdp_frame) < 0))
303daa5cdc3SDavid Ahern return NULL;
304daa5cdc3SDavid Ahern
305c0048cffSJesper Dangaard Brouer /* rxq only valid until napi_schedule ends, convert to xdp_mem_info */
306c0048cffSJesper Dangaard Brouer xdp_frame->mem = xdp->rxq->mem;
307c0048cffSJesper Dangaard Brouer
308c0048cffSJesper Dangaard Brouer return xdp_frame;
309c0048cffSJesper Dangaard Brouer }
310c0048cffSJesper Dangaard Brouer
311bf25146aSEelco Chaudron void __xdp_return(void *data, struct xdp_mem_info *mem, bool napi_direct,
312bf25146aSEelco Chaudron struct xdp_buff *xdp);
31303993094SJesper Dangaard Brouer void xdp_return_frame(struct xdp_frame *xdpf);
314389ab7f0SJesper Dangaard Brouer void xdp_return_frame_rx_napi(struct xdp_frame *xdpf);
315c497176cSBjörn Töpel void xdp_return_buff(struct xdp_buff *xdp);
31689653987SLorenzo Bianconi void xdp_flush_frame_bulk(struct xdp_frame_bulk *bq);
31789653987SLorenzo Bianconi void xdp_return_frame_bulk(struct xdp_frame *xdpf,
31889653987SLorenzo Bianconi struct xdp_frame_bulk *bq);
3195ab073ffSJesper Dangaard Brouer
xdp_get_frame_len(struct xdp_frame * xdpf)3205142239aSLorenzo Bianconi static __always_inline unsigned int xdp_get_frame_len(struct xdp_frame *xdpf)
3215142239aSLorenzo Bianconi {
3225142239aSLorenzo Bianconi struct skb_shared_info *sinfo;
3235142239aSLorenzo Bianconi unsigned int len = xdpf->len;
3245142239aSLorenzo Bianconi
3255142239aSLorenzo Bianconi if (likely(!xdp_frame_has_frags(xdpf)))
3265142239aSLorenzo Bianconi goto out;
3275142239aSLorenzo Bianconi
3285142239aSLorenzo Bianconi sinfo = xdp_get_shared_info_from_frame(xdpf);
3295142239aSLorenzo Bianconi len += sinfo->xdp_frags_size;
3305142239aSLorenzo Bianconi out:
3315142239aSLorenzo Bianconi return len;
3325142239aSLorenzo Bianconi }
3335142239aSLorenzo Bianconi
334bf25146aSEelco Chaudron int __xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq,
335bf25146aSEelco Chaudron struct net_device *dev, u32 queue_index,
336bf25146aSEelco Chaudron unsigned int napi_id, u32 frag_size);
337bf25146aSEelco Chaudron static inline int
xdp_rxq_info_reg(struct xdp_rxq_info * xdp_rxq,struct net_device * dev,u32 queue_index,unsigned int napi_id)338bf25146aSEelco Chaudron xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq,
339bf25146aSEelco Chaudron struct net_device *dev, u32 queue_index,
340bf25146aSEelco Chaudron unsigned int napi_id)
341bf25146aSEelco Chaudron {
342bf25146aSEelco Chaudron return __xdp_rxq_info_reg(xdp_rxq, dev, queue_index, napi_id, 0);
343bf25146aSEelco Chaudron }
344bf25146aSEelco Chaudron
345aecd67b6SJesper Dangaard Brouer void xdp_rxq_info_unreg(struct xdp_rxq_info *xdp_rxq);
346aecd67b6SJesper Dangaard Brouer void xdp_rxq_info_unused(struct xdp_rxq_info *xdp_rxq);
347c0124f32SJesper Dangaard Brouer bool xdp_rxq_info_is_reg(struct xdp_rxq_info *xdp_rxq);
3485ab073ffSJesper Dangaard Brouer int xdp_rxq_info_reg_mem_model(struct xdp_rxq_info *xdp_rxq,
3495ab073ffSJesper Dangaard Brouer enum xdp_mem_type type, void *allocator);
350dce5bd61SBjörn Töpel void xdp_rxq_info_unreg_mem_model(struct xdp_rxq_info *xdp_rxq);
3514a48ef70SToke Høiland-Jørgensen int xdp_reg_mem_model(struct xdp_mem_info *mem,
3524a48ef70SToke Høiland-Jørgensen enum xdp_mem_type type, void *allocator);
3534a48ef70SToke Høiland-Jørgensen void xdp_unreg_mem_model(struct xdp_mem_info *mem);
354aecd67b6SJesper Dangaard Brouer
355106ca27fSJesper Dangaard Brouer /* Drivers not supporting XDP metadata can use this helper, which
356106ca27fSJesper Dangaard Brouer * rejects any room expansion for metadata as a result.
357106ca27fSJesper Dangaard Brouer */
358106ca27fSJesper Dangaard Brouer static __always_inline void
xdp_set_data_meta_invalid(struct xdp_buff * xdp)359106ca27fSJesper Dangaard Brouer xdp_set_data_meta_invalid(struct xdp_buff *xdp)
360106ca27fSJesper Dangaard Brouer {
361106ca27fSJesper Dangaard Brouer xdp->data_meta = xdp->data + 1;
362106ca27fSJesper Dangaard Brouer }
363106ca27fSJesper Dangaard Brouer
364106ca27fSJesper Dangaard Brouer static __always_inline bool
xdp_data_meta_unsupported(const struct xdp_buff * xdp)365106ca27fSJesper Dangaard Brouer xdp_data_meta_unsupported(const struct xdp_buff *xdp)
366106ca27fSJesper Dangaard Brouer {
367106ca27fSJesper Dangaard Brouer return unlikely(xdp->data_meta > xdp->data);
368106ca27fSJesper Dangaard Brouer }
369106ca27fSJesper Dangaard Brouer
xdp_metalen_invalid(unsigned long metalen)3707445cf31SZvi Effron static inline bool xdp_metalen_invalid(unsigned long metalen)
3717445cf31SZvi Effron {
3727445cf31SZvi Effron return (metalen & (sizeof(__u32) - 1)) || (metalen > 32);
3737445cf31SZvi Effron }
3747445cf31SZvi Effron
37505296620SJakub Kicinski struct xdp_attachment_info {
37605296620SJakub Kicinski struct bpf_prog *prog;
37705296620SJakub Kicinski u32 flags;
37805296620SJakub Kicinski };
37905296620SJakub Kicinski
38005296620SJakub Kicinski struct netdev_bpf;
38105296620SJakub Kicinski void xdp_attachment_setup(struct xdp_attachment_info *info,
38205296620SJakub Kicinski struct netdev_bpf *bpf);
38305296620SJakub Kicinski
38489653987SLorenzo Bianconi #define DEV_MAP_BULK_SIZE XDP_BULK_QUEUE_SIZE
385788f87acSIoana Ciornei
3863d76a4d3SStanislav Fomichev #define XDP_METADATA_KFUNC_xxx \
3873d76a4d3SStanislav Fomichev XDP_METADATA_KFUNC(XDP_METADATA_KFUNC_RX_TIMESTAMP, \
3883d76a4d3SStanislav Fomichev bpf_xdp_metadata_rx_timestamp) \
3893d76a4d3SStanislav Fomichev XDP_METADATA_KFUNC(XDP_METADATA_KFUNC_RX_HASH, \
3903d76a4d3SStanislav Fomichev bpf_xdp_metadata_rx_hash) \
3913d76a4d3SStanislav Fomichev
3923d76a4d3SStanislav Fomichev enum {
3933d76a4d3SStanislav Fomichev #define XDP_METADATA_KFUNC(name, _) name,
3943d76a4d3SStanislav Fomichev XDP_METADATA_KFUNC_xxx
3953d76a4d3SStanislav Fomichev #undef XDP_METADATA_KFUNC
3963d76a4d3SStanislav Fomichev MAX_XDP_METADATA_KFUNC,
3973d76a4d3SStanislav Fomichev };
3983d76a4d3SStanislav Fomichev
3990cd917a4SJesper Dangaard Brouer enum xdp_rss_hash_type {
4000cd917a4SJesper Dangaard Brouer /* First part: Individual bits for L3/L4 types */
4010cd917a4SJesper Dangaard Brouer XDP_RSS_L3_IPV4 = BIT(0),
4020cd917a4SJesper Dangaard Brouer XDP_RSS_L3_IPV6 = BIT(1),
4030cd917a4SJesper Dangaard Brouer
4040cd917a4SJesper Dangaard Brouer /* The fixed (L3) IPv4 and IPv6 headers can both be followed by
4050cd917a4SJesper Dangaard Brouer * variable/dynamic headers, IPv4 called Options and IPv6 called
4060cd917a4SJesper Dangaard Brouer * Extension Headers. HW RSS type can contain this info.
4070cd917a4SJesper Dangaard Brouer */
4080cd917a4SJesper Dangaard Brouer XDP_RSS_L3_DYNHDR = BIT(2),
4090cd917a4SJesper Dangaard Brouer
4100cd917a4SJesper Dangaard Brouer /* When RSS hash covers L4 then drivers MUST set XDP_RSS_L4 bit in
4110cd917a4SJesper Dangaard Brouer * addition to the protocol specific bit. This ease interaction with
4120cd917a4SJesper Dangaard Brouer * SKBs and avoids reserving a fixed mask for future L4 protocol bits.
4130cd917a4SJesper Dangaard Brouer */
4140cd917a4SJesper Dangaard Brouer XDP_RSS_L4 = BIT(3), /* L4 based hash, proto can be unknown */
4150cd917a4SJesper Dangaard Brouer XDP_RSS_L4_TCP = BIT(4),
4160cd917a4SJesper Dangaard Brouer XDP_RSS_L4_UDP = BIT(5),
4170cd917a4SJesper Dangaard Brouer XDP_RSS_L4_SCTP = BIT(6),
4180cd917a4SJesper Dangaard Brouer XDP_RSS_L4_IPSEC = BIT(7), /* L4 based hash include IPSEC SPI */
4190cd917a4SJesper Dangaard Brouer
4200cd917a4SJesper Dangaard Brouer /* Second part: RSS hash type combinations used for driver HW mapping */
4210cd917a4SJesper Dangaard Brouer XDP_RSS_TYPE_NONE = 0,
4220cd917a4SJesper Dangaard Brouer XDP_RSS_TYPE_L2 = XDP_RSS_TYPE_NONE,
4230cd917a4SJesper Dangaard Brouer
4240cd917a4SJesper Dangaard Brouer XDP_RSS_TYPE_L3_IPV4 = XDP_RSS_L3_IPV4,
4250cd917a4SJesper Dangaard Brouer XDP_RSS_TYPE_L3_IPV6 = XDP_RSS_L3_IPV6,
4260cd917a4SJesper Dangaard Brouer XDP_RSS_TYPE_L3_IPV4_OPT = XDP_RSS_L3_IPV4 | XDP_RSS_L3_DYNHDR,
4270cd917a4SJesper Dangaard Brouer XDP_RSS_TYPE_L3_IPV6_EX = XDP_RSS_L3_IPV6 | XDP_RSS_L3_DYNHDR,
4280cd917a4SJesper Dangaard Brouer
4290cd917a4SJesper Dangaard Brouer XDP_RSS_TYPE_L4_ANY = XDP_RSS_L4,
4300cd917a4SJesper Dangaard Brouer XDP_RSS_TYPE_L4_IPV4_TCP = XDP_RSS_L3_IPV4 | XDP_RSS_L4 | XDP_RSS_L4_TCP,
4310cd917a4SJesper Dangaard Brouer XDP_RSS_TYPE_L4_IPV4_UDP = XDP_RSS_L3_IPV4 | XDP_RSS_L4 | XDP_RSS_L4_UDP,
4320cd917a4SJesper Dangaard Brouer XDP_RSS_TYPE_L4_IPV4_SCTP = XDP_RSS_L3_IPV4 | XDP_RSS_L4 | XDP_RSS_L4_SCTP,
43367f245c2SJesper Dangaard Brouer XDP_RSS_TYPE_L4_IPV4_IPSEC = XDP_RSS_L3_IPV4 | XDP_RSS_L4 | XDP_RSS_L4_IPSEC,
4340cd917a4SJesper Dangaard Brouer
4350cd917a4SJesper Dangaard Brouer XDP_RSS_TYPE_L4_IPV6_TCP = XDP_RSS_L3_IPV6 | XDP_RSS_L4 | XDP_RSS_L4_TCP,
4360cd917a4SJesper Dangaard Brouer XDP_RSS_TYPE_L4_IPV6_UDP = XDP_RSS_L3_IPV6 | XDP_RSS_L4 | XDP_RSS_L4_UDP,
4370cd917a4SJesper Dangaard Brouer XDP_RSS_TYPE_L4_IPV6_SCTP = XDP_RSS_L3_IPV6 | XDP_RSS_L4 | XDP_RSS_L4_SCTP,
43867f245c2SJesper Dangaard Brouer XDP_RSS_TYPE_L4_IPV6_IPSEC = XDP_RSS_L3_IPV6 | XDP_RSS_L4 | XDP_RSS_L4_IPSEC,
4390cd917a4SJesper Dangaard Brouer
4400cd917a4SJesper Dangaard Brouer XDP_RSS_TYPE_L4_IPV6_TCP_EX = XDP_RSS_TYPE_L4_IPV6_TCP | XDP_RSS_L3_DYNHDR,
4410cd917a4SJesper Dangaard Brouer XDP_RSS_TYPE_L4_IPV6_UDP_EX = XDP_RSS_TYPE_L4_IPV6_UDP | XDP_RSS_L3_DYNHDR,
4420cd917a4SJesper Dangaard Brouer XDP_RSS_TYPE_L4_IPV6_SCTP_EX = XDP_RSS_TYPE_L4_IPV6_SCTP | XDP_RSS_L3_DYNHDR,
4430cd917a4SJesper Dangaard Brouer };
4440cd917a4SJesper Dangaard Brouer
445*680ee045SJakub Kicinski struct xdp_metadata_ops {
446*680ee045SJakub Kicinski int (*xmo_rx_timestamp)(const struct xdp_md *ctx, u64 *timestamp);
447*680ee045SJakub Kicinski int (*xmo_rx_hash)(const struct xdp_md *ctx, u32 *hash,
448*680ee045SJakub Kicinski enum xdp_rss_hash_type *rss_type);
449*680ee045SJakub Kicinski };
450*680ee045SJakub Kicinski
4513d76a4d3SStanislav Fomichev #ifdef CONFIG_NET
4523d76a4d3SStanislav Fomichev u32 bpf_xdp_metadata_kfunc_id(int id);
4533d76a4d3SStanislav Fomichev bool bpf_dev_bound_kfunc_id(u32 btf_id);
454f85949f9SLorenzo Bianconi void xdp_set_features_flag(struct net_device *dev, xdp_features_t val);
45566c0e13aSMarek Majtyka void xdp_features_set_redirect_target(struct net_device *dev, bool support_sg);
45666c0e13aSMarek Majtyka void xdp_features_clear_redirect_target(struct net_device *dev);
4573d76a4d3SStanislav Fomichev #else
bpf_xdp_metadata_kfunc_id(int id)4583d76a4d3SStanislav Fomichev static inline u32 bpf_xdp_metadata_kfunc_id(int id) { return 0; }
bpf_dev_bound_kfunc_id(u32 btf_id)4593d76a4d3SStanislav Fomichev static inline bool bpf_dev_bound_kfunc_id(u32 btf_id) { return false; }
46066c0e13aSMarek Majtyka
46166c0e13aSMarek Majtyka static inline void
xdp_set_features_flag(struct net_device * dev,xdp_features_t val)462f85949f9SLorenzo Bianconi xdp_set_features_flag(struct net_device *dev, xdp_features_t val)
463f85949f9SLorenzo Bianconi {
464f85949f9SLorenzo Bianconi }
465f85949f9SLorenzo Bianconi
466f85949f9SLorenzo Bianconi static inline void
xdp_features_set_redirect_target(struct net_device * dev,bool support_sg)46766c0e13aSMarek Majtyka xdp_features_set_redirect_target(struct net_device *dev, bool support_sg)
46866c0e13aSMarek Majtyka {
46966c0e13aSMarek Majtyka }
47066c0e13aSMarek Majtyka
47166c0e13aSMarek Majtyka static inline void
xdp_features_clear_redirect_target(struct net_device * dev)47266c0e13aSMarek Majtyka xdp_features_clear_redirect_target(struct net_device *dev)
47366c0e13aSMarek Majtyka {
47466c0e13aSMarek Majtyka }
4753d76a4d3SStanislav Fomichev #endif
4763d76a4d3SStanislav Fomichev
xdp_clear_features_flag(struct net_device * dev)477f85949f9SLorenzo Bianconi static inline void xdp_clear_features_flag(struct net_device *dev)
478f85949f9SLorenzo Bianconi {
479f85949f9SLorenzo Bianconi xdp_set_features_flag(dev, 0);
480f85949f9SLorenzo Bianconi }
481f85949f9SLorenzo Bianconi
bpf_prog_run_xdp(const struct bpf_prog * prog,struct xdp_buff * xdp)482*680ee045SJakub Kicinski static __always_inline u32 bpf_prog_run_xdp(const struct bpf_prog *prog,
483*680ee045SJakub Kicinski struct xdp_buff *xdp)
484*680ee045SJakub Kicinski {
485*680ee045SJakub Kicinski /* Driver XDP hooks are invoked within a single NAPI poll cycle and thus
486*680ee045SJakub Kicinski * under local_bh_disable(), which provides the needed RCU protection
487*680ee045SJakub Kicinski * for accessing map entries.
488*680ee045SJakub Kicinski */
489*680ee045SJakub Kicinski u32 act = __bpf_prog_run(prog, xdp, BPF_DISPATCHER_FUNC(xdp));
490*680ee045SJakub Kicinski
491*680ee045SJakub Kicinski if (static_branch_unlikely(&bpf_master_redirect_enabled_key)) {
492*680ee045SJakub Kicinski if (act == XDP_TX && netif_is_bond_slave(xdp->rxq->dev))
493*680ee045SJakub Kicinski act = xdp_master_redirect(xdp);
494*680ee045SJakub Kicinski }
495*680ee045SJakub Kicinski
496*680ee045SJakub Kicinski return act;
497*680ee045SJakub Kicinski }
498aecd67b6SJesper Dangaard Brouer #endif /* __LINUX_NET_XDP_H__ */
499