1ddc64d0aSThomas Gleixner /* SPDX-License-Identifier: GPL-2.0-only */ 2aecd67b6SJesper Dangaard Brouer /* include/net/xdp.h 3aecd67b6SJesper Dangaard Brouer * 4aecd67b6SJesper Dangaard Brouer * Copyright (c) 2017 Jesper Dangaard Brouer, Red Hat Inc. 5aecd67b6SJesper Dangaard Brouer */ 6aecd67b6SJesper Dangaard Brouer #ifndef __LINUX_NET_XDP_H__ 7aecd67b6SJesper Dangaard Brouer #define __LINUX_NET_XDP_H__ 8aecd67b6SJesper Dangaard Brouer 9f95f0f95SJesper Dangaard Brouer #include <linux/skbuff.h> /* skb_shared_info */ 10f95f0f95SJesper Dangaard Brouer 11aecd67b6SJesper Dangaard Brouer /** 12aecd67b6SJesper Dangaard Brouer * DOC: XDP RX-queue information 13aecd67b6SJesper Dangaard Brouer * 14aecd67b6SJesper Dangaard Brouer * The XDP RX-queue info (xdp_rxq_info) is associated with the driver 15aecd67b6SJesper Dangaard Brouer * level RX-ring queues. It is information that is specific to how 16aecd67b6SJesper Dangaard Brouer * the driver have configured a given RX-ring queue. 17aecd67b6SJesper Dangaard Brouer * 18*6bbc7103SKev Jackson * Each xdp_buff frame received in the driver carries a (pointer) 19aecd67b6SJesper Dangaard Brouer * reference to this xdp_rxq_info structure. This provides the XDP 20aecd67b6SJesper Dangaard Brouer * data-path read-access to RX-info for both kernel and bpf-side 21aecd67b6SJesper Dangaard Brouer * (limited subset). 22aecd67b6SJesper Dangaard Brouer * 23aecd67b6SJesper Dangaard Brouer * For now, direct access is only safe while running in NAPI/softirq 24*6bbc7103SKev Jackson * context. Contents are read-mostly and must not be updated during 25aecd67b6SJesper Dangaard Brouer * driver NAPI/softirq poll. 26aecd67b6SJesper Dangaard Brouer * 27aecd67b6SJesper Dangaard Brouer * The driver usage API is a register and unregister API. 28aecd67b6SJesper Dangaard Brouer * 29aecd67b6SJesper Dangaard Brouer * The struct is not directly tied to the XDP prog. A new XDP prog 30aecd67b6SJesper Dangaard Brouer * can be attached as long as it doesn't change the underlying 31aecd67b6SJesper Dangaard Brouer * RX-ring. If the RX-ring does change significantly, the NIC driver 32aecd67b6SJesper Dangaard Brouer * naturally need to stop the RX-ring before purging and reallocating 33*6bbc7103SKev Jackson * memory. In that process the driver MUST call unregister (which 34*6bbc7103SKev Jackson * also applies for driver shutdown and unload). The register API is 35aecd67b6SJesper Dangaard Brouer * also mandatory during RX-ring setup. 36aecd67b6SJesper Dangaard Brouer */ 37aecd67b6SJesper Dangaard Brouer 385ab073ffSJesper Dangaard Brouer enum xdp_mem_type { 395ab073ffSJesper Dangaard Brouer MEM_TYPE_PAGE_SHARED = 0, /* Split-page refcnt based model */ 405ab073ffSJesper Dangaard Brouer MEM_TYPE_PAGE_ORDER0, /* Orig XDP full page model */ 4157d0a1c1SJesper Dangaard Brouer MEM_TYPE_PAGE_POOL, 422b43470aSBjörn Töpel MEM_TYPE_XSK_BUFF_POOL, 435ab073ffSJesper Dangaard Brouer MEM_TYPE_MAX, 445ab073ffSJesper Dangaard Brouer }; 455ab073ffSJesper Dangaard Brouer 4642b33468SJesper Dangaard Brouer /* XDP flags for ndo_xdp_xmit */ 4742b33468SJesper Dangaard Brouer #define XDP_XMIT_FLUSH (1U << 0) /* doorbell signal consumer */ 4842b33468SJesper Dangaard Brouer #define XDP_XMIT_FLAGS_MASK XDP_XMIT_FLUSH 4942b33468SJesper Dangaard Brouer 505ab073ffSJesper Dangaard Brouer struct xdp_mem_info { 515ab073ffSJesper Dangaard Brouer u32 type; /* enum xdp_mem_type, but known size type */ 528d5d8852SJesper Dangaard Brouer u32 id; 535ab073ffSJesper Dangaard Brouer }; 545ab073ffSJesper Dangaard Brouer 5557d0a1c1SJesper Dangaard Brouer struct page_pool; 5657d0a1c1SJesper Dangaard Brouer 57aecd67b6SJesper Dangaard Brouer struct xdp_rxq_info { 58aecd67b6SJesper Dangaard Brouer struct net_device *dev; 59aecd67b6SJesper Dangaard Brouer u32 queue_index; 60aecd67b6SJesper Dangaard Brouer u32 reg_state; 615ab073ffSJesper Dangaard Brouer struct xdp_mem_info mem; 62b02e5a0eSBjörn Töpel unsigned int napi_id; 63aecd67b6SJesper Dangaard Brouer } ____cacheline_aligned; /* perf critical, avoid false-sharing */ 64aecd67b6SJesper Dangaard Brouer 6564b59025SDavid Ahern struct xdp_txq_info { 6664b59025SDavid Ahern struct net_device *dev; 6764b59025SDavid Ahern }; 6864b59025SDavid Ahern 69106ca27fSJesper Dangaard Brouer struct xdp_buff { 70106ca27fSJesper Dangaard Brouer void *data; 71106ca27fSJesper Dangaard Brouer void *data_end; 72106ca27fSJesper Dangaard Brouer void *data_meta; 73106ca27fSJesper Dangaard Brouer void *data_hard_start; 74106ca27fSJesper Dangaard Brouer struct xdp_rxq_info *rxq; 7564b59025SDavid Ahern struct xdp_txq_info *txq; 76f95f0f95SJesper Dangaard Brouer u32 frame_sz; /* frame size to deduce data_hard_end/reserved tailroom*/ 77106ca27fSJesper Dangaard Brouer }; 785ab073ffSJesper Dangaard Brouer 7943b5169dSLorenzo Bianconi static __always_inline void 8043b5169dSLorenzo Bianconi xdp_init_buff(struct xdp_buff *xdp, u32 frame_sz, struct xdp_rxq_info *rxq) 8143b5169dSLorenzo Bianconi { 8243b5169dSLorenzo Bianconi xdp->frame_sz = frame_sz; 8343b5169dSLorenzo Bianconi xdp->rxq = rxq; 8443b5169dSLorenzo Bianconi } 8543b5169dSLorenzo Bianconi 86be9df4afSLorenzo Bianconi static __always_inline void 87be9df4afSLorenzo Bianconi xdp_prepare_buff(struct xdp_buff *xdp, unsigned char *hard_start, 88be9df4afSLorenzo Bianconi int headroom, int data_len, const bool meta_valid) 89be9df4afSLorenzo Bianconi { 90be9df4afSLorenzo Bianconi unsigned char *data = hard_start + headroom; 91be9df4afSLorenzo Bianconi 92be9df4afSLorenzo Bianconi xdp->data_hard_start = hard_start; 93be9df4afSLorenzo Bianconi xdp->data = data; 94be9df4afSLorenzo Bianconi xdp->data_end = data + data_len; 95be9df4afSLorenzo Bianconi xdp->data_meta = meta_valid ? data : data + 1; 96be9df4afSLorenzo Bianconi } 97be9df4afSLorenzo Bianconi 98f95f0f95SJesper Dangaard Brouer /* Reserve memory area at end-of data area. 99f95f0f95SJesper Dangaard Brouer * 100f95f0f95SJesper Dangaard Brouer * This macro reserves tailroom in the XDP buffer by limiting the 101f95f0f95SJesper Dangaard Brouer * XDP/BPF data access to data_hard_end. Notice same area (and size) 102f95f0f95SJesper Dangaard Brouer * is used for XDP_PASS, when constructing the SKB via build_skb(). 103f95f0f95SJesper Dangaard Brouer */ 104f95f0f95SJesper Dangaard Brouer #define xdp_data_hard_end(xdp) \ 105f95f0f95SJesper Dangaard Brouer ((xdp)->data_hard_start + (xdp)->frame_sz - \ 106f95f0f95SJesper Dangaard Brouer SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) 107f95f0f95SJesper Dangaard Brouer 1082f0bc54bSLorenzo Bianconi static inline struct skb_shared_info * 1092f0bc54bSLorenzo Bianconi xdp_get_shared_info_from_buff(struct xdp_buff *xdp) 1102f0bc54bSLorenzo Bianconi { 1112f0bc54bSLorenzo Bianconi return (struct skb_shared_info *)xdp_data_hard_end(xdp); 1122f0bc54bSLorenzo Bianconi } 1132f0bc54bSLorenzo Bianconi 114c0048cffSJesper Dangaard Brouer struct xdp_frame { 115c0048cffSJesper Dangaard Brouer void *data; 116c0048cffSJesper Dangaard Brouer u16 len; 117c0048cffSJesper Dangaard Brouer u16 headroom; 11834cc0b33SJesper Dangaard Brouer u32 metasize:8; 11934cc0b33SJesper Dangaard Brouer u32 frame_sz:24; 120c0048cffSJesper Dangaard Brouer /* Lifetime of xdp_rxq_info is limited to NAPI/enqueue time, 121c0048cffSJesper Dangaard Brouer * while mem info is valid on remote CPU. 122c0048cffSJesper Dangaard Brouer */ 123c0048cffSJesper Dangaard Brouer struct xdp_mem_info mem; 12470280ed9SJesper Dangaard Brouer struct net_device *dev_rx; /* used by cpumap */ 125c0048cffSJesper Dangaard Brouer }; 126c0048cffSJesper Dangaard Brouer 12789653987SLorenzo Bianconi #define XDP_BULK_QUEUE_SIZE 16 12889653987SLorenzo Bianconi struct xdp_frame_bulk { 12989653987SLorenzo Bianconi int count; 13089653987SLorenzo Bianconi void *xa; 13189653987SLorenzo Bianconi void *q[XDP_BULK_QUEUE_SIZE]; 13289653987SLorenzo Bianconi }; 13389653987SLorenzo Bianconi 13489653987SLorenzo Bianconi static __always_inline void xdp_frame_bulk_init(struct xdp_frame_bulk *bq) 13589653987SLorenzo Bianconi { 13689653987SLorenzo Bianconi /* bq->count will be zero'ed when bq->xa gets updated */ 13789653987SLorenzo Bianconi bq->xa = NULL; 13889653987SLorenzo Bianconi } 139dee72f8aSDavid S. Miller 1402f0bc54bSLorenzo Bianconi static inline struct skb_shared_info * 1412f0bc54bSLorenzo Bianconi xdp_get_shared_info_from_frame(struct xdp_frame *frame) 1422f0bc54bSLorenzo Bianconi { 1432f0bc54bSLorenzo Bianconi void *data_hard_start = frame->data - frame->headroom - sizeof(*frame); 1442f0bc54bSLorenzo Bianconi 1452f0bc54bSLorenzo Bianconi return (struct skb_shared_info *)(data_hard_start + frame->frame_sz - 1462f0bc54bSLorenzo Bianconi SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); 1472f0bc54bSLorenzo Bianconi } 1482f0bc54bSLorenzo Bianconi 14992164774SLorenzo Bianconi struct xdp_cpumap_stats { 15028b1520eSLorenzo Bianconi unsigned int redirect; 15192164774SLorenzo Bianconi unsigned int pass; 15292164774SLorenzo Bianconi unsigned int drop; 15392164774SLorenzo Bianconi }; 15492164774SLorenzo Bianconi 155a8d5b4abSToshiaki Makita /* Clear kernel pointers in xdp_frame */ 156a8d5b4abSToshiaki Makita static inline void xdp_scrub_frame(struct xdp_frame *frame) 157a8d5b4abSToshiaki Makita { 158a8d5b4abSToshiaki Makita frame->data = NULL; 159a8d5b4abSToshiaki Makita frame->dev_rx = NULL; 160a8d5b4abSToshiaki Makita } 161a8d5b4abSToshiaki Makita 16234cc0b33SJesper Dangaard Brouer /* Avoids inlining WARN macro in fast-path */ 16334cc0b33SJesper Dangaard Brouer void xdp_warn(const char *msg, const char *func, const int line); 16434cc0b33SJesper Dangaard Brouer #define XDP_WARN(msg) xdp_warn(msg, __func__, __LINE__) 16534cc0b33SJesper Dangaard Brouer 166b0d1beefSBjörn Töpel struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp); 16797a0e1eaSLorenzo Bianconi struct sk_buff *__xdp_build_skb_from_frame(struct xdp_frame *xdpf, 16897a0e1eaSLorenzo Bianconi struct sk_buff *skb, 16997a0e1eaSLorenzo Bianconi struct net_device *dev); 17089f479f0SLorenzo Bianconi struct sk_buff *xdp_build_skb_from_frame(struct xdp_frame *xdpf, 17189f479f0SLorenzo Bianconi struct net_device *dev); 17265e6dcf7SLorenzo Bianconi int xdp_alloc_skb_bulk(void **skbs, int n_skb, gfp_t gfp); 173e624d4edSHangbin Liu struct xdp_frame *xdpf_clone(struct xdp_frame *xdpf); 174b0d1beefSBjörn Töpel 175fc379872SLorenzo Bianconi static inline 176fc379872SLorenzo Bianconi void xdp_convert_frame_to_buff(struct xdp_frame *frame, struct xdp_buff *xdp) 177fc379872SLorenzo Bianconi { 178fc379872SLorenzo Bianconi xdp->data_hard_start = frame->data - frame->headroom - sizeof(*frame); 179fc379872SLorenzo Bianconi xdp->data = frame->data; 180fc379872SLorenzo Bianconi xdp->data_end = frame->data + frame->len; 181fc379872SLorenzo Bianconi xdp->data_meta = frame->data - frame->metasize; 182fc379872SLorenzo Bianconi xdp->frame_sz = frame->frame_sz; 183fc379872SLorenzo Bianconi } 184fc379872SLorenzo Bianconi 185c0048cffSJesper Dangaard Brouer static inline 186daa5cdc3SDavid Ahern int xdp_update_frame_from_buff(struct xdp_buff *xdp, 187daa5cdc3SDavid Ahern struct xdp_frame *xdp_frame) 188c0048cffSJesper Dangaard Brouer { 189daa5cdc3SDavid Ahern int metasize, headroom; 19002b55e56SBjörn Töpel 191c0048cffSJesper Dangaard Brouer /* Assure headroom is available for storing info */ 192c0048cffSJesper Dangaard Brouer headroom = xdp->data - xdp->data_hard_start; 193c0048cffSJesper Dangaard Brouer metasize = xdp->data - xdp->data_meta; 194c0048cffSJesper Dangaard Brouer metasize = metasize > 0 ? metasize : 0; 195c0048cffSJesper Dangaard Brouer if (unlikely((headroom - metasize) < sizeof(*xdp_frame))) 196daa5cdc3SDavid Ahern return -ENOSPC; 197c0048cffSJesper Dangaard Brouer 19834cc0b33SJesper Dangaard Brouer /* Catch if driver didn't reserve tailroom for skb_shared_info */ 19934cc0b33SJesper Dangaard Brouer if (unlikely(xdp->data_end > xdp_data_hard_end(xdp))) { 20034cc0b33SJesper Dangaard Brouer XDP_WARN("Driver BUG: missing reserved tailroom"); 201daa5cdc3SDavid Ahern return -ENOSPC; 20234cc0b33SJesper Dangaard Brouer } 20334cc0b33SJesper Dangaard Brouer 204c0048cffSJesper Dangaard Brouer xdp_frame->data = xdp->data; 205c0048cffSJesper Dangaard Brouer xdp_frame->len = xdp->data_end - xdp->data; 206c0048cffSJesper Dangaard Brouer xdp_frame->headroom = headroom - sizeof(*xdp_frame); 207c0048cffSJesper Dangaard Brouer xdp_frame->metasize = metasize; 20834cc0b33SJesper Dangaard Brouer xdp_frame->frame_sz = xdp->frame_sz; 209c0048cffSJesper Dangaard Brouer 210daa5cdc3SDavid Ahern return 0; 211daa5cdc3SDavid Ahern } 212daa5cdc3SDavid Ahern 213daa5cdc3SDavid Ahern /* Convert xdp_buff to xdp_frame */ 214daa5cdc3SDavid Ahern static inline 215daa5cdc3SDavid Ahern struct xdp_frame *xdp_convert_buff_to_frame(struct xdp_buff *xdp) 216daa5cdc3SDavid Ahern { 217daa5cdc3SDavid Ahern struct xdp_frame *xdp_frame; 218daa5cdc3SDavid Ahern 219daa5cdc3SDavid Ahern if (xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL) 220daa5cdc3SDavid Ahern return xdp_convert_zc_to_xdp_frame(xdp); 221daa5cdc3SDavid Ahern 222daa5cdc3SDavid Ahern /* Store info in top of packet */ 223daa5cdc3SDavid Ahern xdp_frame = xdp->data_hard_start; 224daa5cdc3SDavid Ahern if (unlikely(xdp_update_frame_from_buff(xdp, xdp_frame) < 0)) 225daa5cdc3SDavid Ahern return NULL; 226daa5cdc3SDavid Ahern 227c0048cffSJesper Dangaard Brouer /* rxq only valid until napi_schedule ends, convert to xdp_mem_info */ 228c0048cffSJesper Dangaard Brouer xdp_frame->mem = xdp->rxq->mem; 229c0048cffSJesper Dangaard Brouer 230c0048cffSJesper Dangaard Brouer return xdp_frame; 231c0048cffSJesper Dangaard Brouer } 232c0048cffSJesper Dangaard Brouer 23303993094SJesper Dangaard Brouer void xdp_return_frame(struct xdp_frame *xdpf); 234389ab7f0SJesper Dangaard Brouer void xdp_return_frame_rx_napi(struct xdp_frame *xdpf); 235c497176cSBjörn Töpel void xdp_return_buff(struct xdp_buff *xdp); 23689653987SLorenzo Bianconi void xdp_flush_frame_bulk(struct xdp_frame_bulk *bq); 23789653987SLorenzo Bianconi void xdp_return_frame_bulk(struct xdp_frame *xdpf, 23889653987SLorenzo Bianconi struct xdp_frame_bulk *bq); 2395ab073ffSJesper Dangaard Brouer 2406bf071bfSJesper Dangaard Brouer /* When sending xdp_frame into the network stack, then there is no 2416bf071bfSJesper Dangaard Brouer * return point callback, which is needed to release e.g. DMA-mapping 2426bf071bfSJesper Dangaard Brouer * resources with page_pool. Thus, have explicit function to release 2436bf071bfSJesper Dangaard Brouer * frame resources. 2446bf071bfSJesper Dangaard Brouer */ 2456bf071bfSJesper Dangaard Brouer void __xdp_release_frame(void *data, struct xdp_mem_info *mem); 2466bf071bfSJesper Dangaard Brouer static inline void xdp_release_frame(struct xdp_frame *xdpf) 2476bf071bfSJesper Dangaard Brouer { 2486bf071bfSJesper Dangaard Brouer struct xdp_mem_info *mem = &xdpf->mem; 2496bf071bfSJesper Dangaard Brouer 2506bf071bfSJesper Dangaard Brouer /* Curr only page_pool needs this */ 2516bf071bfSJesper Dangaard Brouer if (mem->type == MEM_TYPE_PAGE_POOL) 2526bf071bfSJesper Dangaard Brouer __xdp_release_frame(xdpf->data, mem); 2536bf071bfSJesper Dangaard Brouer } 2546bf071bfSJesper Dangaard Brouer 255aecd67b6SJesper Dangaard Brouer int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq, 256b02e5a0eSBjörn Töpel struct net_device *dev, u32 queue_index, unsigned int napi_id); 257aecd67b6SJesper Dangaard Brouer void xdp_rxq_info_unreg(struct xdp_rxq_info *xdp_rxq); 258aecd67b6SJesper Dangaard Brouer void xdp_rxq_info_unused(struct xdp_rxq_info *xdp_rxq); 259c0124f32SJesper Dangaard Brouer bool xdp_rxq_info_is_reg(struct xdp_rxq_info *xdp_rxq); 2605ab073ffSJesper Dangaard Brouer int xdp_rxq_info_reg_mem_model(struct xdp_rxq_info *xdp_rxq, 2615ab073ffSJesper Dangaard Brouer enum xdp_mem_type type, void *allocator); 262dce5bd61SBjörn Töpel void xdp_rxq_info_unreg_mem_model(struct xdp_rxq_info *xdp_rxq); 263aecd67b6SJesper Dangaard Brouer 264106ca27fSJesper Dangaard Brouer /* Drivers not supporting XDP metadata can use this helper, which 265106ca27fSJesper Dangaard Brouer * rejects any room expansion for metadata as a result. 266106ca27fSJesper Dangaard Brouer */ 267106ca27fSJesper Dangaard Brouer static __always_inline void 268106ca27fSJesper Dangaard Brouer xdp_set_data_meta_invalid(struct xdp_buff *xdp) 269106ca27fSJesper Dangaard Brouer { 270106ca27fSJesper Dangaard Brouer xdp->data_meta = xdp->data + 1; 271106ca27fSJesper Dangaard Brouer } 272106ca27fSJesper Dangaard Brouer 273106ca27fSJesper Dangaard Brouer static __always_inline bool 274106ca27fSJesper Dangaard Brouer xdp_data_meta_unsupported(const struct xdp_buff *xdp) 275106ca27fSJesper Dangaard Brouer { 276106ca27fSJesper Dangaard Brouer return unlikely(xdp->data_meta > xdp->data); 277106ca27fSJesper Dangaard Brouer } 278106ca27fSJesper Dangaard Brouer 2797445cf31SZvi Effron static inline bool xdp_metalen_invalid(unsigned long metalen) 2807445cf31SZvi Effron { 2817445cf31SZvi Effron return (metalen & (sizeof(__u32) - 1)) || (metalen > 32); 2827445cf31SZvi Effron } 2837445cf31SZvi Effron 28405296620SJakub Kicinski struct xdp_attachment_info { 28505296620SJakub Kicinski struct bpf_prog *prog; 28605296620SJakub Kicinski u32 flags; 28705296620SJakub Kicinski }; 28805296620SJakub Kicinski 28905296620SJakub Kicinski struct netdev_bpf; 29005296620SJakub Kicinski void xdp_attachment_setup(struct xdp_attachment_info *info, 29105296620SJakub Kicinski struct netdev_bpf *bpf); 29205296620SJakub Kicinski 29389653987SLorenzo Bianconi #define DEV_MAP_BULK_SIZE XDP_BULK_QUEUE_SIZE 294788f87acSIoana Ciornei 295aecd67b6SJesper Dangaard Brouer #endif /* __LINUX_NET_XDP_H__ */ 296