1ddc64d0aSThomas Gleixner /* SPDX-License-Identifier: GPL-2.0-only */ 2aecd67b6SJesper Dangaard Brouer /* include/net/xdp.h 3aecd67b6SJesper Dangaard Brouer * 4aecd67b6SJesper Dangaard Brouer * Copyright (c) 2017 Jesper Dangaard Brouer, Red Hat Inc. 5aecd67b6SJesper Dangaard Brouer */ 6aecd67b6SJesper Dangaard Brouer #ifndef __LINUX_NET_XDP_H__ 7aecd67b6SJesper Dangaard Brouer #define __LINUX_NET_XDP_H__ 8aecd67b6SJesper Dangaard Brouer 9f95f0f95SJesper Dangaard Brouer #include <linux/skbuff.h> /* skb_shared_info */ 10f95f0f95SJesper Dangaard Brouer 11aecd67b6SJesper Dangaard Brouer /** 12aecd67b6SJesper Dangaard Brouer * DOC: XDP RX-queue information 13aecd67b6SJesper Dangaard Brouer * 14aecd67b6SJesper Dangaard Brouer * The XDP RX-queue info (xdp_rxq_info) is associated with the driver 15aecd67b6SJesper Dangaard Brouer * level RX-ring queues. It is information that is specific to how 16aecd67b6SJesper Dangaard Brouer * the driver have configured a given RX-ring queue. 17aecd67b6SJesper Dangaard Brouer * 18aecd67b6SJesper Dangaard Brouer * Each xdp_buff frame received in the driver carry a (pointer) 19aecd67b6SJesper Dangaard Brouer * reference to this xdp_rxq_info structure. This provides the XDP 20aecd67b6SJesper Dangaard Brouer * data-path read-access to RX-info for both kernel and bpf-side 21aecd67b6SJesper Dangaard Brouer * (limited subset). 22aecd67b6SJesper Dangaard Brouer * 23aecd67b6SJesper Dangaard Brouer * For now, direct access is only safe while running in NAPI/softirq 24aecd67b6SJesper Dangaard Brouer * context. Contents is read-mostly and must not be updated during 25aecd67b6SJesper Dangaard Brouer * driver NAPI/softirq poll. 26aecd67b6SJesper Dangaard Brouer * 27aecd67b6SJesper Dangaard Brouer * The driver usage API is a register and unregister API. 28aecd67b6SJesper Dangaard Brouer * 29aecd67b6SJesper Dangaard Brouer * The struct is not directly tied to the XDP prog. A new XDP prog 30aecd67b6SJesper Dangaard Brouer * can be attached as long as it doesn't change the underlying 31aecd67b6SJesper Dangaard Brouer * RX-ring. If the RX-ring does change significantly, the NIC driver 32aecd67b6SJesper Dangaard Brouer * naturally need to stop the RX-ring before purging and reallocating 33aecd67b6SJesper Dangaard Brouer * memory. In that process the driver MUST call unregistor (which 34aecd67b6SJesper Dangaard Brouer * also apply for driver shutdown and unload). The register API is 35aecd67b6SJesper Dangaard Brouer * also mandatory during RX-ring setup. 36aecd67b6SJesper Dangaard Brouer */ 37aecd67b6SJesper Dangaard Brouer 385ab073ffSJesper Dangaard Brouer enum xdp_mem_type { 395ab073ffSJesper Dangaard Brouer MEM_TYPE_PAGE_SHARED = 0, /* Split-page refcnt based model */ 405ab073ffSJesper Dangaard Brouer MEM_TYPE_PAGE_ORDER0, /* Orig XDP full page model */ 4157d0a1c1SJesper Dangaard Brouer MEM_TYPE_PAGE_POOL, 422b43470aSBjörn Töpel MEM_TYPE_XSK_BUFF_POOL, 435ab073ffSJesper Dangaard Brouer MEM_TYPE_MAX, 445ab073ffSJesper Dangaard Brouer }; 455ab073ffSJesper Dangaard Brouer 4642b33468SJesper Dangaard Brouer /* XDP flags for ndo_xdp_xmit */ 4742b33468SJesper Dangaard Brouer #define XDP_XMIT_FLUSH (1U << 0) /* doorbell signal consumer */ 4842b33468SJesper Dangaard Brouer #define XDP_XMIT_FLAGS_MASK XDP_XMIT_FLUSH 4942b33468SJesper Dangaard Brouer 505ab073ffSJesper Dangaard Brouer struct xdp_mem_info { 515ab073ffSJesper Dangaard Brouer u32 type; /* enum xdp_mem_type, but known size type */ 528d5d8852SJesper Dangaard Brouer u32 id; 535ab073ffSJesper Dangaard Brouer }; 545ab073ffSJesper Dangaard Brouer 5557d0a1c1SJesper Dangaard Brouer struct page_pool; 5657d0a1c1SJesper Dangaard Brouer 57aecd67b6SJesper Dangaard Brouer struct xdp_rxq_info { 58aecd67b6SJesper Dangaard Brouer struct net_device *dev; 59aecd67b6SJesper Dangaard Brouer u32 queue_index; 60aecd67b6SJesper Dangaard Brouer u32 reg_state; 615ab073ffSJesper Dangaard Brouer struct xdp_mem_info mem; 62b02e5a0eSBjörn Töpel unsigned int napi_id; 63aecd67b6SJesper Dangaard Brouer } ____cacheline_aligned; /* perf critical, avoid false-sharing */ 64aecd67b6SJesper Dangaard Brouer 6564b59025SDavid Ahern struct xdp_txq_info { 6664b59025SDavid Ahern struct net_device *dev; 6764b59025SDavid Ahern }; 6864b59025SDavid Ahern 69106ca27fSJesper Dangaard Brouer struct xdp_buff { 70106ca27fSJesper Dangaard Brouer void *data; 71106ca27fSJesper Dangaard Brouer void *data_end; 72106ca27fSJesper Dangaard Brouer void *data_meta; 73106ca27fSJesper Dangaard Brouer void *data_hard_start; 74106ca27fSJesper Dangaard Brouer struct xdp_rxq_info *rxq; 7564b59025SDavid Ahern struct xdp_txq_info *txq; 76f95f0f95SJesper Dangaard Brouer u32 frame_sz; /* frame size to deduce data_hard_end/reserved tailroom*/ 77106ca27fSJesper Dangaard Brouer }; 785ab073ffSJesper Dangaard Brouer 7943b5169dSLorenzo Bianconi static __always_inline void 8043b5169dSLorenzo Bianconi xdp_init_buff(struct xdp_buff *xdp, u32 frame_sz, struct xdp_rxq_info *rxq) 8143b5169dSLorenzo Bianconi { 8243b5169dSLorenzo Bianconi xdp->frame_sz = frame_sz; 8343b5169dSLorenzo Bianconi xdp->rxq = rxq; 8443b5169dSLorenzo Bianconi } 8543b5169dSLorenzo Bianconi 86be9df4afSLorenzo Bianconi static __always_inline void 87be9df4afSLorenzo Bianconi xdp_prepare_buff(struct xdp_buff *xdp, unsigned char *hard_start, 88be9df4afSLorenzo Bianconi int headroom, int data_len, const bool meta_valid) 89be9df4afSLorenzo Bianconi { 90be9df4afSLorenzo Bianconi unsigned char *data = hard_start + headroom; 91be9df4afSLorenzo Bianconi 92be9df4afSLorenzo Bianconi xdp->data_hard_start = hard_start; 93be9df4afSLorenzo Bianconi xdp->data = data; 94be9df4afSLorenzo Bianconi xdp->data_end = data + data_len; 95be9df4afSLorenzo Bianconi xdp->data_meta = meta_valid ? data : data + 1; 96be9df4afSLorenzo Bianconi } 97be9df4afSLorenzo Bianconi 98f95f0f95SJesper Dangaard Brouer /* Reserve memory area at end-of data area. 99f95f0f95SJesper Dangaard Brouer * 100f95f0f95SJesper Dangaard Brouer * This macro reserves tailroom in the XDP buffer by limiting the 101f95f0f95SJesper Dangaard Brouer * XDP/BPF data access to data_hard_end. Notice same area (and size) 102f95f0f95SJesper Dangaard Brouer * is used for XDP_PASS, when constructing the SKB via build_skb(). 103f95f0f95SJesper Dangaard Brouer */ 104f95f0f95SJesper Dangaard Brouer #define xdp_data_hard_end(xdp) \ 105f95f0f95SJesper Dangaard Brouer ((xdp)->data_hard_start + (xdp)->frame_sz - \ 106f95f0f95SJesper Dangaard Brouer SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) 107f95f0f95SJesper Dangaard Brouer 1082f0bc54bSLorenzo Bianconi static inline struct skb_shared_info * 1092f0bc54bSLorenzo Bianconi xdp_get_shared_info_from_buff(struct xdp_buff *xdp) 1102f0bc54bSLorenzo Bianconi { 1112f0bc54bSLorenzo Bianconi return (struct skb_shared_info *)xdp_data_hard_end(xdp); 1122f0bc54bSLorenzo Bianconi } 1132f0bc54bSLorenzo Bianconi 114c0048cffSJesper Dangaard Brouer struct xdp_frame { 115c0048cffSJesper Dangaard Brouer void *data; 116c0048cffSJesper Dangaard Brouer u16 len; 117c0048cffSJesper Dangaard Brouer u16 headroom; 11834cc0b33SJesper Dangaard Brouer u32 metasize:8; 11934cc0b33SJesper Dangaard Brouer u32 frame_sz:24; 120c0048cffSJesper Dangaard Brouer /* Lifetime of xdp_rxq_info is limited to NAPI/enqueue time, 121c0048cffSJesper Dangaard Brouer * while mem info is valid on remote CPU. 122c0048cffSJesper Dangaard Brouer */ 123c0048cffSJesper Dangaard Brouer struct xdp_mem_info mem; 12470280ed9SJesper Dangaard Brouer struct net_device *dev_rx; /* used by cpumap */ 125c0048cffSJesper Dangaard Brouer }; 126c0048cffSJesper Dangaard Brouer 12789653987SLorenzo Bianconi #define XDP_BULK_QUEUE_SIZE 16 12889653987SLorenzo Bianconi struct xdp_frame_bulk { 12989653987SLorenzo Bianconi int count; 13089653987SLorenzo Bianconi void *xa; 13189653987SLorenzo Bianconi void *q[XDP_BULK_QUEUE_SIZE]; 13289653987SLorenzo Bianconi }; 13389653987SLorenzo Bianconi 13489653987SLorenzo Bianconi static __always_inline void xdp_frame_bulk_init(struct xdp_frame_bulk *bq) 13589653987SLorenzo Bianconi { 13689653987SLorenzo Bianconi /* bq->count will be zero'ed when bq->xa gets updated */ 13789653987SLorenzo Bianconi bq->xa = NULL; 13889653987SLorenzo Bianconi } 139dee72f8aSDavid S. Miller 1402f0bc54bSLorenzo Bianconi static inline struct skb_shared_info * 1412f0bc54bSLorenzo Bianconi xdp_get_shared_info_from_frame(struct xdp_frame *frame) 1422f0bc54bSLorenzo Bianconi { 1432f0bc54bSLorenzo Bianconi void *data_hard_start = frame->data - frame->headroom - sizeof(*frame); 1442f0bc54bSLorenzo Bianconi 1452f0bc54bSLorenzo Bianconi return (struct skb_shared_info *)(data_hard_start + frame->frame_sz - 1462f0bc54bSLorenzo Bianconi SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); 1472f0bc54bSLorenzo Bianconi } 1482f0bc54bSLorenzo Bianconi 14992164774SLorenzo Bianconi struct xdp_cpumap_stats { 15028b1520eSLorenzo Bianconi unsigned int redirect; 15192164774SLorenzo Bianconi unsigned int pass; 15292164774SLorenzo Bianconi unsigned int drop; 15392164774SLorenzo Bianconi }; 15492164774SLorenzo Bianconi 155a8d5b4abSToshiaki Makita /* Clear kernel pointers in xdp_frame */ 156a8d5b4abSToshiaki Makita static inline void xdp_scrub_frame(struct xdp_frame *frame) 157a8d5b4abSToshiaki Makita { 158a8d5b4abSToshiaki Makita frame->data = NULL; 159a8d5b4abSToshiaki Makita frame->dev_rx = NULL; 160a8d5b4abSToshiaki Makita } 161a8d5b4abSToshiaki Makita 16234cc0b33SJesper Dangaard Brouer /* Avoids inlining WARN macro in fast-path */ 16334cc0b33SJesper Dangaard Brouer void xdp_warn(const char *msg, const char *func, const int line); 16434cc0b33SJesper Dangaard Brouer #define XDP_WARN(msg) xdp_warn(msg, __func__, __LINE__) 16534cc0b33SJesper Dangaard Brouer 166b0d1beefSBjörn Töpel struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp); 167*97a0e1eaSLorenzo Bianconi struct sk_buff *__xdp_build_skb_from_frame(struct xdp_frame *xdpf, 168*97a0e1eaSLorenzo Bianconi struct sk_buff *skb, 169*97a0e1eaSLorenzo Bianconi struct net_device *dev); 170b0d1beefSBjörn Töpel 171fc379872SLorenzo Bianconi static inline 172fc379872SLorenzo Bianconi void xdp_convert_frame_to_buff(struct xdp_frame *frame, struct xdp_buff *xdp) 173fc379872SLorenzo Bianconi { 174fc379872SLorenzo Bianconi xdp->data_hard_start = frame->data - frame->headroom - sizeof(*frame); 175fc379872SLorenzo Bianconi xdp->data = frame->data; 176fc379872SLorenzo Bianconi xdp->data_end = frame->data + frame->len; 177fc379872SLorenzo Bianconi xdp->data_meta = frame->data - frame->metasize; 178fc379872SLorenzo Bianconi xdp->frame_sz = frame->frame_sz; 179fc379872SLorenzo Bianconi } 180fc379872SLorenzo Bianconi 181c0048cffSJesper Dangaard Brouer static inline 182daa5cdc3SDavid Ahern int xdp_update_frame_from_buff(struct xdp_buff *xdp, 183daa5cdc3SDavid Ahern struct xdp_frame *xdp_frame) 184c0048cffSJesper Dangaard Brouer { 185daa5cdc3SDavid Ahern int metasize, headroom; 18602b55e56SBjörn Töpel 187c0048cffSJesper Dangaard Brouer /* Assure headroom is available for storing info */ 188c0048cffSJesper Dangaard Brouer headroom = xdp->data - xdp->data_hard_start; 189c0048cffSJesper Dangaard Brouer metasize = xdp->data - xdp->data_meta; 190c0048cffSJesper Dangaard Brouer metasize = metasize > 0 ? metasize : 0; 191c0048cffSJesper Dangaard Brouer if (unlikely((headroom - metasize) < sizeof(*xdp_frame))) 192daa5cdc3SDavid Ahern return -ENOSPC; 193c0048cffSJesper Dangaard Brouer 19434cc0b33SJesper Dangaard Brouer /* Catch if driver didn't reserve tailroom for skb_shared_info */ 19534cc0b33SJesper Dangaard Brouer if (unlikely(xdp->data_end > xdp_data_hard_end(xdp))) { 19634cc0b33SJesper Dangaard Brouer XDP_WARN("Driver BUG: missing reserved tailroom"); 197daa5cdc3SDavid Ahern return -ENOSPC; 19834cc0b33SJesper Dangaard Brouer } 19934cc0b33SJesper Dangaard Brouer 200c0048cffSJesper Dangaard Brouer xdp_frame->data = xdp->data; 201c0048cffSJesper Dangaard Brouer xdp_frame->len = xdp->data_end - xdp->data; 202c0048cffSJesper Dangaard Brouer xdp_frame->headroom = headroom - sizeof(*xdp_frame); 203c0048cffSJesper Dangaard Brouer xdp_frame->metasize = metasize; 20434cc0b33SJesper Dangaard Brouer xdp_frame->frame_sz = xdp->frame_sz; 205c0048cffSJesper Dangaard Brouer 206daa5cdc3SDavid Ahern return 0; 207daa5cdc3SDavid Ahern } 208daa5cdc3SDavid Ahern 209daa5cdc3SDavid Ahern /* Convert xdp_buff to xdp_frame */ 210daa5cdc3SDavid Ahern static inline 211daa5cdc3SDavid Ahern struct xdp_frame *xdp_convert_buff_to_frame(struct xdp_buff *xdp) 212daa5cdc3SDavid Ahern { 213daa5cdc3SDavid Ahern struct xdp_frame *xdp_frame; 214daa5cdc3SDavid Ahern 215daa5cdc3SDavid Ahern if (xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL) 216daa5cdc3SDavid Ahern return xdp_convert_zc_to_xdp_frame(xdp); 217daa5cdc3SDavid Ahern 218daa5cdc3SDavid Ahern /* Store info in top of packet */ 219daa5cdc3SDavid Ahern xdp_frame = xdp->data_hard_start; 220daa5cdc3SDavid Ahern if (unlikely(xdp_update_frame_from_buff(xdp, xdp_frame) < 0)) 221daa5cdc3SDavid Ahern return NULL; 222daa5cdc3SDavid Ahern 223c0048cffSJesper Dangaard Brouer /* rxq only valid until napi_schedule ends, convert to xdp_mem_info */ 224c0048cffSJesper Dangaard Brouer xdp_frame->mem = xdp->rxq->mem; 225c0048cffSJesper Dangaard Brouer 226c0048cffSJesper Dangaard Brouer return xdp_frame; 227c0048cffSJesper Dangaard Brouer } 228c0048cffSJesper Dangaard Brouer 22903993094SJesper Dangaard Brouer void xdp_return_frame(struct xdp_frame *xdpf); 230389ab7f0SJesper Dangaard Brouer void xdp_return_frame_rx_napi(struct xdp_frame *xdpf); 231c497176cSBjörn Töpel void xdp_return_buff(struct xdp_buff *xdp); 23289653987SLorenzo Bianconi void xdp_flush_frame_bulk(struct xdp_frame_bulk *bq); 23389653987SLorenzo Bianconi void xdp_return_frame_bulk(struct xdp_frame *xdpf, 23489653987SLorenzo Bianconi struct xdp_frame_bulk *bq); 2355ab073ffSJesper Dangaard Brouer 2366bf071bfSJesper Dangaard Brouer /* When sending xdp_frame into the network stack, then there is no 2376bf071bfSJesper Dangaard Brouer * return point callback, which is needed to release e.g. DMA-mapping 2386bf071bfSJesper Dangaard Brouer * resources with page_pool. Thus, have explicit function to release 2396bf071bfSJesper Dangaard Brouer * frame resources. 2406bf071bfSJesper Dangaard Brouer */ 2416bf071bfSJesper Dangaard Brouer void __xdp_release_frame(void *data, struct xdp_mem_info *mem); 2426bf071bfSJesper Dangaard Brouer static inline void xdp_release_frame(struct xdp_frame *xdpf) 2436bf071bfSJesper Dangaard Brouer { 2446bf071bfSJesper Dangaard Brouer struct xdp_mem_info *mem = &xdpf->mem; 2456bf071bfSJesper Dangaard Brouer 2466bf071bfSJesper Dangaard Brouer /* Curr only page_pool needs this */ 2476bf071bfSJesper Dangaard Brouer if (mem->type == MEM_TYPE_PAGE_POOL) 2486bf071bfSJesper Dangaard Brouer __xdp_release_frame(xdpf->data, mem); 2496bf071bfSJesper Dangaard Brouer } 2506bf071bfSJesper Dangaard Brouer 251aecd67b6SJesper Dangaard Brouer int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq, 252b02e5a0eSBjörn Töpel struct net_device *dev, u32 queue_index, unsigned int napi_id); 253aecd67b6SJesper Dangaard Brouer void xdp_rxq_info_unreg(struct xdp_rxq_info *xdp_rxq); 254aecd67b6SJesper Dangaard Brouer void xdp_rxq_info_unused(struct xdp_rxq_info *xdp_rxq); 255c0124f32SJesper Dangaard Brouer bool xdp_rxq_info_is_reg(struct xdp_rxq_info *xdp_rxq); 2565ab073ffSJesper Dangaard Brouer int xdp_rxq_info_reg_mem_model(struct xdp_rxq_info *xdp_rxq, 2575ab073ffSJesper Dangaard Brouer enum xdp_mem_type type, void *allocator); 258dce5bd61SBjörn Töpel void xdp_rxq_info_unreg_mem_model(struct xdp_rxq_info *xdp_rxq); 259aecd67b6SJesper Dangaard Brouer 260106ca27fSJesper Dangaard Brouer /* Drivers not supporting XDP metadata can use this helper, which 261106ca27fSJesper Dangaard Brouer * rejects any room expansion for metadata as a result. 262106ca27fSJesper Dangaard Brouer */ 263106ca27fSJesper Dangaard Brouer static __always_inline void 264106ca27fSJesper Dangaard Brouer xdp_set_data_meta_invalid(struct xdp_buff *xdp) 265106ca27fSJesper Dangaard Brouer { 266106ca27fSJesper Dangaard Brouer xdp->data_meta = xdp->data + 1; 267106ca27fSJesper Dangaard Brouer } 268106ca27fSJesper Dangaard Brouer 269106ca27fSJesper Dangaard Brouer static __always_inline bool 270106ca27fSJesper Dangaard Brouer xdp_data_meta_unsupported(const struct xdp_buff *xdp) 271106ca27fSJesper Dangaard Brouer { 272106ca27fSJesper Dangaard Brouer return unlikely(xdp->data_meta > xdp->data); 273106ca27fSJesper Dangaard Brouer } 274106ca27fSJesper Dangaard Brouer 27505296620SJakub Kicinski struct xdp_attachment_info { 27605296620SJakub Kicinski struct bpf_prog *prog; 27705296620SJakub Kicinski u32 flags; 27805296620SJakub Kicinski }; 27905296620SJakub Kicinski 28005296620SJakub Kicinski struct netdev_bpf; 28105296620SJakub Kicinski void xdp_attachment_setup(struct xdp_attachment_info *info, 28205296620SJakub Kicinski struct netdev_bpf *bpf); 28305296620SJakub Kicinski 28489653987SLorenzo Bianconi #define DEV_MAP_BULK_SIZE XDP_BULK_QUEUE_SIZE 285788f87acSIoana Ciornei 286aecd67b6SJesper Dangaard Brouer #endif /* __LINUX_NET_XDP_H__ */ 287