1ddc64d0aSThomas Gleixner /* SPDX-License-Identifier: GPL-2.0-only */ 2aecd67b6SJesper Dangaard Brouer /* include/net/xdp.h 3aecd67b6SJesper Dangaard Brouer * 4aecd67b6SJesper Dangaard Brouer * Copyright (c) 2017 Jesper Dangaard Brouer, Red Hat Inc. 5aecd67b6SJesper Dangaard Brouer */ 6aecd67b6SJesper Dangaard Brouer #ifndef __LINUX_NET_XDP_H__ 7aecd67b6SJesper Dangaard Brouer #define __LINUX_NET_XDP_H__ 8aecd67b6SJesper Dangaard Brouer 9aecd67b6SJesper Dangaard Brouer /** 10aecd67b6SJesper Dangaard Brouer * DOC: XDP RX-queue information 11aecd67b6SJesper Dangaard Brouer * 12aecd67b6SJesper Dangaard Brouer * The XDP RX-queue info (xdp_rxq_info) is associated with the driver 13aecd67b6SJesper Dangaard Brouer * level RX-ring queues. It is information that is specific to how 14aecd67b6SJesper Dangaard Brouer * the driver have configured a given RX-ring queue. 15aecd67b6SJesper Dangaard Brouer * 16aecd67b6SJesper Dangaard Brouer * Each xdp_buff frame received in the driver carry a (pointer) 17aecd67b6SJesper Dangaard Brouer * reference to this xdp_rxq_info structure. This provides the XDP 18aecd67b6SJesper Dangaard Brouer * data-path read-access to RX-info for both kernel and bpf-side 19aecd67b6SJesper Dangaard Brouer * (limited subset). 20aecd67b6SJesper Dangaard Brouer * 21aecd67b6SJesper Dangaard Brouer * For now, direct access is only safe while running in NAPI/softirq 22aecd67b6SJesper Dangaard Brouer * context. Contents is read-mostly and must not be updated during 23aecd67b6SJesper Dangaard Brouer * driver NAPI/softirq poll. 24aecd67b6SJesper Dangaard Brouer * 25aecd67b6SJesper Dangaard Brouer * The driver usage API is a register and unregister API. 26aecd67b6SJesper Dangaard Brouer * 27aecd67b6SJesper Dangaard Brouer * The struct is not directly tied to the XDP prog. A new XDP prog 28aecd67b6SJesper Dangaard Brouer * can be attached as long as it doesn't change the underlying 29aecd67b6SJesper Dangaard Brouer * RX-ring. If the RX-ring does change significantly, the NIC driver 30aecd67b6SJesper Dangaard Brouer * naturally need to stop the RX-ring before purging and reallocating 31aecd67b6SJesper Dangaard Brouer * memory. In that process the driver MUST call unregistor (which 32aecd67b6SJesper Dangaard Brouer * also apply for driver shutdown and unload). The register API is 33aecd67b6SJesper Dangaard Brouer * also mandatory during RX-ring setup. 34aecd67b6SJesper Dangaard Brouer */ 35aecd67b6SJesper Dangaard Brouer 365ab073ffSJesper Dangaard Brouer enum xdp_mem_type { 375ab073ffSJesper Dangaard Brouer MEM_TYPE_PAGE_SHARED = 0, /* Split-page refcnt based model */ 385ab073ffSJesper Dangaard Brouer MEM_TYPE_PAGE_ORDER0, /* Orig XDP full page model */ 3957d0a1c1SJesper Dangaard Brouer MEM_TYPE_PAGE_POOL, 4002b55e56SBjörn Töpel MEM_TYPE_ZERO_COPY, 415ab073ffSJesper Dangaard Brouer MEM_TYPE_MAX, 425ab073ffSJesper Dangaard Brouer }; 435ab073ffSJesper Dangaard Brouer 4442b33468SJesper Dangaard Brouer /* XDP flags for ndo_xdp_xmit */ 4542b33468SJesper Dangaard Brouer #define XDP_XMIT_FLUSH (1U << 0) /* doorbell signal consumer */ 4642b33468SJesper Dangaard Brouer #define XDP_XMIT_FLAGS_MASK XDP_XMIT_FLUSH 4742b33468SJesper Dangaard Brouer 485ab073ffSJesper Dangaard Brouer struct xdp_mem_info { 495ab073ffSJesper Dangaard Brouer u32 type; /* enum xdp_mem_type, but known size type */ 508d5d8852SJesper Dangaard Brouer u32 id; 515ab073ffSJesper Dangaard Brouer }; 525ab073ffSJesper Dangaard Brouer 5357d0a1c1SJesper Dangaard Brouer struct page_pool; 5457d0a1c1SJesper Dangaard Brouer 5502b55e56SBjörn Töpel struct zero_copy_allocator { 5602b55e56SBjörn Töpel void (*free)(struct zero_copy_allocator *zca, unsigned long handle); 5702b55e56SBjörn Töpel }; 5802b55e56SBjörn Töpel 59aecd67b6SJesper Dangaard Brouer struct xdp_rxq_info { 60aecd67b6SJesper Dangaard Brouer struct net_device *dev; 61aecd67b6SJesper Dangaard Brouer u32 queue_index; 62aecd67b6SJesper Dangaard Brouer u32 reg_state; 635ab073ffSJesper Dangaard Brouer struct xdp_mem_info mem; 64aecd67b6SJesper Dangaard Brouer } ____cacheline_aligned; /* perf critical, avoid false-sharing */ 65aecd67b6SJesper Dangaard Brouer 66106ca27fSJesper Dangaard Brouer struct xdp_buff { 67106ca27fSJesper Dangaard Brouer void *data; 68106ca27fSJesper Dangaard Brouer void *data_end; 69106ca27fSJesper Dangaard Brouer void *data_meta; 70106ca27fSJesper Dangaard Brouer void *data_hard_start; 7102b55e56SBjörn Töpel unsigned long handle; 72106ca27fSJesper Dangaard Brouer struct xdp_rxq_info *rxq; 73106ca27fSJesper Dangaard Brouer }; 745ab073ffSJesper Dangaard Brouer 75c0048cffSJesper Dangaard Brouer struct xdp_frame { 76c0048cffSJesper Dangaard Brouer void *data; 77c0048cffSJesper Dangaard Brouer u16 len; 78c0048cffSJesper Dangaard Brouer u16 headroom; 79c0048cffSJesper Dangaard Brouer u16 metasize; 80c0048cffSJesper Dangaard Brouer /* Lifetime of xdp_rxq_info is limited to NAPI/enqueue time, 81c0048cffSJesper Dangaard Brouer * while mem info is valid on remote CPU. 82c0048cffSJesper Dangaard Brouer */ 83c0048cffSJesper Dangaard Brouer struct xdp_mem_info mem; 8470280ed9SJesper Dangaard Brouer struct net_device *dev_rx; /* used by cpumap */ 85c0048cffSJesper Dangaard Brouer }; 86c0048cffSJesper Dangaard Brouer 87a8d5b4abSToshiaki Makita /* Clear kernel pointers in xdp_frame */ 88a8d5b4abSToshiaki Makita static inline void xdp_scrub_frame(struct xdp_frame *frame) 89a8d5b4abSToshiaki Makita { 90a8d5b4abSToshiaki Makita frame->data = NULL; 91a8d5b4abSToshiaki Makita frame->dev_rx = NULL; 92a8d5b4abSToshiaki Makita } 93a8d5b4abSToshiaki Makita 94b0d1beefSBjörn Töpel struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp); 95b0d1beefSBjörn Töpel 96c0048cffSJesper Dangaard Brouer /* Convert xdp_buff to xdp_frame */ 97c0048cffSJesper Dangaard Brouer static inline 98c0048cffSJesper Dangaard Brouer struct xdp_frame *convert_to_xdp_frame(struct xdp_buff *xdp) 99c0048cffSJesper Dangaard Brouer { 100c0048cffSJesper Dangaard Brouer struct xdp_frame *xdp_frame; 101c0048cffSJesper Dangaard Brouer int metasize; 102c0048cffSJesper Dangaard Brouer int headroom; 103c0048cffSJesper Dangaard Brouer 10402b55e56SBjörn Töpel if (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY) 105b0d1beefSBjörn Töpel return xdp_convert_zc_to_xdp_frame(xdp); 10602b55e56SBjörn Töpel 107c0048cffSJesper Dangaard Brouer /* Assure headroom is available for storing info */ 108c0048cffSJesper Dangaard Brouer headroom = xdp->data - xdp->data_hard_start; 109c0048cffSJesper Dangaard Brouer metasize = xdp->data - xdp->data_meta; 110c0048cffSJesper Dangaard Brouer metasize = metasize > 0 ? metasize : 0; 111c0048cffSJesper Dangaard Brouer if (unlikely((headroom - metasize) < sizeof(*xdp_frame))) 112c0048cffSJesper Dangaard Brouer return NULL; 113c0048cffSJesper Dangaard Brouer 114c0048cffSJesper Dangaard Brouer /* Store info in top of packet */ 115c0048cffSJesper Dangaard Brouer xdp_frame = xdp->data_hard_start; 116c0048cffSJesper Dangaard Brouer 117c0048cffSJesper Dangaard Brouer xdp_frame->data = xdp->data; 118c0048cffSJesper Dangaard Brouer xdp_frame->len = xdp->data_end - xdp->data; 119c0048cffSJesper Dangaard Brouer xdp_frame->headroom = headroom - sizeof(*xdp_frame); 120c0048cffSJesper Dangaard Brouer xdp_frame->metasize = metasize; 121c0048cffSJesper Dangaard Brouer 122c0048cffSJesper Dangaard Brouer /* rxq only valid until napi_schedule ends, convert to xdp_mem_info */ 123c0048cffSJesper Dangaard Brouer xdp_frame->mem = xdp->rxq->mem; 124c0048cffSJesper Dangaard Brouer 125c0048cffSJesper Dangaard Brouer return xdp_frame; 126c0048cffSJesper Dangaard Brouer } 127c0048cffSJesper Dangaard Brouer 12803993094SJesper Dangaard Brouer void xdp_return_frame(struct xdp_frame *xdpf); 129389ab7f0SJesper Dangaard Brouer void xdp_return_frame_rx_napi(struct xdp_frame *xdpf); 130c497176cSBjörn Töpel void xdp_return_buff(struct xdp_buff *xdp); 1315ab073ffSJesper Dangaard Brouer 1326bf071bfSJesper Dangaard Brouer /* When sending xdp_frame into the network stack, then there is no 1336bf071bfSJesper Dangaard Brouer * return point callback, which is needed to release e.g. DMA-mapping 1346bf071bfSJesper Dangaard Brouer * resources with page_pool. Thus, have explicit function to release 1356bf071bfSJesper Dangaard Brouer * frame resources. 1366bf071bfSJesper Dangaard Brouer */ 1376bf071bfSJesper Dangaard Brouer void __xdp_release_frame(void *data, struct xdp_mem_info *mem); 1386bf071bfSJesper Dangaard Brouer static inline void xdp_release_frame(struct xdp_frame *xdpf) 1396bf071bfSJesper Dangaard Brouer { 1406bf071bfSJesper Dangaard Brouer struct xdp_mem_info *mem = &xdpf->mem; 1416bf071bfSJesper Dangaard Brouer 1426bf071bfSJesper Dangaard Brouer /* Curr only page_pool needs this */ 1436bf071bfSJesper Dangaard Brouer if (mem->type == MEM_TYPE_PAGE_POOL) 1446bf071bfSJesper Dangaard Brouer __xdp_release_frame(xdpf->data, mem); 1456bf071bfSJesper Dangaard Brouer } 1466bf071bfSJesper Dangaard Brouer 147aecd67b6SJesper Dangaard Brouer int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq, 148aecd67b6SJesper Dangaard Brouer struct net_device *dev, u32 queue_index); 149aecd67b6SJesper Dangaard Brouer void xdp_rxq_info_unreg(struct xdp_rxq_info *xdp_rxq); 150aecd67b6SJesper Dangaard Brouer void xdp_rxq_info_unused(struct xdp_rxq_info *xdp_rxq); 151c0124f32SJesper Dangaard Brouer bool xdp_rxq_info_is_reg(struct xdp_rxq_info *xdp_rxq); 1525ab073ffSJesper Dangaard Brouer int xdp_rxq_info_reg_mem_model(struct xdp_rxq_info *xdp_rxq, 1535ab073ffSJesper Dangaard Brouer enum xdp_mem_type type, void *allocator); 154dce5bd61SBjörn Töpel void xdp_rxq_info_unreg_mem_model(struct xdp_rxq_info *xdp_rxq); 155aecd67b6SJesper Dangaard Brouer 156106ca27fSJesper Dangaard Brouer /* Drivers not supporting XDP metadata can use this helper, which 157106ca27fSJesper Dangaard Brouer * rejects any room expansion for metadata as a result. 158106ca27fSJesper Dangaard Brouer */ 159106ca27fSJesper Dangaard Brouer static __always_inline void 160106ca27fSJesper Dangaard Brouer xdp_set_data_meta_invalid(struct xdp_buff *xdp) 161106ca27fSJesper Dangaard Brouer { 162106ca27fSJesper Dangaard Brouer xdp->data_meta = xdp->data + 1; 163106ca27fSJesper Dangaard Brouer } 164106ca27fSJesper Dangaard Brouer 165106ca27fSJesper Dangaard Brouer static __always_inline bool 166106ca27fSJesper Dangaard Brouer xdp_data_meta_unsupported(const struct xdp_buff *xdp) 167106ca27fSJesper Dangaard Brouer { 168106ca27fSJesper Dangaard Brouer return unlikely(xdp->data_meta > xdp->data); 169106ca27fSJesper Dangaard Brouer } 170106ca27fSJesper Dangaard Brouer 17105296620SJakub Kicinski struct xdp_attachment_info { 17205296620SJakub Kicinski struct bpf_prog *prog; 17305296620SJakub Kicinski u32 flags; 17405296620SJakub Kicinski }; 17505296620SJakub Kicinski 17605296620SJakub Kicinski struct netdev_bpf; 17705296620SJakub Kicinski int xdp_attachment_query(struct xdp_attachment_info *info, 17805296620SJakub Kicinski struct netdev_bpf *bpf); 17905296620SJakub Kicinski bool xdp_attachment_flags_ok(struct xdp_attachment_info *info, 18005296620SJakub Kicinski struct netdev_bpf *bpf); 18105296620SJakub Kicinski void xdp_attachment_setup(struct xdp_attachment_info *info, 18205296620SJakub Kicinski struct netdev_bpf *bpf); 18305296620SJakub Kicinski 184aecd67b6SJesper Dangaard Brouer #endif /* __LINUX_NET_XDP_H__ */ 185