1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* AF_XDP internal functions 3 * Copyright(c) 2018 Intel Corporation. 4 */ 5 6 #ifndef _LINUX_XDP_SOCK_H 7 #define _LINUX_XDP_SOCK_H 8 9 #include <linux/workqueue.h> 10 #include <linux/if_xdp.h> 11 #include <linux/mutex.h> 12 #include <linux/spinlock.h> 13 #include <linux/mm.h> 14 #include <net/sock.h> 15 16 struct net_device; 17 struct xsk_queue; 18 19 /* Masks for xdp_umem_page flags. 20 * The low 12-bits of the addr will be 0 since this is the page address, so we 21 * can use them for flags. 22 */ 23 #define XSK_NEXT_PG_CONTIG_SHIFT 0 24 #define XSK_NEXT_PG_CONTIG_MASK (1ULL << XSK_NEXT_PG_CONTIG_SHIFT) 25 26 struct xdp_umem_page { 27 void *addr; 28 dma_addr_t dma; 29 }; 30 31 struct xdp_umem_fq_reuse { 32 u32 nentries; 33 u32 length; 34 u64 handles[]; 35 }; 36 37 /* Flags for the umem flags field. 38 * 39 * The NEED_WAKEUP flag is 1 due to the reuse of the flags field for public 40 * flags. See inlude/uapi/include/linux/if_xdp.h. 41 */ 42 #define XDP_UMEM_USES_NEED_WAKEUP (1 << 1) 43 44 struct xdp_umem { 45 struct xsk_queue *fq; 46 struct xsk_queue *cq; 47 struct xdp_umem_page *pages; 48 u64 chunk_mask; 49 u64 size; 50 u32 headroom; 51 u32 chunk_size_nohr; 52 struct user_struct *user; 53 unsigned long address; 54 refcount_t users; 55 struct work_struct work; 56 struct page **pgs; 57 u32 npgs; 58 u16 queue_id; 59 u8 need_wakeup; 60 u8 flags; 61 int id; 62 struct net_device *dev; 63 struct xdp_umem_fq_reuse *fq_reuse; 64 bool zc; 65 spinlock_t xsk_list_lock; 66 struct list_head xsk_list; 67 }; 68 69 /* Nodes are linked in the struct xdp_sock map_list field, and used to 70 * track which maps a certain socket reside in. 71 */ 72 struct xsk_map; 73 struct xsk_map_node { 74 struct list_head node; 75 struct xsk_map *map; 76 struct xdp_sock **map_entry; 77 }; 78 79 struct xdp_sock { 80 /* struct sock must be the first member of struct xdp_sock */ 81 struct sock sk; 82 struct xsk_queue *rx; 83 struct net_device *dev; 84 struct xdp_umem *umem; 85 struct list_head flush_node; 86 u16 queue_id; 87 bool zc; 88 enum { 89 XSK_READY = 0, 90 XSK_BOUND, 91 XSK_UNBOUND, 92 } state; 93 /* Protects multiple processes in the control path */ 94 struct mutex mutex; 95 struct xsk_queue *tx ____cacheline_aligned_in_smp; 96 struct list_head list; 97 /* Mutual exclusion of NAPI TX thread and sendmsg error paths 98 * in the SKB destructor callback. 99 */ 100 spinlock_t tx_completion_lock; 101 /* Protects generic receive. */ 102 spinlock_t rx_lock; 103 u64 rx_dropped; 104 struct list_head map_list; 105 /* Protects map_list */ 106 spinlock_t map_list_lock; 107 }; 108 109 struct xdp_buff; 110 #ifdef CONFIG_XDP_SOCKETS 111 int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp); 112 int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp); 113 void xsk_flush(struct xdp_sock *xs); 114 bool xsk_is_setup_for_bpf_map(struct xdp_sock *xs); 115 /* Used from netdev driver */ 116 bool xsk_umem_has_addrs(struct xdp_umem *umem, u32 cnt); 117 u64 *xsk_umem_peek_addr(struct xdp_umem *umem, u64 *addr); 118 void xsk_umem_discard_addr(struct xdp_umem *umem); 119 void xsk_umem_complete_tx(struct xdp_umem *umem, u32 nb_entries); 120 bool xsk_umem_consume_tx(struct xdp_umem *umem, struct xdp_desc *desc); 121 void xsk_umem_consume_tx_done(struct xdp_umem *umem); 122 struct xdp_umem_fq_reuse *xsk_reuseq_prepare(u32 nentries); 123 struct xdp_umem_fq_reuse *xsk_reuseq_swap(struct xdp_umem *umem, 124 struct xdp_umem_fq_reuse *newq); 125 void xsk_reuseq_free(struct xdp_umem_fq_reuse *rq); 126 struct xdp_umem *xdp_get_umem_from_qid(struct net_device *dev, u16 queue_id); 127 void xsk_set_rx_need_wakeup(struct xdp_umem *umem); 128 void xsk_set_tx_need_wakeup(struct xdp_umem *umem); 129 void xsk_clear_rx_need_wakeup(struct xdp_umem *umem); 130 void xsk_clear_tx_need_wakeup(struct xdp_umem *umem); 131 bool xsk_umem_uses_need_wakeup(struct xdp_umem *umem); 132 133 void xsk_map_try_sock_delete(struct xsk_map *map, struct xdp_sock *xs, 134 struct xdp_sock **map_entry); 135 int xsk_map_inc(struct xsk_map *map); 136 void xsk_map_put(struct xsk_map *map); 137 138 static inline u64 xsk_umem_extract_addr(u64 addr) 139 { 140 return addr & XSK_UNALIGNED_BUF_ADDR_MASK; 141 } 142 143 static inline u64 xsk_umem_extract_offset(u64 addr) 144 { 145 return addr >> XSK_UNALIGNED_BUF_OFFSET_SHIFT; 146 } 147 148 static inline u64 xsk_umem_add_offset_to_addr(u64 addr) 149 { 150 return xsk_umem_extract_addr(addr) + xsk_umem_extract_offset(addr); 151 } 152 153 static inline char *xdp_umem_get_data(struct xdp_umem *umem, u64 addr) 154 { 155 unsigned long page_addr; 156 157 addr = xsk_umem_add_offset_to_addr(addr); 158 page_addr = (unsigned long)umem->pages[addr >> PAGE_SHIFT].addr; 159 160 return (char *)(page_addr & PAGE_MASK) + (addr & ~PAGE_MASK); 161 } 162 163 static inline dma_addr_t xdp_umem_get_dma(struct xdp_umem *umem, u64 addr) 164 { 165 addr = xsk_umem_add_offset_to_addr(addr); 166 167 return umem->pages[addr >> PAGE_SHIFT].dma + (addr & ~PAGE_MASK); 168 } 169 170 /* Reuse-queue aware version of FILL queue helpers */ 171 static inline bool xsk_umem_has_addrs_rq(struct xdp_umem *umem, u32 cnt) 172 { 173 struct xdp_umem_fq_reuse *rq = umem->fq_reuse; 174 175 if (rq->length >= cnt) 176 return true; 177 178 return xsk_umem_has_addrs(umem, cnt - rq->length); 179 } 180 181 static inline u64 *xsk_umem_peek_addr_rq(struct xdp_umem *umem, u64 *addr) 182 { 183 struct xdp_umem_fq_reuse *rq = umem->fq_reuse; 184 185 if (!rq->length) 186 return xsk_umem_peek_addr(umem, addr); 187 188 *addr = rq->handles[rq->length - 1]; 189 return addr; 190 } 191 192 static inline void xsk_umem_discard_addr_rq(struct xdp_umem *umem) 193 { 194 struct xdp_umem_fq_reuse *rq = umem->fq_reuse; 195 196 if (!rq->length) 197 xsk_umem_discard_addr(umem); 198 else 199 rq->length--; 200 } 201 202 static inline void xsk_umem_fq_reuse(struct xdp_umem *umem, u64 addr) 203 { 204 struct xdp_umem_fq_reuse *rq = umem->fq_reuse; 205 206 rq->handles[rq->length++] = addr; 207 } 208 209 /* Handle the offset appropriately depending on aligned or unaligned mode. 210 * For unaligned mode, we store the offset in the upper 16-bits of the address. 211 * For aligned mode, we simply add the offset to the address. 212 */ 213 static inline u64 xsk_umem_adjust_offset(struct xdp_umem *umem, u64 address, 214 u64 offset) 215 { 216 if (umem->flags & XDP_UMEM_UNALIGNED_CHUNK_FLAG) 217 return address + (offset << XSK_UNALIGNED_BUF_OFFSET_SHIFT); 218 else 219 return address + offset; 220 } 221 #else 222 static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) 223 { 224 return -ENOTSUPP; 225 } 226 227 static inline int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) 228 { 229 return -ENOTSUPP; 230 } 231 232 static inline void xsk_flush(struct xdp_sock *xs) 233 { 234 } 235 236 static inline bool xsk_is_setup_for_bpf_map(struct xdp_sock *xs) 237 { 238 return false; 239 } 240 241 static inline bool xsk_umem_has_addrs(struct xdp_umem *umem, u32 cnt) 242 { 243 return false; 244 } 245 246 static inline u64 *xsk_umem_peek_addr(struct xdp_umem *umem, u64 *addr) 247 { 248 return NULL; 249 } 250 251 static inline void xsk_umem_discard_addr(struct xdp_umem *umem) 252 { 253 } 254 255 static inline void xsk_umem_complete_tx(struct xdp_umem *umem, u32 nb_entries) 256 { 257 } 258 259 static inline bool xsk_umem_consume_tx(struct xdp_umem *umem, 260 struct xdp_desc *desc) 261 { 262 return false; 263 } 264 265 static inline void xsk_umem_consume_tx_done(struct xdp_umem *umem) 266 { 267 } 268 269 static inline struct xdp_umem_fq_reuse *xsk_reuseq_prepare(u32 nentries) 270 { 271 return NULL; 272 } 273 274 static inline struct xdp_umem_fq_reuse *xsk_reuseq_swap( 275 struct xdp_umem *umem, 276 struct xdp_umem_fq_reuse *newq) 277 { 278 return NULL; 279 } 280 static inline void xsk_reuseq_free(struct xdp_umem_fq_reuse *rq) 281 { 282 } 283 284 static inline struct xdp_umem *xdp_get_umem_from_qid(struct net_device *dev, 285 u16 queue_id) 286 { 287 return NULL; 288 } 289 290 static inline u64 xsk_umem_extract_addr(u64 addr) 291 { 292 return 0; 293 } 294 295 static inline u64 xsk_umem_extract_offset(u64 addr) 296 { 297 return 0; 298 } 299 300 static inline u64 xsk_umem_add_offset_to_addr(u64 addr) 301 { 302 return 0; 303 } 304 305 static inline char *xdp_umem_get_data(struct xdp_umem *umem, u64 addr) 306 { 307 return NULL; 308 } 309 310 static inline dma_addr_t xdp_umem_get_dma(struct xdp_umem *umem, u64 addr) 311 { 312 return 0; 313 } 314 315 static inline bool xsk_umem_has_addrs_rq(struct xdp_umem *umem, u32 cnt) 316 { 317 return false; 318 } 319 320 static inline u64 *xsk_umem_peek_addr_rq(struct xdp_umem *umem, u64 *addr) 321 { 322 return NULL; 323 } 324 325 static inline void xsk_umem_discard_addr_rq(struct xdp_umem *umem) 326 { 327 } 328 329 static inline void xsk_umem_fq_reuse(struct xdp_umem *umem, u64 addr) 330 { 331 } 332 333 static inline void xsk_set_rx_need_wakeup(struct xdp_umem *umem) 334 { 335 } 336 337 static inline void xsk_set_tx_need_wakeup(struct xdp_umem *umem) 338 { 339 } 340 341 static inline void xsk_clear_rx_need_wakeup(struct xdp_umem *umem) 342 { 343 } 344 345 static inline void xsk_clear_tx_need_wakeup(struct xdp_umem *umem) 346 { 347 } 348 349 static inline bool xsk_umem_uses_need_wakeup(struct xdp_umem *umem) 350 { 351 return false; 352 } 353 354 static inline u64 xsk_umem_adjust_offset(struct xdp_umem *umem, u64 handle, 355 u64 offset) 356 { 357 return 0; 358 } 359 360 #endif /* CONFIG_XDP_SOCKETS */ 361 362 #endif /* _LINUX_XDP_SOCK_H */ 363