1 // SPDX-License-Identifier: GPL-2.0-only 2 /* net/core/xdp.c 3 * 4 * Copyright (c) 2017 Jesper Dangaard Brouer, Red Hat Inc. 5 */ 6 #include <linux/bpf.h> 7 #include <linux/filter.h> 8 #include <linux/types.h> 9 #include <linux/mm.h> 10 #include <linux/netdevice.h> 11 #include <linux/slab.h> 12 #include <linux/idr.h> 13 #include <linux/rhashtable.h> 14 #include <net/page_pool.h> 15 16 #include <net/xdp.h> 17 #include <net/xdp_priv.h> /* struct xdp_mem_allocator */ 18 #include <trace/events/xdp.h> 19 20 #define REG_STATE_NEW 0x0 21 #define REG_STATE_REGISTERED 0x1 22 #define REG_STATE_UNREGISTERED 0x2 23 #define REG_STATE_UNUSED 0x3 24 25 static DEFINE_IDA(mem_id_pool); 26 static DEFINE_MUTEX(mem_id_lock); 27 #define MEM_ID_MAX 0xFFFE 28 #define MEM_ID_MIN 1 29 static int mem_id_next = MEM_ID_MIN; 30 31 static bool mem_id_init; /* false */ 32 static struct rhashtable *mem_id_ht; 33 34 static u32 xdp_mem_id_hashfn(const void *data, u32 len, u32 seed) 35 { 36 const u32 *k = data; 37 const u32 key = *k; 38 39 BUILD_BUG_ON(FIELD_SIZEOF(struct xdp_mem_allocator, mem.id) 40 != sizeof(u32)); 41 42 /* Use cyclic increasing ID as direct hash key */ 43 return key; 44 } 45 46 static int xdp_mem_id_cmp(struct rhashtable_compare_arg *arg, 47 const void *ptr) 48 { 49 const struct xdp_mem_allocator *xa = ptr; 50 u32 mem_id = *(u32 *)arg->key; 51 52 return xa->mem.id != mem_id; 53 } 54 55 static const struct rhashtable_params mem_id_rht_params = { 56 .nelem_hint = 64, 57 .head_offset = offsetof(struct xdp_mem_allocator, node), 58 .key_offset = offsetof(struct xdp_mem_allocator, mem.id), 59 .key_len = FIELD_SIZEOF(struct xdp_mem_allocator, mem.id), 60 .max_size = MEM_ID_MAX, 61 .min_size = 8, 62 .automatic_shrinking = true, 63 .hashfn = xdp_mem_id_hashfn, 64 .obj_cmpfn = xdp_mem_id_cmp, 65 }; 66 67 static void __xdp_mem_allocator_rcu_free(struct rcu_head *rcu) 68 { 69 struct xdp_mem_allocator *xa; 70 71 xa = container_of(rcu, struct xdp_mem_allocator, rcu); 72 73 /* Allocator have indicated safe to remove before this is called */ 74 if (xa->mem.type == MEM_TYPE_PAGE_POOL) 75 page_pool_free(xa->page_pool); 76 77 /* Allow this ID to be reused */ 78 ida_simple_remove(&mem_id_pool, xa->mem.id); 79 80 /* Poison memory */ 81 xa->mem.id = 0xFFFF; 82 xa->mem.type = 0xF0F0; 83 xa->allocator = (void *)0xDEAD9001; 84 85 kfree(xa); 86 } 87 88 static bool __mem_id_disconnect(int id, bool force) 89 { 90 struct xdp_mem_allocator *xa; 91 bool safe_to_remove = true; 92 93 mutex_lock(&mem_id_lock); 94 95 xa = rhashtable_lookup_fast(mem_id_ht, &id, mem_id_rht_params); 96 if (!xa) { 97 mutex_unlock(&mem_id_lock); 98 WARN(1, "Request remove non-existing id(%d), driver bug?", id); 99 return true; 100 } 101 xa->disconnect_cnt++; 102 103 /* Detects in-flight packet-pages for page_pool */ 104 if (xa->mem.type == MEM_TYPE_PAGE_POOL) 105 safe_to_remove = page_pool_request_shutdown(xa->page_pool); 106 107 trace_mem_disconnect(xa, safe_to_remove, force); 108 109 if ((safe_to_remove || force) && 110 !rhashtable_remove_fast(mem_id_ht, &xa->node, mem_id_rht_params)) 111 call_rcu(&xa->rcu, __xdp_mem_allocator_rcu_free); 112 113 mutex_unlock(&mem_id_lock); 114 return (safe_to_remove|force); 115 } 116 117 #define DEFER_TIME (msecs_to_jiffies(1000)) 118 #define DEFER_WARN_INTERVAL (30 * HZ) 119 #define DEFER_MAX_RETRIES 120 120 121 static void mem_id_disconnect_defer_retry(struct work_struct *wq) 122 { 123 struct delayed_work *dwq = to_delayed_work(wq); 124 struct xdp_mem_allocator *xa = container_of(dwq, typeof(*xa), defer_wq); 125 bool force = false; 126 127 if (xa->disconnect_cnt > DEFER_MAX_RETRIES) 128 force = true; 129 130 if (__mem_id_disconnect(xa->mem.id, force)) 131 return; 132 133 /* Periodic warning */ 134 if (time_after_eq(jiffies, xa->defer_warn)) { 135 int sec = (s32)((u32)jiffies - (u32)xa->defer_start) / HZ; 136 137 pr_warn("%s() stalled mem.id=%u shutdown %d attempts %d sec\n", 138 __func__, xa->mem.id, xa->disconnect_cnt, sec); 139 xa->defer_warn = jiffies + DEFER_WARN_INTERVAL; 140 } 141 142 /* Still not ready to be disconnected, retry later */ 143 schedule_delayed_work(&xa->defer_wq, DEFER_TIME); 144 } 145 146 void xdp_rxq_info_unreg_mem_model(struct xdp_rxq_info *xdp_rxq) 147 { 148 struct xdp_mem_allocator *xa; 149 int id = xdp_rxq->mem.id; 150 151 if (xdp_rxq->reg_state != REG_STATE_REGISTERED) { 152 WARN(1, "Missing register, driver bug"); 153 return; 154 } 155 156 if (xdp_rxq->mem.type != MEM_TYPE_PAGE_POOL && 157 xdp_rxq->mem.type != MEM_TYPE_ZERO_COPY) { 158 return; 159 } 160 161 if (id == 0) 162 return; 163 164 if (__mem_id_disconnect(id, false)) 165 return; 166 167 /* Could not disconnect, defer new disconnect attempt to later */ 168 mutex_lock(&mem_id_lock); 169 170 xa = rhashtable_lookup_fast(mem_id_ht, &id, mem_id_rht_params); 171 if (!xa) { 172 mutex_unlock(&mem_id_lock); 173 return; 174 } 175 xa->defer_start = jiffies; 176 xa->defer_warn = jiffies + DEFER_WARN_INTERVAL; 177 178 INIT_DELAYED_WORK(&xa->defer_wq, mem_id_disconnect_defer_retry); 179 mutex_unlock(&mem_id_lock); 180 schedule_delayed_work(&xa->defer_wq, DEFER_TIME); 181 } 182 EXPORT_SYMBOL_GPL(xdp_rxq_info_unreg_mem_model); 183 184 /* This unregister operation will also cleanup and destroy the 185 * allocator. The page_pool_free() operation is first called when it's 186 * safe to remove, possibly deferred to a workqueue. 187 */ 188 void xdp_rxq_info_unreg(struct xdp_rxq_info *xdp_rxq) 189 { 190 /* Simplify driver cleanup code paths, allow unreg "unused" */ 191 if (xdp_rxq->reg_state == REG_STATE_UNUSED) 192 return; 193 194 WARN(!(xdp_rxq->reg_state == REG_STATE_REGISTERED), "Driver BUG"); 195 196 xdp_rxq_info_unreg_mem_model(xdp_rxq); 197 198 xdp_rxq->reg_state = REG_STATE_UNREGISTERED; 199 xdp_rxq->dev = NULL; 200 201 /* Reset mem info to defaults */ 202 xdp_rxq->mem.id = 0; 203 xdp_rxq->mem.type = 0; 204 } 205 EXPORT_SYMBOL_GPL(xdp_rxq_info_unreg); 206 207 static void xdp_rxq_info_init(struct xdp_rxq_info *xdp_rxq) 208 { 209 memset(xdp_rxq, 0, sizeof(*xdp_rxq)); 210 } 211 212 /* Returns 0 on success, negative on failure */ 213 int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq, 214 struct net_device *dev, u32 queue_index) 215 { 216 if (xdp_rxq->reg_state == REG_STATE_UNUSED) { 217 WARN(1, "Driver promised not to register this"); 218 return -EINVAL; 219 } 220 221 if (xdp_rxq->reg_state == REG_STATE_REGISTERED) { 222 WARN(1, "Missing unregister, handled but fix driver"); 223 xdp_rxq_info_unreg(xdp_rxq); 224 } 225 226 if (!dev) { 227 WARN(1, "Missing net_device from driver"); 228 return -ENODEV; 229 } 230 231 /* State either UNREGISTERED or NEW */ 232 xdp_rxq_info_init(xdp_rxq); 233 xdp_rxq->dev = dev; 234 xdp_rxq->queue_index = queue_index; 235 236 xdp_rxq->reg_state = REG_STATE_REGISTERED; 237 return 0; 238 } 239 EXPORT_SYMBOL_GPL(xdp_rxq_info_reg); 240 241 void xdp_rxq_info_unused(struct xdp_rxq_info *xdp_rxq) 242 { 243 xdp_rxq->reg_state = REG_STATE_UNUSED; 244 } 245 EXPORT_SYMBOL_GPL(xdp_rxq_info_unused); 246 247 bool xdp_rxq_info_is_reg(struct xdp_rxq_info *xdp_rxq) 248 { 249 return (xdp_rxq->reg_state == REG_STATE_REGISTERED); 250 } 251 EXPORT_SYMBOL_GPL(xdp_rxq_info_is_reg); 252 253 static int __mem_id_init_hash_table(void) 254 { 255 struct rhashtable *rht; 256 int ret; 257 258 if (unlikely(mem_id_init)) 259 return 0; 260 261 rht = kzalloc(sizeof(*rht), GFP_KERNEL); 262 if (!rht) 263 return -ENOMEM; 264 265 ret = rhashtable_init(rht, &mem_id_rht_params); 266 if (ret < 0) { 267 kfree(rht); 268 return ret; 269 } 270 mem_id_ht = rht; 271 smp_mb(); /* mutex lock should provide enough pairing */ 272 mem_id_init = true; 273 274 return 0; 275 } 276 277 /* Allocate a cyclic ID that maps to allocator pointer. 278 * See: https://www.kernel.org/doc/html/latest/core-api/idr.html 279 * 280 * Caller must lock mem_id_lock. 281 */ 282 static int __mem_id_cyclic_get(gfp_t gfp) 283 { 284 int retries = 1; 285 int id; 286 287 again: 288 id = ida_simple_get(&mem_id_pool, mem_id_next, MEM_ID_MAX, gfp); 289 if (id < 0) { 290 if (id == -ENOSPC) { 291 /* Cyclic allocator, reset next id */ 292 if (retries--) { 293 mem_id_next = MEM_ID_MIN; 294 goto again; 295 } 296 } 297 return id; /* errno */ 298 } 299 mem_id_next = id + 1; 300 301 return id; 302 } 303 304 static bool __is_supported_mem_type(enum xdp_mem_type type) 305 { 306 if (type == MEM_TYPE_PAGE_POOL) 307 return is_page_pool_compiled_in(); 308 309 if (type >= MEM_TYPE_MAX) 310 return false; 311 312 return true; 313 } 314 315 int xdp_rxq_info_reg_mem_model(struct xdp_rxq_info *xdp_rxq, 316 enum xdp_mem_type type, void *allocator) 317 { 318 struct xdp_mem_allocator *xdp_alloc; 319 gfp_t gfp = GFP_KERNEL; 320 int id, errno, ret; 321 void *ptr; 322 323 if (xdp_rxq->reg_state != REG_STATE_REGISTERED) { 324 WARN(1, "Missing register, driver bug"); 325 return -EFAULT; 326 } 327 328 if (!__is_supported_mem_type(type)) 329 return -EOPNOTSUPP; 330 331 xdp_rxq->mem.type = type; 332 333 if (!allocator) { 334 if (type == MEM_TYPE_PAGE_POOL || type == MEM_TYPE_ZERO_COPY) 335 return -EINVAL; /* Setup time check page_pool req */ 336 return 0; 337 } 338 339 /* Delay init of rhashtable to save memory if feature isn't used */ 340 if (!mem_id_init) { 341 mutex_lock(&mem_id_lock); 342 ret = __mem_id_init_hash_table(); 343 mutex_unlock(&mem_id_lock); 344 if (ret < 0) { 345 WARN_ON(1); 346 return ret; 347 } 348 } 349 350 xdp_alloc = kzalloc(sizeof(*xdp_alloc), gfp); 351 if (!xdp_alloc) 352 return -ENOMEM; 353 354 mutex_lock(&mem_id_lock); 355 id = __mem_id_cyclic_get(gfp); 356 if (id < 0) { 357 errno = id; 358 goto err; 359 } 360 xdp_rxq->mem.id = id; 361 xdp_alloc->mem = xdp_rxq->mem; 362 xdp_alloc->allocator = allocator; 363 364 /* Insert allocator into ID lookup table */ 365 ptr = rhashtable_insert_slow(mem_id_ht, &id, &xdp_alloc->node); 366 if (IS_ERR(ptr)) { 367 ida_simple_remove(&mem_id_pool, xdp_rxq->mem.id); 368 xdp_rxq->mem.id = 0; 369 errno = PTR_ERR(ptr); 370 goto err; 371 } 372 373 if (type == MEM_TYPE_PAGE_POOL) 374 page_pool_get(xdp_alloc->page_pool); 375 376 mutex_unlock(&mem_id_lock); 377 378 trace_mem_connect(xdp_alloc, xdp_rxq); 379 return 0; 380 err: 381 mutex_unlock(&mem_id_lock); 382 kfree(xdp_alloc); 383 return errno; 384 } 385 EXPORT_SYMBOL_GPL(xdp_rxq_info_reg_mem_model); 386 387 /* XDP RX runs under NAPI protection, and in different delivery error 388 * scenarios (e.g. queue full), it is possible to return the xdp_frame 389 * while still leveraging this protection. The @napi_direct boolian 390 * is used for those calls sites. Thus, allowing for faster recycling 391 * of xdp_frames/pages in those cases. 392 */ 393 static void __xdp_return(void *data, struct xdp_mem_info *mem, bool napi_direct, 394 unsigned long handle) 395 { 396 struct xdp_mem_allocator *xa; 397 struct page *page; 398 399 switch (mem->type) { 400 case MEM_TYPE_PAGE_POOL: 401 rcu_read_lock(); 402 /* mem->id is valid, checked in xdp_rxq_info_reg_mem_model() */ 403 xa = rhashtable_lookup(mem_id_ht, &mem->id, mem_id_rht_params); 404 page = virt_to_head_page(data); 405 if (likely(xa)) { 406 napi_direct &= !xdp_return_frame_no_direct(); 407 page_pool_put_page(xa->page_pool, page, napi_direct); 408 } else { 409 /* Hopefully stack show who to blame for late return */ 410 WARN_ONCE(1, "page_pool gone mem.id=%d", mem->id); 411 trace_mem_return_failed(mem, page); 412 put_page(page); 413 } 414 rcu_read_unlock(); 415 break; 416 case MEM_TYPE_PAGE_SHARED: 417 page_frag_free(data); 418 break; 419 case MEM_TYPE_PAGE_ORDER0: 420 page = virt_to_page(data); /* Assumes order0 page*/ 421 put_page(page); 422 break; 423 case MEM_TYPE_ZERO_COPY: 424 /* NB! Only valid from an xdp_buff! */ 425 rcu_read_lock(); 426 /* mem->id is valid, checked in xdp_rxq_info_reg_mem_model() */ 427 xa = rhashtable_lookup(mem_id_ht, &mem->id, mem_id_rht_params); 428 xa->zc_alloc->free(xa->zc_alloc, handle); 429 rcu_read_unlock(); 430 default: 431 /* Not possible, checked in xdp_rxq_info_reg_mem_model() */ 432 break; 433 } 434 } 435 436 void xdp_return_frame(struct xdp_frame *xdpf) 437 { 438 __xdp_return(xdpf->data, &xdpf->mem, false, 0); 439 } 440 EXPORT_SYMBOL_GPL(xdp_return_frame); 441 442 void xdp_return_frame_rx_napi(struct xdp_frame *xdpf) 443 { 444 __xdp_return(xdpf->data, &xdpf->mem, true, 0); 445 } 446 EXPORT_SYMBOL_GPL(xdp_return_frame_rx_napi); 447 448 void xdp_return_buff(struct xdp_buff *xdp) 449 { 450 __xdp_return(xdp->data, &xdp->rxq->mem, true, xdp->handle); 451 } 452 EXPORT_SYMBOL_GPL(xdp_return_buff); 453 454 /* Only called for MEM_TYPE_PAGE_POOL see xdp.h */ 455 void __xdp_release_frame(void *data, struct xdp_mem_info *mem) 456 { 457 struct xdp_mem_allocator *xa; 458 struct page *page; 459 460 rcu_read_lock(); 461 xa = rhashtable_lookup(mem_id_ht, &mem->id, mem_id_rht_params); 462 page = virt_to_head_page(data); 463 if (xa) 464 page_pool_release_page(xa->page_pool, page); 465 rcu_read_unlock(); 466 } 467 EXPORT_SYMBOL_GPL(__xdp_release_frame); 468 469 int xdp_attachment_query(struct xdp_attachment_info *info, 470 struct netdev_bpf *bpf) 471 { 472 bpf->prog_id = info->prog ? info->prog->aux->id : 0; 473 bpf->prog_flags = info->prog ? info->flags : 0; 474 return 0; 475 } 476 EXPORT_SYMBOL_GPL(xdp_attachment_query); 477 478 bool xdp_attachment_flags_ok(struct xdp_attachment_info *info, 479 struct netdev_bpf *bpf) 480 { 481 if (info->prog && (bpf->flags ^ info->flags) & XDP_FLAGS_MODES) { 482 NL_SET_ERR_MSG(bpf->extack, 483 "program loaded with different flags"); 484 return false; 485 } 486 return true; 487 } 488 EXPORT_SYMBOL_GPL(xdp_attachment_flags_ok); 489 490 void xdp_attachment_setup(struct xdp_attachment_info *info, 491 struct netdev_bpf *bpf) 492 { 493 if (info->prog) 494 bpf_prog_put(info->prog); 495 info->prog = bpf->prog; 496 info->flags = bpf->flags; 497 } 498 EXPORT_SYMBOL_GPL(xdp_attachment_setup); 499 500 struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp) 501 { 502 unsigned int metasize, totsize; 503 void *addr, *data_to_copy; 504 struct xdp_frame *xdpf; 505 struct page *page; 506 507 /* Clone into a MEM_TYPE_PAGE_ORDER0 xdp_frame. */ 508 metasize = xdp_data_meta_unsupported(xdp) ? 0 : 509 xdp->data - xdp->data_meta; 510 totsize = xdp->data_end - xdp->data + metasize; 511 512 if (sizeof(*xdpf) + totsize > PAGE_SIZE) 513 return NULL; 514 515 page = dev_alloc_page(); 516 if (!page) 517 return NULL; 518 519 addr = page_to_virt(page); 520 xdpf = addr; 521 memset(xdpf, 0, sizeof(*xdpf)); 522 523 addr += sizeof(*xdpf); 524 data_to_copy = metasize ? xdp->data_meta : xdp->data; 525 memcpy(addr, data_to_copy, totsize); 526 527 xdpf->data = addr + metasize; 528 xdpf->len = totsize - metasize; 529 xdpf->headroom = 0; 530 xdpf->metasize = metasize; 531 xdpf->mem.type = MEM_TYPE_PAGE_ORDER0; 532 533 xdp_return_buff(xdp); 534 return xdpf; 535 } 536 EXPORT_SYMBOL_GPL(xdp_convert_zc_to_xdp_frame); 537