1 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 2 * 3 * This program is free software; you can redistribute it and/or 4 * modify it under the terms of version 2 of the GNU General Public 5 * License as published by the Free Software Foundation. 6 * 7 * This program is distributed in the hope that it will be useful, but 8 * WITHOUT ANY WARRANTY; without even the implied warranty of 9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 10 * General Public License for more details. 11 */ 12 #include <linux/bpf.h> 13 #include <linux/bpf_trace.h> 14 #include <linux/syscalls.h> 15 #include <linux/slab.h> 16 #include <linux/sched/signal.h> 17 #include <linux/vmalloc.h> 18 #include <linux/mmzone.h> 19 #include <linux/anon_inodes.h> 20 #include <linux/file.h> 21 #include <linux/license.h> 22 #include <linux/filter.h> 23 #include <linux/version.h> 24 #include <linux/kernel.h> 25 #include <linux/idr.h> 26 #include <linux/cred.h> 27 #include <linux/timekeeping.h> 28 #include <linux/ctype.h> 29 30 #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY || \ 31 (map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \ 32 (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \ 33 (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) 34 #define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) 35 #define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_HASH(map)) 36 37 #define BPF_OBJ_FLAG_MASK (BPF_F_RDONLY | BPF_F_WRONLY) 38 39 DEFINE_PER_CPU(int, bpf_prog_active); 40 static DEFINE_IDR(prog_idr); 41 static DEFINE_SPINLOCK(prog_idr_lock); 42 static DEFINE_IDR(map_idr); 43 static DEFINE_SPINLOCK(map_idr_lock); 44 45 int sysctl_unprivileged_bpf_disabled __read_mostly; 46 47 static const struct bpf_map_ops * const bpf_map_types[] = { 48 #define BPF_PROG_TYPE(_id, _ops) 49 #define BPF_MAP_TYPE(_id, _ops) \ 50 [_id] = &_ops, 51 #include <linux/bpf_types.h> 52 #undef BPF_PROG_TYPE 53 #undef BPF_MAP_TYPE 54 }; 55 56 /* 57 * If we're handed a bigger struct than we know of, ensure all the unknown bits 58 * are 0 - i.e. new user-space does not rely on any kernel feature extensions 59 * we don't know about yet. 60 * 61 * There is a ToCToU between this function call and the following 62 * copy_from_user() call. However, this is not a concern since this function is 63 * meant to be a future-proofing of bits. 64 */ 65 static int check_uarg_tail_zero(void __user *uaddr, 66 size_t expected_size, 67 size_t actual_size) 68 { 69 unsigned char __user *addr; 70 unsigned char __user *end; 71 unsigned char val; 72 int err; 73 74 if (unlikely(actual_size > PAGE_SIZE)) /* silly large */ 75 return -E2BIG; 76 77 if (unlikely(!access_ok(VERIFY_READ, uaddr, actual_size))) 78 return -EFAULT; 79 80 if (actual_size <= expected_size) 81 return 0; 82 83 addr = uaddr + expected_size; 84 end = uaddr + actual_size; 85 86 for (; addr < end; addr++) { 87 err = get_user(val, addr); 88 if (err) 89 return err; 90 if (val) 91 return -E2BIG; 92 } 93 94 return 0; 95 } 96 97 const struct bpf_map_ops bpf_map_offload_ops = { 98 .map_alloc = bpf_map_offload_map_alloc, 99 .map_free = bpf_map_offload_map_free, 100 }; 101 102 static struct bpf_map *find_and_alloc_map(union bpf_attr *attr) 103 { 104 const struct bpf_map_ops *ops; 105 struct bpf_map *map; 106 int err; 107 108 if (attr->map_type >= ARRAY_SIZE(bpf_map_types)) 109 return ERR_PTR(-EINVAL); 110 ops = bpf_map_types[attr->map_type]; 111 if (!ops) 112 return ERR_PTR(-EINVAL); 113 114 if (ops->map_alloc_check) { 115 err = ops->map_alloc_check(attr); 116 if (err) 117 return ERR_PTR(err); 118 } 119 if (attr->map_ifindex) 120 ops = &bpf_map_offload_ops; 121 map = ops->map_alloc(attr); 122 if (IS_ERR(map)) 123 return map; 124 map->ops = ops; 125 map->map_type = attr->map_type; 126 return map; 127 } 128 129 void *bpf_map_area_alloc(size_t size, int numa_node) 130 { 131 /* We definitely need __GFP_NORETRY, so OOM killer doesn't 132 * trigger under memory pressure as we really just want to 133 * fail instead. 134 */ 135 const gfp_t flags = __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO; 136 void *area; 137 138 if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) { 139 area = kmalloc_node(size, GFP_USER | flags, numa_node); 140 if (area != NULL) 141 return area; 142 } 143 144 return __vmalloc_node_flags_caller(size, numa_node, GFP_KERNEL | flags, 145 __builtin_return_address(0)); 146 } 147 148 void bpf_map_area_free(void *area) 149 { 150 kvfree(area); 151 } 152 153 void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr) 154 { 155 map->map_type = attr->map_type; 156 map->key_size = attr->key_size; 157 map->value_size = attr->value_size; 158 map->max_entries = attr->max_entries; 159 map->map_flags = attr->map_flags; 160 map->numa_node = bpf_map_attr_numa_node(attr); 161 } 162 163 int bpf_map_precharge_memlock(u32 pages) 164 { 165 struct user_struct *user = get_current_user(); 166 unsigned long memlock_limit, cur; 167 168 memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 169 cur = atomic_long_read(&user->locked_vm); 170 free_uid(user); 171 if (cur + pages > memlock_limit) 172 return -EPERM; 173 return 0; 174 } 175 176 static int bpf_map_charge_memlock(struct bpf_map *map) 177 { 178 struct user_struct *user = get_current_user(); 179 unsigned long memlock_limit; 180 181 memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 182 183 atomic_long_add(map->pages, &user->locked_vm); 184 185 if (atomic_long_read(&user->locked_vm) > memlock_limit) { 186 atomic_long_sub(map->pages, &user->locked_vm); 187 free_uid(user); 188 return -EPERM; 189 } 190 map->user = user; 191 return 0; 192 } 193 194 static void bpf_map_uncharge_memlock(struct bpf_map *map) 195 { 196 struct user_struct *user = map->user; 197 198 atomic_long_sub(map->pages, &user->locked_vm); 199 free_uid(user); 200 } 201 202 static int bpf_map_alloc_id(struct bpf_map *map) 203 { 204 int id; 205 206 idr_preload(GFP_KERNEL); 207 spin_lock_bh(&map_idr_lock); 208 id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC); 209 if (id > 0) 210 map->id = id; 211 spin_unlock_bh(&map_idr_lock); 212 idr_preload_end(); 213 214 if (WARN_ON_ONCE(!id)) 215 return -ENOSPC; 216 217 return id > 0 ? 0 : id; 218 } 219 220 void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock) 221 { 222 unsigned long flags; 223 224 /* Offloaded maps are removed from the IDR store when their device 225 * disappears - even if someone holds an fd to them they are unusable, 226 * the memory is gone, all ops will fail; they are simply waiting for 227 * refcnt to drop to be freed. 228 */ 229 if (!map->id) 230 return; 231 232 if (do_idr_lock) 233 spin_lock_irqsave(&map_idr_lock, flags); 234 else 235 __acquire(&map_idr_lock); 236 237 idr_remove(&map_idr, map->id); 238 map->id = 0; 239 240 if (do_idr_lock) 241 spin_unlock_irqrestore(&map_idr_lock, flags); 242 else 243 __release(&map_idr_lock); 244 } 245 246 /* called from workqueue */ 247 static void bpf_map_free_deferred(struct work_struct *work) 248 { 249 struct bpf_map *map = container_of(work, struct bpf_map, work); 250 251 bpf_map_uncharge_memlock(map); 252 security_bpf_map_free(map); 253 /* implementation dependent freeing */ 254 map->ops->map_free(map); 255 } 256 257 static void bpf_map_put_uref(struct bpf_map *map) 258 { 259 if (atomic_dec_and_test(&map->usercnt)) { 260 if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) 261 bpf_fd_array_map_clear(map); 262 } 263 } 264 265 /* decrement map refcnt and schedule it for freeing via workqueue 266 * (unrelying map implementation ops->map_free() might sleep) 267 */ 268 static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock) 269 { 270 if (atomic_dec_and_test(&map->refcnt)) { 271 /* bpf_map_free_id() must be called first */ 272 bpf_map_free_id(map, do_idr_lock); 273 INIT_WORK(&map->work, bpf_map_free_deferred); 274 schedule_work(&map->work); 275 } 276 } 277 278 void bpf_map_put(struct bpf_map *map) 279 { 280 __bpf_map_put(map, true); 281 } 282 283 void bpf_map_put_with_uref(struct bpf_map *map) 284 { 285 bpf_map_put_uref(map); 286 bpf_map_put(map); 287 } 288 289 static int bpf_map_release(struct inode *inode, struct file *filp) 290 { 291 struct bpf_map *map = filp->private_data; 292 293 if (map->ops->map_release) 294 map->ops->map_release(map, filp); 295 296 bpf_map_put_with_uref(map); 297 return 0; 298 } 299 300 #ifdef CONFIG_PROC_FS 301 static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp) 302 { 303 const struct bpf_map *map = filp->private_data; 304 const struct bpf_array *array; 305 u32 owner_prog_type = 0; 306 u32 owner_jited = 0; 307 308 if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) { 309 array = container_of(map, struct bpf_array, map); 310 owner_prog_type = array->owner_prog_type; 311 owner_jited = array->owner_jited; 312 } 313 314 seq_printf(m, 315 "map_type:\t%u\n" 316 "key_size:\t%u\n" 317 "value_size:\t%u\n" 318 "max_entries:\t%u\n" 319 "map_flags:\t%#x\n" 320 "memlock:\t%llu\n", 321 map->map_type, 322 map->key_size, 323 map->value_size, 324 map->max_entries, 325 map->map_flags, 326 map->pages * 1ULL << PAGE_SHIFT); 327 328 if (owner_prog_type) { 329 seq_printf(m, "owner_prog_type:\t%u\n", 330 owner_prog_type); 331 seq_printf(m, "owner_jited:\t%u\n", 332 owner_jited); 333 } 334 } 335 #endif 336 337 static ssize_t bpf_dummy_read(struct file *filp, char __user *buf, size_t siz, 338 loff_t *ppos) 339 { 340 /* We need this handler such that alloc_file() enables 341 * f_mode with FMODE_CAN_READ. 342 */ 343 return -EINVAL; 344 } 345 346 static ssize_t bpf_dummy_write(struct file *filp, const char __user *buf, 347 size_t siz, loff_t *ppos) 348 { 349 /* We need this handler such that alloc_file() enables 350 * f_mode with FMODE_CAN_WRITE. 351 */ 352 return -EINVAL; 353 } 354 355 const struct file_operations bpf_map_fops = { 356 #ifdef CONFIG_PROC_FS 357 .show_fdinfo = bpf_map_show_fdinfo, 358 #endif 359 .release = bpf_map_release, 360 .read = bpf_dummy_read, 361 .write = bpf_dummy_write, 362 }; 363 364 int bpf_map_new_fd(struct bpf_map *map, int flags) 365 { 366 int ret; 367 368 ret = security_bpf_map(map, OPEN_FMODE(flags)); 369 if (ret < 0) 370 return ret; 371 372 return anon_inode_getfd("bpf-map", &bpf_map_fops, map, 373 flags | O_CLOEXEC); 374 } 375 376 int bpf_get_file_flag(int flags) 377 { 378 if ((flags & BPF_F_RDONLY) && (flags & BPF_F_WRONLY)) 379 return -EINVAL; 380 if (flags & BPF_F_RDONLY) 381 return O_RDONLY; 382 if (flags & BPF_F_WRONLY) 383 return O_WRONLY; 384 return O_RDWR; 385 } 386 387 /* helper macro to check that unused fields 'union bpf_attr' are zero */ 388 #define CHECK_ATTR(CMD) \ 389 memchr_inv((void *) &attr->CMD##_LAST_FIELD + \ 390 sizeof(attr->CMD##_LAST_FIELD), 0, \ 391 sizeof(*attr) - \ 392 offsetof(union bpf_attr, CMD##_LAST_FIELD) - \ 393 sizeof(attr->CMD##_LAST_FIELD)) != NULL 394 395 /* dst and src must have at least BPF_OBJ_NAME_LEN number of bytes. 396 * Return 0 on success and < 0 on error. 397 */ 398 static int bpf_obj_name_cpy(char *dst, const char *src) 399 { 400 const char *end = src + BPF_OBJ_NAME_LEN; 401 402 memset(dst, 0, BPF_OBJ_NAME_LEN); 403 404 /* Copy all isalnum() and '_' char */ 405 while (src < end && *src) { 406 if (!isalnum(*src) && *src != '_') 407 return -EINVAL; 408 *dst++ = *src++; 409 } 410 411 /* No '\0' found in BPF_OBJ_NAME_LEN number of bytes */ 412 if (src == end) 413 return -EINVAL; 414 415 return 0; 416 } 417 418 #define BPF_MAP_CREATE_LAST_FIELD map_ifindex 419 /* called via syscall */ 420 static int map_create(union bpf_attr *attr) 421 { 422 int numa_node = bpf_map_attr_numa_node(attr); 423 struct bpf_map *map; 424 int f_flags; 425 int err; 426 427 err = CHECK_ATTR(BPF_MAP_CREATE); 428 if (err) 429 return -EINVAL; 430 431 f_flags = bpf_get_file_flag(attr->map_flags); 432 if (f_flags < 0) 433 return f_flags; 434 435 if (numa_node != NUMA_NO_NODE && 436 ((unsigned int)numa_node >= nr_node_ids || 437 !node_online(numa_node))) 438 return -EINVAL; 439 440 /* find map type and init map: hashtable vs rbtree vs bloom vs ... */ 441 map = find_and_alloc_map(attr); 442 if (IS_ERR(map)) 443 return PTR_ERR(map); 444 445 err = bpf_obj_name_cpy(map->name, attr->map_name); 446 if (err) 447 goto free_map_nouncharge; 448 449 atomic_set(&map->refcnt, 1); 450 atomic_set(&map->usercnt, 1); 451 452 err = security_bpf_map_alloc(map); 453 if (err) 454 goto free_map_nouncharge; 455 456 err = bpf_map_charge_memlock(map); 457 if (err) 458 goto free_map_sec; 459 460 err = bpf_map_alloc_id(map); 461 if (err) 462 goto free_map; 463 464 err = bpf_map_new_fd(map, f_flags); 465 if (err < 0) { 466 /* failed to allocate fd. 467 * bpf_map_put() is needed because the above 468 * bpf_map_alloc_id() has published the map 469 * to the userspace and the userspace may 470 * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID. 471 */ 472 bpf_map_put(map); 473 return err; 474 } 475 476 trace_bpf_map_create(map, err); 477 return err; 478 479 free_map: 480 bpf_map_uncharge_memlock(map); 481 free_map_sec: 482 security_bpf_map_free(map); 483 free_map_nouncharge: 484 map->ops->map_free(map); 485 return err; 486 } 487 488 /* if error is returned, fd is released. 489 * On success caller should complete fd access with matching fdput() 490 */ 491 struct bpf_map *__bpf_map_get(struct fd f) 492 { 493 if (!f.file) 494 return ERR_PTR(-EBADF); 495 if (f.file->f_op != &bpf_map_fops) { 496 fdput(f); 497 return ERR_PTR(-EINVAL); 498 } 499 500 return f.file->private_data; 501 } 502 503 /* prog's and map's refcnt limit */ 504 #define BPF_MAX_REFCNT 32768 505 506 struct bpf_map *bpf_map_inc(struct bpf_map *map, bool uref) 507 { 508 if (atomic_inc_return(&map->refcnt) > BPF_MAX_REFCNT) { 509 atomic_dec(&map->refcnt); 510 return ERR_PTR(-EBUSY); 511 } 512 if (uref) 513 atomic_inc(&map->usercnt); 514 return map; 515 } 516 517 struct bpf_map *bpf_map_get_with_uref(u32 ufd) 518 { 519 struct fd f = fdget(ufd); 520 struct bpf_map *map; 521 522 map = __bpf_map_get(f); 523 if (IS_ERR(map)) 524 return map; 525 526 map = bpf_map_inc(map, true); 527 fdput(f); 528 529 return map; 530 } 531 532 /* map_idr_lock should have been held */ 533 static struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map, 534 bool uref) 535 { 536 int refold; 537 538 refold = __atomic_add_unless(&map->refcnt, 1, 0); 539 540 if (refold >= BPF_MAX_REFCNT) { 541 __bpf_map_put(map, false); 542 return ERR_PTR(-EBUSY); 543 } 544 545 if (!refold) 546 return ERR_PTR(-ENOENT); 547 548 if (uref) 549 atomic_inc(&map->usercnt); 550 551 return map; 552 } 553 554 int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value) 555 { 556 return -ENOTSUPP; 557 } 558 559 /* last field in 'union bpf_attr' used by this command */ 560 #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD value 561 562 static int map_lookup_elem(union bpf_attr *attr) 563 { 564 void __user *ukey = u64_to_user_ptr(attr->key); 565 void __user *uvalue = u64_to_user_ptr(attr->value); 566 int ufd = attr->map_fd; 567 struct bpf_map *map; 568 void *key, *value, *ptr; 569 u32 value_size; 570 struct fd f; 571 int err; 572 573 if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM)) 574 return -EINVAL; 575 576 f = fdget(ufd); 577 map = __bpf_map_get(f); 578 if (IS_ERR(map)) 579 return PTR_ERR(map); 580 581 if (!(f.file->f_mode & FMODE_CAN_READ)) { 582 err = -EPERM; 583 goto err_put; 584 } 585 586 key = memdup_user(ukey, map->key_size); 587 if (IS_ERR(key)) { 588 err = PTR_ERR(key); 589 goto err_put; 590 } 591 592 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 593 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 594 map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) 595 value_size = round_up(map->value_size, 8) * num_possible_cpus(); 596 else if (IS_FD_MAP(map)) 597 value_size = sizeof(u32); 598 else 599 value_size = map->value_size; 600 601 err = -ENOMEM; 602 value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 603 if (!value) 604 goto free_key; 605 606 if (bpf_map_is_dev_bound(map)) { 607 err = bpf_map_offload_lookup_elem(map, key, value); 608 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 609 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 610 err = bpf_percpu_hash_copy(map, key, value); 611 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 612 err = bpf_percpu_array_copy(map, key, value); 613 } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) { 614 err = bpf_stackmap_copy(map, key, value); 615 } else if (IS_FD_ARRAY(map)) { 616 err = bpf_fd_array_map_lookup_elem(map, key, value); 617 } else if (IS_FD_HASH(map)) { 618 err = bpf_fd_htab_map_lookup_elem(map, key, value); 619 } else { 620 rcu_read_lock(); 621 ptr = map->ops->map_lookup_elem(map, key); 622 if (ptr) 623 memcpy(value, ptr, value_size); 624 rcu_read_unlock(); 625 err = ptr ? 0 : -ENOENT; 626 } 627 628 if (err) 629 goto free_value; 630 631 err = -EFAULT; 632 if (copy_to_user(uvalue, value, value_size) != 0) 633 goto free_value; 634 635 trace_bpf_map_lookup_elem(map, ufd, key, value); 636 err = 0; 637 638 free_value: 639 kfree(value); 640 free_key: 641 kfree(key); 642 err_put: 643 fdput(f); 644 return err; 645 } 646 647 #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags 648 649 static int map_update_elem(union bpf_attr *attr) 650 { 651 void __user *ukey = u64_to_user_ptr(attr->key); 652 void __user *uvalue = u64_to_user_ptr(attr->value); 653 int ufd = attr->map_fd; 654 struct bpf_map *map; 655 void *key, *value; 656 u32 value_size; 657 struct fd f; 658 int err; 659 660 if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM)) 661 return -EINVAL; 662 663 f = fdget(ufd); 664 map = __bpf_map_get(f); 665 if (IS_ERR(map)) 666 return PTR_ERR(map); 667 668 if (!(f.file->f_mode & FMODE_CAN_WRITE)) { 669 err = -EPERM; 670 goto err_put; 671 } 672 673 key = memdup_user(ukey, map->key_size); 674 if (IS_ERR(key)) { 675 err = PTR_ERR(key); 676 goto err_put; 677 } 678 679 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 680 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 681 map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) 682 value_size = round_up(map->value_size, 8) * num_possible_cpus(); 683 else 684 value_size = map->value_size; 685 686 err = -ENOMEM; 687 value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 688 if (!value) 689 goto free_key; 690 691 err = -EFAULT; 692 if (copy_from_user(value, uvalue, value_size) != 0) 693 goto free_value; 694 695 /* Need to create a kthread, thus must support schedule */ 696 if (bpf_map_is_dev_bound(map)) { 697 err = bpf_map_offload_update_elem(map, key, value, attr->flags); 698 goto out; 699 } else if (map->map_type == BPF_MAP_TYPE_CPUMAP) { 700 err = map->ops->map_update_elem(map, key, value, attr->flags); 701 goto out; 702 } 703 704 /* must increment bpf_prog_active to avoid kprobe+bpf triggering from 705 * inside bpf map update or delete otherwise deadlocks are possible 706 */ 707 preempt_disable(); 708 __this_cpu_inc(bpf_prog_active); 709 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 710 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 711 err = bpf_percpu_hash_update(map, key, value, attr->flags); 712 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 713 err = bpf_percpu_array_update(map, key, value, attr->flags); 714 } else if (IS_FD_ARRAY(map)) { 715 rcu_read_lock(); 716 err = bpf_fd_array_map_update_elem(map, f.file, key, value, 717 attr->flags); 718 rcu_read_unlock(); 719 } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) { 720 rcu_read_lock(); 721 err = bpf_fd_htab_map_update_elem(map, f.file, key, value, 722 attr->flags); 723 rcu_read_unlock(); 724 } else { 725 rcu_read_lock(); 726 err = map->ops->map_update_elem(map, key, value, attr->flags); 727 rcu_read_unlock(); 728 } 729 __this_cpu_dec(bpf_prog_active); 730 preempt_enable(); 731 out: 732 if (!err) 733 trace_bpf_map_update_elem(map, ufd, key, value); 734 free_value: 735 kfree(value); 736 free_key: 737 kfree(key); 738 err_put: 739 fdput(f); 740 return err; 741 } 742 743 #define BPF_MAP_DELETE_ELEM_LAST_FIELD key 744 745 static int map_delete_elem(union bpf_attr *attr) 746 { 747 void __user *ukey = u64_to_user_ptr(attr->key); 748 int ufd = attr->map_fd; 749 struct bpf_map *map; 750 struct fd f; 751 void *key; 752 int err; 753 754 if (CHECK_ATTR(BPF_MAP_DELETE_ELEM)) 755 return -EINVAL; 756 757 f = fdget(ufd); 758 map = __bpf_map_get(f); 759 if (IS_ERR(map)) 760 return PTR_ERR(map); 761 762 if (!(f.file->f_mode & FMODE_CAN_WRITE)) { 763 err = -EPERM; 764 goto err_put; 765 } 766 767 key = memdup_user(ukey, map->key_size); 768 if (IS_ERR(key)) { 769 err = PTR_ERR(key); 770 goto err_put; 771 } 772 773 if (bpf_map_is_dev_bound(map)) { 774 err = bpf_map_offload_delete_elem(map, key); 775 goto out; 776 } 777 778 preempt_disable(); 779 __this_cpu_inc(bpf_prog_active); 780 rcu_read_lock(); 781 err = map->ops->map_delete_elem(map, key); 782 rcu_read_unlock(); 783 __this_cpu_dec(bpf_prog_active); 784 preempt_enable(); 785 out: 786 if (!err) 787 trace_bpf_map_delete_elem(map, ufd, key); 788 kfree(key); 789 err_put: 790 fdput(f); 791 return err; 792 } 793 794 /* last field in 'union bpf_attr' used by this command */ 795 #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key 796 797 static int map_get_next_key(union bpf_attr *attr) 798 { 799 void __user *ukey = u64_to_user_ptr(attr->key); 800 void __user *unext_key = u64_to_user_ptr(attr->next_key); 801 int ufd = attr->map_fd; 802 struct bpf_map *map; 803 void *key, *next_key; 804 struct fd f; 805 int err; 806 807 if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY)) 808 return -EINVAL; 809 810 f = fdget(ufd); 811 map = __bpf_map_get(f); 812 if (IS_ERR(map)) 813 return PTR_ERR(map); 814 815 if (!(f.file->f_mode & FMODE_CAN_READ)) { 816 err = -EPERM; 817 goto err_put; 818 } 819 820 if (ukey) { 821 key = memdup_user(ukey, map->key_size); 822 if (IS_ERR(key)) { 823 err = PTR_ERR(key); 824 goto err_put; 825 } 826 } else { 827 key = NULL; 828 } 829 830 err = -ENOMEM; 831 next_key = kmalloc(map->key_size, GFP_USER); 832 if (!next_key) 833 goto free_key; 834 835 if (bpf_map_is_dev_bound(map)) { 836 err = bpf_map_offload_get_next_key(map, key, next_key); 837 goto out; 838 } 839 840 rcu_read_lock(); 841 err = map->ops->map_get_next_key(map, key, next_key); 842 rcu_read_unlock(); 843 out: 844 if (err) 845 goto free_next_key; 846 847 err = -EFAULT; 848 if (copy_to_user(unext_key, next_key, map->key_size) != 0) 849 goto free_next_key; 850 851 trace_bpf_map_next_key(map, ufd, key, next_key); 852 err = 0; 853 854 free_next_key: 855 kfree(next_key); 856 free_key: 857 kfree(key); 858 err_put: 859 fdput(f); 860 return err; 861 } 862 863 static const struct bpf_prog_ops * const bpf_prog_types[] = { 864 #define BPF_PROG_TYPE(_id, _name) \ 865 [_id] = & _name ## _prog_ops, 866 #define BPF_MAP_TYPE(_id, _ops) 867 #include <linux/bpf_types.h> 868 #undef BPF_PROG_TYPE 869 #undef BPF_MAP_TYPE 870 }; 871 872 static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog) 873 { 874 if (type >= ARRAY_SIZE(bpf_prog_types) || !bpf_prog_types[type]) 875 return -EINVAL; 876 877 if (!bpf_prog_is_dev_bound(prog->aux)) 878 prog->aux->ops = bpf_prog_types[type]; 879 else 880 prog->aux->ops = &bpf_offload_prog_ops; 881 prog->type = type; 882 return 0; 883 } 884 885 /* drop refcnt on maps used by eBPF program and free auxilary data */ 886 static void free_used_maps(struct bpf_prog_aux *aux) 887 { 888 int i; 889 890 for (i = 0; i < aux->used_map_cnt; i++) 891 bpf_map_put(aux->used_maps[i]); 892 893 kfree(aux->used_maps); 894 } 895 896 int __bpf_prog_charge(struct user_struct *user, u32 pages) 897 { 898 unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 899 unsigned long user_bufs; 900 901 if (user) { 902 user_bufs = atomic_long_add_return(pages, &user->locked_vm); 903 if (user_bufs > memlock_limit) { 904 atomic_long_sub(pages, &user->locked_vm); 905 return -EPERM; 906 } 907 } 908 909 return 0; 910 } 911 912 void __bpf_prog_uncharge(struct user_struct *user, u32 pages) 913 { 914 if (user) 915 atomic_long_sub(pages, &user->locked_vm); 916 } 917 918 static int bpf_prog_charge_memlock(struct bpf_prog *prog) 919 { 920 struct user_struct *user = get_current_user(); 921 int ret; 922 923 ret = __bpf_prog_charge(user, prog->pages); 924 if (ret) { 925 free_uid(user); 926 return ret; 927 } 928 929 prog->aux->user = user; 930 return 0; 931 } 932 933 static void bpf_prog_uncharge_memlock(struct bpf_prog *prog) 934 { 935 struct user_struct *user = prog->aux->user; 936 937 __bpf_prog_uncharge(user, prog->pages); 938 free_uid(user); 939 } 940 941 static int bpf_prog_alloc_id(struct bpf_prog *prog) 942 { 943 int id; 944 945 idr_preload(GFP_KERNEL); 946 spin_lock_bh(&prog_idr_lock); 947 id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC); 948 if (id > 0) 949 prog->aux->id = id; 950 spin_unlock_bh(&prog_idr_lock); 951 idr_preload_end(); 952 953 /* id is in [1, INT_MAX) */ 954 if (WARN_ON_ONCE(!id)) 955 return -ENOSPC; 956 957 return id > 0 ? 0 : id; 958 } 959 960 void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock) 961 { 962 /* cBPF to eBPF migrations are currently not in the idr store. 963 * Offloaded programs are removed from the store when their device 964 * disappears - even if someone grabs an fd to them they are unusable, 965 * simply waiting for refcnt to drop to be freed. 966 */ 967 if (!prog->aux->id) 968 return; 969 970 if (do_idr_lock) 971 spin_lock_bh(&prog_idr_lock); 972 else 973 __acquire(&prog_idr_lock); 974 975 idr_remove(&prog_idr, prog->aux->id); 976 prog->aux->id = 0; 977 978 if (do_idr_lock) 979 spin_unlock_bh(&prog_idr_lock); 980 else 981 __release(&prog_idr_lock); 982 } 983 984 static void __bpf_prog_put_rcu(struct rcu_head *rcu) 985 { 986 struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu); 987 988 free_used_maps(aux); 989 bpf_prog_uncharge_memlock(aux->prog); 990 security_bpf_prog_free(aux); 991 bpf_prog_free(aux->prog); 992 } 993 994 static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock) 995 { 996 if (atomic_dec_and_test(&prog->aux->refcnt)) { 997 int i; 998 999 trace_bpf_prog_put_rcu(prog); 1000 /* bpf_prog_free_id() must be called first */ 1001 bpf_prog_free_id(prog, do_idr_lock); 1002 1003 for (i = 0; i < prog->aux->func_cnt; i++) 1004 bpf_prog_kallsyms_del(prog->aux->func[i]); 1005 bpf_prog_kallsyms_del(prog); 1006 1007 call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); 1008 } 1009 } 1010 1011 void bpf_prog_put(struct bpf_prog *prog) 1012 { 1013 __bpf_prog_put(prog, true); 1014 } 1015 EXPORT_SYMBOL_GPL(bpf_prog_put); 1016 1017 static int bpf_prog_release(struct inode *inode, struct file *filp) 1018 { 1019 struct bpf_prog *prog = filp->private_data; 1020 1021 bpf_prog_put(prog); 1022 return 0; 1023 } 1024 1025 #ifdef CONFIG_PROC_FS 1026 static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp) 1027 { 1028 const struct bpf_prog *prog = filp->private_data; 1029 char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; 1030 1031 bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); 1032 seq_printf(m, 1033 "prog_type:\t%u\n" 1034 "prog_jited:\t%u\n" 1035 "prog_tag:\t%s\n" 1036 "memlock:\t%llu\n", 1037 prog->type, 1038 prog->jited, 1039 prog_tag, 1040 prog->pages * 1ULL << PAGE_SHIFT); 1041 } 1042 #endif 1043 1044 const struct file_operations bpf_prog_fops = { 1045 #ifdef CONFIG_PROC_FS 1046 .show_fdinfo = bpf_prog_show_fdinfo, 1047 #endif 1048 .release = bpf_prog_release, 1049 .read = bpf_dummy_read, 1050 .write = bpf_dummy_write, 1051 }; 1052 1053 int bpf_prog_new_fd(struct bpf_prog *prog) 1054 { 1055 int ret; 1056 1057 ret = security_bpf_prog(prog); 1058 if (ret < 0) 1059 return ret; 1060 1061 return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog, 1062 O_RDWR | O_CLOEXEC); 1063 } 1064 1065 static struct bpf_prog *____bpf_prog_get(struct fd f) 1066 { 1067 if (!f.file) 1068 return ERR_PTR(-EBADF); 1069 if (f.file->f_op != &bpf_prog_fops) { 1070 fdput(f); 1071 return ERR_PTR(-EINVAL); 1072 } 1073 1074 return f.file->private_data; 1075 } 1076 1077 struct bpf_prog *bpf_prog_add(struct bpf_prog *prog, int i) 1078 { 1079 if (atomic_add_return(i, &prog->aux->refcnt) > BPF_MAX_REFCNT) { 1080 atomic_sub(i, &prog->aux->refcnt); 1081 return ERR_PTR(-EBUSY); 1082 } 1083 return prog; 1084 } 1085 EXPORT_SYMBOL_GPL(bpf_prog_add); 1086 1087 void bpf_prog_sub(struct bpf_prog *prog, int i) 1088 { 1089 /* Only to be used for undoing previous bpf_prog_add() in some 1090 * error path. We still know that another entity in our call 1091 * path holds a reference to the program, thus atomic_sub() can 1092 * be safely used in such cases! 1093 */ 1094 WARN_ON(atomic_sub_return(i, &prog->aux->refcnt) == 0); 1095 } 1096 EXPORT_SYMBOL_GPL(bpf_prog_sub); 1097 1098 struct bpf_prog *bpf_prog_inc(struct bpf_prog *prog) 1099 { 1100 return bpf_prog_add(prog, 1); 1101 } 1102 EXPORT_SYMBOL_GPL(bpf_prog_inc); 1103 1104 /* prog_idr_lock should have been held */ 1105 struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog) 1106 { 1107 int refold; 1108 1109 refold = __atomic_add_unless(&prog->aux->refcnt, 1, 0); 1110 1111 if (refold >= BPF_MAX_REFCNT) { 1112 __bpf_prog_put(prog, false); 1113 return ERR_PTR(-EBUSY); 1114 } 1115 1116 if (!refold) 1117 return ERR_PTR(-ENOENT); 1118 1119 return prog; 1120 } 1121 EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero); 1122 1123 bool bpf_prog_get_ok(struct bpf_prog *prog, 1124 enum bpf_prog_type *attach_type, bool attach_drv) 1125 { 1126 /* not an attachment, just a refcount inc, always allow */ 1127 if (!attach_type) 1128 return true; 1129 1130 if (prog->type != *attach_type) 1131 return false; 1132 if (bpf_prog_is_dev_bound(prog->aux) && !attach_drv) 1133 return false; 1134 1135 return true; 1136 } 1137 1138 static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *attach_type, 1139 bool attach_drv) 1140 { 1141 struct fd f = fdget(ufd); 1142 struct bpf_prog *prog; 1143 1144 prog = ____bpf_prog_get(f); 1145 if (IS_ERR(prog)) 1146 return prog; 1147 if (!bpf_prog_get_ok(prog, attach_type, attach_drv)) { 1148 prog = ERR_PTR(-EINVAL); 1149 goto out; 1150 } 1151 1152 prog = bpf_prog_inc(prog); 1153 out: 1154 fdput(f); 1155 return prog; 1156 } 1157 1158 struct bpf_prog *bpf_prog_get(u32 ufd) 1159 { 1160 return __bpf_prog_get(ufd, NULL, false); 1161 } 1162 1163 struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type, 1164 bool attach_drv) 1165 { 1166 struct bpf_prog *prog = __bpf_prog_get(ufd, &type, attach_drv); 1167 1168 if (!IS_ERR(prog)) 1169 trace_bpf_prog_get_type(prog); 1170 return prog; 1171 } 1172 EXPORT_SYMBOL_GPL(bpf_prog_get_type_dev); 1173 1174 /* Initially all BPF programs could be loaded w/o specifying 1175 * expected_attach_type. Later for some of them specifying expected_attach_type 1176 * at load time became required so that program could be validated properly. 1177 * Programs of types that are allowed to be loaded both w/ and w/o (for 1178 * backward compatibility) expected_attach_type, should have the default attach 1179 * type assigned to expected_attach_type for the latter case, so that it can be 1180 * validated later at attach time. 1181 * 1182 * bpf_prog_load_fixup_attach_type() sets expected_attach_type in @attr if 1183 * prog type requires it but has some attach types that have to be backward 1184 * compatible. 1185 */ 1186 static void bpf_prog_load_fixup_attach_type(union bpf_attr *attr) 1187 { 1188 switch (attr->prog_type) { 1189 case BPF_PROG_TYPE_CGROUP_SOCK: 1190 /* Unfortunately BPF_ATTACH_TYPE_UNSPEC enumeration doesn't 1191 * exist so checking for non-zero is the way to go here. 1192 */ 1193 if (!attr->expected_attach_type) 1194 attr->expected_attach_type = 1195 BPF_CGROUP_INET_SOCK_CREATE; 1196 break; 1197 } 1198 } 1199 1200 static int 1201 bpf_prog_load_check_attach_type(enum bpf_prog_type prog_type, 1202 enum bpf_attach_type expected_attach_type) 1203 { 1204 switch (prog_type) { 1205 case BPF_PROG_TYPE_CGROUP_SOCK: 1206 switch (expected_attach_type) { 1207 case BPF_CGROUP_INET_SOCK_CREATE: 1208 case BPF_CGROUP_INET4_POST_BIND: 1209 case BPF_CGROUP_INET6_POST_BIND: 1210 return 0; 1211 default: 1212 return -EINVAL; 1213 } 1214 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 1215 switch (expected_attach_type) { 1216 case BPF_CGROUP_INET4_BIND: 1217 case BPF_CGROUP_INET6_BIND: 1218 case BPF_CGROUP_INET4_CONNECT: 1219 case BPF_CGROUP_INET6_CONNECT: 1220 return 0; 1221 default: 1222 return -EINVAL; 1223 } 1224 default: 1225 return 0; 1226 } 1227 } 1228 1229 /* last field in 'union bpf_attr' used by this command */ 1230 #define BPF_PROG_LOAD_LAST_FIELD expected_attach_type 1231 1232 static int bpf_prog_load(union bpf_attr *attr) 1233 { 1234 enum bpf_prog_type type = attr->prog_type; 1235 struct bpf_prog *prog; 1236 int err; 1237 char license[128]; 1238 bool is_gpl; 1239 1240 if (CHECK_ATTR(BPF_PROG_LOAD)) 1241 return -EINVAL; 1242 1243 if (attr->prog_flags & ~BPF_F_STRICT_ALIGNMENT) 1244 return -EINVAL; 1245 1246 /* copy eBPF program license from user space */ 1247 if (strncpy_from_user(license, u64_to_user_ptr(attr->license), 1248 sizeof(license) - 1) < 0) 1249 return -EFAULT; 1250 license[sizeof(license) - 1] = 0; 1251 1252 /* eBPF programs must be GPL compatible to use GPL-ed functions */ 1253 is_gpl = license_is_gpl_compatible(license); 1254 1255 if (attr->insn_cnt == 0 || attr->insn_cnt > BPF_MAXINSNS) 1256 return -E2BIG; 1257 1258 if (type == BPF_PROG_TYPE_KPROBE && 1259 attr->kern_version != LINUX_VERSION_CODE) 1260 return -EINVAL; 1261 1262 if (type != BPF_PROG_TYPE_SOCKET_FILTER && 1263 type != BPF_PROG_TYPE_CGROUP_SKB && 1264 !capable(CAP_SYS_ADMIN)) 1265 return -EPERM; 1266 1267 bpf_prog_load_fixup_attach_type(attr); 1268 if (bpf_prog_load_check_attach_type(type, attr->expected_attach_type)) 1269 return -EINVAL; 1270 1271 /* plain bpf_prog allocation */ 1272 prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER); 1273 if (!prog) 1274 return -ENOMEM; 1275 1276 prog->expected_attach_type = attr->expected_attach_type; 1277 1278 prog->aux->offload_requested = !!attr->prog_ifindex; 1279 1280 err = security_bpf_prog_alloc(prog->aux); 1281 if (err) 1282 goto free_prog_nouncharge; 1283 1284 err = bpf_prog_charge_memlock(prog); 1285 if (err) 1286 goto free_prog_sec; 1287 1288 prog->len = attr->insn_cnt; 1289 1290 err = -EFAULT; 1291 if (copy_from_user(prog->insns, u64_to_user_ptr(attr->insns), 1292 bpf_prog_insn_size(prog)) != 0) 1293 goto free_prog; 1294 1295 prog->orig_prog = NULL; 1296 prog->jited = 0; 1297 1298 atomic_set(&prog->aux->refcnt, 1); 1299 prog->gpl_compatible = is_gpl ? 1 : 0; 1300 1301 if (bpf_prog_is_dev_bound(prog->aux)) { 1302 err = bpf_prog_offload_init(prog, attr); 1303 if (err) 1304 goto free_prog; 1305 } 1306 1307 /* find program type: socket_filter vs tracing_filter */ 1308 err = find_prog_type(type, prog); 1309 if (err < 0) 1310 goto free_prog; 1311 1312 prog->aux->load_time = ktime_get_boot_ns(); 1313 err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name); 1314 if (err) 1315 goto free_prog; 1316 1317 /* run eBPF verifier */ 1318 err = bpf_check(&prog, attr); 1319 if (err < 0) 1320 goto free_used_maps; 1321 1322 /* eBPF program is ready to be JITed */ 1323 if (!prog->bpf_func) 1324 prog = bpf_prog_select_runtime(prog, &err); 1325 if (err < 0) 1326 goto free_used_maps; 1327 1328 err = bpf_prog_alloc_id(prog); 1329 if (err) 1330 goto free_used_maps; 1331 1332 err = bpf_prog_new_fd(prog); 1333 if (err < 0) { 1334 /* failed to allocate fd. 1335 * bpf_prog_put() is needed because the above 1336 * bpf_prog_alloc_id() has published the prog 1337 * to the userspace and the userspace may 1338 * have refcnt-ed it through BPF_PROG_GET_FD_BY_ID. 1339 */ 1340 bpf_prog_put(prog); 1341 return err; 1342 } 1343 1344 bpf_prog_kallsyms_add(prog); 1345 trace_bpf_prog_load(prog, err); 1346 return err; 1347 1348 free_used_maps: 1349 free_used_maps(prog->aux); 1350 free_prog: 1351 bpf_prog_uncharge_memlock(prog); 1352 free_prog_sec: 1353 security_bpf_prog_free(prog->aux); 1354 free_prog_nouncharge: 1355 bpf_prog_free(prog); 1356 return err; 1357 } 1358 1359 #define BPF_OBJ_LAST_FIELD file_flags 1360 1361 static int bpf_obj_pin(const union bpf_attr *attr) 1362 { 1363 if (CHECK_ATTR(BPF_OBJ) || attr->file_flags != 0) 1364 return -EINVAL; 1365 1366 return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname)); 1367 } 1368 1369 static int bpf_obj_get(const union bpf_attr *attr) 1370 { 1371 if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0 || 1372 attr->file_flags & ~BPF_OBJ_FLAG_MASK) 1373 return -EINVAL; 1374 1375 return bpf_obj_get_user(u64_to_user_ptr(attr->pathname), 1376 attr->file_flags); 1377 } 1378 1379 struct bpf_raw_tracepoint { 1380 struct bpf_raw_event_map *btp; 1381 struct bpf_prog *prog; 1382 }; 1383 1384 static int bpf_raw_tracepoint_release(struct inode *inode, struct file *filp) 1385 { 1386 struct bpf_raw_tracepoint *raw_tp = filp->private_data; 1387 1388 if (raw_tp->prog) { 1389 bpf_probe_unregister(raw_tp->btp, raw_tp->prog); 1390 bpf_prog_put(raw_tp->prog); 1391 } 1392 kfree(raw_tp); 1393 return 0; 1394 } 1395 1396 static const struct file_operations bpf_raw_tp_fops = { 1397 .release = bpf_raw_tracepoint_release, 1398 .read = bpf_dummy_read, 1399 .write = bpf_dummy_write, 1400 }; 1401 1402 #define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd 1403 1404 static int bpf_raw_tracepoint_open(const union bpf_attr *attr) 1405 { 1406 struct bpf_raw_tracepoint *raw_tp; 1407 struct bpf_raw_event_map *btp; 1408 struct bpf_prog *prog; 1409 char tp_name[128]; 1410 int tp_fd, err; 1411 1412 if (strncpy_from_user(tp_name, u64_to_user_ptr(attr->raw_tracepoint.name), 1413 sizeof(tp_name) - 1) < 0) 1414 return -EFAULT; 1415 tp_name[sizeof(tp_name) - 1] = 0; 1416 1417 btp = bpf_find_raw_tracepoint(tp_name); 1418 if (!btp) 1419 return -ENOENT; 1420 1421 raw_tp = kzalloc(sizeof(*raw_tp), GFP_USER); 1422 if (!raw_tp) 1423 return -ENOMEM; 1424 raw_tp->btp = btp; 1425 1426 prog = bpf_prog_get_type(attr->raw_tracepoint.prog_fd, 1427 BPF_PROG_TYPE_RAW_TRACEPOINT); 1428 if (IS_ERR(prog)) { 1429 err = PTR_ERR(prog); 1430 goto out_free_tp; 1431 } 1432 1433 err = bpf_probe_register(raw_tp->btp, prog); 1434 if (err) 1435 goto out_put_prog; 1436 1437 raw_tp->prog = prog; 1438 tp_fd = anon_inode_getfd("bpf-raw-tracepoint", &bpf_raw_tp_fops, raw_tp, 1439 O_CLOEXEC); 1440 if (tp_fd < 0) { 1441 bpf_probe_unregister(raw_tp->btp, prog); 1442 err = tp_fd; 1443 goto out_put_prog; 1444 } 1445 return tp_fd; 1446 1447 out_put_prog: 1448 bpf_prog_put(prog); 1449 out_free_tp: 1450 kfree(raw_tp); 1451 return err; 1452 } 1453 1454 #ifdef CONFIG_CGROUP_BPF 1455 1456 static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog, 1457 enum bpf_attach_type attach_type) 1458 { 1459 switch (prog->type) { 1460 case BPF_PROG_TYPE_CGROUP_SOCK: 1461 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 1462 return attach_type == prog->expected_attach_type ? 0 : -EINVAL; 1463 default: 1464 return 0; 1465 } 1466 } 1467 1468 #define BPF_PROG_ATTACH_LAST_FIELD attach_flags 1469 1470 static int sockmap_get_from_fd(const union bpf_attr *attr, 1471 int type, bool attach) 1472 { 1473 struct bpf_prog *prog = NULL; 1474 int ufd = attr->target_fd; 1475 struct bpf_map *map; 1476 struct fd f; 1477 int err; 1478 1479 f = fdget(ufd); 1480 map = __bpf_map_get(f); 1481 if (IS_ERR(map)) 1482 return PTR_ERR(map); 1483 1484 if (attach) { 1485 prog = bpf_prog_get_type(attr->attach_bpf_fd, type); 1486 if (IS_ERR(prog)) { 1487 fdput(f); 1488 return PTR_ERR(prog); 1489 } 1490 } 1491 1492 err = sock_map_prog(map, prog, attr->attach_type); 1493 if (err) { 1494 fdput(f); 1495 if (prog) 1496 bpf_prog_put(prog); 1497 return err; 1498 } 1499 1500 fdput(f); 1501 return 0; 1502 } 1503 1504 #define BPF_F_ATTACH_MASK \ 1505 (BPF_F_ALLOW_OVERRIDE | BPF_F_ALLOW_MULTI) 1506 1507 static int bpf_prog_attach(const union bpf_attr *attr) 1508 { 1509 enum bpf_prog_type ptype; 1510 struct bpf_prog *prog; 1511 struct cgroup *cgrp; 1512 int ret; 1513 1514 if (!capable(CAP_NET_ADMIN)) 1515 return -EPERM; 1516 1517 if (CHECK_ATTR(BPF_PROG_ATTACH)) 1518 return -EINVAL; 1519 1520 if (attr->attach_flags & ~BPF_F_ATTACH_MASK) 1521 return -EINVAL; 1522 1523 switch (attr->attach_type) { 1524 case BPF_CGROUP_INET_INGRESS: 1525 case BPF_CGROUP_INET_EGRESS: 1526 ptype = BPF_PROG_TYPE_CGROUP_SKB; 1527 break; 1528 case BPF_CGROUP_INET_SOCK_CREATE: 1529 case BPF_CGROUP_INET4_POST_BIND: 1530 case BPF_CGROUP_INET6_POST_BIND: 1531 ptype = BPF_PROG_TYPE_CGROUP_SOCK; 1532 break; 1533 case BPF_CGROUP_INET4_BIND: 1534 case BPF_CGROUP_INET6_BIND: 1535 case BPF_CGROUP_INET4_CONNECT: 1536 case BPF_CGROUP_INET6_CONNECT: 1537 ptype = BPF_PROG_TYPE_CGROUP_SOCK_ADDR; 1538 break; 1539 case BPF_CGROUP_SOCK_OPS: 1540 ptype = BPF_PROG_TYPE_SOCK_OPS; 1541 break; 1542 case BPF_CGROUP_DEVICE: 1543 ptype = BPF_PROG_TYPE_CGROUP_DEVICE; 1544 break; 1545 case BPF_SK_MSG_VERDICT: 1546 return sockmap_get_from_fd(attr, BPF_PROG_TYPE_SK_MSG, true); 1547 case BPF_SK_SKB_STREAM_PARSER: 1548 case BPF_SK_SKB_STREAM_VERDICT: 1549 return sockmap_get_from_fd(attr, BPF_PROG_TYPE_SK_SKB, true); 1550 default: 1551 return -EINVAL; 1552 } 1553 1554 prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); 1555 if (IS_ERR(prog)) 1556 return PTR_ERR(prog); 1557 1558 if (bpf_prog_attach_check_attach_type(prog, attr->attach_type)) { 1559 bpf_prog_put(prog); 1560 return -EINVAL; 1561 } 1562 1563 cgrp = cgroup_get_from_fd(attr->target_fd); 1564 if (IS_ERR(cgrp)) { 1565 bpf_prog_put(prog); 1566 return PTR_ERR(cgrp); 1567 } 1568 1569 ret = cgroup_bpf_attach(cgrp, prog, attr->attach_type, 1570 attr->attach_flags); 1571 if (ret) 1572 bpf_prog_put(prog); 1573 cgroup_put(cgrp); 1574 1575 return ret; 1576 } 1577 1578 #define BPF_PROG_DETACH_LAST_FIELD attach_type 1579 1580 static int bpf_prog_detach(const union bpf_attr *attr) 1581 { 1582 enum bpf_prog_type ptype; 1583 struct bpf_prog *prog; 1584 struct cgroup *cgrp; 1585 int ret; 1586 1587 if (!capable(CAP_NET_ADMIN)) 1588 return -EPERM; 1589 1590 if (CHECK_ATTR(BPF_PROG_DETACH)) 1591 return -EINVAL; 1592 1593 switch (attr->attach_type) { 1594 case BPF_CGROUP_INET_INGRESS: 1595 case BPF_CGROUP_INET_EGRESS: 1596 ptype = BPF_PROG_TYPE_CGROUP_SKB; 1597 break; 1598 case BPF_CGROUP_INET_SOCK_CREATE: 1599 case BPF_CGROUP_INET4_POST_BIND: 1600 case BPF_CGROUP_INET6_POST_BIND: 1601 ptype = BPF_PROG_TYPE_CGROUP_SOCK; 1602 break; 1603 case BPF_CGROUP_INET4_BIND: 1604 case BPF_CGROUP_INET6_BIND: 1605 case BPF_CGROUP_INET4_CONNECT: 1606 case BPF_CGROUP_INET6_CONNECT: 1607 ptype = BPF_PROG_TYPE_CGROUP_SOCK_ADDR; 1608 break; 1609 case BPF_CGROUP_SOCK_OPS: 1610 ptype = BPF_PROG_TYPE_SOCK_OPS; 1611 break; 1612 case BPF_CGROUP_DEVICE: 1613 ptype = BPF_PROG_TYPE_CGROUP_DEVICE; 1614 break; 1615 case BPF_SK_MSG_VERDICT: 1616 return sockmap_get_from_fd(attr, BPF_PROG_TYPE_SK_MSG, false); 1617 case BPF_SK_SKB_STREAM_PARSER: 1618 case BPF_SK_SKB_STREAM_VERDICT: 1619 return sockmap_get_from_fd(attr, BPF_PROG_TYPE_SK_SKB, false); 1620 default: 1621 return -EINVAL; 1622 } 1623 1624 cgrp = cgroup_get_from_fd(attr->target_fd); 1625 if (IS_ERR(cgrp)) 1626 return PTR_ERR(cgrp); 1627 1628 prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); 1629 if (IS_ERR(prog)) 1630 prog = NULL; 1631 1632 ret = cgroup_bpf_detach(cgrp, prog, attr->attach_type, 0); 1633 if (prog) 1634 bpf_prog_put(prog); 1635 cgroup_put(cgrp); 1636 return ret; 1637 } 1638 1639 #define BPF_PROG_QUERY_LAST_FIELD query.prog_cnt 1640 1641 static int bpf_prog_query(const union bpf_attr *attr, 1642 union bpf_attr __user *uattr) 1643 { 1644 struct cgroup *cgrp; 1645 int ret; 1646 1647 if (!capable(CAP_NET_ADMIN)) 1648 return -EPERM; 1649 if (CHECK_ATTR(BPF_PROG_QUERY)) 1650 return -EINVAL; 1651 if (attr->query.query_flags & ~BPF_F_QUERY_EFFECTIVE) 1652 return -EINVAL; 1653 1654 switch (attr->query.attach_type) { 1655 case BPF_CGROUP_INET_INGRESS: 1656 case BPF_CGROUP_INET_EGRESS: 1657 case BPF_CGROUP_INET_SOCK_CREATE: 1658 case BPF_CGROUP_INET4_BIND: 1659 case BPF_CGROUP_INET6_BIND: 1660 case BPF_CGROUP_INET4_POST_BIND: 1661 case BPF_CGROUP_INET6_POST_BIND: 1662 case BPF_CGROUP_INET4_CONNECT: 1663 case BPF_CGROUP_INET6_CONNECT: 1664 case BPF_CGROUP_SOCK_OPS: 1665 case BPF_CGROUP_DEVICE: 1666 break; 1667 default: 1668 return -EINVAL; 1669 } 1670 cgrp = cgroup_get_from_fd(attr->query.target_fd); 1671 if (IS_ERR(cgrp)) 1672 return PTR_ERR(cgrp); 1673 ret = cgroup_bpf_query(cgrp, attr, uattr); 1674 cgroup_put(cgrp); 1675 return ret; 1676 } 1677 #endif /* CONFIG_CGROUP_BPF */ 1678 1679 #define BPF_PROG_TEST_RUN_LAST_FIELD test.duration 1680 1681 static int bpf_prog_test_run(const union bpf_attr *attr, 1682 union bpf_attr __user *uattr) 1683 { 1684 struct bpf_prog *prog; 1685 int ret = -ENOTSUPP; 1686 1687 if (!capable(CAP_SYS_ADMIN)) 1688 return -EPERM; 1689 if (CHECK_ATTR(BPF_PROG_TEST_RUN)) 1690 return -EINVAL; 1691 1692 prog = bpf_prog_get(attr->test.prog_fd); 1693 if (IS_ERR(prog)) 1694 return PTR_ERR(prog); 1695 1696 if (prog->aux->ops->test_run) 1697 ret = prog->aux->ops->test_run(prog, attr, uattr); 1698 1699 bpf_prog_put(prog); 1700 return ret; 1701 } 1702 1703 #define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id 1704 1705 static int bpf_obj_get_next_id(const union bpf_attr *attr, 1706 union bpf_attr __user *uattr, 1707 struct idr *idr, 1708 spinlock_t *lock) 1709 { 1710 u32 next_id = attr->start_id; 1711 int err = 0; 1712 1713 if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX) 1714 return -EINVAL; 1715 1716 if (!capable(CAP_SYS_ADMIN)) 1717 return -EPERM; 1718 1719 next_id++; 1720 spin_lock_bh(lock); 1721 if (!idr_get_next(idr, &next_id)) 1722 err = -ENOENT; 1723 spin_unlock_bh(lock); 1724 1725 if (!err) 1726 err = put_user(next_id, &uattr->next_id); 1727 1728 return err; 1729 } 1730 1731 #define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id 1732 1733 static int bpf_prog_get_fd_by_id(const union bpf_attr *attr) 1734 { 1735 struct bpf_prog *prog; 1736 u32 id = attr->prog_id; 1737 int fd; 1738 1739 if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID)) 1740 return -EINVAL; 1741 1742 if (!capable(CAP_SYS_ADMIN)) 1743 return -EPERM; 1744 1745 spin_lock_bh(&prog_idr_lock); 1746 prog = idr_find(&prog_idr, id); 1747 if (prog) 1748 prog = bpf_prog_inc_not_zero(prog); 1749 else 1750 prog = ERR_PTR(-ENOENT); 1751 spin_unlock_bh(&prog_idr_lock); 1752 1753 if (IS_ERR(prog)) 1754 return PTR_ERR(prog); 1755 1756 fd = bpf_prog_new_fd(prog); 1757 if (fd < 0) 1758 bpf_prog_put(prog); 1759 1760 return fd; 1761 } 1762 1763 #define BPF_MAP_GET_FD_BY_ID_LAST_FIELD open_flags 1764 1765 static int bpf_map_get_fd_by_id(const union bpf_attr *attr) 1766 { 1767 struct bpf_map *map; 1768 u32 id = attr->map_id; 1769 int f_flags; 1770 int fd; 1771 1772 if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID) || 1773 attr->open_flags & ~BPF_OBJ_FLAG_MASK) 1774 return -EINVAL; 1775 1776 if (!capable(CAP_SYS_ADMIN)) 1777 return -EPERM; 1778 1779 f_flags = bpf_get_file_flag(attr->open_flags); 1780 if (f_flags < 0) 1781 return f_flags; 1782 1783 spin_lock_bh(&map_idr_lock); 1784 map = idr_find(&map_idr, id); 1785 if (map) 1786 map = bpf_map_inc_not_zero(map, true); 1787 else 1788 map = ERR_PTR(-ENOENT); 1789 spin_unlock_bh(&map_idr_lock); 1790 1791 if (IS_ERR(map)) 1792 return PTR_ERR(map); 1793 1794 fd = bpf_map_new_fd(map, f_flags); 1795 if (fd < 0) 1796 bpf_map_put(map); 1797 1798 return fd; 1799 } 1800 1801 static const struct bpf_map *bpf_map_from_imm(const struct bpf_prog *prog, 1802 unsigned long addr) 1803 { 1804 int i; 1805 1806 for (i = 0; i < prog->aux->used_map_cnt; i++) 1807 if (prog->aux->used_maps[i] == (void *)addr) 1808 return prog->aux->used_maps[i]; 1809 return NULL; 1810 } 1811 1812 static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog) 1813 { 1814 const struct bpf_map *map; 1815 struct bpf_insn *insns; 1816 u64 imm; 1817 int i; 1818 1819 insns = kmemdup(prog->insnsi, bpf_prog_insn_size(prog), 1820 GFP_USER); 1821 if (!insns) 1822 return insns; 1823 1824 for (i = 0; i < prog->len; i++) { 1825 if (insns[i].code == (BPF_JMP | BPF_TAIL_CALL)) { 1826 insns[i].code = BPF_JMP | BPF_CALL; 1827 insns[i].imm = BPF_FUNC_tail_call; 1828 /* fall-through */ 1829 } 1830 if (insns[i].code == (BPF_JMP | BPF_CALL) || 1831 insns[i].code == (BPF_JMP | BPF_CALL_ARGS)) { 1832 if (insns[i].code == (BPF_JMP | BPF_CALL_ARGS)) 1833 insns[i].code = BPF_JMP | BPF_CALL; 1834 if (!bpf_dump_raw_ok()) 1835 insns[i].imm = 0; 1836 continue; 1837 } 1838 1839 if (insns[i].code != (BPF_LD | BPF_IMM | BPF_DW)) 1840 continue; 1841 1842 imm = ((u64)insns[i + 1].imm << 32) | (u32)insns[i].imm; 1843 map = bpf_map_from_imm(prog, imm); 1844 if (map) { 1845 insns[i].src_reg = BPF_PSEUDO_MAP_FD; 1846 insns[i].imm = map->id; 1847 insns[i + 1].imm = 0; 1848 continue; 1849 } 1850 1851 if (!bpf_dump_raw_ok() && 1852 imm == (unsigned long)prog->aux) { 1853 insns[i].imm = 0; 1854 insns[i + 1].imm = 0; 1855 continue; 1856 } 1857 } 1858 1859 return insns; 1860 } 1861 1862 static int bpf_prog_get_info_by_fd(struct bpf_prog *prog, 1863 const union bpf_attr *attr, 1864 union bpf_attr __user *uattr) 1865 { 1866 struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info); 1867 struct bpf_prog_info info = {}; 1868 u32 info_len = attr->info.info_len; 1869 char __user *uinsns; 1870 u32 ulen; 1871 int err; 1872 1873 err = check_uarg_tail_zero(uinfo, sizeof(info), info_len); 1874 if (err) 1875 return err; 1876 info_len = min_t(u32, sizeof(info), info_len); 1877 1878 if (copy_from_user(&info, uinfo, info_len)) 1879 return -EFAULT; 1880 1881 info.type = prog->type; 1882 info.id = prog->aux->id; 1883 info.load_time = prog->aux->load_time; 1884 info.created_by_uid = from_kuid_munged(current_user_ns(), 1885 prog->aux->user->uid); 1886 1887 memcpy(info.tag, prog->tag, sizeof(prog->tag)); 1888 memcpy(info.name, prog->aux->name, sizeof(prog->aux->name)); 1889 1890 ulen = info.nr_map_ids; 1891 info.nr_map_ids = prog->aux->used_map_cnt; 1892 ulen = min_t(u32, info.nr_map_ids, ulen); 1893 if (ulen) { 1894 u32 __user *user_map_ids = u64_to_user_ptr(info.map_ids); 1895 u32 i; 1896 1897 for (i = 0; i < ulen; i++) 1898 if (put_user(prog->aux->used_maps[i]->id, 1899 &user_map_ids[i])) 1900 return -EFAULT; 1901 } 1902 1903 if (!capable(CAP_SYS_ADMIN)) { 1904 info.jited_prog_len = 0; 1905 info.xlated_prog_len = 0; 1906 goto done; 1907 } 1908 1909 ulen = info.xlated_prog_len; 1910 info.xlated_prog_len = bpf_prog_insn_size(prog); 1911 if (info.xlated_prog_len && ulen) { 1912 struct bpf_insn *insns_sanitized; 1913 bool fault; 1914 1915 if (prog->blinded && !bpf_dump_raw_ok()) { 1916 info.xlated_prog_insns = 0; 1917 goto done; 1918 } 1919 insns_sanitized = bpf_insn_prepare_dump(prog); 1920 if (!insns_sanitized) 1921 return -ENOMEM; 1922 uinsns = u64_to_user_ptr(info.xlated_prog_insns); 1923 ulen = min_t(u32, info.xlated_prog_len, ulen); 1924 fault = copy_to_user(uinsns, insns_sanitized, ulen); 1925 kfree(insns_sanitized); 1926 if (fault) 1927 return -EFAULT; 1928 } 1929 1930 if (bpf_prog_is_dev_bound(prog->aux)) { 1931 err = bpf_prog_offload_info_fill(&info, prog); 1932 if (err) 1933 return err; 1934 goto done; 1935 } 1936 1937 /* NOTE: the following code is supposed to be skipped for offload. 1938 * bpf_prog_offload_info_fill() is the place to fill similar fields 1939 * for offload. 1940 */ 1941 ulen = info.jited_prog_len; 1942 info.jited_prog_len = prog->jited_len; 1943 if (info.jited_prog_len && ulen) { 1944 if (bpf_dump_raw_ok()) { 1945 uinsns = u64_to_user_ptr(info.jited_prog_insns); 1946 ulen = min_t(u32, info.jited_prog_len, ulen); 1947 if (copy_to_user(uinsns, prog->bpf_func, ulen)) 1948 return -EFAULT; 1949 } else { 1950 info.jited_prog_insns = 0; 1951 } 1952 } 1953 1954 done: 1955 if (copy_to_user(uinfo, &info, info_len) || 1956 put_user(info_len, &uattr->info.info_len)) 1957 return -EFAULT; 1958 1959 return 0; 1960 } 1961 1962 static int bpf_map_get_info_by_fd(struct bpf_map *map, 1963 const union bpf_attr *attr, 1964 union bpf_attr __user *uattr) 1965 { 1966 struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info); 1967 struct bpf_map_info info = {}; 1968 u32 info_len = attr->info.info_len; 1969 int err; 1970 1971 err = check_uarg_tail_zero(uinfo, sizeof(info), info_len); 1972 if (err) 1973 return err; 1974 info_len = min_t(u32, sizeof(info), info_len); 1975 1976 info.type = map->map_type; 1977 info.id = map->id; 1978 info.key_size = map->key_size; 1979 info.value_size = map->value_size; 1980 info.max_entries = map->max_entries; 1981 info.map_flags = map->map_flags; 1982 memcpy(info.name, map->name, sizeof(map->name)); 1983 1984 if (bpf_map_is_dev_bound(map)) { 1985 err = bpf_map_offload_info_fill(&info, map); 1986 if (err) 1987 return err; 1988 } 1989 1990 if (copy_to_user(uinfo, &info, info_len) || 1991 put_user(info_len, &uattr->info.info_len)) 1992 return -EFAULT; 1993 1994 return 0; 1995 } 1996 1997 #define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info 1998 1999 static int bpf_obj_get_info_by_fd(const union bpf_attr *attr, 2000 union bpf_attr __user *uattr) 2001 { 2002 int ufd = attr->info.bpf_fd; 2003 struct fd f; 2004 int err; 2005 2006 if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD)) 2007 return -EINVAL; 2008 2009 f = fdget(ufd); 2010 if (!f.file) 2011 return -EBADFD; 2012 2013 if (f.file->f_op == &bpf_prog_fops) 2014 err = bpf_prog_get_info_by_fd(f.file->private_data, attr, 2015 uattr); 2016 else if (f.file->f_op == &bpf_map_fops) 2017 err = bpf_map_get_info_by_fd(f.file->private_data, attr, 2018 uattr); 2019 else 2020 err = -EINVAL; 2021 2022 fdput(f); 2023 return err; 2024 } 2025 2026 SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size) 2027 { 2028 union bpf_attr attr = {}; 2029 int err; 2030 2031 if (sysctl_unprivileged_bpf_disabled && !capable(CAP_SYS_ADMIN)) 2032 return -EPERM; 2033 2034 err = check_uarg_tail_zero(uattr, sizeof(attr), size); 2035 if (err) 2036 return err; 2037 size = min_t(u32, size, sizeof(attr)); 2038 2039 /* copy attributes from user space, may be less than sizeof(bpf_attr) */ 2040 if (copy_from_user(&attr, uattr, size) != 0) 2041 return -EFAULT; 2042 2043 err = security_bpf(cmd, &attr, size); 2044 if (err < 0) 2045 return err; 2046 2047 switch (cmd) { 2048 case BPF_MAP_CREATE: 2049 err = map_create(&attr); 2050 break; 2051 case BPF_MAP_LOOKUP_ELEM: 2052 err = map_lookup_elem(&attr); 2053 break; 2054 case BPF_MAP_UPDATE_ELEM: 2055 err = map_update_elem(&attr); 2056 break; 2057 case BPF_MAP_DELETE_ELEM: 2058 err = map_delete_elem(&attr); 2059 break; 2060 case BPF_MAP_GET_NEXT_KEY: 2061 err = map_get_next_key(&attr); 2062 break; 2063 case BPF_PROG_LOAD: 2064 err = bpf_prog_load(&attr); 2065 break; 2066 case BPF_OBJ_PIN: 2067 err = bpf_obj_pin(&attr); 2068 break; 2069 case BPF_OBJ_GET: 2070 err = bpf_obj_get(&attr); 2071 break; 2072 #ifdef CONFIG_CGROUP_BPF 2073 case BPF_PROG_ATTACH: 2074 err = bpf_prog_attach(&attr); 2075 break; 2076 case BPF_PROG_DETACH: 2077 err = bpf_prog_detach(&attr); 2078 break; 2079 case BPF_PROG_QUERY: 2080 err = bpf_prog_query(&attr, uattr); 2081 break; 2082 #endif 2083 case BPF_PROG_TEST_RUN: 2084 err = bpf_prog_test_run(&attr, uattr); 2085 break; 2086 case BPF_PROG_GET_NEXT_ID: 2087 err = bpf_obj_get_next_id(&attr, uattr, 2088 &prog_idr, &prog_idr_lock); 2089 break; 2090 case BPF_MAP_GET_NEXT_ID: 2091 err = bpf_obj_get_next_id(&attr, uattr, 2092 &map_idr, &map_idr_lock); 2093 break; 2094 case BPF_PROG_GET_FD_BY_ID: 2095 err = bpf_prog_get_fd_by_id(&attr); 2096 break; 2097 case BPF_MAP_GET_FD_BY_ID: 2098 err = bpf_map_get_fd_by_id(&attr); 2099 break; 2100 case BPF_OBJ_GET_INFO_BY_FD: 2101 err = bpf_obj_get_info_by_fd(&attr, uattr); 2102 break; 2103 case BPF_RAW_TRACEPOINT_OPEN: 2104 err = bpf_raw_tracepoint_open(&attr); 2105 break; 2106 default: 2107 err = -EINVAL; 2108 break; 2109 } 2110 2111 return err; 2112 } 2113