1 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 2 * 3 * This program is free software; you can redistribute it and/or 4 * modify it under the terms of version 2 of the GNU General Public 5 * License as published by the Free Software Foundation. 6 * 7 * This program is distributed in the hope that it will be useful, but 8 * WITHOUT ANY WARRANTY; without even the implied warranty of 9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 10 * General Public License for more details. 11 */ 12 #include <linux/bpf.h> 13 #include <linux/bpf_trace.h> 14 #include <linux/syscalls.h> 15 #include <linux/slab.h> 16 #include <linux/sched/signal.h> 17 #include <linux/vmalloc.h> 18 #include <linux/mmzone.h> 19 #include <linux/anon_inodes.h> 20 #include <linux/file.h> 21 #include <linux/license.h> 22 #include <linux/filter.h> 23 #include <linux/version.h> 24 #include <linux/kernel.h> 25 #include <linux/idr.h> 26 27 #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY || \ 28 (map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \ 29 (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \ 30 (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) 31 #define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) 32 #define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_HASH(map)) 33 34 DEFINE_PER_CPU(int, bpf_prog_active); 35 static DEFINE_IDR(prog_idr); 36 static DEFINE_SPINLOCK(prog_idr_lock); 37 static DEFINE_IDR(map_idr); 38 static DEFINE_SPINLOCK(map_idr_lock); 39 40 int sysctl_unprivileged_bpf_disabled __read_mostly; 41 42 static const struct bpf_map_ops * const bpf_map_types[] = { 43 #define BPF_PROG_TYPE(_id, _ops) 44 #define BPF_MAP_TYPE(_id, _ops) \ 45 [_id] = &_ops, 46 #include <linux/bpf_types.h> 47 #undef BPF_PROG_TYPE 48 #undef BPF_MAP_TYPE 49 }; 50 51 /* 52 * If we're handed a bigger struct than we know of, ensure all the unknown bits 53 * are 0 - i.e. new user-space does not rely on any kernel feature extensions 54 * we don't know about yet. 55 * 56 * There is a ToCToU between this function call and the following 57 * copy_from_user() call. However, this is not a concern since this function is 58 * meant to be a future-proofing of bits. 59 */ 60 static int check_uarg_tail_zero(void __user *uaddr, 61 size_t expected_size, 62 size_t actual_size) 63 { 64 unsigned char __user *addr; 65 unsigned char __user *end; 66 unsigned char val; 67 int err; 68 69 if (unlikely(actual_size > PAGE_SIZE)) /* silly large */ 70 return -E2BIG; 71 72 if (unlikely(!access_ok(VERIFY_READ, uaddr, actual_size))) 73 return -EFAULT; 74 75 if (actual_size <= expected_size) 76 return 0; 77 78 addr = uaddr + expected_size; 79 end = uaddr + actual_size; 80 81 for (; addr < end; addr++) { 82 err = get_user(val, addr); 83 if (err) 84 return err; 85 if (val) 86 return -E2BIG; 87 } 88 89 return 0; 90 } 91 92 static struct bpf_map *find_and_alloc_map(union bpf_attr *attr) 93 { 94 struct bpf_map *map; 95 96 if (attr->map_type >= ARRAY_SIZE(bpf_map_types) || 97 !bpf_map_types[attr->map_type]) 98 return ERR_PTR(-EINVAL); 99 100 map = bpf_map_types[attr->map_type]->map_alloc(attr); 101 if (IS_ERR(map)) 102 return map; 103 map->ops = bpf_map_types[attr->map_type]; 104 map->map_type = attr->map_type; 105 return map; 106 } 107 108 void *bpf_map_area_alloc(size_t size, int numa_node) 109 { 110 /* We definitely need __GFP_NORETRY, so OOM killer doesn't 111 * trigger under memory pressure as we really just want to 112 * fail instead. 113 */ 114 const gfp_t flags = __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO; 115 void *area; 116 117 if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) { 118 area = kmalloc_node(size, GFP_USER | flags, numa_node); 119 if (area != NULL) 120 return area; 121 } 122 123 return __vmalloc_node_flags_caller(size, numa_node, GFP_KERNEL | flags, 124 __builtin_return_address(0)); 125 } 126 127 void bpf_map_area_free(void *area) 128 { 129 kvfree(area); 130 } 131 132 int bpf_map_precharge_memlock(u32 pages) 133 { 134 struct user_struct *user = get_current_user(); 135 unsigned long memlock_limit, cur; 136 137 memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 138 cur = atomic_long_read(&user->locked_vm); 139 free_uid(user); 140 if (cur + pages > memlock_limit) 141 return -EPERM; 142 return 0; 143 } 144 145 static int bpf_map_charge_memlock(struct bpf_map *map) 146 { 147 struct user_struct *user = get_current_user(); 148 unsigned long memlock_limit; 149 150 memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 151 152 atomic_long_add(map->pages, &user->locked_vm); 153 154 if (atomic_long_read(&user->locked_vm) > memlock_limit) { 155 atomic_long_sub(map->pages, &user->locked_vm); 156 free_uid(user); 157 return -EPERM; 158 } 159 map->user = user; 160 return 0; 161 } 162 163 static void bpf_map_uncharge_memlock(struct bpf_map *map) 164 { 165 struct user_struct *user = map->user; 166 167 atomic_long_sub(map->pages, &user->locked_vm); 168 free_uid(user); 169 } 170 171 static int bpf_map_alloc_id(struct bpf_map *map) 172 { 173 int id; 174 175 spin_lock_bh(&map_idr_lock); 176 id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC); 177 if (id > 0) 178 map->id = id; 179 spin_unlock_bh(&map_idr_lock); 180 181 if (WARN_ON_ONCE(!id)) 182 return -ENOSPC; 183 184 return id > 0 ? 0 : id; 185 } 186 187 static void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock) 188 { 189 if (do_idr_lock) 190 spin_lock_bh(&map_idr_lock); 191 else 192 __acquire(&map_idr_lock); 193 194 idr_remove(&map_idr, map->id); 195 196 if (do_idr_lock) 197 spin_unlock_bh(&map_idr_lock); 198 else 199 __release(&map_idr_lock); 200 } 201 202 /* called from workqueue */ 203 static void bpf_map_free_deferred(struct work_struct *work) 204 { 205 struct bpf_map *map = container_of(work, struct bpf_map, work); 206 207 bpf_map_uncharge_memlock(map); 208 /* implementation dependent freeing */ 209 map->ops->map_free(map); 210 } 211 212 static void bpf_map_put_uref(struct bpf_map *map) 213 { 214 if (atomic_dec_and_test(&map->usercnt)) { 215 if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) 216 bpf_fd_array_map_clear(map); 217 } 218 } 219 220 /* decrement map refcnt and schedule it for freeing via workqueue 221 * (unrelying map implementation ops->map_free() might sleep) 222 */ 223 static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock) 224 { 225 if (atomic_dec_and_test(&map->refcnt)) { 226 /* bpf_map_free_id() must be called first */ 227 bpf_map_free_id(map, do_idr_lock); 228 INIT_WORK(&map->work, bpf_map_free_deferred); 229 schedule_work(&map->work); 230 } 231 } 232 233 void bpf_map_put(struct bpf_map *map) 234 { 235 __bpf_map_put(map, true); 236 } 237 238 void bpf_map_put_with_uref(struct bpf_map *map) 239 { 240 bpf_map_put_uref(map); 241 bpf_map_put(map); 242 } 243 244 static int bpf_map_release(struct inode *inode, struct file *filp) 245 { 246 struct bpf_map *map = filp->private_data; 247 248 if (map->ops->map_release) 249 map->ops->map_release(map, filp); 250 251 bpf_map_put_with_uref(map); 252 return 0; 253 } 254 255 #ifdef CONFIG_PROC_FS 256 static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp) 257 { 258 const struct bpf_map *map = filp->private_data; 259 const struct bpf_array *array; 260 u32 owner_prog_type = 0; 261 u32 owner_jited = 0; 262 263 if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) { 264 array = container_of(map, struct bpf_array, map); 265 owner_prog_type = array->owner_prog_type; 266 owner_jited = array->owner_jited; 267 } 268 269 seq_printf(m, 270 "map_type:\t%u\n" 271 "key_size:\t%u\n" 272 "value_size:\t%u\n" 273 "max_entries:\t%u\n" 274 "map_flags:\t%#x\n" 275 "memlock:\t%llu\n", 276 map->map_type, 277 map->key_size, 278 map->value_size, 279 map->max_entries, 280 map->map_flags, 281 map->pages * 1ULL << PAGE_SHIFT); 282 283 if (owner_prog_type) { 284 seq_printf(m, "owner_prog_type:\t%u\n", 285 owner_prog_type); 286 seq_printf(m, "owner_jited:\t%u\n", 287 owner_jited); 288 } 289 } 290 #endif 291 292 static const struct file_operations bpf_map_fops = { 293 #ifdef CONFIG_PROC_FS 294 .show_fdinfo = bpf_map_show_fdinfo, 295 #endif 296 .release = bpf_map_release, 297 }; 298 299 int bpf_map_new_fd(struct bpf_map *map) 300 { 301 return anon_inode_getfd("bpf-map", &bpf_map_fops, map, 302 O_RDWR | O_CLOEXEC); 303 } 304 305 /* helper macro to check that unused fields 'union bpf_attr' are zero */ 306 #define CHECK_ATTR(CMD) \ 307 memchr_inv((void *) &attr->CMD##_LAST_FIELD + \ 308 sizeof(attr->CMD##_LAST_FIELD), 0, \ 309 sizeof(*attr) - \ 310 offsetof(union bpf_attr, CMD##_LAST_FIELD) - \ 311 sizeof(attr->CMD##_LAST_FIELD)) != NULL 312 313 #define BPF_MAP_CREATE_LAST_FIELD numa_node 314 /* called via syscall */ 315 static int map_create(union bpf_attr *attr) 316 { 317 int numa_node = bpf_map_attr_numa_node(attr); 318 struct bpf_map *map; 319 int err; 320 321 err = CHECK_ATTR(BPF_MAP_CREATE); 322 if (err) 323 return -EINVAL; 324 325 if (numa_node != NUMA_NO_NODE && 326 ((unsigned int)numa_node >= nr_node_ids || 327 !node_online(numa_node))) 328 return -EINVAL; 329 330 /* find map type and init map: hashtable vs rbtree vs bloom vs ... */ 331 map = find_and_alloc_map(attr); 332 if (IS_ERR(map)) 333 return PTR_ERR(map); 334 335 atomic_set(&map->refcnt, 1); 336 atomic_set(&map->usercnt, 1); 337 338 err = bpf_map_charge_memlock(map); 339 if (err) 340 goto free_map_nouncharge; 341 342 err = bpf_map_alloc_id(map); 343 if (err) 344 goto free_map; 345 346 err = bpf_map_new_fd(map); 347 if (err < 0) { 348 /* failed to allocate fd. 349 * bpf_map_put() is needed because the above 350 * bpf_map_alloc_id() has published the map 351 * to the userspace and the userspace may 352 * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID. 353 */ 354 bpf_map_put(map); 355 return err; 356 } 357 358 trace_bpf_map_create(map, err); 359 return err; 360 361 free_map: 362 bpf_map_uncharge_memlock(map); 363 free_map_nouncharge: 364 map->ops->map_free(map); 365 return err; 366 } 367 368 /* if error is returned, fd is released. 369 * On success caller should complete fd access with matching fdput() 370 */ 371 struct bpf_map *__bpf_map_get(struct fd f) 372 { 373 if (!f.file) 374 return ERR_PTR(-EBADF); 375 if (f.file->f_op != &bpf_map_fops) { 376 fdput(f); 377 return ERR_PTR(-EINVAL); 378 } 379 380 return f.file->private_data; 381 } 382 383 /* prog's and map's refcnt limit */ 384 #define BPF_MAX_REFCNT 32768 385 386 struct bpf_map *bpf_map_inc(struct bpf_map *map, bool uref) 387 { 388 if (atomic_inc_return(&map->refcnt) > BPF_MAX_REFCNT) { 389 atomic_dec(&map->refcnt); 390 return ERR_PTR(-EBUSY); 391 } 392 if (uref) 393 atomic_inc(&map->usercnt); 394 return map; 395 } 396 397 struct bpf_map *bpf_map_get_with_uref(u32 ufd) 398 { 399 struct fd f = fdget(ufd); 400 struct bpf_map *map; 401 402 map = __bpf_map_get(f); 403 if (IS_ERR(map)) 404 return map; 405 406 map = bpf_map_inc(map, true); 407 fdput(f); 408 409 return map; 410 } 411 412 /* map_idr_lock should have been held */ 413 static struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map, 414 bool uref) 415 { 416 int refold; 417 418 refold = __atomic_add_unless(&map->refcnt, 1, 0); 419 420 if (refold >= BPF_MAX_REFCNT) { 421 __bpf_map_put(map, false); 422 return ERR_PTR(-EBUSY); 423 } 424 425 if (!refold) 426 return ERR_PTR(-ENOENT); 427 428 if (uref) 429 atomic_inc(&map->usercnt); 430 431 return map; 432 } 433 434 int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value) 435 { 436 return -ENOTSUPP; 437 } 438 439 /* last field in 'union bpf_attr' used by this command */ 440 #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD value 441 442 static int map_lookup_elem(union bpf_attr *attr) 443 { 444 void __user *ukey = u64_to_user_ptr(attr->key); 445 void __user *uvalue = u64_to_user_ptr(attr->value); 446 int ufd = attr->map_fd; 447 struct bpf_map *map; 448 void *key, *value, *ptr; 449 u32 value_size; 450 struct fd f; 451 int err; 452 453 if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM)) 454 return -EINVAL; 455 456 f = fdget(ufd); 457 map = __bpf_map_get(f); 458 if (IS_ERR(map)) 459 return PTR_ERR(map); 460 461 key = memdup_user(ukey, map->key_size); 462 if (IS_ERR(key)) { 463 err = PTR_ERR(key); 464 goto err_put; 465 } 466 467 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 468 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 469 map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) 470 value_size = round_up(map->value_size, 8) * num_possible_cpus(); 471 else if (IS_FD_MAP(map)) 472 value_size = sizeof(u32); 473 else 474 value_size = map->value_size; 475 476 err = -ENOMEM; 477 value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 478 if (!value) 479 goto free_key; 480 481 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 482 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 483 err = bpf_percpu_hash_copy(map, key, value); 484 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 485 err = bpf_percpu_array_copy(map, key, value); 486 } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) { 487 err = bpf_stackmap_copy(map, key, value); 488 } else if (IS_FD_ARRAY(map)) { 489 err = bpf_fd_array_map_lookup_elem(map, key, value); 490 } else if (IS_FD_HASH(map)) { 491 err = bpf_fd_htab_map_lookup_elem(map, key, value); 492 } else { 493 rcu_read_lock(); 494 ptr = map->ops->map_lookup_elem(map, key); 495 if (ptr) 496 memcpy(value, ptr, value_size); 497 rcu_read_unlock(); 498 err = ptr ? 0 : -ENOENT; 499 } 500 501 if (err) 502 goto free_value; 503 504 err = -EFAULT; 505 if (copy_to_user(uvalue, value, value_size) != 0) 506 goto free_value; 507 508 trace_bpf_map_lookup_elem(map, ufd, key, value); 509 err = 0; 510 511 free_value: 512 kfree(value); 513 free_key: 514 kfree(key); 515 err_put: 516 fdput(f); 517 return err; 518 } 519 520 #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags 521 522 static int map_update_elem(union bpf_attr *attr) 523 { 524 void __user *ukey = u64_to_user_ptr(attr->key); 525 void __user *uvalue = u64_to_user_ptr(attr->value); 526 int ufd = attr->map_fd; 527 struct bpf_map *map; 528 void *key, *value; 529 u32 value_size; 530 struct fd f; 531 int err; 532 533 if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM)) 534 return -EINVAL; 535 536 f = fdget(ufd); 537 map = __bpf_map_get(f); 538 if (IS_ERR(map)) 539 return PTR_ERR(map); 540 541 key = memdup_user(ukey, map->key_size); 542 if (IS_ERR(key)) { 543 err = PTR_ERR(key); 544 goto err_put; 545 } 546 547 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 548 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 549 map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) 550 value_size = round_up(map->value_size, 8) * num_possible_cpus(); 551 else 552 value_size = map->value_size; 553 554 err = -ENOMEM; 555 value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 556 if (!value) 557 goto free_key; 558 559 err = -EFAULT; 560 if (copy_from_user(value, uvalue, value_size) != 0) 561 goto free_value; 562 563 /* must increment bpf_prog_active to avoid kprobe+bpf triggering from 564 * inside bpf map update or delete otherwise deadlocks are possible 565 */ 566 preempt_disable(); 567 __this_cpu_inc(bpf_prog_active); 568 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 569 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 570 err = bpf_percpu_hash_update(map, key, value, attr->flags); 571 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 572 err = bpf_percpu_array_update(map, key, value, attr->flags); 573 } else if (map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || 574 map->map_type == BPF_MAP_TYPE_PROG_ARRAY || 575 map->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || 576 map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) { 577 rcu_read_lock(); 578 err = bpf_fd_array_map_update_elem(map, f.file, key, value, 579 attr->flags); 580 rcu_read_unlock(); 581 } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) { 582 rcu_read_lock(); 583 err = bpf_fd_htab_map_update_elem(map, f.file, key, value, 584 attr->flags); 585 rcu_read_unlock(); 586 } else { 587 rcu_read_lock(); 588 err = map->ops->map_update_elem(map, key, value, attr->flags); 589 rcu_read_unlock(); 590 } 591 __this_cpu_dec(bpf_prog_active); 592 preempt_enable(); 593 594 if (!err) 595 trace_bpf_map_update_elem(map, ufd, key, value); 596 free_value: 597 kfree(value); 598 free_key: 599 kfree(key); 600 err_put: 601 fdput(f); 602 return err; 603 } 604 605 #define BPF_MAP_DELETE_ELEM_LAST_FIELD key 606 607 static int map_delete_elem(union bpf_attr *attr) 608 { 609 void __user *ukey = u64_to_user_ptr(attr->key); 610 int ufd = attr->map_fd; 611 struct bpf_map *map; 612 struct fd f; 613 void *key; 614 int err; 615 616 if (CHECK_ATTR(BPF_MAP_DELETE_ELEM)) 617 return -EINVAL; 618 619 f = fdget(ufd); 620 map = __bpf_map_get(f); 621 if (IS_ERR(map)) 622 return PTR_ERR(map); 623 624 key = memdup_user(ukey, map->key_size); 625 if (IS_ERR(key)) { 626 err = PTR_ERR(key); 627 goto err_put; 628 } 629 630 preempt_disable(); 631 __this_cpu_inc(bpf_prog_active); 632 rcu_read_lock(); 633 err = map->ops->map_delete_elem(map, key); 634 rcu_read_unlock(); 635 __this_cpu_dec(bpf_prog_active); 636 preempt_enable(); 637 638 if (!err) 639 trace_bpf_map_delete_elem(map, ufd, key); 640 kfree(key); 641 err_put: 642 fdput(f); 643 return err; 644 } 645 646 /* last field in 'union bpf_attr' used by this command */ 647 #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key 648 649 static int map_get_next_key(union bpf_attr *attr) 650 { 651 void __user *ukey = u64_to_user_ptr(attr->key); 652 void __user *unext_key = u64_to_user_ptr(attr->next_key); 653 int ufd = attr->map_fd; 654 struct bpf_map *map; 655 void *key, *next_key; 656 struct fd f; 657 int err; 658 659 if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY)) 660 return -EINVAL; 661 662 f = fdget(ufd); 663 map = __bpf_map_get(f); 664 if (IS_ERR(map)) 665 return PTR_ERR(map); 666 667 if (ukey) { 668 key = memdup_user(ukey, map->key_size); 669 if (IS_ERR(key)) { 670 err = PTR_ERR(key); 671 goto err_put; 672 } 673 } else { 674 key = NULL; 675 } 676 677 err = -ENOMEM; 678 next_key = kmalloc(map->key_size, GFP_USER); 679 if (!next_key) 680 goto free_key; 681 682 rcu_read_lock(); 683 err = map->ops->map_get_next_key(map, key, next_key); 684 rcu_read_unlock(); 685 if (err) 686 goto free_next_key; 687 688 err = -EFAULT; 689 if (copy_to_user(unext_key, next_key, map->key_size) != 0) 690 goto free_next_key; 691 692 trace_bpf_map_next_key(map, ufd, key, next_key); 693 err = 0; 694 695 free_next_key: 696 kfree(next_key); 697 free_key: 698 kfree(key); 699 err_put: 700 fdput(f); 701 return err; 702 } 703 704 static const struct bpf_verifier_ops * const bpf_prog_types[] = { 705 #define BPF_PROG_TYPE(_id, _ops) \ 706 [_id] = &_ops, 707 #define BPF_MAP_TYPE(_id, _ops) 708 #include <linux/bpf_types.h> 709 #undef BPF_PROG_TYPE 710 #undef BPF_MAP_TYPE 711 }; 712 713 static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog) 714 { 715 if (type >= ARRAY_SIZE(bpf_prog_types) || !bpf_prog_types[type]) 716 return -EINVAL; 717 718 prog->aux->ops = bpf_prog_types[type]; 719 prog->type = type; 720 return 0; 721 } 722 723 /* drop refcnt on maps used by eBPF program and free auxilary data */ 724 static void free_used_maps(struct bpf_prog_aux *aux) 725 { 726 int i; 727 728 for (i = 0; i < aux->used_map_cnt; i++) 729 bpf_map_put(aux->used_maps[i]); 730 731 kfree(aux->used_maps); 732 } 733 734 int __bpf_prog_charge(struct user_struct *user, u32 pages) 735 { 736 unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 737 unsigned long user_bufs; 738 739 if (user) { 740 user_bufs = atomic_long_add_return(pages, &user->locked_vm); 741 if (user_bufs > memlock_limit) { 742 atomic_long_sub(pages, &user->locked_vm); 743 return -EPERM; 744 } 745 } 746 747 return 0; 748 } 749 750 void __bpf_prog_uncharge(struct user_struct *user, u32 pages) 751 { 752 if (user) 753 atomic_long_sub(pages, &user->locked_vm); 754 } 755 756 static int bpf_prog_charge_memlock(struct bpf_prog *prog) 757 { 758 struct user_struct *user = get_current_user(); 759 int ret; 760 761 ret = __bpf_prog_charge(user, prog->pages); 762 if (ret) { 763 free_uid(user); 764 return ret; 765 } 766 767 prog->aux->user = user; 768 return 0; 769 } 770 771 static void bpf_prog_uncharge_memlock(struct bpf_prog *prog) 772 { 773 struct user_struct *user = prog->aux->user; 774 775 __bpf_prog_uncharge(user, prog->pages); 776 free_uid(user); 777 } 778 779 static int bpf_prog_alloc_id(struct bpf_prog *prog) 780 { 781 int id; 782 783 spin_lock_bh(&prog_idr_lock); 784 id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC); 785 if (id > 0) 786 prog->aux->id = id; 787 spin_unlock_bh(&prog_idr_lock); 788 789 /* id is in [1, INT_MAX) */ 790 if (WARN_ON_ONCE(!id)) 791 return -ENOSPC; 792 793 return id > 0 ? 0 : id; 794 } 795 796 static void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock) 797 { 798 /* cBPF to eBPF migrations are currently not in the idr store. */ 799 if (!prog->aux->id) 800 return; 801 802 if (do_idr_lock) 803 spin_lock_bh(&prog_idr_lock); 804 else 805 __acquire(&prog_idr_lock); 806 807 idr_remove(&prog_idr, prog->aux->id); 808 809 if (do_idr_lock) 810 spin_unlock_bh(&prog_idr_lock); 811 else 812 __release(&prog_idr_lock); 813 } 814 815 static void __bpf_prog_put_rcu(struct rcu_head *rcu) 816 { 817 struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu); 818 819 free_used_maps(aux); 820 bpf_prog_uncharge_memlock(aux->prog); 821 bpf_prog_free(aux->prog); 822 } 823 824 static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock) 825 { 826 if (atomic_dec_and_test(&prog->aux->refcnt)) { 827 trace_bpf_prog_put_rcu(prog); 828 /* bpf_prog_free_id() must be called first */ 829 bpf_prog_free_id(prog, do_idr_lock); 830 bpf_prog_kallsyms_del(prog); 831 call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); 832 } 833 } 834 835 void bpf_prog_put(struct bpf_prog *prog) 836 { 837 __bpf_prog_put(prog, true); 838 } 839 EXPORT_SYMBOL_GPL(bpf_prog_put); 840 841 static int bpf_prog_release(struct inode *inode, struct file *filp) 842 { 843 struct bpf_prog *prog = filp->private_data; 844 845 bpf_prog_put(prog); 846 return 0; 847 } 848 849 #ifdef CONFIG_PROC_FS 850 static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp) 851 { 852 const struct bpf_prog *prog = filp->private_data; 853 char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; 854 855 bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); 856 seq_printf(m, 857 "prog_type:\t%u\n" 858 "prog_jited:\t%u\n" 859 "prog_tag:\t%s\n" 860 "memlock:\t%llu\n", 861 prog->type, 862 prog->jited, 863 prog_tag, 864 prog->pages * 1ULL << PAGE_SHIFT); 865 } 866 #endif 867 868 static const struct file_operations bpf_prog_fops = { 869 #ifdef CONFIG_PROC_FS 870 .show_fdinfo = bpf_prog_show_fdinfo, 871 #endif 872 .release = bpf_prog_release, 873 }; 874 875 int bpf_prog_new_fd(struct bpf_prog *prog) 876 { 877 return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog, 878 O_RDWR | O_CLOEXEC); 879 } 880 881 static struct bpf_prog *____bpf_prog_get(struct fd f) 882 { 883 if (!f.file) 884 return ERR_PTR(-EBADF); 885 if (f.file->f_op != &bpf_prog_fops) { 886 fdput(f); 887 return ERR_PTR(-EINVAL); 888 } 889 890 return f.file->private_data; 891 } 892 893 struct bpf_prog *bpf_prog_add(struct bpf_prog *prog, int i) 894 { 895 if (atomic_add_return(i, &prog->aux->refcnt) > BPF_MAX_REFCNT) { 896 atomic_sub(i, &prog->aux->refcnt); 897 return ERR_PTR(-EBUSY); 898 } 899 return prog; 900 } 901 EXPORT_SYMBOL_GPL(bpf_prog_add); 902 903 void bpf_prog_sub(struct bpf_prog *prog, int i) 904 { 905 /* Only to be used for undoing previous bpf_prog_add() in some 906 * error path. We still know that another entity in our call 907 * path holds a reference to the program, thus atomic_sub() can 908 * be safely used in such cases! 909 */ 910 WARN_ON(atomic_sub_return(i, &prog->aux->refcnt) == 0); 911 } 912 EXPORT_SYMBOL_GPL(bpf_prog_sub); 913 914 struct bpf_prog *bpf_prog_inc(struct bpf_prog *prog) 915 { 916 return bpf_prog_add(prog, 1); 917 } 918 EXPORT_SYMBOL_GPL(bpf_prog_inc); 919 920 /* prog_idr_lock should have been held */ 921 struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog) 922 { 923 int refold; 924 925 refold = __atomic_add_unless(&prog->aux->refcnt, 1, 0); 926 927 if (refold >= BPF_MAX_REFCNT) { 928 __bpf_prog_put(prog, false); 929 return ERR_PTR(-EBUSY); 930 } 931 932 if (!refold) 933 return ERR_PTR(-ENOENT); 934 935 return prog; 936 } 937 EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero); 938 939 static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *type) 940 { 941 struct fd f = fdget(ufd); 942 struct bpf_prog *prog; 943 944 prog = ____bpf_prog_get(f); 945 if (IS_ERR(prog)) 946 return prog; 947 if (type && prog->type != *type) { 948 prog = ERR_PTR(-EINVAL); 949 goto out; 950 } 951 952 prog = bpf_prog_inc(prog); 953 out: 954 fdput(f); 955 return prog; 956 } 957 958 struct bpf_prog *bpf_prog_get(u32 ufd) 959 { 960 return __bpf_prog_get(ufd, NULL); 961 } 962 963 struct bpf_prog *bpf_prog_get_type(u32 ufd, enum bpf_prog_type type) 964 { 965 struct bpf_prog *prog = __bpf_prog_get(ufd, &type); 966 967 if (!IS_ERR(prog)) 968 trace_bpf_prog_get_type(prog); 969 return prog; 970 } 971 EXPORT_SYMBOL_GPL(bpf_prog_get_type); 972 973 /* last field in 'union bpf_attr' used by this command */ 974 #define BPF_PROG_LOAD_LAST_FIELD prog_flags 975 976 static int bpf_prog_load(union bpf_attr *attr) 977 { 978 enum bpf_prog_type type = attr->prog_type; 979 struct bpf_prog *prog; 980 int err; 981 char license[128]; 982 bool is_gpl; 983 984 if (CHECK_ATTR(BPF_PROG_LOAD)) 985 return -EINVAL; 986 987 if (attr->prog_flags & ~BPF_F_STRICT_ALIGNMENT) 988 return -EINVAL; 989 990 /* copy eBPF program license from user space */ 991 if (strncpy_from_user(license, u64_to_user_ptr(attr->license), 992 sizeof(license) - 1) < 0) 993 return -EFAULT; 994 license[sizeof(license) - 1] = 0; 995 996 /* eBPF programs must be GPL compatible to use GPL-ed functions */ 997 is_gpl = license_is_gpl_compatible(license); 998 999 if (attr->insn_cnt == 0 || attr->insn_cnt > BPF_MAXINSNS) 1000 return -E2BIG; 1001 1002 if (type == BPF_PROG_TYPE_KPROBE && 1003 attr->kern_version != LINUX_VERSION_CODE) 1004 return -EINVAL; 1005 1006 if (type != BPF_PROG_TYPE_SOCKET_FILTER && 1007 type != BPF_PROG_TYPE_CGROUP_SKB && 1008 !capable(CAP_SYS_ADMIN)) 1009 return -EPERM; 1010 1011 /* plain bpf_prog allocation */ 1012 prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER); 1013 if (!prog) 1014 return -ENOMEM; 1015 1016 err = bpf_prog_charge_memlock(prog); 1017 if (err) 1018 goto free_prog_nouncharge; 1019 1020 prog->len = attr->insn_cnt; 1021 1022 err = -EFAULT; 1023 if (copy_from_user(prog->insns, u64_to_user_ptr(attr->insns), 1024 bpf_prog_insn_size(prog)) != 0) 1025 goto free_prog; 1026 1027 prog->orig_prog = NULL; 1028 prog->jited = 0; 1029 1030 atomic_set(&prog->aux->refcnt, 1); 1031 prog->gpl_compatible = is_gpl ? 1 : 0; 1032 1033 /* find program type: socket_filter vs tracing_filter */ 1034 err = find_prog_type(type, prog); 1035 if (err < 0) 1036 goto free_prog; 1037 1038 /* run eBPF verifier */ 1039 err = bpf_check(&prog, attr); 1040 if (err < 0) 1041 goto free_used_maps; 1042 1043 /* eBPF program is ready to be JITed */ 1044 prog = bpf_prog_select_runtime(prog, &err); 1045 if (err < 0) 1046 goto free_used_maps; 1047 1048 err = bpf_prog_alloc_id(prog); 1049 if (err) 1050 goto free_used_maps; 1051 1052 err = bpf_prog_new_fd(prog); 1053 if (err < 0) { 1054 /* failed to allocate fd. 1055 * bpf_prog_put() is needed because the above 1056 * bpf_prog_alloc_id() has published the prog 1057 * to the userspace and the userspace may 1058 * have refcnt-ed it through BPF_PROG_GET_FD_BY_ID. 1059 */ 1060 bpf_prog_put(prog); 1061 return err; 1062 } 1063 1064 bpf_prog_kallsyms_add(prog); 1065 trace_bpf_prog_load(prog, err); 1066 return err; 1067 1068 free_used_maps: 1069 free_used_maps(prog->aux); 1070 free_prog: 1071 bpf_prog_uncharge_memlock(prog); 1072 free_prog_nouncharge: 1073 bpf_prog_free(prog); 1074 return err; 1075 } 1076 1077 #define BPF_OBJ_LAST_FIELD bpf_fd 1078 1079 static int bpf_obj_pin(const union bpf_attr *attr) 1080 { 1081 if (CHECK_ATTR(BPF_OBJ)) 1082 return -EINVAL; 1083 1084 return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname)); 1085 } 1086 1087 static int bpf_obj_get(const union bpf_attr *attr) 1088 { 1089 if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0) 1090 return -EINVAL; 1091 1092 return bpf_obj_get_user(u64_to_user_ptr(attr->pathname)); 1093 } 1094 1095 #ifdef CONFIG_CGROUP_BPF 1096 1097 #define BPF_PROG_ATTACH_LAST_FIELD attach_flags 1098 1099 static int sockmap_get_from_fd(const union bpf_attr *attr, bool attach) 1100 { 1101 struct bpf_prog *prog = NULL; 1102 int ufd = attr->target_fd; 1103 struct bpf_map *map; 1104 struct fd f; 1105 int err; 1106 1107 f = fdget(ufd); 1108 map = __bpf_map_get(f); 1109 if (IS_ERR(map)) 1110 return PTR_ERR(map); 1111 1112 if (attach) { 1113 prog = bpf_prog_get_type(attr->attach_bpf_fd, 1114 BPF_PROG_TYPE_SK_SKB); 1115 if (IS_ERR(prog)) { 1116 fdput(f); 1117 return PTR_ERR(prog); 1118 } 1119 } 1120 1121 err = sock_map_prog(map, prog, attr->attach_type); 1122 if (err) { 1123 fdput(f); 1124 if (prog) 1125 bpf_prog_put(prog); 1126 return err; 1127 } 1128 1129 fdput(f); 1130 return 0; 1131 } 1132 1133 static int bpf_prog_attach(const union bpf_attr *attr) 1134 { 1135 enum bpf_prog_type ptype; 1136 struct bpf_prog *prog; 1137 struct cgroup *cgrp; 1138 int ret; 1139 1140 if (!capable(CAP_NET_ADMIN)) 1141 return -EPERM; 1142 1143 if (CHECK_ATTR(BPF_PROG_ATTACH)) 1144 return -EINVAL; 1145 1146 if (attr->attach_flags & ~BPF_F_ALLOW_OVERRIDE) 1147 return -EINVAL; 1148 1149 switch (attr->attach_type) { 1150 case BPF_CGROUP_INET_INGRESS: 1151 case BPF_CGROUP_INET_EGRESS: 1152 ptype = BPF_PROG_TYPE_CGROUP_SKB; 1153 break; 1154 case BPF_CGROUP_INET_SOCK_CREATE: 1155 ptype = BPF_PROG_TYPE_CGROUP_SOCK; 1156 break; 1157 case BPF_CGROUP_SOCK_OPS: 1158 ptype = BPF_PROG_TYPE_SOCK_OPS; 1159 break; 1160 case BPF_SK_SKB_STREAM_PARSER: 1161 case BPF_SK_SKB_STREAM_VERDICT: 1162 return sockmap_get_from_fd(attr, true); 1163 default: 1164 return -EINVAL; 1165 } 1166 1167 prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); 1168 if (IS_ERR(prog)) 1169 return PTR_ERR(prog); 1170 1171 cgrp = cgroup_get_from_fd(attr->target_fd); 1172 if (IS_ERR(cgrp)) { 1173 bpf_prog_put(prog); 1174 return PTR_ERR(cgrp); 1175 } 1176 1177 ret = cgroup_bpf_update(cgrp, prog, attr->attach_type, 1178 attr->attach_flags & BPF_F_ALLOW_OVERRIDE); 1179 if (ret) 1180 bpf_prog_put(prog); 1181 cgroup_put(cgrp); 1182 1183 return ret; 1184 } 1185 1186 #define BPF_PROG_DETACH_LAST_FIELD attach_type 1187 1188 static int bpf_prog_detach(const union bpf_attr *attr) 1189 { 1190 struct cgroup *cgrp; 1191 int ret; 1192 1193 if (!capable(CAP_NET_ADMIN)) 1194 return -EPERM; 1195 1196 if (CHECK_ATTR(BPF_PROG_DETACH)) 1197 return -EINVAL; 1198 1199 switch (attr->attach_type) { 1200 case BPF_CGROUP_INET_INGRESS: 1201 case BPF_CGROUP_INET_EGRESS: 1202 case BPF_CGROUP_INET_SOCK_CREATE: 1203 case BPF_CGROUP_SOCK_OPS: 1204 cgrp = cgroup_get_from_fd(attr->target_fd); 1205 if (IS_ERR(cgrp)) 1206 return PTR_ERR(cgrp); 1207 1208 ret = cgroup_bpf_update(cgrp, NULL, attr->attach_type, false); 1209 cgroup_put(cgrp); 1210 break; 1211 case BPF_SK_SKB_STREAM_PARSER: 1212 case BPF_SK_SKB_STREAM_VERDICT: 1213 ret = sockmap_get_from_fd(attr, false); 1214 break; 1215 default: 1216 return -EINVAL; 1217 } 1218 1219 return ret; 1220 } 1221 1222 #endif /* CONFIG_CGROUP_BPF */ 1223 1224 #define BPF_PROG_TEST_RUN_LAST_FIELD test.duration 1225 1226 static int bpf_prog_test_run(const union bpf_attr *attr, 1227 union bpf_attr __user *uattr) 1228 { 1229 struct bpf_prog *prog; 1230 int ret = -ENOTSUPP; 1231 1232 if (CHECK_ATTR(BPF_PROG_TEST_RUN)) 1233 return -EINVAL; 1234 1235 prog = bpf_prog_get(attr->test.prog_fd); 1236 if (IS_ERR(prog)) 1237 return PTR_ERR(prog); 1238 1239 if (prog->aux->ops->test_run) 1240 ret = prog->aux->ops->test_run(prog, attr, uattr); 1241 1242 bpf_prog_put(prog); 1243 return ret; 1244 } 1245 1246 #define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id 1247 1248 static int bpf_obj_get_next_id(const union bpf_attr *attr, 1249 union bpf_attr __user *uattr, 1250 struct idr *idr, 1251 spinlock_t *lock) 1252 { 1253 u32 next_id = attr->start_id; 1254 int err = 0; 1255 1256 if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX) 1257 return -EINVAL; 1258 1259 if (!capable(CAP_SYS_ADMIN)) 1260 return -EPERM; 1261 1262 next_id++; 1263 spin_lock_bh(lock); 1264 if (!idr_get_next(idr, &next_id)) 1265 err = -ENOENT; 1266 spin_unlock_bh(lock); 1267 1268 if (!err) 1269 err = put_user(next_id, &uattr->next_id); 1270 1271 return err; 1272 } 1273 1274 #define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id 1275 1276 static int bpf_prog_get_fd_by_id(const union bpf_attr *attr) 1277 { 1278 struct bpf_prog *prog; 1279 u32 id = attr->prog_id; 1280 int fd; 1281 1282 if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID)) 1283 return -EINVAL; 1284 1285 if (!capable(CAP_SYS_ADMIN)) 1286 return -EPERM; 1287 1288 spin_lock_bh(&prog_idr_lock); 1289 prog = idr_find(&prog_idr, id); 1290 if (prog) 1291 prog = bpf_prog_inc_not_zero(prog); 1292 else 1293 prog = ERR_PTR(-ENOENT); 1294 spin_unlock_bh(&prog_idr_lock); 1295 1296 if (IS_ERR(prog)) 1297 return PTR_ERR(prog); 1298 1299 fd = bpf_prog_new_fd(prog); 1300 if (fd < 0) 1301 bpf_prog_put(prog); 1302 1303 return fd; 1304 } 1305 1306 #define BPF_MAP_GET_FD_BY_ID_LAST_FIELD map_id 1307 1308 static int bpf_map_get_fd_by_id(const union bpf_attr *attr) 1309 { 1310 struct bpf_map *map; 1311 u32 id = attr->map_id; 1312 int fd; 1313 1314 if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID)) 1315 return -EINVAL; 1316 1317 if (!capable(CAP_SYS_ADMIN)) 1318 return -EPERM; 1319 1320 spin_lock_bh(&map_idr_lock); 1321 map = idr_find(&map_idr, id); 1322 if (map) 1323 map = bpf_map_inc_not_zero(map, true); 1324 else 1325 map = ERR_PTR(-ENOENT); 1326 spin_unlock_bh(&map_idr_lock); 1327 1328 if (IS_ERR(map)) 1329 return PTR_ERR(map); 1330 1331 fd = bpf_map_new_fd(map); 1332 if (fd < 0) 1333 bpf_map_put(map); 1334 1335 return fd; 1336 } 1337 1338 static int bpf_prog_get_info_by_fd(struct bpf_prog *prog, 1339 const union bpf_attr *attr, 1340 union bpf_attr __user *uattr) 1341 { 1342 struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info); 1343 struct bpf_prog_info info = {}; 1344 u32 info_len = attr->info.info_len; 1345 char __user *uinsns; 1346 u32 ulen; 1347 int err; 1348 1349 err = check_uarg_tail_zero(uinfo, sizeof(info), info_len); 1350 if (err) 1351 return err; 1352 info_len = min_t(u32, sizeof(info), info_len); 1353 1354 if (copy_from_user(&info, uinfo, info_len)) 1355 return -EFAULT; 1356 1357 info.type = prog->type; 1358 info.id = prog->aux->id; 1359 1360 memcpy(info.tag, prog->tag, sizeof(prog->tag)); 1361 1362 if (!capable(CAP_SYS_ADMIN)) { 1363 info.jited_prog_len = 0; 1364 info.xlated_prog_len = 0; 1365 goto done; 1366 } 1367 1368 ulen = info.jited_prog_len; 1369 info.jited_prog_len = prog->jited_len; 1370 if (info.jited_prog_len && ulen) { 1371 uinsns = u64_to_user_ptr(info.jited_prog_insns); 1372 ulen = min_t(u32, info.jited_prog_len, ulen); 1373 if (copy_to_user(uinsns, prog->bpf_func, ulen)) 1374 return -EFAULT; 1375 } 1376 1377 ulen = info.xlated_prog_len; 1378 info.xlated_prog_len = bpf_prog_insn_size(prog); 1379 if (info.xlated_prog_len && ulen) { 1380 uinsns = u64_to_user_ptr(info.xlated_prog_insns); 1381 ulen = min_t(u32, info.xlated_prog_len, ulen); 1382 if (copy_to_user(uinsns, prog->insnsi, ulen)) 1383 return -EFAULT; 1384 } 1385 1386 done: 1387 if (copy_to_user(uinfo, &info, info_len) || 1388 put_user(info_len, &uattr->info.info_len)) 1389 return -EFAULT; 1390 1391 return 0; 1392 } 1393 1394 static int bpf_map_get_info_by_fd(struct bpf_map *map, 1395 const union bpf_attr *attr, 1396 union bpf_attr __user *uattr) 1397 { 1398 struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info); 1399 struct bpf_map_info info = {}; 1400 u32 info_len = attr->info.info_len; 1401 int err; 1402 1403 err = check_uarg_tail_zero(uinfo, sizeof(info), info_len); 1404 if (err) 1405 return err; 1406 info_len = min_t(u32, sizeof(info), info_len); 1407 1408 info.type = map->map_type; 1409 info.id = map->id; 1410 info.key_size = map->key_size; 1411 info.value_size = map->value_size; 1412 info.max_entries = map->max_entries; 1413 info.map_flags = map->map_flags; 1414 1415 if (copy_to_user(uinfo, &info, info_len) || 1416 put_user(info_len, &uattr->info.info_len)) 1417 return -EFAULT; 1418 1419 return 0; 1420 } 1421 1422 #define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info 1423 1424 static int bpf_obj_get_info_by_fd(const union bpf_attr *attr, 1425 union bpf_attr __user *uattr) 1426 { 1427 int ufd = attr->info.bpf_fd; 1428 struct fd f; 1429 int err; 1430 1431 if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD)) 1432 return -EINVAL; 1433 1434 f = fdget(ufd); 1435 if (!f.file) 1436 return -EBADFD; 1437 1438 if (f.file->f_op == &bpf_prog_fops) 1439 err = bpf_prog_get_info_by_fd(f.file->private_data, attr, 1440 uattr); 1441 else if (f.file->f_op == &bpf_map_fops) 1442 err = bpf_map_get_info_by_fd(f.file->private_data, attr, 1443 uattr); 1444 else 1445 err = -EINVAL; 1446 1447 fdput(f); 1448 return err; 1449 } 1450 1451 SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size) 1452 { 1453 union bpf_attr attr = {}; 1454 int err; 1455 1456 if (!capable(CAP_SYS_ADMIN) && sysctl_unprivileged_bpf_disabled) 1457 return -EPERM; 1458 1459 err = check_uarg_tail_zero(uattr, sizeof(attr), size); 1460 if (err) 1461 return err; 1462 size = min_t(u32, size, sizeof(attr)); 1463 1464 /* copy attributes from user space, may be less than sizeof(bpf_attr) */ 1465 if (copy_from_user(&attr, uattr, size) != 0) 1466 return -EFAULT; 1467 1468 switch (cmd) { 1469 case BPF_MAP_CREATE: 1470 err = map_create(&attr); 1471 break; 1472 case BPF_MAP_LOOKUP_ELEM: 1473 err = map_lookup_elem(&attr); 1474 break; 1475 case BPF_MAP_UPDATE_ELEM: 1476 err = map_update_elem(&attr); 1477 break; 1478 case BPF_MAP_DELETE_ELEM: 1479 err = map_delete_elem(&attr); 1480 break; 1481 case BPF_MAP_GET_NEXT_KEY: 1482 err = map_get_next_key(&attr); 1483 break; 1484 case BPF_PROG_LOAD: 1485 err = bpf_prog_load(&attr); 1486 break; 1487 case BPF_OBJ_PIN: 1488 err = bpf_obj_pin(&attr); 1489 break; 1490 case BPF_OBJ_GET: 1491 err = bpf_obj_get(&attr); 1492 break; 1493 #ifdef CONFIG_CGROUP_BPF 1494 case BPF_PROG_ATTACH: 1495 err = bpf_prog_attach(&attr); 1496 break; 1497 case BPF_PROG_DETACH: 1498 err = bpf_prog_detach(&attr); 1499 break; 1500 #endif 1501 case BPF_PROG_TEST_RUN: 1502 err = bpf_prog_test_run(&attr, uattr); 1503 break; 1504 case BPF_PROG_GET_NEXT_ID: 1505 err = bpf_obj_get_next_id(&attr, uattr, 1506 &prog_idr, &prog_idr_lock); 1507 break; 1508 case BPF_MAP_GET_NEXT_ID: 1509 err = bpf_obj_get_next_id(&attr, uattr, 1510 &map_idr, &map_idr_lock); 1511 break; 1512 case BPF_PROG_GET_FD_BY_ID: 1513 err = bpf_prog_get_fd_by_id(&attr); 1514 break; 1515 case BPF_MAP_GET_FD_BY_ID: 1516 err = bpf_map_get_fd_by_id(&attr); 1517 break; 1518 case BPF_OBJ_GET_INFO_BY_FD: 1519 err = bpf_obj_get_info_by_fd(&attr, uattr); 1520 break; 1521 default: 1522 err = -EINVAL; 1523 break; 1524 } 1525 1526 return err; 1527 } 1528