1 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 2 * 3 * This program is free software; you can redistribute it and/or 4 * modify it under the terms of version 2 of the GNU General Public 5 * License as published by the Free Software Foundation. 6 * 7 * This program is distributed in the hope that it will be useful, but 8 * WITHOUT ANY WARRANTY; without even the implied warranty of 9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 10 * General Public License for more details. 11 */ 12 #include <linux/bpf.h> 13 #include <linux/bpf_trace.h> 14 #include <linux/syscalls.h> 15 #include <linux/slab.h> 16 #include <linux/sched/signal.h> 17 #include <linux/vmalloc.h> 18 #include <linux/mmzone.h> 19 #include <linux/anon_inodes.h> 20 #include <linux/file.h> 21 #include <linux/license.h> 22 #include <linux/filter.h> 23 #include <linux/version.h> 24 #include <linux/kernel.h> 25 #include <linux/idr.h> 26 27 #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY || \ 28 (map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \ 29 (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \ 30 (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) 31 #define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) 32 #define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_HASH(map)) 33 34 DEFINE_PER_CPU(int, bpf_prog_active); 35 static DEFINE_IDR(prog_idr); 36 static DEFINE_SPINLOCK(prog_idr_lock); 37 static DEFINE_IDR(map_idr); 38 static DEFINE_SPINLOCK(map_idr_lock); 39 40 int sysctl_unprivileged_bpf_disabled __read_mostly; 41 42 static const struct bpf_map_ops * const bpf_map_types[] = { 43 #define BPF_PROG_TYPE(_id, _ops) 44 #define BPF_MAP_TYPE(_id, _ops) \ 45 [_id] = &_ops, 46 #include <linux/bpf_types.h> 47 #undef BPF_PROG_TYPE 48 #undef BPF_MAP_TYPE 49 }; 50 51 /* 52 * If we're handed a bigger struct than we know of, ensure all the unknown bits 53 * are 0 - i.e. new user-space does not rely on any kernel feature extensions 54 * we don't know about yet. 55 * 56 * There is a ToCToU between this function call and the following 57 * copy_from_user() call. However, this is not a concern since this function is 58 * meant to be a future-proofing of bits. 59 */ 60 static int check_uarg_tail_zero(void __user *uaddr, 61 size_t expected_size, 62 size_t actual_size) 63 { 64 unsigned char __user *addr; 65 unsigned char __user *end; 66 unsigned char val; 67 int err; 68 69 if (unlikely(actual_size > PAGE_SIZE)) /* silly large */ 70 return -E2BIG; 71 72 if (unlikely(!access_ok(VERIFY_READ, uaddr, actual_size))) 73 return -EFAULT; 74 75 if (actual_size <= expected_size) 76 return 0; 77 78 addr = uaddr + expected_size; 79 end = uaddr + actual_size; 80 81 for (; addr < end; addr++) { 82 err = get_user(val, addr); 83 if (err) 84 return err; 85 if (val) 86 return -E2BIG; 87 } 88 89 return 0; 90 } 91 92 static struct bpf_map *find_and_alloc_map(union bpf_attr *attr) 93 { 94 struct bpf_map *map; 95 96 if (attr->map_type >= ARRAY_SIZE(bpf_map_types) || 97 !bpf_map_types[attr->map_type]) 98 return ERR_PTR(-EINVAL); 99 100 map = bpf_map_types[attr->map_type]->map_alloc(attr); 101 if (IS_ERR(map)) 102 return map; 103 map->ops = bpf_map_types[attr->map_type]; 104 map->map_type = attr->map_type; 105 return map; 106 } 107 108 void *bpf_map_area_alloc(size_t size, int numa_node) 109 { 110 /* We definitely need __GFP_NORETRY, so OOM killer doesn't 111 * trigger under memory pressure as we really just want to 112 * fail instead. 113 */ 114 const gfp_t flags = __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO; 115 void *area; 116 117 if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) { 118 area = kmalloc_node(size, GFP_USER | flags, numa_node); 119 if (area != NULL) 120 return area; 121 } 122 123 return __vmalloc_node_flags_caller(size, numa_node, GFP_KERNEL | flags, 124 __builtin_return_address(0)); 125 } 126 127 void bpf_map_area_free(void *area) 128 { 129 kvfree(area); 130 } 131 132 int bpf_map_precharge_memlock(u32 pages) 133 { 134 struct user_struct *user = get_current_user(); 135 unsigned long memlock_limit, cur; 136 137 memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 138 cur = atomic_long_read(&user->locked_vm); 139 free_uid(user); 140 if (cur + pages > memlock_limit) 141 return -EPERM; 142 return 0; 143 } 144 145 static int bpf_map_charge_memlock(struct bpf_map *map) 146 { 147 struct user_struct *user = get_current_user(); 148 unsigned long memlock_limit; 149 150 memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 151 152 atomic_long_add(map->pages, &user->locked_vm); 153 154 if (atomic_long_read(&user->locked_vm) > memlock_limit) { 155 atomic_long_sub(map->pages, &user->locked_vm); 156 free_uid(user); 157 return -EPERM; 158 } 159 map->user = user; 160 return 0; 161 } 162 163 static void bpf_map_uncharge_memlock(struct bpf_map *map) 164 { 165 struct user_struct *user = map->user; 166 167 atomic_long_sub(map->pages, &user->locked_vm); 168 free_uid(user); 169 } 170 171 static int bpf_map_alloc_id(struct bpf_map *map) 172 { 173 int id; 174 175 spin_lock_bh(&map_idr_lock); 176 id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC); 177 if (id > 0) 178 map->id = id; 179 spin_unlock_bh(&map_idr_lock); 180 181 if (WARN_ON_ONCE(!id)) 182 return -ENOSPC; 183 184 return id > 0 ? 0 : id; 185 } 186 187 static void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock) 188 { 189 unsigned long flags; 190 191 if (do_idr_lock) 192 spin_lock_irqsave(&map_idr_lock, flags); 193 else 194 __acquire(&map_idr_lock); 195 196 idr_remove(&map_idr, map->id); 197 198 if (do_idr_lock) 199 spin_unlock_irqrestore(&map_idr_lock, flags); 200 else 201 __release(&map_idr_lock); 202 } 203 204 /* called from workqueue */ 205 static void bpf_map_free_deferred(struct work_struct *work) 206 { 207 struct bpf_map *map = container_of(work, struct bpf_map, work); 208 209 bpf_map_uncharge_memlock(map); 210 /* implementation dependent freeing */ 211 map->ops->map_free(map); 212 } 213 214 static void bpf_map_put_uref(struct bpf_map *map) 215 { 216 if (atomic_dec_and_test(&map->usercnt)) { 217 if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) 218 bpf_fd_array_map_clear(map); 219 } 220 } 221 222 /* decrement map refcnt and schedule it for freeing via workqueue 223 * (unrelying map implementation ops->map_free() might sleep) 224 */ 225 static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock) 226 { 227 if (atomic_dec_and_test(&map->refcnt)) { 228 /* bpf_map_free_id() must be called first */ 229 bpf_map_free_id(map, do_idr_lock); 230 INIT_WORK(&map->work, bpf_map_free_deferred); 231 schedule_work(&map->work); 232 } 233 } 234 235 void bpf_map_put(struct bpf_map *map) 236 { 237 __bpf_map_put(map, true); 238 } 239 240 void bpf_map_put_with_uref(struct bpf_map *map) 241 { 242 bpf_map_put_uref(map); 243 bpf_map_put(map); 244 } 245 246 static int bpf_map_release(struct inode *inode, struct file *filp) 247 { 248 struct bpf_map *map = filp->private_data; 249 250 if (map->ops->map_release) 251 map->ops->map_release(map, filp); 252 253 bpf_map_put_with_uref(map); 254 return 0; 255 } 256 257 #ifdef CONFIG_PROC_FS 258 static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp) 259 { 260 const struct bpf_map *map = filp->private_data; 261 const struct bpf_array *array; 262 u32 owner_prog_type = 0; 263 u32 owner_jited = 0; 264 265 if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) { 266 array = container_of(map, struct bpf_array, map); 267 owner_prog_type = array->owner_prog_type; 268 owner_jited = array->owner_jited; 269 } 270 271 seq_printf(m, 272 "map_type:\t%u\n" 273 "key_size:\t%u\n" 274 "value_size:\t%u\n" 275 "max_entries:\t%u\n" 276 "map_flags:\t%#x\n" 277 "memlock:\t%llu\n", 278 map->map_type, 279 map->key_size, 280 map->value_size, 281 map->max_entries, 282 map->map_flags, 283 map->pages * 1ULL << PAGE_SHIFT); 284 285 if (owner_prog_type) { 286 seq_printf(m, "owner_prog_type:\t%u\n", 287 owner_prog_type); 288 seq_printf(m, "owner_jited:\t%u\n", 289 owner_jited); 290 } 291 } 292 #endif 293 294 static const struct file_operations bpf_map_fops = { 295 #ifdef CONFIG_PROC_FS 296 .show_fdinfo = bpf_map_show_fdinfo, 297 #endif 298 .release = bpf_map_release, 299 }; 300 301 int bpf_map_new_fd(struct bpf_map *map) 302 { 303 return anon_inode_getfd("bpf-map", &bpf_map_fops, map, 304 O_RDWR | O_CLOEXEC); 305 } 306 307 /* helper macro to check that unused fields 'union bpf_attr' are zero */ 308 #define CHECK_ATTR(CMD) \ 309 memchr_inv((void *) &attr->CMD##_LAST_FIELD + \ 310 sizeof(attr->CMD##_LAST_FIELD), 0, \ 311 sizeof(*attr) - \ 312 offsetof(union bpf_attr, CMD##_LAST_FIELD) - \ 313 sizeof(attr->CMD##_LAST_FIELD)) != NULL 314 315 #define BPF_MAP_CREATE_LAST_FIELD numa_node 316 /* called via syscall */ 317 static int map_create(union bpf_attr *attr) 318 { 319 int numa_node = bpf_map_attr_numa_node(attr); 320 struct bpf_map *map; 321 int err; 322 323 err = CHECK_ATTR(BPF_MAP_CREATE); 324 if (err) 325 return -EINVAL; 326 327 if (numa_node != NUMA_NO_NODE && 328 ((unsigned int)numa_node >= nr_node_ids || 329 !node_online(numa_node))) 330 return -EINVAL; 331 332 /* find map type and init map: hashtable vs rbtree vs bloom vs ... */ 333 map = find_and_alloc_map(attr); 334 if (IS_ERR(map)) 335 return PTR_ERR(map); 336 337 atomic_set(&map->refcnt, 1); 338 atomic_set(&map->usercnt, 1); 339 340 err = bpf_map_charge_memlock(map); 341 if (err) 342 goto free_map_nouncharge; 343 344 err = bpf_map_alloc_id(map); 345 if (err) 346 goto free_map; 347 348 err = bpf_map_new_fd(map); 349 if (err < 0) { 350 /* failed to allocate fd. 351 * bpf_map_put() is needed because the above 352 * bpf_map_alloc_id() has published the map 353 * to the userspace and the userspace may 354 * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID. 355 */ 356 bpf_map_put(map); 357 return err; 358 } 359 360 trace_bpf_map_create(map, err); 361 return err; 362 363 free_map: 364 bpf_map_uncharge_memlock(map); 365 free_map_nouncharge: 366 map->ops->map_free(map); 367 return err; 368 } 369 370 /* if error is returned, fd is released. 371 * On success caller should complete fd access with matching fdput() 372 */ 373 struct bpf_map *__bpf_map_get(struct fd f) 374 { 375 if (!f.file) 376 return ERR_PTR(-EBADF); 377 if (f.file->f_op != &bpf_map_fops) { 378 fdput(f); 379 return ERR_PTR(-EINVAL); 380 } 381 382 return f.file->private_data; 383 } 384 385 /* prog's and map's refcnt limit */ 386 #define BPF_MAX_REFCNT 32768 387 388 struct bpf_map *bpf_map_inc(struct bpf_map *map, bool uref) 389 { 390 if (atomic_inc_return(&map->refcnt) > BPF_MAX_REFCNT) { 391 atomic_dec(&map->refcnt); 392 return ERR_PTR(-EBUSY); 393 } 394 if (uref) 395 atomic_inc(&map->usercnt); 396 return map; 397 } 398 399 struct bpf_map *bpf_map_get_with_uref(u32 ufd) 400 { 401 struct fd f = fdget(ufd); 402 struct bpf_map *map; 403 404 map = __bpf_map_get(f); 405 if (IS_ERR(map)) 406 return map; 407 408 map = bpf_map_inc(map, true); 409 fdput(f); 410 411 return map; 412 } 413 414 /* map_idr_lock should have been held */ 415 static struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map, 416 bool uref) 417 { 418 int refold; 419 420 refold = __atomic_add_unless(&map->refcnt, 1, 0); 421 422 if (refold >= BPF_MAX_REFCNT) { 423 __bpf_map_put(map, false); 424 return ERR_PTR(-EBUSY); 425 } 426 427 if (!refold) 428 return ERR_PTR(-ENOENT); 429 430 if (uref) 431 atomic_inc(&map->usercnt); 432 433 return map; 434 } 435 436 int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value) 437 { 438 return -ENOTSUPP; 439 } 440 441 /* last field in 'union bpf_attr' used by this command */ 442 #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD value 443 444 static int map_lookup_elem(union bpf_attr *attr) 445 { 446 void __user *ukey = u64_to_user_ptr(attr->key); 447 void __user *uvalue = u64_to_user_ptr(attr->value); 448 int ufd = attr->map_fd; 449 struct bpf_map *map; 450 void *key, *value, *ptr; 451 u32 value_size; 452 struct fd f; 453 int err; 454 455 if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM)) 456 return -EINVAL; 457 458 f = fdget(ufd); 459 map = __bpf_map_get(f); 460 if (IS_ERR(map)) 461 return PTR_ERR(map); 462 463 key = memdup_user(ukey, map->key_size); 464 if (IS_ERR(key)) { 465 err = PTR_ERR(key); 466 goto err_put; 467 } 468 469 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 470 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 471 map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) 472 value_size = round_up(map->value_size, 8) * num_possible_cpus(); 473 else if (IS_FD_MAP(map)) 474 value_size = sizeof(u32); 475 else 476 value_size = map->value_size; 477 478 err = -ENOMEM; 479 value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 480 if (!value) 481 goto free_key; 482 483 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 484 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 485 err = bpf_percpu_hash_copy(map, key, value); 486 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 487 err = bpf_percpu_array_copy(map, key, value); 488 } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) { 489 err = bpf_stackmap_copy(map, key, value); 490 } else if (IS_FD_ARRAY(map)) { 491 err = bpf_fd_array_map_lookup_elem(map, key, value); 492 } else if (IS_FD_HASH(map)) { 493 err = bpf_fd_htab_map_lookup_elem(map, key, value); 494 } else { 495 rcu_read_lock(); 496 ptr = map->ops->map_lookup_elem(map, key); 497 if (ptr) 498 memcpy(value, ptr, value_size); 499 rcu_read_unlock(); 500 err = ptr ? 0 : -ENOENT; 501 } 502 503 if (err) 504 goto free_value; 505 506 err = -EFAULT; 507 if (copy_to_user(uvalue, value, value_size) != 0) 508 goto free_value; 509 510 trace_bpf_map_lookup_elem(map, ufd, key, value); 511 err = 0; 512 513 free_value: 514 kfree(value); 515 free_key: 516 kfree(key); 517 err_put: 518 fdput(f); 519 return err; 520 } 521 522 #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags 523 524 static int map_update_elem(union bpf_attr *attr) 525 { 526 void __user *ukey = u64_to_user_ptr(attr->key); 527 void __user *uvalue = u64_to_user_ptr(attr->value); 528 int ufd = attr->map_fd; 529 struct bpf_map *map; 530 void *key, *value; 531 u32 value_size; 532 struct fd f; 533 int err; 534 535 if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM)) 536 return -EINVAL; 537 538 f = fdget(ufd); 539 map = __bpf_map_get(f); 540 if (IS_ERR(map)) 541 return PTR_ERR(map); 542 543 key = memdup_user(ukey, map->key_size); 544 if (IS_ERR(key)) { 545 err = PTR_ERR(key); 546 goto err_put; 547 } 548 549 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 550 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 551 map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) 552 value_size = round_up(map->value_size, 8) * num_possible_cpus(); 553 else 554 value_size = map->value_size; 555 556 err = -ENOMEM; 557 value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 558 if (!value) 559 goto free_key; 560 561 err = -EFAULT; 562 if (copy_from_user(value, uvalue, value_size) != 0) 563 goto free_value; 564 565 /* must increment bpf_prog_active to avoid kprobe+bpf triggering from 566 * inside bpf map update or delete otherwise deadlocks are possible 567 */ 568 preempt_disable(); 569 __this_cpu_inc(bpf_prog_active); 570 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 571 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 572 err = bpf_percpu_hash_update(map, key, value, attr->flags); 573 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 574 err = bpf_percpu_array_update(map, key, value, attr->flags); 575 } else if (map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || 576 map->map_type == BPF_MAP_TYPE_PROG_ARRAY || 577 map->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || 578 map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) { 579 rcu_read_lock(); 580 err = bpf_fd_array_map_update_elem(map, f.file, key, value, 581 attr->flags); 582 rcu_read_unlock(); 583 } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) { 584 rcu_read_lock(); 585 err = bpf_fd_htab_map_update_elem(map, f.file, key, value, 586 attr->flags); 587 rcu_read_unlock(); 588 } else { 589 rcu_read_lock(); 590 err = map->ops->map_update_elem(map, key, value, attr->flags); 591 rcu_read_unlock(); 592 } 593 __this_cpu_dec(bpf_prog_active); 594 preempt_enable(); 595 596 if (!err) 597 trace_bpf_map_update_elem(map, ufd, key, value); 598 free_value: 599 kfree(value); 600 free_key: 601 kfree(key); 602 err_put: 603 fdput(f); 604 return err; 605 } 606 607 #define BPF_MAP_DELETE_ELEM_LAST_FIELD key 608 609 static int map_delete_elem(union bpf_attr *attr) 610 { 611 void __user *ukey = u64_to_user_ptr(attr->key); 612 int ufd = attr->map_fd; 613 struct bpf_map *map; 614 struct fd f; 615 void *key; 616 int err; 617 618 if (CHECK_ATTR(BPF_MAP_DELETE_ELEM)) 619 return -EINVAL; 620 621 f = fdget(ufd); 622 map = __bpf_map_get(f); 623 if (IS_ERR(map)) 624 return PTR_ERR(map); 625 626 key = memdup_user(ukey, map->key_size); 627 if (IS_ERR(key)) { 628 err = PTR_ERR(key); 629 goto err_put; 630 } 631 632 preempt_disable(); 633 __this_cpu_inc(bpf_prog_active); 634 rcu_read_lock(); 635 err = map->ops->map_delete_elem(map, key); 636 rcu_read_unlock(); 637 __this_cpu_dec(bpf_prog_active); 638 preempt_enable(); 639 640 if (!err) 641 trace_bpf_map_delete_elem(map, ufd, key); 642 kfree(key); 643 err_put: 644 fdput(f); 645 return err; 646 } 647 648 /* last field in 'union bpf_attr' used by this command */ 649 #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key 650 651 static int map_get_next_key(union bpf_attr *attr) 652 { 653 void __user *ukey = u64_to_user_ptr(attr->key); 654 void __user *unext_key = u64_to_user_ptr(attr->next_key); 655 int ufd = attr->map_fd; 656 struct bpf_map *map; 657 void *key, *next_key; 658 struct fd f; 659 int err; 660 661 if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY)) 662 return -EINVAL; 663 664 f = fdget(ufd); 665 map = __bpf_map_get(f); 666 if (IS_ERR(map)) 667 return PTR_ERR(map); 668 669 if (ukey) { 670 key = memdup_user(ukey, map->key_size); 671 if (IS_ERR(key)) { 672 err = PTR_ERR(key); 673 goto err_put; 674 } 675 } else { 676 key = NULL; 677 } 678 679 err = -ENOMEM; 680 next_key = kmalloc(map->key_size, GFP_USER); 681 if (!next_key) 682 goto free_key; 683 684 rcu_read_lock(); 685 err = map->ops->map_get_next_key(map, key, next_key); 686 rcu_read_unlock(); 687 if (err) 688 goto free_next_key; 689 690 err = -EFAULT; 691 if (copy_to_user(unext_key, next_key, map->key_size) != 0) 692 goto free_next_key; 693 694 trace_bpf_map_next_key(map, ufd, key, next_key); 695 err = 0; 696 697 free_next_key: 698 kfree(next_key); 699 free_key: 700 kfree(key); 701 err_put: 702 fdput(f); 703 return err; 704 } 705 706 static const struct bpf_verifier_ops * const bpf_prog_types[] = { 707 #define BPF_PROG_TYPE(_id, _ops) \ 708 [_id] = &_ops, 709 #define BPF_MAP_TYPE(_id, _ops) 710 #include <linux/bpf_types.h> 711 #undef BPF_PROG_TYPE 712 #undef BPF_MAP_TYPE 713 }; 714 715 static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog) 716 { 717 if (type >= ARRAY_SIZE(bpf_prog_types) || !bpf_prog_types[type]) 718 return -EINVAL; 719 720 prog->aux->ops = bpf_prog_types[type]; 721 prog->type = type; 722 return 0; 723 } 724 725 /* drop refcnt on maps used by eBPF program and free auxilary data */ 726 static void free_used_maps(struct bpf_prog_aux *aux) 727 { 728 int i; 729 730 for (i = 0; i < aux->used_map_cnt; i++) 731 bpf_map_put(aux->used_maps[i]); 732 733 kfree(aux->used_maps); 734 } 735 736 int __bpf_prog_charge(struct user_struct *user, u32 pages) 737 { 738 unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 739 unsigned long user_bufs; 740 741 if (user) { 742 user_bufs = atomic_long_add_return(pages, &user->locked_vm); 743 if (user_bufs > memlock_limit) { 744 atomic_long_sub(pages, &user->locked_vm); 745 return -EPERM; 746 } 747 } 748 749 return 0; 750 } 751 752 void __bpf_prog_uncharge(struct user_struct *user, u32 pages) 753 { 754 if (user) 755 atomic_long_sub(pages, &user->locked_vm); 756 } 757 758 static int bpf_prog_charge_memlock(struct bpf_prog *prog) 759 { 760 struct user_struct *user = get_current_user(); 761 int ret; 762 763 ret = __bpf_prog_charge(user, prog->pages); 764 if (ret) { 765 free_uid(user); 766 return ret; 767 } 768 769 prog->aux->user = user; 770 return 0; 771 } 772 773 static void bpf_prog_uncharge_memlock(struct bpf_prog *prog) 774 { 775 struct user_struct *user = prog->aux->user; 776 777 __bpf_prog_uncharge(user, prog->pages); 778 free_uid(user); 779 } 780 781 static int bpf_prog_alloc_id(struct bpf_prog *prog) 782 { 783 int id; 784 785 spin_lock_bh(&prog_idr_lock); 786 id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC); 787 if (id > 0) 788 prog->aux->id = id; 789 spin_unlock_bh(&prog_idr_lock); 790 791 /* id is in [1, INT_MAX) */ 792 if (WARN_ON_ONCE(!id)) 793 return -ENOSPC; 794 795 return id > 0 ? 0 : id; 796 } 797 798 static void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock) 799 { 800 /* cBPF to eBPF migrations are currently not in the idr store. */ 801 if (!prog->aux->id) 802 return; 803 804 if (do_idr_lock) 805 spin_lock_bh(&prog_idr_lock); 806 else 807 __acquire(&prog_idr_lock); 808 809 idr_remove(&prog_idr, prog->aux->id); 810 811 if (do_idr_lock) 812 spin_unlock_bh(&prog_idr_lock); 813 else 814 __release(&prog_idr_lock); 815 } 816 817 static void __bpf_prog_put_rcu(struct rcu_head *rcu) 818 { 819 struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu); 820 821 free_used_maps(aux); 822 bpf_prog_uncharge_memlock(aux->prog); 823 bpf_prog_free(aux->prog); 824 } 825 826 static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock) 827 { 828 if (atomic_dec_and_test(&prog->aux->refcnt)) { 829 trace_bpf_prog_put_rcu(prog); 830 /* bpf_prog_free_id() must be called first */ 831 bpf_prog_free_id(prog, do_idr_lock); 832 bpf_prog_kallsyms_del(prog); 833 call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); 834 } 835 } 836 837 void bpf_prog_put(struct bpf_prog *prog) 838 { 839 __bpf_prog_put(prog, true); 840 } 841 EXPORT_SYMBOL_GPL(bpf_prog_put); 842 843 static int bpf_prog_release(struct inode *inode, struct file *filp) 844 { 845 struct bpf_prog *prog = filp->private_data; 846 847 bpf_prog_put(prog); 848 return 0; 849 } 850 851 #ifdef CONFIG_PROC_FS 852 static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp) 853 { 854 const struct bpf_prog *prog = filp->private_data; 855 char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; 856 857 bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); 858 seq_printf(m, 859 "prog_type:\t%u\n" 860 "prog_jited:\t%u\n" 861 "prog_tag:\t%s\n" 862 "memlock:\t%llu\n", 863 prog->type, 864 prog->jited, 865 prog_tag, 866 prog->pages * 1ULL << PAGE_SHIFT); 867 } 868 #endif 869 870 static const struct file_operations bpf_prog_fops = { 871 #ifdef CONFIG_PROC_FS 872 .show_fdinfo = bpf_prog_show_fdinfo, 873 #endif 874 .release = bpf_prog_release, 875 }; 876 877 int bpf_prog_new_fd(struct bpf_prog *prog) 878 { 879 return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog, 880 O_RDWR | O_CLOEXEC); 881 } 882 883 static struct bpf_prog *____bpf_prog_get(struct fd f) 884 { 885 if (!f.file) 886 return ERR_PTR(-EBADF); 887 if (f.file->f_op != &bpf_prog_fops) { 888 fdput(f); 889 return ERR_PTR(-EINVAL); 890 } 891 892 return f.file->private_data; 893 } 894 895 struct bpf_prog *bpf_prog_add(struct bpf_prog *prog, int i) 896 { 897 if (atomic_add_return(i, &prog->aux->refcnt) > BPF_MAX_REFCNT) { 898 atomic_sub(i, &prog->aux->refcnt); 899 return ERR_PTR(-EBUSY); 900 } 901 return prog; 902 } 903 EXPORT_SYMBOL_GPL(bpf_prog_add); 904 905 void bpf_prog_sub(struct bpf_prog *prog, int i) 906 { 907 /* Only to be used for undoing previous bpf_prog_add() in some 908 * error path. We still know that another entity in our call 909 * path holds a reference to the program, thus atomic_sub() can 910 * be safely used in such cases! 911 */ 912 WARN_ON(atomic_sub_return(i, &prog->aux->refcnt) == 0); 913 } 914 EXPORT_SYMBOL_GPL(bpf_prog_sub); 915 916 struct bpf_prog *bpf_prog_inc(struct bpf_prog *prog) 917 { 918 return bpf_prog_add(prog, 1); 919 } 920 EXPORT_SYMBOL_GPL(bpf_prog_inc); 921 922 /* prog_idr_lock should have been held */ 923 struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog) 924 { 925 int refold; 926 927 refold = __atomic_add_unless(&prog->aux->refcnt, 1, 0); 928 929 if (refold >= BPF_MAX_REFCNT) { 930 __bpf_prog_put(prog, false); 931 return ERR_PTR(-EBUSY); 932 } 933 934 if (!refold) 935 return ERR_PTR(-ENOENT); 936 937 return prog; 938 } 939 EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero); 940 941 static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *type) 942 { 943 struct fd f = fdget(ufd); 944 struct bpf_prog *prog; 945 946 prog = ____bpf_prog_get(f); 947 if (IS_ERR(prog)) 948 return prog; 949 if (type && prog->type != *type) { 950 prog = ERR_PTR(-EINVAL); 951 goto out; 952 } 953 954 prog = bpf_prog_inc(prog); 955 out: 956 fdput(f); 957 return prog; 958 } 959 960 struct bpf_prog *bpf_prog_get(u32 ufd) 961 { 962 return __bpf_prog_get(ufd, NULL); 963 } 964 965 struct bpf_prog *bpf_prog_get_type(u32 ufd, enum bpf_prog_type type) 966 { 967 struct bpf_prog *prog = __bpf_prog_get(ufd, &type); 968 969 if (!IS_ERR(prog)) 970 trace_bpf_prog_get_type(prog); 971 return prog; 972 } 973 EXPORT_SYMBOL_GPL(bpf_prog_get_type); 974 975 /* last field in 'union bpf_attr' used by this command */ 976 #define BPF_PROG_LOAD_LAST_FIELD prog_flags 977 978 static int bpf_prog_load(union bpf_attr *attr) 979 { 980 enum bpf_prog_type type = attr->prog_type; 981 struct bpf_prog *prog; 982 int err; 983 char license[128]; 984 bool is_gpl; 985 986 if (CHECK_ATTR(BPF_PROG_LOAD)) 987 return -EINVAL; 988 989 if (attr->prog_flags & ~BPF_F_STRICT_ALIGNMENT) 990 return -EINVAL; 991 992 /* copy eBPF program license from user space */ 993 if (strncpy_from_user(license, u64_to_user_ptr(attr->license), 994 sizeof(license) - 1) < 0) 995 return -EFAULT; 996 license[sizeof(license) - 1] = 0; 997 998 /* eBPF programs must be GPL compatible to use GPL-ed functions */ 999 is_gpl = license_is_gpl_compatible(license); 1000 1001 if (attr->insn_cnt == 0 || attr->insn_cnt > BPF_MAXINSNS) 1002 return -E2BIG; 1003 1004 if (type == BPF_PROG_TYPE_KPROBE && 1005 attr->kern_version != LINUX_VERSION_CODE) 1006 return -EINVAL; 1007 1008 if (type != BPF_PROG_TYPE_SOCKET_FILTER && 1009 type != BPF_PROG_TYPE_CGROUP_SKB && 1010 !capable(CAP_SYS_ADMIN)) 1011 return -EPERM; 1012 1013 /* plain bpf_prog allocation */ 1014 prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER); 1015 if (!prog) 1016 return -ENOMEM; 1017 1018 err = bpf_prog_charge_memlock(prog); 1019 if (err) 1020 goto free_prog_nouncharge; 1021 1022 prog->len = attr->insn_cnt; 1023 1024 err = -EFAULT; 1025 if (copy_from_user(prog->insns, u64_to_user_ptr(attr->insns), 1026 bpf_prog_insn_size(prog)) != 0) 1027 goto free_prog; 1028 1029 prog->orig_prog = NULL; 1030 prog->jited = 0; 1031 1032 atomic_set(&prog->aux->refcnt, 1); 1033 prog->gpl_compatible = is_gpl ? 1 : 0; 1034 1035 /* find program type: socket_filter vs tracing_filter */ 1036 err = find_prog_type(type, prog); 1037 if (err < 0) 1038 goto free_prog; 1039 1040 /* run eBPF verifier */ 1041 err = bpf_check(&prog, attr); 1042 if (err < 0) 1043 goto free_used_maps; 1044 1045 /* eBPF program is ready to be JITed */ 1046 prog = bpf_prog_select_runtime(prog, &err); 1047 if (err < 0) 1048 goto free_used_maps; 1049 1050 err = bpf_prog_alloc_id(prog); 1051 if (err) 1052 goto free_used_maps; 1053 1054 err = bpf_prog_new_fd(prog); 1055 if (err < 0) { 1056 /* failed to allocate fd. 1057 * bpf_prog_put() is needed because the above 1058 * bpf_prog_alloc_id() has published the prog 1059 * to the userspace and the userspace may 1060 * have refcnt-ed it through BPF_PROG_GET_FD_BY_ID. 1061 */ 1062 bpf_prog_put(prog); 1063 return err; 1064 } 1065 1066 bpf_prog_kallsyms_add(prog); 1067 trace_bpf_prog_load(prog, err); 1068 return err; 1069 1070 free_used_maps: 1071 free_used_maps(prog->aux); 1072 free_prog: 1073 bpf_prog_uncharge_memlock(prog); 1074 free_prog_nouncharge: 1075 bpf_prog_free(prog); 1076 return err; 1077 } 1078 1079 #define BPF_OBJ_LAST_FIELD bpf_fd 1080 1081 static int bpf_obj_pin(const union bpf_attr *attr) 1082 { 1083 if (CHECK_ATTR(BPF_OBJ)) 1084 return -EINVAL; 1085 1086 return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname)); 1087 } 1088 1089 static int bpf_obj_get(const union bpf_attr *attr) 1090 { 1091 if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0) 1092 return -EINVAL; 1093 1094 return bpf_obj_get_user(u64_to_user_ptr(attr->pathname)); 1095 } 1096 1097 #ifdef CONFIG_CGROUP_BPF 1098 1099 #define BPF_PROG_ATTACH_LAST_FIELD attach_flags 1100 1101 static int sockmap_get_from_fd(const union bpf_attr *attr, bool attach) 1102 { 1103 struct bpf_prog *prog = NULL; 1104 int ufd = attr->target_fd; 1105 struct bpf_map *map; 1106 struct fd f; 1107 int err; 1108 1109 f = fdget(ufd); 1110 map = __bpf_map_get(f); 1111 if (IS_ERR(map)) 1112 return PTR_ERR(map); 1113 1114 if (attach) { 1115 prog = bpf_prog_get_type(attr->attach_bpf_fd, 1116 BPF_PROG_TYPE_SK_SKB); 1117 if (IS_ERR(prog)) { 1118 fdput(f); 1119 return PTR_ERR(prog); 1120 } 1121 } 1122 1123 err = sock_map_prog(map, prog, attr->attach_type); 1124 if (err) { 1125 fdput(f); 1126 if (prog) 1127 bpf_prog_put(prog); 1128 return err; 1129 } 1130 1131 fdput(f); 1132 return 0; 1133 } 1134 1135 static int bpf_prog_attach(const union bpf_attr *attr) 1136 { 1137 enum bpf_prog_type ptype; 1138 struct bpf_prog *prog; 1139 struct cgroup *cgrp; 1140 int ret; 1141 1142 if (!capable(CAP_NET_ADMIN)) 1143 return -EPERM; 1144 1145 if (CHECK_ATTR(BPF_PROG_ATTACH)) 1146 return -EINVAL; 1147 1148 if (attr->attach_flags & ~BPF_F_ALLOW_OVERRIDE) 1149 return -EINVAL; 1150 1151 switch (attr->attach_type) { 1152 case BPF_CGROUP_INET_INGRESS: 1153 case BPF_CGROUP_INET_EGRESS: 1154 ptype = BPF_PROG_TYPE_CGROUP_SKB; 1155 break; 1156 case BPF_CGROUP_INET_SOCK_CREATE: 1157 ptype = BPF_PROG_TYPE_CGROUP_SOCK; 1158 break; 1159 case BPF_CGROUP_SOCK_OPS: 1160 ptype = BPF_PROG_TYPE_SOCK_OPS; 1161 break; 1162 case BPF_SK_SKB_STREAM_PARSER: 1163 case BPF_SK_SKB_STREAM_VERDICT: 1164 return sockmap_get_from_fd(attr, true); 1165 default: 1166 return -EINVAL; 1167 } 1168 1169 prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); 1170 if (IS_ERR(prog)) 1171 return PTR_ERR(prog); 1172 1173 cgrp = cgroup_get_from_fd(attr->target_fd); 1174 if (IS_ERR(cgrp)) { 1175 bpf_prog_put(prog); 1176 return PTR_ERR(cgrp); 1177 } 1178 1179 ret = cgroup_bpf_update(cgrp, prog, attr->attach_type, 1180 attr->attach_flags & BPF_F_ALLOW_OVERRIDE); 1181 if (ret) 1182 bpf_prog_put(prog); 1183 cgroup_put(cgrp); 1184 1185 return ret; 1186 } 1187 1188 #define BPF_PROG_DETACH_LAST_FIELD attach_type 1189 1190 static int bpf_prog_detach(const union bpf_attr *attr) 1191 { 1192 struct cgroup *cgrp; 1193 int ret; 1194 1195 if (!capable(CAP_NET_ADMIN)) 1196 return -EPERM; 1197 1198 if (CHECK_ATTR(BPF_PROG_DETACH)) 1199 return -EINVAL; 1200 1201 switch (attr->attach_type) { 1202 case BPF_CGROUP_INET_INGRESS: 1203 case BPF_CGROUP_INET_EGRESS: 1204 case BPF_CGROUP_INET_SOCK_CREATE: 1205 case BPF_CGROUP_SOCK_OPS: 1206 cgrp = cgroup_get_from_fd(attr->target_fd); 1207 if (IS_ERR(cgrp)) 1208 return PTR_ERR(cgrp); 1209 1210 ret = cgroup_bpf_update(cgrp, NULL, attr->attach_type, false); 1211 cgroup_put(cgrp); 1212 break; 1213 case BPF_SK_SKB_STREAM_PARSER: 1214 case BPF_SK_SKB_STREAM_VERDICT: 1215 ret = sockmap_get_from_fd(attr, false); 1216 break; 1217 default: 1218 return -EINVAL; 1219 } 1220 1221 return ret; 1222 } 1223 1224 #endif /* CONFIG_CGROUP_BPF */ 1225 1226 #define BPF_PROG_TEST_RUN_LAST_FIELD test.duration 1227 1228 static int bpf_prog_test_run(const union bpf_attr *attr, 1229 union bpf_attr __user *uattr) 1230 { 1231 struct bpf_prog *prog; 1232 int ret = -ENOTSUPP; 1233 1234 if (CHECK_ATTR(BPF_PROG_TEST_RUN)) 1235 return -EINVAL; 1236 1237 prog = bpf_prog_get(attr->test.prog_fd); 1238 if (IS_ERR(prog)) 1239 return PTR_ERR(prog); 1240 1241 if (prog->aux->ops->test_run) 1242 ret = prog->aux->ops->test_run(prog, attr, uattr); 1243 1244 bpf_prog_put(prog); 1245 return ret; 1246 } 1247 1248 #define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id 1249 1250 static int bpf_obj_get_next_id(const union bpf_attr *attr, 1251 union bpf_attr __user *uattr, 1252 struct idr *idr, 1253 spinlock_t *lock) 1254 { 1255 u32 next_id = attr->start_id; 1256 int err = 0; 1257 1258 if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX) 1259 return -EINVAL; 1260 1261 if (!capable(CAP_SYS_ADMIN)) 1262 return -EPERM; 1263 1264 next_id++; 1265 spin_lock_bh(lock); 1266 if (!idr_get_next(idr, &next_id)) 1267 err = -ENOENT; 1268 spin_unlock_bh(lock); 1269 1270 if (!err) 1271 err = put_user(next_id, &uattr->next_id); 1272 1273 return err; 1274 } 1275 1276 #define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id 1277 1278 static int bpf_prog_get_fd_by_id(const union bpf_attr *attr) 1279 { 1280 struct bpf_prog *prog; 1281 u32 id = attr->prog_id; 1282 int fd; 1283 1284 if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID)) 1285 return -EINVAL; 1286 1287 if (!capable(CAP_SYS_ADMIN)) 1288 return -EPERM; 1289 1290 spin_lock_bh(&prog_idr_lock); 1291 prog = idr_find(&prog_idr, id); 1292 if (prog) 1293 prog = bpf_prog_inc_not_zero(prog); 1294 else 1295 prog = ERR_PTR(-ENOENT); 1296 spin_unlock_bh(&prog_idr_lock); 1297 1298 if (IS_ERR(prog)) 1299 return PTR_ERR(prog); 1300 1301 fd = bpf_prog_new_fd(prog); 1302 if (fd < 0) 1303 bpf_prog_put(prog); 1304 1305 return fd; 1306 } 1307 1308 #define BPF_MAP_GET_FD_BY_ID_LAST_FIELD map_id 1309 1310 static int bpf_map_get_fd_by_id(const union bpf_attr *attr) 1311 { 1312 struct bpf_map *map; 1313 u32 id = attr->map_id; 1314 int fd; 1315 1316 if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID)) 1317 return -EINVAL; 1318 1319 if (!capable(CAP_SYS_ADMIN)) 1320 return -EPERM; 1321 1322 spin_lock_bh(&map_idr_lock); 1323 map = idr_find(&map_idr, id); 1324 if (map) 1325 map = bpf_map_inc_not_zero(map, true); 1326 else 1327 map = ERR_PTR(-ENOENT); 1328 spin_unlock_bh(&map_idr_lock); 1329 1330 if (IS_ERR(map)) 1331 return PTR_ERR(map); 1332 1333 fd = bpf_map_new_fd(map); 1334 if (fd < 0) 1335 bpf_map_put(map); 1336 1337 return fd; 1338 } 1339 1340 static int bpf_prog_get_info_by_fd(struct bpf_prog *prog, 1341 const union bpf_attr *attr, 1342 union bpf_attr __user *uattr) 1343 { 1344 struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info); 1345 struct bpf_prog_info info = {}; 1346 u32 info_len = attr->info.info_len; 1347 char __user *uinsns; 1348 u32 ulen; 1349 int err; 1350 1351 err = check_uarg_tail_zero(uinfo, sizeof(info), info_len); 1352 if (err) 1353 return err; 1354 info_len = min_t(u32, sizeof(info), info_len); 1355 1356 if (copy_from_user(&info, uinfo, info_len)) 1357 return -EFAULT; 1358 1359 info.type = prog->type; 1360 info.id = prog->aux->id; 1361 1362 memcpy(info.tag, prog->tag, sizeof(prog->tag)); 1363 1364 if (!capable(CAP_SYS_ADMIN)) { 1365 info.jited_prog_len = 0; 1366 info.xlated_prog_len = 0; 1367 goto done; 1368 } 1369 1370 ulen = info.jited_prog_len; 1371 info.jited_prog_len = prog->jited_len; 1372 if (info.jited_prog_len && ulen) { 1373 uinsns = u64_to_user_ptr(info.jited_prog_insns); 1374 ulen = min_t(u32, info.jited_prog_len, ulen); 1375 if (copy_to_user(uinsns, prog->bpf_func, ulen)) 1376 return -EFAULT; 1377 } 1378 1379 ulen = info.xlated_prog_len; 1380 info.xlated_prog_len = bpf_prog_insn_size(prog); 1381 if (info.xlated_prog_len && ulen) { 1382 uinsns = u64_to_user_ptr(info.xlated_prog_insns); 1383 ulen = min_t(u32, info.xlated_prog_len, ulen); 1384 if (copy_to_user(uinsns, prog->insnsi, ulen)) 1385 return -EFAULT; 1386 } 1387 1388 done: 1389 if (copy_to_user(uinfo, &info, info_len) || 1390 put_user(info_len, &uattr->info.info_len)) 1391 return -EFAULT; 1392 1393 return 0; 1394 } 1395 1396 static int bpf_map_get_info_by_fd(struct bpf_map *map, 1397 const union bpf_attr *attr, 1398 union bpf_attr __user *uattr) 1399 { 1400 struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info); 1401 struct bpf_map_info info = {}; 1402 u32 info_len = attr->info.info_len; 1403 int err; 1404 1405 err = check_uarg_tail_zero(uinfo, sizeof(info), info_len); 1406 if (err) 1407 return err; 1408 info_len = min_t(u32, sizeof(info), info_len); 1409 1410 info.type = map->map_type; 1411 info.id = map->id; 1412 info.key_size = map->key_size; 1413 info.value_size = map->value_size; 1414 info.max_entries = map->max_entries; 1415 info.map_flags = map->map_flags; 1416 1417 if (copy_to_user(uinfo, &info, info_len) || 1418 put_user(info_len, &uattr->info.info_len)) 1419 return -EFAULT; 1420 1421 return 0; 1422 } 1423 1424 #define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info 1425 1426 static int bpf_obj_get_info_by_fd(const union bpf_attr *attr, 1427 union bpf_attr __user *uattr) 1428 { 1429 int ufd = attr->info.bpf_fd; 1430 struct fd f; 1431 int err; 1432 1433 if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD)) 1434 return -EINVAL; 1435 1436 f = fdget(ufd); 1437 if (!f.file) 1438 return -EBADFD; 1439 1440 if (f.file->f_op == &bpf_prog_fops) 1441 err = bpf_prog_get_info_by_fd(f.file->private_data, attr, 1442 uattr); 1443 else if (f.file->f_op == &bpf_map_fops) 1444 err = bpf_map_get_info_by_fd(f.file->private_data, attr, 1445 uattr); 1446 else 1447 err = -EINVAL; 1448 1449 fdput(f); 1450 return err; 1451 } 1452 1453 SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size) 1454 { 1455 union bpf_attr attr = {}; 1456 int err; 1457 1458 if (!capable(CAP_SYS_ADMIN) && sysctl_unprivileged_bpf_disabled) 1459 return -EPERM; 1460 1461 err = check_uarg_tail_zero(uattr, sizeof(attr), size); 1462 if (err) 1463 return err; 1464 size = min_t(u32, size, sizeof(attr)); 1465 1466 /* copy attributes from user space, may be less than sizeof(bpf_attr) */ 1467 if (copy_from_user(&attr, uattr, size) != 0) 1468 return -EFAULT; 1469 1470 switch (cmd) { 1471 case BPF_MAP_CREATE: 1472 err = map_create(&attr); 1473 break; 1474 case BPF_MAP_LOOKUP_ELEM: 1475 err = map_lookup_elem(&attr); 1476 break; 1477 case BPF_MAP_UPDATE_ELEM: 1478 err = map_update_elem(&attr); 1479 break; 1480 case BPF_MAP_DELETE_ELEM: 1481 err = map_delete_elem(&attr); 1482 break; 1483 case BPF_MAP_GET_NEXT_KEY: 1484 err = map_get_next_key(&attr); 1485 break; 1486 case BPF_PROG_LOAD: 1487 err = bpf_prog_load(&attr); 1488 break; 1489 case BPF_OBJ_PIN: 1490 err = bpf_obj_pin(&attr); 1491 break; 1492 case BPF_OBJ_GET: 1493 err = bpf_obj_get(&attr); 1494 break; 1495 #ifdef CONFIG_CGROUP_BPF 1496 case BPF_PROG_ATTACH: 1497 err = bpf_prog_attach(&attr); 1498 break; 1499 case BPF_PROG_DETACH: 1500 err = bpf_prog_detach(&attr); 1501 break; 1502 #endif 1503 case BPF_PROG_TEST_RUN: 1504 err = bpf_prog_test_run(&attr, uattr); 1505 break; 1506 case BPF_PROG_GET_NEXT_ID: 1507 err = bpf_obj_get_next_id(&attr, uattr, 1508 &prog_idr, &prog_idr_lock); 1509 break; 1510 case BPF_MAP_GET_NEXT_ID: 1511 err = bpf_obj_get_next_id(&attr, uattr, 1512 &map_idr, &map_idr_lock); 1513 break; 1514 case BPF_PROG_GET_FD_BY_ID: 1515 err = bpf_prog_get_fd_by_id(&attr); 1516 break; 1517 case BPF_MAP_GET_FD_BY_ID: 1518 err = bpf_map_get_fd_by_id(&attr); 1519 break; 1520 case BPF_OBJ_GET_INFO_BY_FD: 1521 err = bpf_obj_get_info_by_fd(&attr, uattr); 1522 break; 1523 default: 1524 err = -EINVAL; 1525 break; 1526 } 1527 1528 return err; 1529 } 1530