1 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 2 * 3 * This program is free software; you can redistribute it and/or 4 * modify it under the terms of version 2 of the GNU General Public 5 * License as published by the Free Software Foundation. 6 * 7 * This program is distributed in the hope that it will be useful, but 8 * WITHOUT ANY WARRANTY; without even the implied warranty of 9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 10 * General Public License for more details. 11 */ 12 #include <linux/bpf.h> 13 #include <linux/bpf_trace.h> 14 #include <linux/syscalls.h> 15 #include <linux/slab.h> 16 #include <linux/sched/signal.h> 17 #include <linux/vmalloc.h> 18 #include <linux/mmzone.h> 19 #include <linux/anon_inodes.h> 20 #include <linux/file.h> 21 #include <linux/license.h> 22 #include <linux/filter.h> 23 #include <linux/version.h> 24 #include <linux/kernel.h> 25 26 DEFINE_PER_CPU(int, bpf_prog_active); 27 28 int sysctl_unprivileged_bpf_disabled __read_mostly; 29 30 static LIST_HEAD(bpf_map_types); 31 32 static struct bpf_map *find_and_alloc_map(union bpf_attr *attr) 33 { 34 struct bpf_map_type_list *tl; 35 struct bpf_map *map; 36 37 list_for_each_entry(tl, &bpf_map_types, list_node) { 38 if (tl->type == attr->map_type) { 39 map = tl->ops->map_alloc(attr); 40 if (IS_ERR(map)) 41 return map; 42 map->ops = tl->ops; 43 map->map_type = attr->map_type; 44 return map; 45 } 46 } 47 return ERR_PTR(-EINVAL); 48 } 49 50 /* boot time registration of different map implementations */ 51 void bpf_register_map_type(struct bpf_map_type_list *tl) 52 { 53 list_add(&tl->list_node, &bpf_map_types); 54 } 55 56 void *bpf_map_area_alloc(size_t size) 57 { 58 /* We definitely need __GFP_NORETRY, so OOM killer doesn't 59 * trigger under memory pressure as we really just want to 60 * fail instead. 61 */ 62 const gfp_t flags = __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO; 63 void *area; 64 65 if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) { 66 area = kmalloc(size, GFP_USER | flags); 67 if (area != NULL) 68 return area; 69 } 70 71 return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM | flags, 72 PAGE_KERNEL); 73 } 74 75 void bpf_map_area_free(void *area) 76 { 77 kvfree(area); 78 } 79 80 int bpf_map_precharge_memlock(u32 pages) 81 { 82 struct user_struct *user = get_current_user(); 83 unsigned long memlock_limit, cur; 84 85 memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 86 cur = atomic_long_read(&user->locked_vm); 87 free_uid(user); 88 if (cur + pages > memlock_limit) 89 return -EPERM; 90 return 0; 91 } 92 93 static int bpf_map_charge_memlock(struct bpf_map *map) 94 { 95 struct user_struct *user = get_current_user(); 96 unsigned long memlock_limit; 97 98 memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 99 100 atomic_long_add(map->pages, &user->locked_vm); 101 102 if (atomic_long_read(&user->locked_vm) > memlock_limit) { 103 atomic_long_sub(map->pages, &user->locked_vm); 104 free_uid(user); 105 return -EPERM; 106 } 107 map->user = user; 108 return 0; 109 } 110 111 static void bpf_map_uncharge_memlock(struct bpf_map *map) 112 { 113 struct user_struct *user = map->user; 114 115 atomic_long_sub(map->pages, &user->locked_vm); 116 free_uid(user); 117 } 118 119 /* called from workqueue */ 120 static void bpf_map_free_deferred(struct work_struct *work) 121 { 122 struct bpf_map *map = container_of(work, struct bpf_map, work); 123 124 bpf_map_uncharge_memlock(map); 125 /* implementation dependent freeing */ 126 map->ops->map_free(map); 127 } 128 129 static void bpf_map_put_uref(struct bpf_map *map) 130 { 131 if (atomic_dec_and_test(&map->usercnt)) { 132 if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) 133 bpf_fd_array_map_clear(map); 134 } 135 } 136 137 /* decrement map refcnt and schedule it for freeing via workqueue 138 * (unrelying map implementation ops->map_free() might sleep) 139 */ 140 void bpf_map_put(struct bpf_map *map) 141 { 142 if (atomic_dec_and_test(&map->refcnt)) { 143 INIT_WORK(&map->work, bpf_map_free_deferred); 144 schedule_work(&map->work); 145 } 146 } 147 148 void bpf_map_put_with_uref(struct bpf_map *map) 149 { 150 bpf_map_put_uref(map); 151 bpf_map_put(map); 152 } 153 154 static int bpf_map_release(struct inode *inode, struct file *filp) 155 { 156 struct bpf_map *map = filp->private_data; 157 158 if (map->ops->map_release) 159 map->ops->map_release(map, filp); 160 161 bpf_map_put_with_uref(map); 162 return 0; 163 } 164 165 #ifdef CONFIG_PROC_FS 166 static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp) 167 { 168 const struct bpf_map *map = filp->private_data; 169 const struct bpf_array *array; 170 u32 owner_prog_type = 0; 171 172 if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) { 173 array = container_of(map, struct bpf_array, map); 174 owner_prog_type = array->owner_prog_type; 175 } 176 177 seq_printf(m, 178 "map_type:\t%u\n" 179 "key_size:\t%u\n" 180 "value_size:\t%u\n" 181 "max_entries:\t%u\n" 182 "map_flags:\t%#x\n" 183 "memlock:\t%llu\n", 184 map->map_type, 185 map->key_size, 186 map->value_size, 187 map->max_entries, 188 map->map_flags, 189 map->pages * 1ULL << PAGE_SHIFT); 190 191 if (owner_prog_type) 192 seq_printf(m, "owner_prog_type:\t%u\n", 193 owner_prog_type); 194 } 195 #endif 196 197 static const struct file_operations bpf_map_fops = { 198 #ifdef CONFIG_PROC_FS 199 .show_fdinfo = bpf_map_show_fdinfo, 200 #endif 201 .release = bpf_map_release, 202 }; 203 204 int bpf_map_new_fd(struct bpf_map *map) 205 { 206 return anon_inode_getfd("bpf-map", &bpf_map_fops, map, 207 O_RDWR | O_CLOEXEC); 208 } 209 210 /* helper macro to check that unused fields 'union bpf_attr' are zero */ 211 #define CHECK_ATTR(CMD) \ 212 memchr_inv((void *) &attr->CMD##_LAST_FIELD + \ 213 sizeof(attr->CMD##_LAST_FIELD), 0, \ 214 sizeof(*attr) - \ 215 offsetof(union bpf_attr, CMD##_LAST_FIELD) - \ 216 sizeof(attr->CMD##_LAST_FIELD)) != NULL 217 218 #define BPF_MAP_CREATE_LAST_FIELD map_flags 219 /* called via syscall */ 220 static int map_create(union bpf_attr *attr) 221 { 222 struct bpf_map *map; 223 int err; 224 225 err = CHECK_ATTR(BPF_MAP_CREATE); 226 if (err) 227 return -EINVAL; 228 229 /* find map type and init map: hashtable vs rbtree vs bloom vs ... */ 230 map = find_and_alloc_map(attr); 231 if (IS_ERR(map)) 232 return PTR_ERR(map); 233 234 atomic_set(&map->refcnt, 1); 235 atomic_set(&map->usercnt, 1); 236 237 err = bpf_map_charge_memlock(map); 238 if (err) 239 goto free_map_nouncharge; 240 241 err = bpf_map_new_fd(map); 242 if (err < 0) 243 /* failed to allocate fd */ 244 goto free_map; 245 246 trace_bpf_map_create(map, err); 247 return err; 248 249 free_map: 250 bpf_map_uncharge_memlock(map); 251 free_map_nouncharge: 252 map->ops->map_free(map); 253 return err; 254 } 255 256 /* if error is returned, fd is released. 257 * On success caller should complete fd access with matching fdput() 258 */ 259 struct bpf_map *__bpf_map_get(struct fd f) 260 { 261 if (!f.file) 262 return ERR_PTR(-EBADF); 263 if (f.file->f_op != &bpf_map_fops) { 264 fdput(f); 265 return ERR_PTR(-EINVAL); 266 } 267 268 return f.file->private_data; 269 } 270 271 /* prog's and map's refcnt limit */ 272 #define BPF_MAX_REFCNT 32768 273 274 struct bpf_map *bpf_map_inc(struct bpf_map *map, bool uref) 275 { 276 if (atomic_inc_return(&map->refcnt) > BPF_MAX_REFCNT) { 277 atomic_dec(&map->refcnt); 278 return ERR_PTR(-EBUSY); 279 } 280 if (uref) 281 atomic_inc(&map->usercnt); 282 return map; 283 } 284 285 struct bpf_map *bpf_map_get_with_uref(u32 ufd) 286 { 287 struct fd f = fdget(ufd); 288 struct bpf_map *map; 289 290 map = __bpf_map_get(f); 291 if (IS_ERR(map)) 292 return map; 293 294 map = bpf_map_inc(map, true); 295 fdput(f); 296 297 return map; 298 } 299 300 int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value) 301 { 302 return -ENOTSUPP; 303 } 304 305 /* last field in 'union bpf_attr' used by this command */ 306 #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD value 307 308 static int map_lookup_elem(union bpf_attr *attr) 309 { 310 void __user *ukey = u64_to_user_ptr(attr->key); 311 void __user *uvalue = u64_to_user_ptr(attr->value); 312 int ufd = attr->map_fd; 313 struct bpf_map *map; 314 void *key, *value, *ptr; 315 u32 value_size; 316 struct fd f; 317 int err; 318 319 if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM)) 320 return -EINVAL; 321 322 f = fdget(ufd); 323 map = __bpf_map_get(f); 324 if (IS_ERR(map)) 325 return PTR_ERR(map); 326 327 err = -ENOMEM; 328 key = kmalloc(map->key_size, GFP_USER); 329 if (!key) 330 goto err_put; 331 332 err = -EFAULT; 333 if (copy_from_user(key, ukey, map->key_size) != 0) 334 goto free_key; 335 336 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 337 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 338 map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) 339 value_size = round_up(map->value_size, 8) * num_possible_cpus(); 340 else 341 value_size = map->value_size; 342 343 err = -ENOMEM; 344 value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 345 if (!value) 346 goto free_key; 347 348 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 349 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 350 err = bpf_percpu_hash_copy(map, key, value); 351 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 352 err = bpf_percpu_array_copy(map, key, value); 353 } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) { 354 err = bpf_stackmap_copy(map, key, value); 355 } else { 356 rcu_read_lock(); 357 ptr = map->ops->map_lookup_elem(map, key); 358 if (ptr) 359 memcpy(value, ptr, value_size); 360 rcu_read_unlock(); 361 err = ptr ? 0 : -ENOENT; 362 } 363 364 if (err) 365 goto free_value; 366 367 err = -EFAULT; 368 if (copy_to_user(uvalue, value, value_size) != 0) 369 goto free_value; 370 371 trace_bpf_map_lookup_elem(map, ufd, key, value); 372 err = 0; 373 374 free_value: 375 kfree(value); 376 free_key: 377 kfree(key); 378 err_put: 379 fdput(f); 380 return err; 381 } 382 383 #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags 384 385 static int map_update_elem(union bpf_attr *attr) 386 { 387 void __user *ukey = u64_to_user_ptr(attr->key); 388 void __user *uvalue = u64_to_user_ptr(attr->value); 389 int ufd = attr->map_fd; 390 struct bpf_map *map; 391 void *key, *value; 392 u32 value_size; 393 struct fd f; 394 int err; 395 396 if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM)) 397 return -EINVAL; 398 399 f = fdget(ufd); 400 map = __bpf_map_get(f); 401 if (IS_ERR(map)) 402 return PTR_ERR(map); 403 404 err = -ENOMEM; 405 key = kmalloc(map->key_size, GFP_USER); 406 if (!key) 407 goto err_put; 408 409 err = -EFAULT; 410 if (copy_from_user(key, ukey, map->key_size) != 0) 411 goto free_key; 412 413 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 414 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 415 map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) 416 value_size = round_up(map->value_size, 8) * num_possible_cpus(); 417 else 418 value_size = map->value_size; 419 420 err = -ENOMEM; 421 value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); 422 if (!value) 423 goto free_key; 424 425 err = -EFAULT; 426 if (copy_from_user(value, uvalue, value_size) != 0) 427 goto free_value; 428 429 /* must increment bpf_prog_active to avoid kprobe+bpf triggering from 430 * inside bpf map update or delete otherwise deadlocks are possible 431 */ 432 preempt_disable(); 433 __this_cpu_inc(bpf_prog_active); 434 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 435 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { 436 err = bpf_percpu_hash_update(map, key, value, attr->flags); 437 } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { 438 err = bpf_percpu_array_update(map, key, value, attr->flags); 439 } else if (map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || 440 map->map_type == BPF_MAP_TYPE_PROG_ARRAY || 441 map->map_type == BPF_MAP_TYPE_CGROUP_ARRAY) { 442 rcu_read_lock(); 443 err = bpf_fd_array_map_update_elem(map, f.file, key, value, 444 attr->flags); 445 rcu_read_unlock(); 446 } else { 447 rcu_read_lock(); 448 err = map->ops->map_update_elem(map, key, value, attr->flags); 449 rcu_read_unlock(); 450 } 451 __this_cpu_dec(bpf_prog_active); 452 preempt_enable(); 453 454 if (!err) 455 trace_bpf_map_update_elem(map, ufd, key, value); 456 free_value: 457 kfree(value); 458 free_key: 459 kfree(key); 460 err_put: 461 fdput(f); 462 return err; 463 } 464 465 #define BPF_MAP_DELETE_ELEM_LAST_FIELD key 466 467 static int map_delete_elem(union bpf_attr *attr) 468 { 469 void __user *ukey = u64_to_user_ptr(attr->key); 470 int ufd = attr->map_fd; 471 struct bpf_map *map; 472 struct fd f; 473 void *key; 474 int err; 475 476 if (CHECK_ATTR(BPF_MAP_DELETE_ELEM)) 477 return -EINVAL; 478 479 f = fdget(ufd); 480 map = __bpf_map_get(f); 481 if (IS_ERR(map)) 482 return PTR_ERR(map); 483 484 err = -ENOMEM; 485 key = kmalloc(map->key_size, GFP_USER); 486 if (!key) 487 goto err_put; 488 489 err = -EFAULT; 490 if (copy_from_user(key, ukey, map->key_size) != 0) 491 goto free_key; 492 493 preempt_disable(); 494 __this_cpu_inc(bpf_prog_active); 495 rcu_read_lock(); 496 err = map->ops->map_delete_elem(map, key); 497 rcu_read_unlock(); 498 __this_cpu_dec(bpf_prog_active); 499 preempt_enable(); 500 501 if (!err) 502 trace_bpf_map_delete_elem(map, ufd, key); 503 free_key: 504 kfree(key); 505 err_put: 506 fdput(f); 507 return err; 508 } 509 510 /* last field in 'union bpf_attr' used by this command */ 511 #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key 512 513 static int map_get_next_key(union bpf_attr *attr) 514 { 515 void __user *ukey = u64_to_user_ptr(attr->key); 516 void __user *unext_key = u64_to_user_ptr(attr->next_key); 517 int ufd = attr->map_fd; 518 struct bpf_map *map; 519 void *key, *next_key; 520 struct fd f; 521 int err; 522 523 if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY)) 524 return -EINVAL; 525 526 f = fdget(ufd); 527 map = __bpf_map_get(f); 528 if (IS_ERR(map)) 529 return PTR_ERR(map); 530 531 err = -ENOMEM; 532 key = kmalloc(map->key_size, GFP_USER); 533 if (!key) 534 goto err_put; 535 536 err = -EFAULT; 537 if (copy_from_user(key, ukey, map->key_size) != 0) 538 goto free_key; 539 540 err = -ENOMEM; 541 next_key = kmalloc(map->key_size, GFP_USER); 542 if (!next_key) 543 goto free_key; 544 545 rcu_read_lock(); 546 err = map->ops->map_get_next_key(map, key, next_key); 547 rcu_read_unlock(); 548 if (err) 549 goto free_next_key; 550 551 err = -EFAULT; 552 if (copy_to_user(unext_key, next_key, map->key_size) != 0) 553 goto free_next_key; 554 555 trace_bpf_map_next_key(map, ufd, key, next_key); 556 err = 0; 557 558 free_next_key: 559 kfree(next_key); 560 free_key: 561 kfree(key); 562 err_put: 563 fdput(f); 564 return err; 565 } 566 567 static LIST_HEAD(bpf_prog_types); 568 569 static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog) 570 { 571 struct bpf_prog_type_list *tl; 572 573 list_for_each_entry(tl, &bpf_prog_types, list_node) { 574 if (tl->type == type) { 575 prog->aux->ops = tl->ops; 576 prog->type = type; 577 return 0; 578 } 579 } 580 581 return -EINVAL; 582 } 583 584 void bpf_register_prog_type(struct bpf_prog_type_list *tl) 585 { 586 list_add(&tl->list_node, &bpf_prog_types); 587 } 588 589 /* fixup insn->imm field of bpf_call instructions: 590 * if (insn->imm == BPF_FUNC_map_lookup_elem) 591 * insn->imm = bpf_map_lookup_elem - __bpf_call_base; 592 * else if (insn->imm == BPF_FUNC_map_update_elem) 593 * insn->imm = bpf_map_update_elem - __bpf_call_base; 594 * else ... 595 * 596 * this function is called after eBPF program passed verification 597 */ 598 static void fixup_bpf_calls(struct bpf_prog *prog) 599 { 600 const struct bpf_func_proto *fn; 601 int i; 602 603 for (i = 0; i < prog->len; i++) { 604 struct bpf_insn *insn = &prog->insnsi[i]; 605 606 if (insn->code == (BPF_JMP | BPF_CALL)) { 607 /* we reach here when program has bpf_call instructions 608 * and it passed bpf_check(), means that 609 * ops->get_func_proto must have been supplied, check it 610 */ 611 BUG_ON(!prog->aux->ops->get_func_proto); 612 613 if (insn->imm == BPF_FUNC_get_route_realm) 614 prog->dst_needed = 1; 615 if (insn->imm == BPF_FUNC_get_prandom_u32) 616 bpf_user_rnd_init_once(); 617 if (insn->imm == BPF_FUNC_xdp_adjust_head) 618 prog->xdp_adjust_head = 1; 619 if (insn->imm == BPF_FUNC_tail_call) { 620 /* If we tail call into other programs, we 621 * cannot make any assumptions since they 622 * can be replaced dynamically during runtime 623 * in the program array. 624 */ 625 prog->cb_access = 1; 626 prog->xdp_adjust_head = 1; 627 628 /* mark bpf_tail_call as different opcode 629 * to avoid conditional branch in 630 * interpeter for every normal call 631 * and to prevent accidental JITing by 632 * JIT compiler that doesn't support 633 * bpf_tail_call yet 634 */ 635 insn->imm = 0; 636 insn->code |= BPF_X; 637 continue; 638 } 639 640 fn = prog->aux->ops->get_func_proto(insn->imm); 641 /* all functions that have prototype and verifier allowed 642 * programs to call them, must be real in-kernel functions 643 */ 644 BUG_ON(!fn->func); 645 insn->imm = fn->func - __bpf_call_base; 646 } 647 } 648 } 649 650 /* drop refcnt on maps used by eBPF program and free auxilary data */ 651 static void free_used_maps(struct bpf_prog_aux *aux) 652 { 653 int i; 654 655 for (i = 0; i < aux->used_map_cnt; i++) 656 bpf_map_put(aux->used_maps[i]); 657 658 kfree(aux->used_maps); 659 } 660 661 int __bpf_prog_charge(struct user_struct *user, u32 pages) 662 { 663 unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 664 unsigned long user_bufs; 665 666 if (user) { 667 user_bufs = atomic_long_add_return(pages, &user->locked_vm); 668 if (user_bufs > memlock_limit) { 669 atomic_long_sub(pages, &user->locked_vm); 670 return -EPERM; 671 } 672 } 673 674 return 0; 675 } 676 677 void __bpf_prog_uncharge(struct user_struct *user, u32 pages) 678 { 679 if (user) 680 atomic_long_sub(pages, &user->locked_vm); 681 } 682 683 static int bpf_prog_charge_memlock(struct bpf_prog *prog) 684 { 685 struct user_struct *user = get_current_user(); 686 int ret; 687 688 ret = __bpf_prog_charge(user, prog->pages); 689 if (ret) { 690 free_uid(user); 691 return ret; 692 } 693 694 prog->aux->user = user; 695 return 0; 696 } 697 698 static void bpf_prog_uncharge_memlock(struct bpf_prog *prog) 699 { 700 struct user_struct *user = prog->aux->user; 701 702 __bpf_prog_uncharge(user, prog->pages); 703 free_uid(user); 704 } 705 706 static void __bpf_prog_put_rcu(struct rcu_head *rcu) 707 { 708 struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu); 709 710 free_used_maps(aux); 711 bpf_prog_uncharge_memlock(aux->prog); 712 bpf_prog_free(aux->prog); 713 } 714 715 void bpf_prog_put(struct bpf_prog *prog) 716 { 717 if (atomic_dec_and_test(&prog->aux->refcnt)) { 718 trace_bpf_prog_put_rcu(prog); 719 bpf_prog_kallsyms_del(prog); 720 call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); 721 } 722 } 723 EXPORT_SYMBOL_GPL(bpf_prog_put); 724 725 static int bpf_prog_release(struct inode *inode, struct file *filp) 726 { 727 struct bpf_prog *prog = filp->private_data; 728 729 bpf_prog_put(prog); 730 return 0; 731 } 732 733 #ifdef CONFIG_PROC_FS 734 static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp) 735 { 736 const struct bpf_prog *prog = filp->private_data; 737 char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; 738 739 bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); 740 seq_printf(m, 741 "prog_type:\t%u\n" 742 "prog_jited:\t%u\n" 743 "prog_tag:\t%s\n" 744 "memlock:\t%llu\n", 745 prog->type, 746 prog->jited, 747 prog_tag, 748 prog->pages * 1ULL << PAGE_SHIFT); 749 } 750 #endif 751 752 static const struct file_operations bpf_prog_fops = { 753 #ifdef CONFIG_PROC_FS 754 .show_fdinfo = bpf_prog_show_fdinfo, 755 #endif 756 .release = bpf_prog_release, 757 }; 758 759 int bpf_prog_new_fd(struct bpf_prog *prog) 760 { 761 return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog, 762 O_RDWR | O_CLOEXEC); 763 } 764 765 static struct bpf_prog *____bpf_prog_get(struct fd f) 766 { 767 if (!f.file) 768 return ERR_PTR(-EBADF); 769 if (f.file->f_op != &bpf_prog_fops) { 770 fdput(f); 771 return ERR_PTR(-EINVAL); 772 } 773 774 return f.file->private_data; 775 } 776 777 struct bpf_prog *bpf_prog_add(struct bpf_prog *prog, int i) 778 { 779 if (atomic_add_return(i, &prog->aux->refcnt) > BPF_MAX_REFCNT) { 780 atomic_sub(i, &prog->aux->refcnt); 781 return ERR_PTR(-EBUSY); 782 } 783 return prog; 784 } 785 EXPORT_SYMBOL_GPL(bpf_prog_add); 786 787 void bpf_prog_sub(struct bpf_prog *prog, int i) 788 { 789 /* Only to be used for undoing previous bpf_prog_add() in some 790 * error path. We still know that another entity in our call 791 * path holds a reference to the program, thus atomic_sub() can 792 * be safely used in such cases! 793 */ 794 WARN_ON(atomic_sub_return(i, &prog->aux->refcnt) == 0); 795 } 796 EXPORT_SYMBOL_GPL(bpf_prog_sub); 797 798 struct bpf_prog *bpf_prog_inc(struct bpf_prog *prog) 799 { 800 return bpf_prog_add(prog, 1); 801 } 802 EXPORT_SYMBOL_GPL(bpf_prog_inc); 803 804 static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *type) 805 { 806 struct fd f = fdget(ufd); 807 struct bpf_prog *prog; 808 809 prog = ____bpf_prog_get(f); 810 if (IS_ERR(prog)) 811 return prog; 812 if (type && prog->type != *type) { 813 prog = ERR_PTR(-EINVAL); 814 goto out; 815 } 816 817 prog = bpf_prog_inc(prog); 818 out: 819 fdput(f); 820 return prog; 821 } 822 823 struct bpf_prog *bpf_prog_get(u32 ufd) 824 { 825 return __bpf_prog_get(ufd, NULL); 826 } 827 828 struct bpf_prog *bpf_prog_get_type(u32 ufd, enum bpf_prog_type type) 829 { 830 struct bpf_prog *prog = __bpf_prog_get(ufd, &type); 831 832 if (!IS_ERR(prog)) 833 trace_bpf_prog_get_type(prog); 834 return prog; 835 } 836 EXPORT_SYMBOL_GPL(bpf_prog_get_type); 837 838 /* last field in 'union bpf_attr' used by this command */ 839 #define BPF_PROG_LOAD_LAST_FIELD kern_version 840 841 static int bpf_prog_load(union bpf_attr *attr) 842 { 843 enum bpf_prog_type type = attr->prog_type; 844 struct bpf_prog *prog; 845 int err; 846 char license[128]; 847 bool is_gpl; 848 849 if (CHECK_ATTR(BPF_PROG_LOAD)) 850 return -EINVAL; 851 852 /* copy eBPF program license from user space */ 853 if (strncpy_from_user(license, u64_to_user_ptr(attr->license), 854 sizeof(license) - 1) < 0) 855 return -EFAULT; 856 license[sizeof(license) - 1] = 0; 857 858 /* eBPF programs must be GPL compatible to use GPL-ed functions */ 859 is_gpl = license_is_gpl_compatible(license); 860 861 if (attr->insn_cnt == 0 || attr->insn_cnt > BPF_MAXINSNS) 862 return -E2BIG; 863 864 if (type == BPF_PROG_TYPE_KPROBE && 865 attr->kern_version != LINUX_VERSION_CODE) 866 return -EINVAL; 867 868 if (type != BPF_PROG_TYPE_SOCKET_FILTER && !capable(CAP_SYS_ADMIN)) 869 return -EPERM; 870 871 /* plain bpf_prog allocation */ 872 prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER); 873 if (!prog) 874 return -ENOMEM; 875 876 err = bpf_prog_charge_memlock(prog); 877 if (err) 878 goto free_prog_nouncharge; 879 880 prog->len = attr->insn_cnt; 881 882 err = -EFAULT; 883 if (copy_from_user(prog->insns, u64_to_user_ptr(attr->insns), 884 bpf_prog_insn_size(prog)) != 0) 885 goto free_prog; 886 887 prog->orig_prog = NULL; 888 prog->jited = 0; 889 890 atomic_set(&prog->aux->refcnt, 1); 891 prog->gpl_compatible = is_gpl ? 1 : 0; 892 893 /* find program type: socket_filter vs tracing_filter */ 894 err = find_prog_type(type, prog); 895 if (err < 0) 896 goto free_prog; 897 898 /* run eBPF verifier */ 899 err = bpf_check(&prog, attr); 900 if (err < 0) 901 goto free_used_maps; 902 903 /* fixup BPF_CALL->imm field */ 904 fixup_bpf_calls(prog); 905 906 /* eBPF program is ready to be JITed */ 907 prog = bpf_prog_select_runtime(prog, &err); 908 if (err < 0) 909 goto free_used_maps; 910 911 err = bpf_prog_new_fd(prog); 912 if (err < 0) 913 /* failed to allocate fd */ 914 goto free_used_maps; 915 916 bpf_prog_kallsyms_add(prog); 917 trace_bpf_prog_load(prog, err); 918 return err; 919 920 free_used_maps: 921 free_used_maps(prog->aux); 922 free_prog: 923 bpf_prog_uncharge_memlock(prog); 924 free_prog_nouncharge: 925 bpf_prog_free(prog); 926 return err; 927 } 928 929 #define BPF_OBJ_LAST_FIELD bpf_fd 930 931 static int bpf_obj_pin(const union bpf_attr *attr) 932 { 933 if (CHECK_ATTR(BPF_OBJ)) 934 return -EINVAL; 935 936 return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname)); 937 } 938 939 static int bpf_obj_get(const union bpf_attr *attr) 940 { 941 if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0) 942 return -EINVAL; 943 944 return bpf_obj_get_user(u64_to_user_ptr(attr->pathname)); 945 } 946 947 #ifdef CONFIG_CGROUP_BPF 948 949 #define BPF_PROG_ATTACH_LAST_FIELD attach_flags 950 951 static int bpf_prog_attach(const union bpf_attr *attr) 952 { 953 enum bpf_prog_type ptype; 954 struct bpf_prog *prog; 955 struct cgroup *cgrp; 956 int ret; 957 958 if (!capable(CAP_NET_ADMIN)) 959 return -EPERM; 960 961 if (CHECK_ATTR(BPF_PROG_ATTACH)) 962 return -EINVAL; 963 964 if (attr->attach_flags & ~BPF_F_ALLOW_OVERRIDE) 965 return -EINVAL; 966 967 switch (attr->attach_type) { 968 case BPF_CGROUP_INET_INGRESS: 969 case BPF_CGROUP_INET_EGRESS: 970 ptype = BPF_PROG_TYPE_CGROUP_SKB; 971 break; 972 case BPF_CGROUP_INET_SOCK_CREATE: 973 ptype = BPF_PROG_TYPE_CGROUP_SOCK; 974 break; 975 default: 976 return -EINVAL; 977 } 978 979 prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); 980 if (IS_ERR(prog)) 981 return PTR_ERR(prog); 982 983 cgrp = cgroup_get_from_fd(attr->target_fd); 984 if (IS_ERR(cgrp)) { 985 bpf_prog_put(prog); 986 return PTR_ERR(cgrp); 987 } 988 989 ret = cgroup_bpf_update(cgrp, prog, attr->attach_type, 990 attr->attach_flags & BPF_F_ALLOW_OVERRIDE); 991 if (ret) 992 bpf_prog_put(prog); 993 cgroup_put(cgrp); 994 995 return ret; 996 } 997 998 #define BPF_PROG_DETACH_LAST_FIELD attach_type 999 1000 static int bpf_prog_detach(const union bpf_attr *attr) 1001 { 1002 struct cgroup *cgrp; 1003 int ret; 1004 1005 if (!capable(CAP_NET_ADMIN)) 1006 return -EPERM; 1007 1008 if (CHECK_ATTR(BPF_PROG_DETACH)) 1009 return -EINVAL; 1010 1011 switch (attr->attach_type) { 1012 case BPF_CGROUP_INET_INGRESS: 1013 case BPF_CGROUP_INET_EGRESS: 1014 case BPF_CGROUP_INET_SOCK_CREATE: 1015 cgrp = cgroup_get_from_fd(attr->target_fd); 1016 if (IS_ERR(cgrp)) 1017 return PTR_ERR(cgrp); 1018 1019 ret = cgroup_bpf_update(cgrp, NULL, attr->attach_type, false); 1020 cgroup_put(cgrp); 1021 break; 1022 1023 default: 1024 return -EINVAL; 1025 } 1026 1027 return ret; 1028 } 1029 #endif /* CONFIG_CGROUP_BPF */ 1030 1031 SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size) 1032 { 1033 union bpf_attr attr = {}; 1034 int err; 1035 1036 if (!capable(CAP_SYS_ADMIN) && sysctl_unprivileged_bpf_disabled) 1037 return -EPERM; 1038 1039 if (!access_ok(VERIFY_READ, uattr, 1)) 1040 return -EFAULT; 1041 1042 if (size > PAGE_SIZE) /* silly large */ 1043 return -E2BIG; 1044 1045 /* If we're handed a bigger struct than we know of, 1046 * ensure all the unknown bits are 0 - i.e. new 1047 * user-space does not rely on any kernel feature 1048 * extensions we dont know about yet. 1049 */ 1050 if (size > sizeof(attr)) { 1051 unsigned char __user *addr; 1052 unsigned char __user *end; 1053 unsigned char val; 1054 1055 addr = (void __user *)uattr + sizeof(attr); 1056 end = (void __user *)uattr + size; 1057 1058 for (; addr < end; addr++) { 1059 err = get_user(val, addr); 1060 if (err) 1061 return err; 1062 if (val) 1063 return -E2BIG; 1064 } 1065 size = sizeof(attr); 1066 } 1067 1068 /* copy attributes from user space, may be less than sizeof(bpf_attr) */ 1069 if (copy_from_user(&attr, uattr, size) != 0) 1070 return -EFAULT; 1071 1072 switch (cmd) { 1073 case BPF_MAP_CREATE: 1074 err = map_create(&attr); 1075 break; 1076 case BPF_MAP_LOOKUP_ELEM: 1077 err = map_lookup_elem(&attr); 1078 break; 1079 case BPF_MAP_UPDATE_ELEM: 1080 err = map_update_elem(&attr); 1081 break; 1082 case BPF_MAP_DELETE_ELEM: 1083 err = map_delete_elem(&attr); 1084 break; 1085 case BPF_MAP_GET_NEXT_KEY: 1086 err = map_get_next_key(&attr); 1087 break; 1088 case BPF_PROG_LOAD: 1089 err = bpf_prog_load(&attr); 1090 break; 1091 case BPF_OBJ_PIN: 1092 err = bpf_obj_pin(&attr); 1093 break; 1094 case BPF_OBJ_GET: 1095 err = bpf_obj_get(&attr); 1096 break; 1097 1098 #ifdef CONFIG_CGROUP_BPF 1099 case BPF_PROG_ATTACH: 1100 err = bpf_prog_attach(&attr); 1101 break; 1102 case BPF_PROG_DETACH: 1103 err = bpf_prog_detach(&attr); 1104 break; 1105 #endif 1106 1107 default: 1108 err = -EINVAL; 1109 break; 1110 } 1111 1112 return err; 1113 } 1114