1 /* 2 * Copyright (C) 2017-2018 Netronome Systems, Inc. 3 * 4 * This software is licensed under the GNU General License Version 2, 5 * June 1991 as shown in the file COPYING in the top-level directory of this 6 * source tree. 7 * 8 * THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" 9 * WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, 10 * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 11 * FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE 12 * OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME 13 * THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 14 */ 15 16 #include <linux/bpf.h> 17 #include <linux/bpf_verifier.h> 18 #include <linux/bug.h> 19 #include <linux/kdev_t.h> 20 #include <linux/list.h> 21 #include <linux/lockdep.h> 22 #include <linux/netdevice.h> 23 #include <linux/printk.h> 24 #include <linux/proc_ns.h> 25 #include <linux/rhashtable.h> 26 #include <linux/rtnetlink.h> 27 #include <linux/rwsem.h> 28 29 /* Protects offdevs, members of bpf_offload_netdev and offload members 30 * of all progs. 31 * RTNL lock cannot be taken when holding this lock. 32 */ 33 static DECLARE_RWSEM(bpf_devs_lock); 34 35 struct bpf_offload_dev { 36 const struct bpf_prog_offload_ops *ops; 37 struct list_head netdevs; 38 void *priv; 39 }; 40 41 struct bpf_offload_netdev { 42 struct rhash_head l; 43 struct net_device *netdev; 44 struct bpf_offload_dev *offdev; 45 struct list_head progs; 46 struct list_head maps; 47 struct list_head offdev_netdevs; 48 }; 49 50 static const struct rhashtable_params offdevs_params = { 51 .nelem_hint = 4, 52 .key_len = sizeof(struct net_device *), 53 .key_offset = offsetof(struct bpf_offload_netdev, netdev), 54 .head_offset = offsetof(struct bpf_offload_netdev, l), 55 .automatic_shrinking = true, 56 }; 57 58 static struct rhashtable offdevs; 59 static bool offdevs_inited; 60 61 static int bpf_dev_offload_check(struct net_device *netdev) 62 { 63 if (!netdev) 64 return -EINVAL; 65 if (!netdev->netdev_ops->ndo_bpf) 66 return -EOPNOTSUPP; 67 return 0; 68 } 69 70 static struct bpf_offload_netdev * 71 bpf_offload_find_netdev(struct net_device *netdev) 72 { 73 lockdep_assert_held(&bpf_devs_lock); 74 75 if (!offdevs_inited) 76 return NULL; 77 return rhashtable_lookup_fast(&offdevs, &netdev, offdevs_params); 78 } 79 80 int bpf_prog_offload_init(struct bpf_prog *prog, union bpf_attr *attr) 81 { 82 struct bpf_offload_netdev *ondev; 83 struct bpf_prog_offload *offload; 84 int err; 85 86 if (attr->prog_type != BPF_PROG_TYPE_SCHED_CLS && 87 attr->prog_type != BPF_PROG_TYPE_XDP) 88 return -EINVAL; 89 90 if (attr->prog_flags) 91 return -EINVAL; 92 93 offload = kzalloc(sizeof(*offload), GFP_USER); 94 if (!offload) 95 return -ENOMEM; 96 97 offload->prog = prog; 98 99 offload->netdev = dev_get_by_index(current->nsproxy->net_ns, 100 attr->prog_ifindex); 101 err = bpf_dev_offload_check(offload->netdev); 102 if (err) 103 goto err_maybe_put; 104 105 down_write(&bpf_devs_lock); 106 ondev = bpf_offload_find_netdev(offload->netdev); 107 if (!ondev) { 108 err = -EINVAL; 109 goto err_unlock; 110 } 111 offload->offdev = ondev->offdev; 112 prog->aux->offload = offload; 113 list_add_tail(&offload->offloads, &ondev->progs); 114 dev_put(offload->netdev); 115 up_write(&bpf_devs_lock); 116 117 return 0; 118 err_unlock: 119 up_write(&bpf_devs_lock); 120 err_maybe_put: 121 if (offload->netdev) 122 dev_put(offload->netdev); 123 kfree(offload); 124 return err; 125 } 126 127 int bpf_prog_offload_verifier_prep(struct bpf_prog *prog) 128 { 129 struct bpf_prog_offload *offload; 130 int ret = -ENODEV; 131 132 down_read(&bpf_devs_lock); 133 offload = prog->aux->offload; 134 if (offload) { 135 ret = offload->offdev->ops->prepare(prog); 136 offload->dev_state = !ret; 137 } 138 up_read(&bpf_devs_lock); 139 140 return ret; 141 } 142 143 int bpf_prog_offload_verify_insn(struct bpf_verifier_env *env, 144 int insn_idx, int prev_insn_idx) 145 { 146 struct bpf_prog_offload *offload; 147 int ret = -ENODEV; 148 149 down_read(&bpf_devs_lock); 150 offload = env->prog->aux->offload; 151 if (offload) 152 ret = offload->offdev->ops->insn_hook(env, insn_idx, 153 prev_insn_idx); 154 up_read(&bpf_devs_lock); 155 156 return ret; 157 } 158 159 int bpf_prog_offload_finalize(struct bpf_verifier_env *env) 160 { 161 struct bpf_prog_offload *offload; 162 int ret = -ENODEV; 163 164 down_read(&bpf_devs_lock); 165 offload = env->prog->aux->offload; 166 if (offload) { 167 if (offload->offdev->ops->finalize) 168 ret = offload->offdev->ops->finalize(env); 169 else 170 ret = 0; 171 } 172 up_read(&bpf_devs_lock); 173 174 return ret; 175 } 176 177 void 178 bpf_prog_offload_replace_insn(struct bpf_verifier_env *env, u32 off, 179 struct bpf_insn *insn) 180 { 181 const struct bpf_prog_offload_ops *ops; 182 struct bpf_prog_offload *offload; 183 int ret = -EOPNOTSUPP; 184 185 down_read(&bpf_devs_lock); 186 offload = env->prog->aux->offload; 187 if (offload) { 188 ops = offload->offdev->ops; 189 if (!offload->opt_failed && ops->replace_insn) 190 ret = ops->replace_insn(env, off, insn); 191 offload->opt_failed |= ret; 192 } 193 up_read(&bpf_devs_lock); 194 } 195 196 void 197 bpf_prog_offload_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) 198 { 199 struct bpf_prog_offload *offload; 200 int ret = -EOPNOTSUPP; 201 202 down_read(&bpf_devs_lock); 203 offload = env->prog->aux->offload; 204 if (offload) { 205 if (!offload->opt_failed && offload->offdev->ops->remove_insns) 206 ret = offload->offdev->ops->remove_insns(env, off, cnt); 207 offload->opt_failed |= ret; 208 } 209 up_read(&bpf_devs_lock); 210 } 211 212 static void __bpf_prog_offload_destroy(struct bpf_prog *prog) 213 { 214 struct bpf_prog_offload *offload = prog->aux->offload; 215 216 if (offload->dev_state) 217 offload->offdev->ops->destroy(prog); 218 219 /* Make sure BPF_PROG_GET_NEXT_ID can't find this dead program */ 220 bpf_prog_free_id(prog, true); 221 222 list_del_init(&offload->offloads); 223 kfree(offload); 224 prog->aux->offload = NULL; 225 } 226 227 void bpf_prog_offload_destroy(struct bpf_prog *prog) 228 { 229 down_write(&bpf_devs_lock); 230 if (prog->aux->offload) 231 __bpf_prog_offload_destroy(prog); 232 up_write(&bpf_devs_lock); 233 } 234 235 static int bpf_prog_offload_translate(struct bpf_prog *prog) 236 { 237 struct bpf_prog_offload *offload; 238 int ret = -ENODEV; 239 240 down_read(&bpf_devs_lock); 241 offload = prog->aux->offload; 242 if (offload) 243 ret = offload->offdev->ops->translate(prog); 244 up_read(&bpf_devs_lock); 245 246 return ret; 247 } 248 249 static unsigned int bpf_prog_warn_on_exec(const void *ctx, 250 const struct bpf_insn *insn) 251 { 252 WARN(1, "attempt to execute device eBPF program on the host!"); 253 return 0; 254 } 255 256 int bpf_prog_offload_compile(struct bpf_prog *prog) 257 { 258 prog->bpf_func = bpf_prog_warn_on_exec; 259 260 return bpf_prog_offload_translate(prog); 261 } 262 263 struct ns_get_path_bpf_prog_args { 264 struct bpf_prog *prog; 265 struct bpf_prog_info *info; 266 }; 267 268 static struct ns_common *bpf_prog_offload_info_fill_ns(void *private_data) 269 { 270 struct ns_get_path_bpf_prog_args *args = private_data; 271 struct bpf_prog_aux *aux = args->prog->aux; 272 struct ns_common *ns; 273 struct net *net; 274 275 rtnl_lock(); 276 down_read(&bpf_devs_lock); 277 278 if (aux->offload) { 279 args->info->ifindex = aux->offload->netdev->ifindex; 280 net = dev_net(aux->offload->netdev); 281 get_net(net); 282 ns = &net->ns; 283 } else { 284 args->info->ifindex = 0; 285 ns = NULL; 286 } 287 288 up_read(&bpf_devs_lock); 289 rtnl_unlock(); 290 291 return ns; 292 } 293 294 int bpf_prog_offload_info_fill(struct bpf_prog_info *info, 295 struct bpf_prog *prog) 296 { 297 struct ns_get_path_bpf_prog_args args = { 298 .prog = prog, 299 .info = info, 300 }; 301 struct bpf_prog_aux *aux = prog->aux; 302 struct inode *ns_inode; 303 struct path ns_path; 304 char __user *uinsns; 305 int res; 306 u32 ulen; 307 308 res = ns_get_path_cb(&ns_path, bpf_prog_offload_info_fill_ns, &args); 309 if (res) { 310 if (!info->ifindex) 311 return -ENODEV; 312 return res; 313 } 314 315 down_read(&bpf_devs_lock); 316 317 if (!aux->offload) { 318 up_read(&bpf_devs_lock); 319 return -ENODEV; 320 } 321 322 ulen = info->jited_prog_len; 323 info->jited_prog_len = aux->offload->jited_len; 324 if (info->jited_prog_len && ulen) { 325 uinsns = u64_to_user_ptr(info->jited_prog_insns); 326 ulen = min_t(u32, info->jited_prog_len, ulen); 327 if (copy_to_user(uinsns, aux->offload->jited_image, ulen)) { 328 up_read(&bpf_devs_lock); 329 return -EFAULT; 330 } 331 } 332 333 up_read(&bpf_devs_lock); 334 335 ns_inode = ns_path.dentry->d_inode; 336 info->netns_dev = new_encode_dev(ns_inode->i_sb->s_dev); 337 info->netns_ino = ns_inode->i_ino; 338 path_put(&ns_path); 339 340 return 0; 341 } 342 343 const struct bpf_prog_ops bpf_offload_prog_ops = { 344 }; 345 346 static int bpf_map_offload_ndo(struct bpf_offloaded_map *offmap, 347 enum bpf_netdev_command cmd) 348 { 349 struct netdev_bpf data = {}; 350 struct net_device *netdev; 351 352 ASSERT_RTNL(); 353 354 data.command = cmd; 355 data.offmap = offmap; 356 /* Caller must make sure netdev is valid */ 357 netdev = offmap->netdev; 358 359 return netdev->netdev_ops->ndo_bpf(netdev, &data); 360 } 361 362 struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr) 363 { 364 struct net *net = current->nsproxy->net_ns; 365 struct bpf_offload_netdev *ondev; 366 struct bpf_offloaded_map *offmap; 367 int err; 368 369 if (!capable(CAP_SYS_ADMIN)) 370 return ERR_PTR(-EPERM); 371 if (attr->map_type != BPF_MAP_TYPE_ARRAY && 372 attr->map_type != BPF_MAP_TYPE_HASH) 373 return ERR_PTR(-EINVAL); 374 375 offmap = bpf_map_area_alloc(sizeof(*offmap), NUMA_NO_NODE); 376 if (!offmap) 377 return ERR_PTR(-ENOMEM); 378 379 bpf_map_init_from_attr(&offmap->map, attr); 380 381 rtnl_lock(); 382 down_write(&bpf_devs_lock); 383 offmap->netdev = __dev_get_by_index(net, attr->map_ifindex); 384 err = bpf_dev_offload_check(offmap->netdev); 385 if (err) 386 goto err_unlock; 387 388 ondev = bpf_offload_find_netdev(offmap->netdev); 389 if (!ondev) { 390 err = -EINVAL; 391 goto err_unlock; 392 } 393 394 err = bpf_map_offload_ndo(offmap, BPF_OFFLOAD_MAP_ALLOC); 395 if (err) 396 goto err_unlock; 397 398 list_add_tail(&offmap->offloads, &ondev->maps); 399 up_write(&bpf_devs_lock); 400 rtnl_unlock(); 401 402 return &offmap->map; 403 404 err_unlock: 405 up_write(&bpf_devs_lock); 406 rtnl_unlock(); 407 bpf_map_area_free(offmap); 408 return ERR_PTR(err); 409 } 410 411 static void __bpf_map_offload_destroy(struct bpf_offloaded_map *offmap) 412 { 413 WARN_ON(bpf_map_offload_ndo(offmap, BPF_OFFLOAD_MAP_FREE)); 414 /* Make sure BPF_MAP_GET_NEXT_ID can't find this dead map */ 415 bpf_map_free_id(&offmap->map, true); 416 list_del_init(&offmap->offloads); 417 offmap->netdev = NULL; 418 } 419 420 void bpf_map_offload_map_free(struct bpf_map *map) 421 { 422 struct bpf_offloaded_map *offmap = map_to_offmap(map); 423 424 rtnl_lock(); 425 down_write(&bpf_devs_lock); 426 if (offmap->netdev) 427 __bpf_map_offload_destroy(offmap); 428 up_write(&bpf_devs_lock); 429 rtnl_unlock(); 430 431 bpf_map_area_free(offmap); 432 } 433 434 int bpf_map_offload_lookup_elem(struct bpf_map *map, void *key, void *value) 435 { 436 struct bpf_offloaded_map *offmap = map_to_offmap(map); 437 int ret = -ENODEV; 438 439 down_read(&bpf_devs_lock); 440 if (offmap->netdev) 441 ret = offmap->dev_ops->map_lookup_elem(offmap, key, value); 442 up_read(&bpf_devs_lock); 443 444 return ret; 445 } 446 447 int bpf_map_offload_update_elem(struct bpf_map *map, 448 void *key, void *value, u64 flags) 449 { 450 struct bpf_offloaded_map *offmap = map_to_offmap(map); 451 int ret = -ENODEV; 452 453 if (unlikely(flags > BPF_EXIST)) 454 return -EINVAL; 455 456 down_read(&bpf_devs_lock); 457 if (offmap->netdev) 458 ret = offmap->dev_ops->map_update_elem(offmap, key, value, 459 flags); 460 up_read(&bpf_devs_lock); 461 462 return ret; 463 } 464 465 int bpf_map_offload_delete_elem(struct bpf_map *map, void *key) 466 { 467 struct bpf_offloaded_map *offmap = map_to_offmap(map); 468 int ret = -ENODEV; 469 470 down_read(&bpf_devs_lock); 471 if (offmap->netdev) 472 ret = offmap->dev_ops->map_delete_elem(offmap, key); 473 up_read(&bpf_devs_lock); 474 475 return ret; 476 } 477 478 int bpf_map_offload_get_next_key(struct bpf_map *map, void *key, void *next_key) 479 { 480 struct bpf_offloaded_map *offmap = map_to_offmap(map); 481 int ret = -ENODEV; 482 483 down_read(&bpf_devs_lock); 484 if (offmap->netdev) 485 ret = offmap->dev_ops->map_get_next_key(offmap, key, next_key); 486 up_read(&bpf_devs_lock); 487 488 return ret; 489 } 490 491 struct ns_get_path_bpf_map_args { 492 struct bpf_offloaded_map *offmap; 493 struct bpf_map_info *info; 494 }; 495 496 static struct ns_common *bpf_map_offload_info_fill_ns(void *private_data) 497 { 498 struct ns_get_path_bpf_map_args *args = private_data; 499 struct ns_common *ns; 500 struct net *net; 501 502 rtnl_lock(); 503 down_read(&bpf_devs_lock); 504 505 if (args->offmap->netdev) { 506 args->info->ifindex = args->offmap->netdev->ifindex; 507 net = dev_net(args->offmap->netdev); 508 get_net(net); 509 ns = &net->ns; 510 } else { 511 args->info->ifindex = 0; 512 ns = NULL; 513 } 514 515 up_read(&bpf_devs_lock); 516 rtnl_unlock(); 517 518 return ns; 519 } 520 521 int bpf_map_offload_info_fill(struct bpf_map_info *info, struct bpf_map *map) 522 { 523 struct ns_get_path_bpf_map_args args = { 524 .offmap = map_to_offmap(map), 525 .info = info, 526 }; 527 struct inode *ns_inode; 528 struct path ns_path; 529 int res; 530 531 res = ns_get_path_cb(&ns_path, bpf_map_offload_info_fill_ns, &args); 532 if (res) { 533 if (!info->ifindex) 534 return -ENODEV; 535 return res; 536 } 537 538 ns_inode = ns_path.dentry->d_inode; 539 info->netns_dev = new_encode_dev(ns_inode->i_sb->s_dev); 540 info->netns_ino = ns_inode->i_ino; 541 path_put(&ns_path); 542 543 return 0; 544 } 545 546 static bool __bpf_offload_dev_match(struct bpf_prog *prog, 547 struct net_device *netdev) 548 { 549 struct bpf_offload_netdev *ondev1, *ondev2; 550 struct bpf_prog_offload *offload; 551 552 if (!bpf_prog_is_dev_bound(prog->aux)) 553 return false; 554 555 offload = prog->aux->offload; 556 if (!offload) 557 return false; 558 if (offload->netdev == netdev) 559 return true; 560 561 ondev1 = bpf_offload_find_netdev(offload->netdev); 562 ondev2 = bpf_offload_find_netdev(netdev); 563 564 return ondev1 && ondev2 && ondev1->offdev == ondev2->offdev; 565 } 566 567 bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev) 568 { 569 bool ret; 570 571 down_read(&bpf_devs_lock); 572 ret = __bpf_offload_dev_match(prog, netdev); 573 up_read(&bpf_devs_lock); 574 575 return ret; 576 } 577 EXPORT_SYMBOL_GPL(bpf_offload_dev_match); 578 579 bool bpf_offload_prog_map_match(struct bpf_prog *prog, struct bpf_map *map) 580 { 581 struct bpf_offloaded_map *offmap; 582 bool ret; 583 584 if (!bpf_map_is_dev_bound(map)) 585 return bpf_map_offload_neutral(map); 586 offmap = map_to_offmap(map); 587 588 down_read(&bpf_devs_lock); 589 ret = __bpf_offload_dev_match(prog, offmap->netdev); 590 up_read(&bpf_devs_lock); 591 592 return ret; 593 } 594 595 int bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev, 596 struct net_device *netdev) 597 { 598 struct bpf_offload_netdev *ondev; 599 int err; 600 601 ondev = kzalloc(sizeof(*ondev), GFP_KERNEL); 602 if (!ondev) 603 return -ENOMEM; 604 605 ondev->netdev = netdev; 606 ondev->offdev = offdev; 607 INIT_LIST_HEAD(&ondev->progs); 608 INIT_LIST_HEAD(&ondev->maps); 609 610 down_write(&bpf_devs_lock); 611 err = rhashtable_insert_fast(&offdevs, &ondev->l, offdevs_params); 612 if (err) { 613 netdev_warn(netdev, "failed to register for BPF offload\n"); 614 goto err_unlock_free; 615 } 616 617 list_add(&ondev->offdev_netdevs, &offdev->netdevs); 618 up_write(&bpf_devs_lock); 619 return 0; 620 621 err_unlock_free: 622 up_write(&bpf_devs_lock); 623 kfree(ondev); 624 return err; 625 } 626 EXPORT_SYMBOL_GPL(bpf_offload_dev_netdev_register); 627 628 void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev, 629 struct net_device *netdev) 630 { 631 struct bpf_offload_netdev *ondev, *altdev; 632 struct bpf_offloaded_map *offmap, *mtmp; 633 struct bpf_prog_offload *offload, *ptmp; 634 635 ASSERT_RTNL(); 636 637 down_write(&bpf_devs_lock); 638 ondev = rhashtable_lookup_fast(&offdevs, &netdev, offdevs_params); 639 if (WARN_ON(!ondev)) 640 goto unlock; 641 642 WARN_ON(rhashtable_remove_fast(&offdevs, &ondev->l, offdevs_params)); 643 list_del(&ondev->offdev_netdevs); 644 645 /* Try to move the objects to another netdev of the device */ 646 altdev = list_first_entry_or_null(&offdev->netdevs, 647 struct bpf_offload_netdev, 648 offdev_netdevs); 649 if (altdev) { 650 list_for_each_entry(offload, &ondev->progs, offloads) 651 offload->netdev = altdev->netdev; 652 list_splice_init(&ondev->progs, &altdev->progs); 653 654 list_for_each_entry(offmap, &ondev->maps, offloads) 655 offmap->netdev = altdev->netdev; 656 list_splice_init(&ondev->maps, &altdev->maps); 657 } else { 658 list_for_each_entry_safe(offload, ptmp, &ondev->progs, offloads) 659 __bpf_prog_offload_destroy(offload->prog); 660 list_for_each_entry_safe(offmap, mtmp, &ondev->maps, offloads) 661 __bpf_map_offload_destroy(offmap); 662 } 663 664 WARN_ON(!list_empty(&ondev->progs)); 665 WARN_ON(!list_empty(&ondev->maps)); 666 kfree(ondev); 667 unlock: 668 up_write(&bpf_devs_lock); 669 } 670 EXPORT_SYMBOL_GPL(bpf_offload_dev_netdev_unregister); 671 672 struct bpf_offload_dev * 673 bpf_offload_dev_create(const struct bpf_prog_offload_ops *ops, void *priv) 674 { 675 struct bpf_offload_dev *offdev; 676 int err; 677 678 down_write(&bpf_devs_lock); 679 if (!offdevs_inited) { 680 err = rhashtable_init(&offdevs, &offdevs_params); 681 if (err) { 682 up_write(&bpf_devs_lock); 683 return ERR_PTR(err); 684 } 685 offdevs_inited = true; 686 } 687 up_write(&bpf_devs_lock); 688 689 offdev = kzalloc(sizeof(*offdev), GFP_KERNEL); 690 if (!offdev) 691 return ERR_PTR(-ENOMEM); 692 693 offdev->ops = ops; 694 offdev->priv = priv; 695 INIT_LIST_HEAD(&offdev->netdevs); 696 697 return offdev; 698 } 699 EXPORT_SYMBOL_GPL(bpf_offload_dev_create); 700 701 void bpf_offload_dev_destroy(struct bpf_offload_dev *offdev) 702 { 703 WARN_ON(!list_empty(&offdev->netdevs)); 704 kfree(offdev); 705 } 706 EXPORT_SYMBOL_GPL(bpf_offload_dev_destroy); 707 708 void *bpf_offload_dev_priv(struct bpf_offload_dev *offdev) 709 { 710 return offdev->priv; 711 } 712 EXPORT_SYMBOL_GPL(bpf_offload_dev_priv); 713