1 /* 2 * Copyright (C) 2017-2018 Netronome Systems, Inc. 3 * 4 * This software is licensed under the GNU General License Version 2, 5 * June 1991 as shown in the file COPYING in the top-level directory of this 6 * source tree. 7 * 8 * THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" 9 * WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, 10 * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 11 * FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE 12 * OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME 13 * THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 14 */ 15 16 #include <linux/bpf.h> 17 #include <linux/bpf_verifier.h> 18 #include <linux/bug.h> 19 #include <linux/kdev_t.h> 20 #include <linux/list.h> 21 #include <linux/lockdep.h> 22 #include <linux/netdevice.h> 23 #include <linux/printk.h> 24 #include <linux/proc_ns.h> 25 #include <linux/rhashtable.h> 26 #include <linux/rtnetlink.h> 27 #include <linux/rwsem.h> 28 29 /* Protects offdevs, members of bpf_offload_netdev and offload members 30 * of all progs. 31 * RTNL lock cannot be taken when holding this lock. 32 */ 33 static DECLARE_RWSEM(bpf_devs_lock); 34 35 struct bpf_offload_dev { 36 const struct bpf_prog_offload_ops *ops; 37 struct list_head netdevs; 38 }; 39 40 struct bpf_offload_netdev { 41 struct rhash_head l; 42 struct net_device *netdev; 43 struct bpf_offload_dev *offdev; 44 struct list_head progs; 45 struct list_head maps; 46 struct list_head offdev_netdevs; 47 }; 48 49 static const struct rhashtable_params offdevs_params = { 50 .nelem_hint = 4, 51 .key_len = sizeof(struct net_device *), 52 .key_offset = offsetof(struct bpf_offload_netdev, netdev), 53 .head_offset = offsetof(struct bpf_offload_netdev, l), 54 .automatic_shrinking = true, 55 }; 56 57 static struct rhashtable offdevs; 58 static bool offdevs_inited; 59 60 static int bpf_dev_offload_check(struct net_device *netdev) 61 { 62 if (!netdev) 63 return -EINVAL; 64 if (!netdev->netdev_ops->ndo_bpf) 65 return -EOPNOTSUPP; 66 return 0; 67 } 68 69 static struct bpf_offload_netdev * 70 bpf_offload_find_netdev(struct net_device *netdev) 71 { 72 lockdep_assert_held(&bpf_devs_lock); 73 74 if (!offdevs_inited) 75 return NULL; 76 return rhashtable_lookup_fast(&offdevs, &netdev, offdevs_params); 77 } 78 79 int bpf_prog_offload_init(struct bpf_prog *prog, union bpf_attr *attr) 80 { 81 struct bpf_offload_netdev *ondev; 82 struct bpf_prog_offload *offload; 83 int err; 84 85 if (attr->prog_type != BPF_PROG_TYPE_SCHED_CLS && 86 attr->prog_type != BPF_PROG_TYPE_XDP) 87 return -EINVAL; 88 89 if (attr->prog_flags) 90 return -EINVAL; 91 92 offload = kzalloc(sizeof(*offload), GFP_USER); 93 if (!offload) 94 return -ENOMEM; 95 96 offload->prog = prog; 97 98 offload->netdev = dev_get_by_index(current->nsproxy->net_ns, 99 attr->prog_ifindex); 100 err = bpf_dev_offload_check(offload->netdev); 101 if (err) 102 goto err_maybe_put; 103 104 down_write(&bpf_devs_lock); 105 ondev = bpf_offload_find_netdev(offload->netdev); 106 if (!ondev) { 107 err = -EINVAL; 108 goto err_unlock; 109 } 110 offload->offdev = ondev->offdev; 111 prog->aux->offload = offload; 112 list_add_tail(&offload->offloads, &ondev->progs); 113 dev_put(offload->netdev); 114 up_write(&bpf_devs_lock); 115 116 return 0; 117 err_unlock: 118 up_write(&bpf_devs_lock); 119 err_maybe_put: 120 if (offload->netdev) 121 dev_put(offload->netdev); 122 kfree(offload); 123 return err; 124 } 125 126 int bpf_prog_offload_verifier_prep(struct bpf_prog *prog) 127 { 128 struct bpf_prog_offload *offload; 129 int ret = -ENODEV; 130 131 down_read(&bpf_devs_lock); 132 offload = prog->aux->offload; 133 if (offload) { 134 ret = offload->offdev->ops->prepare(prog); 135 offload->dev_state = !ret; 136 } 137 up_read(&bpf_devs_lock); 138 139 return ret; 140 } 141 142 int bpf_prog_offload_verify_insn(struct bpf_verifier_env *env, 143 int insn_idx, int prev_insn_idx) 144 { 145 struct bpf_prog_offload *offload; 146 int ret = -ENODEV; 147 148 down_read(&bpf_devs_lock); 149 offload = env->prog->aux->offload; 150 if (offload) 151 ret = offload->offdev->ops->insn_hook(env, insn_idx, 152 prev_insn_idx); 153 up_read(&bpf_devs_lock); 154 155 return ret; 156 } 157 158 int bpf_prog_offload_finalize(struct bpf_verifier_env *env) 159 { 160 struct bpf_prog_offload *offload; 161 int ret = -ENODEV; 162 163 down_read(&bpf_devs_lock); 164 offload = env->prog->aux->offload; 165 if (offload) { 166 if (offload->offdev->ops->finalize) 167 ret = offload->offdev->ops->finalize(env); 168 else 169 ret = 0; 170 } 171 up_read(&bpf_devs_lock); 172 173 return ret; 174 } 175 176 static void __bpf_prog_offload_destroy(struct bpf_prog *prog) 177 { 178 struct bpf_prog_offload *offload = prog->aux->offload; 179 180 if (offload->dev_state) 181 offload->offdev->ops->destroy(prog); 182 183 /* Make sure BPF_PROG_GET_NEXT_ID can't find this dead program */ 184 bpf_prog_free_id(prog, true); 185 186 list_del_init(&offload->offloads); 187 kfree(offload); 188 prog->aux->offload = NULL; 189 } 190 191 void bpf_prog_offload_destroy(struct bpf_prog *prog) 192 { 193 down_write(&bpf_devs_lock); 194 if (prog->aux->offload) 195 __bpf_prog_offload_destroy(prog); 196 up_write(&bpf_devs_lock); 197 } 198 199 static int bpf_prog_offload_translate(struct bpf_prog *prog) 200 { 201 struct bpf_prog_offload *offload; 202 int ret = -ENODEV; 203 204 down_read(&bpf_devs_lock); 205 offload = prog->aux->offload; 206 if (offload) 207 ret = offload->offdev->ops->translate(prog); 208 up_read(&bpf_devs_lock); 209 210 return ret; 211 } 212 213 static unsigned int bpf_prog_warn_on_exec(const void *ctx, 214 const struct bpf_insn *insn) 215 { 216 WARN(1, "attempt to execute device eBPF program on the host!"); 217 return 0; 218 } 219 220 int bpf_prog_offload_compile(struct bpf_prog *prog) 221 { 222 prog->bpf_func = bpf_prog_warn_on_exec; 223 224 return bpf_prog_offload_translate(prog); 225 } 226 227 struct ns_get_path_bpf_prog_args { 228 struct bpf_prog *prog; 229 struct bpf_prog_info *info; 230 }; 231 232 static struct ns_common *bpf_prog_offload_info_fill_ns(void *private_data) 233 { 234 struct ns_get_path_bpf_prog_args *args = private_data; 235 struct bpf_prog_aux *aux = args->prog->aux; 236 struct ns_common *ns; 237 struct net *net; 238 239 rtnl_lock(); 240 down_read(&bpf_devs_lock); 241 242 if (aux->offload) { 243 args->info->ifindex = aux->offload->netdev->ifindex; 244 net = dev_net(aux->offload->netdev); 245 get_net(net); 246 ns = &net->ns; 247 } else { 248 args->info->ifindex = 0; 249 ns = NULL; 250 } 251 252 up_read(&bpf_devs_lock); 253 rtnl_unlock(); 254 255 return ns; 256 } 257 258 int bpf_prog_offload_info_fill(struct bpf_prog_info *info, 259 struct bpf_prog *prog) 260 { 261 struct ns_get_path_bpf_prog_args args = { 262 .prog = prog, 263 .info = info, 264 }; 265 struct bpf_prog_aux *aux = prog->aux; 266 struct inode *ns_inode; 267 struct path ns_path; 268 char __user *uinsns; 269 void *res; 270 u32 ulen; 271 272 res = ns_get_path_cb(&ns_path, bpf_prog_offload_info_fill_ns, &args); 273 if (IS_ERR(res)) { 274 if (!info->ifindex) 275 return -ENODEV; 276 return PTR_ERR(res); 277 } 278 279 down_read(&bpf_devs_lock); 280 281 if (!aux->offload) { 282 up_read(&bpf_devs_lock); 283 return -ENODEV; 284 } 285 286 ulen = info->jited_prog_len; 287 info->jited_prog_len = aux->offload->jited_len; 288 if (info->jited_prog_len & ulen) { 289 uinsns = u64_to_user_ptr(info->jited_prog_insns); 290 ulen = min_t(u32, info->jited_prog_len, ulen); 291 if (copy_to_user(uinsns, aux->offload->jited_image, ulen)) { 292 up_read(&bpf_devs_lock); 293 return -EFAULT; 294 } 295 } 296 297 up_read(&bpf_devs_lock); 298 299 ns_inode = ns_path.dentry->d_inode; 300 info->netns_dev = new_encode_dev(ns_inode->i_sb->s_dev); 301 info->netns_ino = ns_inode->i_ino; 302 path_put(&ns_path); 303 304 return 0; 305 } 306 307 const struct bpf_prog_ops bpf_offload_prog_ops = { 308 }; 309 310 static int bpf_map_offload_ndo(struct bpf_offloaded_map *offmap, 311 enum bpf_netdev_command cmd) 312 { 313 struct netdev_bpf data = {}; 314 struct net_device *netdev; 315 316 ASSERT_RTNL(); 317 318 data.command = cmd; 319 data.offmap = offmap; 320 /* Caller must make sure netdev is valid */ 321 netdev = offmap->netdev; 322 323 return netdev->netdev_ops->ndo_bpf(netdev, &data); 324 } 325 326 struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr) 327 { 328 struct net *net = current->nsproxy->net_ns; 329 struct bpf_offload_netdev *ondev; 330 struct bpf_offloaded_map *offmap; 331 int err; 332 333 if (!capable(CAP_SYS_ADMIN)) 334 return ERR_PTR(-EPERM); 335 if (attr->map_type != BPF_MAP_TYPE_ARRAY && 336 attr->map_type != BPF_MAP_TYPE_HASH) 337 return ERR_PTR(-EINVAL); 338 339 offmap = kzalloc(sizeof(*offmap), GFP_USER); 340 if (!offmap) 341 return ERR_PTR(-ENOMEM); 342 343 bpf_map_init_from_attr(&offmap->map, attr); 344 345 rtnl_lock(); 346 down_write(&bpf_devs_lock); 347 offmap->netdev = __dev_get_by_index(net, attr->map_ifindex); 348 err = bpf_dev_offload_check(offmap->netdev); 349 if (err) 350 goto err_unlock; 351 352 ondev = bpf_offload_find_netdev(offmap->netdev); 353 if (!ondev) { 354 err = -EINVAL; 355 goto err_unlock; 356 } 357 358 err = bpf_map_offload_ndo(offmap, BPF_OFFLOAD_MAP_ALLOC); 359 if (err) 360 goto err_unlock; 361 362 list_add_tail(&offmap->offloads, &ondev->maps); 363 up_write(&bpf_devs_lock); 364 rtnl_unlock(); 365 366 return &offmap->map; 367 368 err_unlock: 369 up_write(&bpf_devs_lock); 370 rtnl_unlock(); 371 kfree(offmap); 372 return ERR_PTR(err); 373 } 374 375 static void __bpf_map_offload_destroy(struct bpf_offloaded_map *offmap) 376 { 377 WARN_ON(bpf_map_offload_ndo(offmap, BPF_OFFLOAD_MAP_FREE)); 378 /* Make sure BPF_MAP_GET_NEXT_ID can't find this dead map */ 379 bpf_map_free_id(&offmap->map, true); 380 list_del_init(&offmap->offloads); 381 offmap->netdev = NULL; 382 } 383 384 void bpf_map_offload_map_free(struct bpf_map *map) 385 { 386 struct bpf_offloaded_map *offmap = map_to_offmap(map); 387 388 rtnl_lock(); 389 down_write(&bpf_devs_lock); 390 if (offmap->netdev) 391 __bpf_map_offload_destroy(offmap); 392 up_write(&bpf_devs_lock); 393 rtnl_unlock(); 394 395 kfree(offmap); 396 } 397 398 int bpf_map_offload_lookup_elem(struct bpf_map *map, void *key, void *value) 399 { 400 struct bpf_offloaded_map *offmap = map_to_offmap(map); 401 int ret = -ENODEV; 402 403 down_read(&bpf_devs_lock); 404 if (offmap->netdev) 405 ret = offmap->dev_ops->map_lookup_elem(offmap, key, value); 406 up_read(&bpf_devs_lock); 407 408 return ret; 409 } 410 411 int bpf_map_offload_update_elem(struct bpf_map *map, 412 void *key, void *value, u64 flags) 413 { 414 struct bpf_offloaded_map *offmap = map_to_offmap(map); 415 int ret = -ENODEV; 416 417 if (unlikely(flags > BPF_EXIST)) 418 return -EINVAL; 419 420 down_read(&bpf_devs_lock); 421 if (offmap->netdev) 422 ret = offmap->dev_ops->map_update_elem(offmap, key, value, 423 flags); 424 up_read(&bpf_devs_lock); 425 426 return ret; 427 } 428 429 int bpf_map_offload_delete_elem(struct bpf_map *map, void *key) 430 { 431 struct bpf_offloaded_map *offmap = map_to_offmap(map); 432 int ret = -ENODEV; 433 434 down_read(&bpf_devs_lock); 435 if (offmap->netdev) 436 ret = offmap->dev_ops->map_delete_elem(offmap, key); 437 up_read(&bpf_devs_lock); 438 439 return ret; 440 } 441 442 int bpf_map_offload_get_next_key(struct bpf_map *map, void *key, void *next_key) 443 { 444 struct bpf_offloaded_map *offmap = map_to_offmap(map); 445 int ret = -ENODEV; 446 447 down_read(&bpf_devs_lock); 448 if (offmap->netdev) 449 ret = offmap->dev_ops->map_get_next_key(offmap, key, next_key); 450 up_read(&bpf_devs_lock); 451 452 return ret; 453 } 454 455 struct ns_get_path_bpf_map_args { 456 struct bpf_offloaded_map *offmap; 457 struct bpf_map_info *info; 458 }; 459 460 static struct ns_common *bpf_map_offload_info_fill_ns(void *private_data) 461 { 462 struct ns_get_path_bpf_map_args *args = private_data; 463 struct ns_common *ns; 464 struct net *net; 465 466 rtnl_lock(); 467 down_read(&bpf_devs_lock); 468 469 if (args->offmap->netdev) { 470 args->info->ifindex = args->offmap->netdev->ifindex; 471 net = dev_net(args->offmap->netdev); 472 get_net(net); 473 ns = &net->ns; 474 } else { 475 args->info->ifindex = 0; 476 ns = NULL; 477 } 478 479 up_read(&bpf_devs_lock); 480 rtnl_unlock(); 481 482 return ns; 483 } 484 485 int bpf_map_offload_info_fill(struct bpf_map_info *info, struct bpf_map *map) 486 { 487 struct ns_get_path_bpf_map_args args = { 488 .offmap = map_to_offmap(map), 489 .info = info, 490 }; 491 struct inode *ns_inode; 492 struct path ns_path; 493 void *res; 494 495 res = ns_get_path_cb(&ns_path, bpf_map_offload_info_fill_ns, &args); 496 if (IS_ERR(res)) { 497 if (!info->ifindex) 498 return -ENODEV; 499 return PTR_ERR(res); 500 } 501 502 ns_inode = ns_path.dentry->d_inode; 503 info->netns_dev = new_encode_dev(ns_inode->i_sb->s_dev); 504 info->netns_ino = ns_inode->i_ino; 505 path_put(&ns_path); 506 507 return 0; 508 } 509 510 static bool __bpf_offload_dev_match(struct bpf_prog *prog, 511 struct net_device *netdev) 512 { 513 struct bpf_offload_netdev *ondev1, *ondev2; 514 struct bpf_prog_offload *offload; 515 516 if (!bpf_prog_is_dev_bound(prog->aux)) 517 return false; 518 519 offload = prog->aux->offload; 520 if (!offload) 521 return false; 522 if (offload->netdev == netdev) 523 return true; 524 525 ondev1 = bpf_offload_find_netdev(offload->netdev); 526 ondev2 = bpf_offload_find_netdev(netdev); 527 528 return ondev1 && ondev2 && ondev1->offdev == ondev2->offdev; 529 } 530 531 bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev) 532 { 533 bool ret; 534 535 down_read(&bpf_devs_lock); 536 ret = __bpf_offload_dev_match(prog, netdev); 537 up_read(&bpf_devs_lock); 538 539 return ret; 540 } 541 EXPORT_SYMBOL_GPL(bpf_offload_dev_match); 542 543 bool bpf_offload_prog_map_match(struct bpf_prog *prog, struct bpf_map *map) 544 { 545 struct bpf_offloaded_map *offmap; 546 bool ret; 547 548 if (!bpf_map_is_dev_bound(map)) 549 return bpf_map_offload_neutral(map); 550 offmap = map_to_offmap(map); 551 552 down_read(&bpf_devs_lock); 553 ret = __bpf_offload_dev_match(prog, offmap->netdev); 554 up_read(&bpf_devs_lock); 555 556 return ret; 557 } 558 559 int bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev, 560 struct net_device *netdev) 561 { 562 struct bpf_offload_netdev *ondev; 563 int err; 564 565 ondev = kzalloc(sizeof(*ondev), GFP_KERNEL); 566 if (!ondev) 567 return -ENOMEM; 568 569 ondev->netdev = netdev; 570 ondev->offdev = offdev; 571 INIT_LIST_HEAD(&ondev->progs); 572 INIT_LIST_HEAD(&ondev->maps); 573 574 down_write(&bpf_devs_lock); 575 err = rhashtable_insert_fast(&offdevs, &ondev->l, offdevs_params); 576 if (err) { 577 netdev_warn(netdev, "failed to register for BPF offload\n"); 578 goto err_unlock_free; 579 } 580 581 list_add(&ondev->offdev_netdevs, &offdev->netdevs); 582 up_write(&bpf_devs_lock); 583 return 0; 584 585 err_unlock_free: 586 up_write(&bpf_devs_lock); 587 kfree(ondev); 588 return err; 589 } 590 EXPORT_SYMBOL_GPL(bpf_offload_dev_netdev_register); 591 592 void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev, 593 struct net_device *netdev) 594 { 595 struct bpf_offload_netdev *ondev, *altdev; 596 struct bpf_offloaded_map *offmap, *mtmp; 597 struct bpf_prog_offload *offload, *ptmp; 598 599 ASSERT_RTNL(); 600 601 down_write(&bpf_devs_lock); 602 ondev = rhashtable_lookup_fast(&offdevs, &netdev, offdevs_params); 603 if (WARN_ON(!ondev)) 604 goto unlock; 605 606 WARN_ON(rhashtable_remove_fast(&offdevs, &ondev->l, offdevs_params)); 607 list_del(&ondev->offdev_netdevs); 608 609 /* Try to move the objects to another netdev of the device */ 610 altdev = list_first_entry_or_null(&offdev->netdevs, 611 struct bpf_offload_netdev, 612 offdev_netdevs); 613 if (altdev) { 614 list_for_each_entry(offload, &ondev->progs, offloads) 615 offload->netdev = altdev->netdev; 616 list_splice_init(&ondev->progs, &altdev->progs); 617 618 list_for_each_entry(offmap, &ondev->maps, offloads) 619 offmap->netdev = altdev->netdev; 620 list_splice_init(&ondev->maps, &altdev->maps); 621 } else { 622 list_for_each_entry_safe(offload, ptmp, &ondev->progs, offloads) 623 __bpf_prog_offload_destroy(offload->prog); 624 list_for_each_entry_safe(offmap, mtmp, &ondev->maps, offloads) 625 __bpf_map_offload_destroy(offmap); 626 } 627 628 WARN_ON(!list_empty(&ondev->progs)); 629 WARN_ON(!list_empty(&ondev->maps)); 630 kfree(ondev); 631 unlock: 632 up_write(&bpf_devs_lock); 633 } 634 EXPORT_SYMBOL_GPL(bpf_offload_dev_netdev_unregister); 635 636 struct bpf_offload_dev * 637 bpf_offload_dev_create(const struct bpf_prog_offload_ops *ops) 638 { 639 struct bpf_offload_dev *offdev; 640 int err; 641 642 down_write(&bpf_devs_lock); 643 if (!offdevs_inited) { 644 err = rhashtable_init(&offdevs, &offdevs_params); 645 if (err) 646 return ERR_PTR(err); 647 offdevs_inited = true; 648 } 649 up_write(&bpf_devs_lock); 650 651 offdev = kzalloc(sizeof(*offdev), GFP_KERNEL); 652 if (!offdev) 653 return ERR_PTR(-ENOMEM); 654 655 offdev->ops = ops; 656 INIT_LIST_HEAD(&offdev->netdevs); 657 658 return offdev; 659 } 660 EXPORT_SYMBOL_GPL(bpf_offload_dev_create); 661 662 void bpf_offload_dev_destroy(struct bpf_offload_dev *offdev) 663 { 664 WARN_ON(!list_empty(&offdev->netdevs)); 665 kfree(offdev); 666 } 667 EXPORT_SYMBOL_GPL(bpf_offload_dev_destroy); 668