1a39e17b2SJakub Kicinski /* 2a39e17b2SJakub Kicinski * Copyright (C) 2017 Netronome Systems, Inc. 3a39e17b2SJakub Kicinski * 4a39e17b2SJakub Kicinski * This software is licensed under the GNU General License Version 2, 5a39e17b2SJakub Kicinski * June 1991 as shown in the file COPYING in the top-level directory of this 6a39e17b2SJakub Kicinski * source tree. 7a39e17b2SJakub Kicinski * 8a39e17b2SJakub Kicinski * THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" 9a39e17b2SJakub Kicinski * WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, 10a39e17b2SJakub Kicinski * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 11a39e17b2SJakub Kicinski * FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE 12a39e17b2SJakub Kicinski * OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME 13a39e17b2SJakub Kicinski * THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 14a39e17b2SJakub Kicinski */ 15a39e17b2SJakub Kicinski 16ab3f0063SJakub Kicinski #include <linux/bpf.h> 17ab3f0063SJakub Kicinski #include <linux/bpf_verifier.h> 18ab3f0063SJakub Kicinski #include <linux/bug.h> 19675fc275SJakub Kicinski #include <linux/kdev_t.h> 20ab3f0063SJakub Kicinski #include <linux/list.h> 21ab3f0063SJakub Kicinski #include <linux/netdevice.h> 22ab3f0063SJakub Kicinski #include <linux/printk.h> 23675fc275SJakub Kicinski #include <linux/proc_ns.h> 24ab3f0063SJakub Kicinski #include <linux/rtnetlink.h> 25e0d3974aSJakub Kicinski #include <linux/rwsem.h> 26ab3f0063SJakub Kicinski 27e0d3974aSJakub Kicinski /* Protects bpf_prog_offload_devs and offload members of all progs. 28e0d3974aSJakub Kicinski * RTNL lock cannot be taken when holding this lock. 29e0d3974aSJakub Kicinski */ 30e0d3974aSJakub Kicinski static DECLARE_RWSEM(bpf_devs_lock); 31ab3f0063SJakub Kicinski static LIST_HEAD(bpf_prog_offload_devs); 32ab3f0063SJakub Kicinski 335bc2d55cSJakub Kicinski static int bpf_dev_offload_check(struct net_device *netdev) 345bc2d55cSJakub Kicinski { 355bc2d55cSJakub Kicinski if (!netdev) 365bc2d55cSJakub Kicinski return -EINVAL; 375bc2d55cSJakub Kicinski if (!netdev->netdev_ops->ndo_bpf) 385bc2d55cSJakub Kicinski return -EOPNOTSUPP; 395bc2d55cSJakub Kicinski return 0; 405bc2d55cSJakub Kicinski } 415bc2d55cSJakub Kicinski 42ab3f0063SJakub Kicinski int bpf_prog_offload_init(struct bpf_prog *prog, union bpf_attr *attr) 43ab3f0063SJakub Kicinski { 440a9c1991SJakub Kicinski struct bpf_prog_offload *offload; 455bc2d55cSJakub Kicinski int err; 46ab3f0063SJakub Kicinski 47649f11dcSJakub Kicinski if (attr->prog_type != BPF_PROG_TYPE_SCHED_CLS && 48649f11dcSJakub Kicinski attr->prog_type != BPF_PROG_TYPE_XDP) 49649f11dcSJakub Kicinski return -EINVAL; 50ab3f0063SJakub Kicinski 51ab3f0063SJakub Kicinski if (attr->prog_flags) 52ab3f0063SJakub Kicinski return -EINVAL; 53ab3f0063SJakub Kicinski 54ab3f0063SJakub Kicinski offload = kzalloc(sizeof(*offload), GFP_USER); 55ab3f0063SJakub Kicinski if (!offload) 56ab3f0063SJakub Kicinski return -ENOMEM; 57ab3f0063SJakub Kicinski 58ab3f0063SJakub Kicinski offload->prog = prog; 59ab3f0063SJakub Kicinski 60e0d3974aSJakub Kicinski offload->netdev = dev_get_by_index(current->nsproxy->net_ns, 61e0d3974aSJakub Kicinski attr->prog_ifindex); 625bc2d55cSJakub Kicinski err = bpf_dev_offload_check(offload->netdev); 635bc2d55cSJakub Kicinski if (err) 645bc2d55cSJakub Kicinski goto err_maybe_put; 65ab3f0063SJakub Kicinski 66e0d3974aSJakub Kicinski down_write(&bpf_devs_lock); 675bc2d55cSJakub Kicinski if (offload->netdev->reg_state != NETREG_REGISTERED) { 685bc2d55cSJakub Kicinski err = -EINVAL; 69e0d3974aSJakub Kicinski goto err_unlock; 705bc2d55cSJakub Kicinski } 71ab3f0063SJakub Kicinski prog->aux->offload = offload; 72ab3f0063SJakub Kicinski list_add_tail(&offload->offloads, &bpf_prog_offload_devs); 73e0d3974aSJakub Kicinski dev_put(offload->netdev); 74e0d3974aSJakub Kicinski up_write(&bpf_devs_lock); 75ab3f0063SJakub Kicinski 76ab3f0063SJakub Kicinski return 0; 77e0d3974aSJakub Kicinski err_unlock: 78e0d3974aSJakub Kicinski up_write(&bpf_devs_lock); 795bc2d55cSJakub Kicinski err_maybe_put: 805bc2d55cSJakub Kicinski if (offload->netdev) 81e0d3974aSJakub Kicinski dev_put(offload->netdev); 82e0d3974aSJakub Kicinski kfree(offload); 835bc2d55cSJakub Kicinski return err; 84ab3f0063SJakub Kicinski } 85ab3f0063SJakub Kicinski 86ab3f0063SJakub Kicinski static int __bpf_offload_ndo(struct bpf_prog *prog, enum bpf_netdev_command cmd, 87ab3f0063SJakub Kicinski struct netdev_bpf *data) 88ab3f0063SJakub Kicinski { 890a9c1991SJakub Kicinski struct bpf_prog_offload *offload = prog->aux->offload; 90ce3b9db4SJakub Kicinski struct net_device *netdev; 91ab3f0063SJakub Kicinski 92ab3f0063SJakub Kicinski ASSERT_RTNL(); 93ab3f0063SJakub Kicinski 94ce3b9db4SJakub Kicinski if (!offload) 95ab3f0063SJakub Kicinski return -ENODEV; 96ce3b9db4SJakub Kicinski netdev = offload->netdev; 97ab3f0063SJakub Kicinski 98ab3f0063SJakub Kicinski data->command = cmd; 99ab3f0063SJakub Kicinski 100ab3f0063SJakub Kicinski return netdev->netdev_ops->ndo_bpf(netdev, data); 101ab3f0063SJakub Kicinski } 102ab3f0063SJakub Kicinski 103ab3f0063SJakub Kicinski int bpf_prog_offload_verifier_prep(struct bpf_verifier_env *env) 104ab3f0063SJakub Kicinski { 105ab3f0063SJakub Kicinski struct netdev_bpf data = {}; 106ab3f0063SJakub Kicinski int err; 107ab3f0063SJakub Kicinski 108ab3f0063SJakub Kicinski data.verifier.prog = env->prog; 109ab3f0063SJakub Kicinski 110ab3f0063SJakub Kicinski rtnl_lock(); 111ab3f0063SJakub Kicinski err = __bpf_offload_ndo(env->prog, BPF_OFFLOAD_VERIFIER_PREP, &data); 112ab3f0063SJakub Kicinski if (err) 113ab3f0063SJakub Kicinski goto exit_unlock; 114ab3f0063SJakub Kicinski 115cae1927cSJakub Kicinski env->prog->aux->offload->dev_ops = data.verifier.ops; 116ab3f0063SJakub Kicinski env->prog->aux->offload->dev_state = true; 117ab3f0063SJakub Kicinski exit_unlock: 118ab3f0063SJakub Kicinski rtnl_unlock(); 119ab3f0063SJakub Kicinski return err; 120ab3f0063SJakub Kicinski } 121ab3f0063SJakub Kicinski 122cae1927cSJakub Kicinski int bpf_prog_offload_verify_insn(struct bpf_verifier_env *env, 123cae1927cSJakub Kicinski int insn_idx, int prev_insn_idx) 124cae1927cSJakub Kicinski { 1250a9c1991SJakub Kicinski struct bpf_prog_offload *offload; 126cae1927cSJakub Kicinski int ret = -ENODEV; 127cae1927cSJakub Kicinski 128cae1927cSJakub Kicinski down_read(&bpf_devs_lock); 129cae1927cSJakub Kicinski offload = env->prog->aux->offload; 130ce3b9db4SJakub Kicinski if (offload) 131cae1927cSJakub Kicinski ret = offload->dev_ops->insn_hook(env, insn_idx, prev_insn_idx); 132cae1927cSJakub Kicinski up_read(&bpf_devs_lock); 133cae1927cSJakub Kicinski 134cae1927cSJakub Kicinski return ret; 135cae1927cSJakub Kicinski } 136cae1927cSJakub Kicinski 137ab3f0063SJakub Kicinski static void __bpf_prog_offload_destroy(struct bpf_prog *prog) 138ab3f0063SJakub Kicinski { 1390a9c1991SJakub Kicinski struct bpf_prog_offload *offload = prog->aux->offload; 140ab3f0063SJakub Kicinski struct netdev_bpf data = {}; 141ab3f0063SJakub Kicinski 142ab3f0063SJakub Kicinski data.offload.prog = prog; 143ab3f0063SJakub Kicinski 144ab3f0063SJakub Kicinski if (offload->dev_state) 145ab3f0063SJakub Kicinski WARN_ON(__bpf_offload_ndo(prog, BPF_OFFLOAD_DESTROY, &data)); 146ab3f0063SJakub Kicinski 147ad8ad79fSJakub Kicinski /* Make sure BPF_PROG_GET_NEXT_ID can't find this dead program */ 148ad8ad79fSJakub Kicinski bpf_prog_free_id(prog, true); 149ad8ad79fSJakub Kicinski 150ab3f0063SJakub Kicinski list_del_init(&offload->offloads); 151ce3b9db4SJakub Kicinski kfree(offload); 152ce3b9db4SJakub Kicinski prog->aux->offload = NULL; 153ab3f0063SJakub Kicinski } 154ab3f0063SJakub Kicinski 155ab3f0063SJakub Kicinski void bpf_prog_offload_destroy(struct bpf_prog *prog) 156ab3f0063SJakub Kicinski { 157ab3f0063SJakub Kicinski rtnl_lock(); 158e0d3974aSJakub Kicinski down_write(&bpf_devs_lock); 159ce3b9db4SJakub Kicinski if (prog->aux->offload) 160ab3f0063SJakub Kicinski __bpf_prog_offload_destroy(prog); 161e0d3974aSJakub Kicinski up_write(&bpf_devs_lock); 162ab3f0063SJakub Kicinski rtnl_unlock(); 163ab3f0063SJakub Kicinski } 164ab3f0063SJakub Kicinski 165ab3f0063SJakub Kicinski static int bpf_prog_offload_translate(struct bpf_prog *prog) 166ab3f0063SJakub Kicinski { 167ab3f0063SJakub Kicinski struct netdev_bpf data = {}; 168ab3f0063SJakub Kicinski int ret; 169ab3f0063SJakub Kicinski 170ab3f0063SJakub Kicinski data.offload.prog = prog; 171ab3f0063SJakub Kicinski 172ab3f0063SJakub Kicinski rtnl_lock(); 173ab3f0063SJakub Kicinski ret = __bpf_offload_ndo(prog, BPF_OFFLOAD_TRANSLATE, &data); 174ab3f0063SJakub Kicinski rtnl_unlock(); 175ab3f0063SJakub Kicinski 176ab3f0063SJakub Kicinski return ret; 177ab3f0063SJakub Kicinski } 178ab3f0063SJakub Kicinski 179ab3f0063SJakub Kicinski static unsigned int bpf_prog_warn_on_exec(const void *ctx, 180ab3f0063SJakub Kicinski const struct bpf_insn *insn) 181ab3f0063SJakub Kicinski { 182ab3f0063SJakub Kicinski WARN(1, "attempt to execute device eBPF program on the host!"); 183ab3f0063SJakub Kicinski return 0; 184ab3f0063SJakub Kicinski } 185ab3f0063SJakub Kicinski 186ab3f0063SJakub Kicinski int bpf_prog_offload_compile(struct bpf_prog *prog) 187ab3f0063SJakub Kicinski { 188ab3f0063SJakub Kicinski prog->bpf_func = bpf_prog_warn_on_exec; 189ab3f0063SJakub Kicinski 190ab3f0063SJakub Kicinski return bpf_prog_offload_translate(prog); 191ab3f0063SJakub Kicinski } 192ab3f0063SJakub Kicinski 193675fc275SJakub Kicinski struct ns_get_path_bpf_prog_args { 194675fc275SJakub Kicinski struct bpf_prog *prog; 195675fc275SJakub Kicinski struct bpf_prog_info *info; 196675fc275SJakub Kicinski }; 197675fc275SJakub Kicinski 198675fc275SJakub Kicinski static struct ns_common *bpf_prog_offload_info_fill_ns(void *private_data) 199675fc275SJakub Kicinski { 200675fc275SJakub Kicinski struct ns_get_path_bpf_prog_args *args = private_data; 201675fc275SJakub Kicinski struct bpf_prog_aux *aux = args->prog->aux; 202675fc275SJakub Kicinski struct ns_common *ns; 203675fc275SJakub Kicinski struct net *net; 204675fc275SJakub Kicinski 205675fc275SJakub Kicinski rtnl_lock(); 206675fc275SJakub Kicinski down_read(&bpf_devs_lock); 207675fc275SJakub Kicinski 208675fc275SJakub Kicinski if (aux->offload) { 209675fc275SJakub Kicinski args->info->ifindex = aux->offload->netdev->ifindex; 210675fc275SJakub Kicinski net = dev_net(aux->offload->netdev); 211675fc275SJakub Kicinski get_net(net); 212675fc275SJakub Kicinski ns = &net->ns; 213675fc275SJakub Kicinski } else { 214675fc275SJakub Kicinski args->info->ifindex = 0; 215675fc275SJakub Kicinski ns = NULL; 216675fc275SJakub Kicinski } 217675fc275SJakub Kicinski 218675fc275SJakub Kicinski up_read(&bpf_devs_lock); 219675fc275SJakub Kicinski rtnl_unlock(); 220675fc275SJakub Kicinski 221675fc275SJakub Kicinski return ns; 222675fc275SJakub Kicinski } 223675fc275SJakub Kicinski 224675fc275SJakub Kicinski int bpf_prog_offload_info_fill(struct bpf_prog_info *info, 225675fc275SJakub Kicinski struct bpf_prog *prog) 226675fc275SJakub Kicinski { 227675fc275SJakub Kicinski struct ns_get_path_bpf_prog_args args = { 228675fc275SJakub Kicinski .prog = prog, 229675fc275SJakub Kicinski .info = info, 230675fc275SJakub Kicinski }; 231675fc275SJakub Kicinski struct inode *ns_inode; 232675fc275SJakub Kicinski struct path ns_path; 233675fc275SJakub Kicinski void *res; 234675fc275SJakub Kicinski 235675fc275SJakub Kicinski res = ns_get_path_cb(&ns_path, bpf_prog_offload_info_fill_ns, &args); 236675fc275SJakub Kicinski if (IS_ERR(res)) { 237675fc275SJakub Kicinski if (!info->ifindex) 238675fc275SJakub Kicinski return -ENODEV; 239675fc275SJakub Kicinski return PTR_ERR(res); 240675fc275SJakub Kicinski } 241675fc275SJakub Kicinski 242675fc275SJakub Kicinski ns_inode = ns_path.dentry->d_inode; 243675fc275SJakub Kicinski info->netns_dev = new_encode_dev(ns_inode->i_sb->s_dev); 244675fc275SJakub Kicinski info->netns_ino = ns_inode->i_ino; 245675fc275SJakub Kicinski path_put(&ns_path); 246675fc275SJakub Kicinski 247675fc275SJakub Kicinski return 0; 248675fc275SJakub Kicinski } 249675fc275SJakub Kicinski 250ab3f0063SJakub Kicinski const struct bpf_prog_ops bpf_offload_prog_ops = { 251ab3f0063SJakub Kicinski }; 252ab3f0063SJakub Kicinski 253ab3f0063SJakub Kicinski static int bpf_offload_notification(struct notifier_block *notifier, 254ab3f0063SJakub Kicinski ulong event, void *ptr) 255ab3f0063SJakub Kicinski { 256ab3f0063SJakub Kicinski struct net_device *netdev = netdev_notifier_info_to_dev(ptr); 2570a9c1991SJakub Kicinski struct bpf_prog_offload *offload, *tmp; 258ab3f0063SJakub Kicinski 259ab3f0063SJakub Kicinski ASSERT_RTNL(); 260ab3f0063SJakub Kicinski 261ab3f0063SJakub Kicinski switch (event) { 262ab3f0063SJakub Kicinski case NETDEV_UNREGISTER: 26362c71b45SJakub Kicinski /* ignore namespace changes */ 26462c71b45SJakub Kicinski if (netdev->reg_state != NETREG_UNREGISTERING) 26562c71b45SJakub Kicinski break; 26662c71b45SJakub Kicinski 267e0d3974aSJakub Kicinski down_write(&bpf_devs_lock); 268ab3f0063SJakub Kicinski list_for_each_entry_safe(offload, tmp, &bpf_prog_offload_devs, 269ab3f0063SJakub Kicinski offloads) { 270ab3f0063SJakub Kicinski if (offload->netdev == netdev) 271ab3f0063SJakub Kicinski __bpf_prog_offload_destroy(offload->prog); 272ab3f0063SJakub Kicinski } 273e0d3974aSJakub Kicinski up_write(&bpf_devs_lock); 274ab3f0063SJakub Kicinski break; 275ab3f0063SJakub Kicinski default: 276ab3f0063SJakub Kicinski break; 277ab3f0063SJakub Kicinski } 278ab3f0063SJakub Kicinski return NOTIFY_OK; 279ab3f0063SJakub Kicinski } 280ab3f0063SJakub Kicinski 281ab3f0063SJakub Kicinski static struct notifier_block bpf_offload_notifier = { 282ab3f0063SJakub Kicinski .notifier_call = bpf_offload_notification, 283ab3f0063SJakub Kicinski }; 284ab3f0063SJakub Kicinski 285ab3f0063SJakub Kicinski static int __init bpf_offload_init(void) 286ab3f0063SJakub Kicinski { 287ab3f0063SJakub Kicinski register_netdevice_notifier(&bpf_offload_notifier); 288ab3f0063SJakub Kicinski return 0; 289ab3f0063SJakub Kicinski } 290ab3f0063SJakub Kicinski 291ab3f0063SJakub Kicinski subsys_initcall(bpf_offload_init); 292