1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2023 Isovalent */ 3 4 #include <linux/bpf.h> 5 #include <linux/bpf_mprog.h> 6 #include <linux/netdevice.h> 7 8 #include <net/tcx.h> 9 10 int tcx_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog) 11 { 12 bool created, ingress = attr->attach_type == BPF_TCX_INGRESS; 13 struct net *net = current->nsproxy->net_ns; 14 struct bpf_mprog_entry *entry, *entry_new; 15 struct bpf_prog *replace_prog = NULL; 16 struct net_device *dev; 17 int ret; 18 19 rtnl_lock(); 20 dev = __dev_get_by_index(net, attr->target_ifindex); 21 if (!dev) { 22 ret = -ENODEV; 23 goto out; 24 } 25 if (attr->attach_flags & BPF_F_REPLACE) { 26 replace_prog = bpf_prog_get_type(attr->replace_bpf_fd, 27 prog->type); 28 if (IS_ERR(replace_prog)) { 29 ret = PTR_ERR(replace_prog); 30 replace_prog = NULL; 31 goto out; 32 } 33 } 34 entry = tcx_entry_fetch_or_create(dev, ingress, &created); 35 if (!entry) { 36 ret = -ENOMEM; 37 goto out; 38 } 39 ret = bpf_mprog_attach(entry, &entry_new, prog, NULL, replace_prog, 40 attr->attach_flags, attr->relative_fd, 41 attr->expected_revision); 42 if (!ret) { 43 if (entry != entry_new) { 44 tcx_entry_update(dev, entry_new, ingress); 45 tcx_entry_sync(); 46 tcx_skeys_inc(ingress); 47 } 48 bpf_mprog_commit(entry); 49 } else if (created) { 50 tcx_entry_free(entry); 51 } 52 out: 53 if (replace_prog) 54 bpf_prog_put(replace_prog); 55 rtnl_unlock(); 56 return ret; 57 } 58 59 int tcx_prog_detach(const union bpf_attr *attr, struct bpf_prog *prog) 60 { 61 bool ingress = attr->attach_type == BPF_TCX_INGRESS; 62 struct net *net = current->nsproxy->net_ns; 63 struct bpf_mprog_entry *entry, *entry_new; 64 struct net_device *dev; 65 int ret; 66 67 rtnl_lock(); 68 dev = __dev_get_by_index(net, attr->target_ifindex); 69 if (!dev) { 70 ret = -ENODEV; 71 goto out; 72 } 73 entry = tcx_entry_fetch(dev, ingress); 74 if (!entry) { 75 ret = -ENOENT; 76 goto out; 77 } 78 ret = bpf_mprog_detach(entry, &entry_new, prog, NULL, attr->attach_flags, 79 attr->relative_fd, attr->expected_revision); 80 if (!ret) { 81 if (!tcx_entry_is_active(entry_new)) 82 entry_new = NULL; 83 tcx_entry_update(dev, entry_new, ingress); 84 tcx_entry_sync(); 85 tcx_skeys_dec(ingress); 86 bpf_mprog_commit(entry); 87 if (!entry_new) 88 tcx_entry_free(entry); 89 } 90 out: 91 rtnl_unlock(); 92 return ret; 93 } 94 95 void tcx_uninstall(struct net_device *dev, bool ingress) 96 { 97 struct bpf_tuple tuple = {}; 98 struct bpf_mprog_entry *entry; 99 struct bpf_mprog_fp *fp; 100 struct bpf_mprog_cp *cp; 101 102 entry = tcx_entry_fetch(dev, ingress); 103 if (!entry) 104 return; 105 tcx_entry_update(dev, NULL, ingress); 106 tcx_entry_sync(); 107 bpf_mprog_foreach_tuple(entry, fp, cp, tuple) { 108 if (tuple.link) 109 tcx_link(tuple.link)->dev = NULL; 110 else 111 bpf_prog_put(tuple.prog); 112 tcx_skeys_dec(ingress); 113 } 114 WARN_ON_ONCE(tcx_entry(entry)->miniq_active); 115 tcx_entry_free(entry); 116 } 117 118 int tcx_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr) 119 { 120 bool ingress = attr->query.attach_type == BPF_TCX_INGRESS; 121 struct net *net = current->nsproxy->net_ns; 122 struct bpf_mprog_entry *entry; 123 struct net_device *dev; 124 int ret; 125 126 rtnl_lock(); 127 dev = __dev_get_by_index(net, attr->query.target_ifindex); 128 if (!dev) { 129 ret = -ENODEV; 130 goto out; 131 } 132 entry = tcx_entry_fetch(dev, ingress); 133 if (!entry) { 134 ret = -ENOENT; 135 goto out; 136 } 137 ret = bpf_mprog_query(attr, uattr, entry); 138 out: 139 rtnl_unlock(); 140 return ret; 141 } 142 143 static int tcx_link_prog_attach(struct bpf_link *link, u32 flags, u32 id_or_fd, 144 u64 revision) 145 { 146 struct tcx_link *tcx = tcx_link(link); 147 bool created, ingress = tcx->location == BPF_TCX_INGRESS; 148 struct bpf_mprog_entry *entry, *entry_new; 149 struct net_device *dev = tcx->dev; 150 int ret; 151 152 ASSERT_RTNL(); 153 entry = tcx_entry_fetch_or_create(dev, ingress, &created); 154 if (!entry) 155 return -ENOMEM; 156 ret = bpf_mprog_attach(entry, &entry_new, link->prog, link, NULL, flags, 157 id_or_fd, revision); 158 if (!ret) { 159 if (entry != entry_new) { 160 tcx_entry_update(dev, entry_new, ingress); 161 tcx_entry_sync(); 162 tcx_skeys_inc(ingress); 163 } 164 bpf_mprog_commit(entry); 165 } else if (created) { 166 tcx_entry_free(entry); 167 } 168 return ret; 169 } 170 171 static void tcx_link_release(struct bpf_link *link) 172 { 173 struct tcx_link *tcx = tcx_link(link); 174 bool ingress = tcx->location == BPF_TCX_INGRESS; 175 struct bpf_mprog_entry *entry, *entry_new; 176 struct net_device *dev; 177 int ret = 0; 178 179 rtnl_lock(); 180 dev = tcx->dev; 181 if (!dev) 182 goto out; 183 entry = tcx_entry_fetch(dev, ingress); 184 if (!entry) { 185 ret = -ENOENT; 186 goto out; 187 } 188 ret = bpf_mprog_detach(entry, &entry_new, link->prog, link, 0, 0, 0); 189 if (!ret) { 190 if (!tcx_entry_is_active(entry_new)) 191 entry_new = NULL; 192 tcx_entry_update(dev, entry_new, ingress); 193 tcx_entry_sync(); 194 tcx_skeys_dec(ingress); 195 bpf_mprog_commit(entry); 196 if (!entry_new) 197 tcx_entry_free(entry); 198 tcx->dev = NULL; 199 } 200 out: 201 WARN_ON_ONCE(ret); 202 rtnl_unlock(); 203 } 204 205 static int tcx_link_update(struct bpf_link *link, struct bpf_prog *nprog, 206 struct bpf_prog *oprog) 207 { 208 struct tcx_link *tcx = tcx_link(link); 209 bool ingress = tcx->location == BPF_TCX_INGRESS; 210 struct bpf_mprog_entry *entry, *entry_new; 211 struct net_device *dev; 212 int ret = 0; 213 214 rtnl_lock(); 215 dev = tcx->dev; 216 if (!dev) { 217 ret = -ENOLINK; 218 goto out; 219 } 220 if (oprog && link->prog != oprog) { 221 ret = -EPERM; 222 goto out; 223 } 224 oprog = link->prog; 225 if (oprog == nprog) { 226 bpf_prog_put(nprog); 227 goto out; 228 } 229 entry = tcx_entry_fetch(dev, ingress); 230 if (!entry) { 231 ret = -ENOENT; 232 goto out; 233 } 234 ret = bpf_mprog_attach(entry, &entry_new, nprog, link, oprog, 235 BPF_F_REPLACE | BPF_F_ID, 236 link->prog->aux->id, 0); 237 if (!ret) { 238 WARN_ON_ONCE(entry != entry_new); 239 oprog = xchg(&link->prog, nprog); 240 bpf_prog_put(oprog); 241 bpf_mprog_commit(entry); 242 } 243 out: 244 rtnl_unlock(); 245 return ret; 246 } 247 248 static void tcx_link_dealloc(struct bpf_link *link) 249 { 250 kfree(tcx_link(link)); 251 } 252 253 static void tcx_link_fdinfo(const struct bpf_link *link, struct seq_file *seq) 254 { 255 const struct tcx_link *tcx = tcx_link_const(link); 256 u32 ifindex = 0; 257 258 rtnl_lock(); 259 if (tcx->dev) 260 ifindex = tcx->dev->ifindex; 261 rtnl_unlock(); 262 263 seq_printf(seq, "ifindex:\t%u\n", ifindex); 264 seq_printf(seq, "attach_type:\t%u (%s)\n", 265 tcx->location, 266 tcx->location == BPF_TCX_INGRESS ? "ingress" : "egress"); 267 } 268 269 static int tcx_link_fill_info(const struct bpf_link *link, 270 struct bpf_link_info *info) 271 { 272 const struct tcx_link *tcx = tcx_link_const(link); 273 u32 ifindex = 0; 274 275 rtnl_lock(); 276 if (tcx->dev) 277 ifindex = tcx->dev->ifindex; 278 rtnl_unlock(); 279 280 info->tcx.ifindex = ifindex; 281 info->tcx.attach_type = tcx->location; 282 return 0; 283 } 284 285 static int tcx_link_detach(struct bpf_link *link) 286 { 287 tcx_link_release(link); 288 return 0; 289 } 290 291 static const struct bpf_link_ops tcx_link_lops = { 292 .release = tcx_link_release, 293 .detach = tcx_link_detach, 294 .dealloc = tcx_link_dealloc, 295 .update_prog = tcx_link_update, 296 .show_fdinfo = tcx_link_fdinfo, 297 .fill_link_info = tcx_link_fill_info, 298 }; 299 300 static int tcx_link_init(struct tcx_link *tcx, 301 struct bpf_link_primer *link_primer, 302 const union bpf_attr *attr, 303 struct net_device *dev, 304 struct bpf_prog *prog) 305 { 306 bpf_link_init(&tcx->link, BPF_LINK_TYPE_TCX, &tcx_link_lops, prog); 307 tcx->location = attr->link_create.attach_type; 308 tcx->dev = dev; 309 return bpf_link_prime(&tcx->link, link_primer); 310 } 311 312 int tcx_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) 313 { 314 struct net *net = current->nsproxy->net_ns; 315 struct bpf_link_primer link_primer; 316 struct net_device *dev; 317 struct tcx_link *tcx; 318 int ret; 319 320 rtnl_lock(); 321 dev = __dev_get_by_index(net, attr->link_create.target_ifindex); 322 if (!dev) { 323 ret = -ENODEV; 324 goto out; 325 } 326 tcx = kzalloc(sizeof(*tcx), GFP_USER); 327 if (!tcx) { 328 ret = -ENOMEM; 329 goto out; 330 } 331 ret = tcx_link_init(tcx, &link_primer, attr, dev, prog); 332 if (ret) { 333 kfree(tcx); 334 goto out; 335 } 336 ret = tcx_link_prog_attach(&tcx->link, attr->link_create.flags, 337 attr->link_create.tcx.relative_fd, 338 attr->link_create.tcx.expected_revision); 339 if (ret) { 340 tcx->dev = NULL; 341 bpf_link_cleanup(&link_primer); 342 goto out; 343 } 344 ret = bpf_link_settle(&link_primer); 345 out: 346 rtnl_unlock(); 347 return ret; 348 } 349