1 /* 2 * Copyright (C) 2016-2018 Netronome Systems, Inc. 3 * 4 * This software is dual licensed under the GNU General License Version 2, 5 * June 1991 as shown in the file COPYING in the top-level directory of this 6 * source tree or the BSD 2-Clause License provided below. You have the 7 * option to license this software under the complete terms of either license. 8 * 9 * The BSD 2-Clause License: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * 1. Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * 2. Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 /* 35 * nfp_net_offload.c 36 * Netronome network device driver: TC offload functions for PF and VF 37 */ 38 39 #define pr_fmt(fmt) "NFP net bpf: " fmt 40 41 #include <linux/bpf.h> 42 #include <linux/kernel.h> 43 #include <linux/netdevice.h> 44 #include <linux/pci.h> 45 #include <linux/jiffies.h> 46 #include <linux/timer.h> 47 #include <linux/list.h> 48 #include <linux/mm.h> 49 50 #include <net/pkt_cls.h> 51 #include <net/tc_act/tc_gact.h> 52 #include <net/tc_act/tc_mirred.h> 53 54 #include "main.h" 55 #include "../nfp_app.h" 56 #include "../nfp_net_ctrl.h" 57 #include "../nfp_net.h" 58 59 static int 60 nfp_map_ptr_record(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog, 61 struct bpf_map *map) 62 { 63 struct nfp_bpf_neutral_map *record; 64 int err; 65 66 /* Map record paths are entered via ndo, update side is protected. */ 67 ASSERT_RTNL(); 68 69 /* Reuse path - other offloaded program is already tracking this map. */ 70 record = rhashtable_lookup_fast(&bpf->maps_neutral, &map, 71 nfp_bpf_maps_neutral_params); 72 if (record) { 73 nfp_prog->map_records[nfp_prog->map_records_cnt++] = record; 74 record->count++; 75 return 0; 76 } 77 78 /* Grab a single ref to the map for our record. The prog destroy ndo 79 * happens after free_used_maps(). 80 */ 81 map = bpf_map_inc(map, false); 82 if (IS_ERR(map)) 83 return PTR_ERR(map); 84 85 record = kmalloc(sizeof(*record), GFP_KERNEL); 86 if (!record) { 87 err = -ENOMEM; 88 goto err_map_put; 89 } 90 91 record->ptr = map; 92 record->count = 1; 93 94 err = rhashtable_insert_fast(&bpf->maps_neutral, &record->l, 95 nfp_bpf_maps_neutral_params); 96 if (err) 97 goto err_free_rec; 98 99 nfp_prog->map_records[nfp_prog->map_records_cnt++] = record; 100 101 return 0; 102 103 err_free_rec: 104 kfree(record); 105 err_map_put: 106 bpf_map_put(map); 107 return err; 108 } 109 110 static void 111 nfp_map_ptrs_forget(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog) 112 { 113 bool freed = false; 114 int i; 115 116 ASSERT_RTNL(); 117 118 for (i = 0; i < nfp_prog->map_records_cnt; i++) { 119 if (--nfp_prog->map_records[i]->count) { 120 nfp_prog->map_records[i] = NULL; 121 continue; 122 } 123 124 WARN_ON(rhashtable_remove_fast(&bpf->maps_neutral, 125 &nfp_prog->map_records[i]->l, 126 nfp_bpf_maps_neutral_params)); 127 freed = true; 128 } 129 130 if (freed) { 131 synchronize_rcu(); 132 133 for (i = 0; i < nfp_prog->map_records_cnt; i++) 134 if (nfp_prog->map_records[i]) { 135 bpf_map_put(nfp_prog->map_records[i]->ptr); 136 kfree(nfp_prog->map_records[i]); 137 } 138 } 139 140 kfree(nfp_prog->map_records); 141 nfp_prog->map_records = NULL; 142 nfp_prog->map_records_cnt = 0; 143 } 144 145 static int 146 nfp_map_ptrs_record(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog, 147 struct bpf_prog *prog) 148 { 149 int i, cnt, err; 150 151 /* Quickly count the maps we will have to remember */ 152 cnt = 0; 153 for (i = 0; i < prog->aux->used_map_cnt; i++) 154 if (bpf_map_offload_neutral(prog->aux->used_maps[i])) 155 cnt++; 156 if (!cnt) 157 return 0; 158 159 nfp_prog->map_records = kmalloc_array(cnt, 160 sizeof(nfp_prog->map_records[0]), 161 GFP_KERNEL); 162 if (!nfp_prog->map_records) 163 return -ENOMEM; 164 165 for (i = 0; i < prog->aux->used_map_cnt; i++) 166 if (bpf_map_offload_neutral(prog->aux->used_maps[i])) { 167 err = nfp_map_ptr_record(bpf, nfp_prog, 168 prog->aux->used_maps[i]); 169 if (err) { 170 nfp_map_ptrs_forget(bpf, nfp_prog); 171 return err; 172 } 173 } 174 WARN_ON(cnt != nfp_prog->map_records_cnt); 175 176 return 0; 177 } 178 179 static int 180 nfp_prog_prepare(struct nfp_prog *nfp_prog, const struct bpf_insn *prog, 181 unsigned int cnt) 182 { 183 struct nfp_insn_meta *meta; 184 unsigned int i; 185 186 for (i = 0; i < cnt; i++) { 187 meta = kzalloc(sizeof(*meta), GFP_KERNEL); 188 if (!meta) 189 return -ENOMEM; 190 191 meta->insn = prog[i]; 192 meta->n = i; 193 194 list_add_tail(&meta->l, &nfp_prog->insns); 195 } 196 197 nfp_bpf_jit_prepare(nfp_prog, cnt); 198 199 return 0; 200 } 201 202 static void nfp_prog_free(struct nfp_prog *nfp_prog) 203 { 204 struct nfp_insn_meta *meta, *tmp; 205 206 list_for_each_entry_safe(meta, tmp, &nfp_prog->insns, l) { 207 list_del(&meta->l); 208 kfree(meta); 209 } 210 kfree(nfp_prog); 211 } 212 213 static int 214 nfp_bpf_verifier_prep(struct nfp_app *app, struct nfp_net *nn, 215 struct netdev_bpf *bpf) 216 { 217 struct bpf_prog *prog = bpf->verifier.prog; 218 struct nfp_prog *nfp_prog; 219 int ret; 220 221 nfp_prog = kzalloc(sizeof(*nfp_prog), GFP_KERNEL); 222 if (!nfp_prog) 223 return -ENOMEM; 224 prog->aux->offload->dev_priv = nfp_prog; 225 226 INIT_LIST_HEAD(&nfp_prog->insns); 227 nfp_prog->type = prog->type; 228 nfp_prog->bpf = app->priv; 229 230 ret = nfp_prog_prepare(nfp_prog, prog->insnsi, prog->len); 231 if (ret) 232 goto err_free; 233 234 nfp_prog->verifier_meta = nfp_prog_first_meta(nfp_prog); 235 bpf->verifier.ops = &nfp_bpf_analyzer_ops; 236 237 return 0; 238 239 err_free: 240 nfp_prog_free(nfp_prog); 241 242 return ret; 243 } 244 245 static int nfp_bpf_translate(struct nfp_net *nn, struct bpf_prog *prog) 246 { 247 struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv; 248 unsigned int stack_size; 249 unsigned int max_instr; 250 int err; 251 252 stack_size = nn_readb(nn, NFP_NET_CFG_BPF_STACK_SZ) * 64; 253 if (prog->aux->stack_depth > stack_size) { 254 nn_info(nn, "stack too large: program %dB > FW stack %dB\n", 255 prog->aux->stack_depth, stack_size); 256 return -EOPNOTSUPP; 257 } 258 nfp_prog->stack_depth = round_up(prog->aux->stack_depth, 4); 259 260 max_instr = nn_readw(nn, NFP_NET_CFG_BPF_MAX_LEN); 261 nfp_prog->__prog_alloc_len = max_instr * sizeof(u64); 262 263 nfp_prog->prog = kvmalloc(nfp_prog->__prog_alloc_len, GFP_KERNEL); 264 if (!nfp_prog->prog) 265 return -ENOMEM; 266 267 err = nfp_bpf_jit(nfp_prog); 268 if (err) 269 return err; 270 271 prog->aux->offload->jited_len = nfp_prog->prog_len * sizeof(u64); 272 prog->aux->offload->jited_image = nfp_prog->prog; 273 274 return nfp_map_ptrs_record(nfp_prog->bpf, nfp_prog, prog); 275 } 276 277 static int nfp_bpf_destroy(struct nfp_net *nn, struct bpf_prog *prog) 278 { 279 struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv; 280 281 kvfree(nfp_prog->prog); 282 nfp_map_ptrs_forget(nfp_prog->bpf, nfp_prog); 283 nfp_prog_free(nfp_prog); 284 285 return 0; 286 } 287 288 /* Atomic engine requires values to be in big endian, we need to byte swap 289 * the value words used with xadd. 290 */ 291 static void nfp_map_bpf_byte_swap(struct nfp_bpf_map *nfp_map, void *value) 292 { 293 u32 *word = value; 294 unsigned int i; 295 296 for (i = 0; i < DIV_ROUND_UP(nfp_map->offmap->map.value_size, 4); i++) 297 if (nfp_map->use_map[i] == NFP_MAP_USE_ATOMIC_CNT) 298 word[i] = (__force u32)cpu_to_be32(word[i]); 299 } 300 301 static int 302 nfp_bpf_map_lookup_entry(struct bpf_offloaded_map *offmap, 303 void *key, void *value) 304 { 305 int err; 306 307 err = nfp_bpf_ctrl_lookup_entry(offmap, key, value); 308 if (err) 309 return err; 310 311 nfp_map_bpf_byte_swap(offmap->dev_priv, value); 312 return 0; 313 } 314 315 static int 316 nfp_bpf_map_update_entry(struct bpf_offloaded_map *offmap, 317 void *key, void *value, u64 flags) 318 { 319 nfp_map_bpf_byte_swap(offmap->dev_priv, value); 320 return nfp_bpf_ctrl_update_entry(offmap, key, value, flags); 321 } 322 323 static int 324 nfp_bpf_map_get_next_key(struct bpf_offloaded_map *offmap, 325 void *key, void *next_key) 326 { 327 if (!key) 328 return nfp_bpf_ctrl_getfirst_entry(offmap, next_key); 329 return nfp_bpf_ctrl_getnext_entry(offmap, key, next_key); 330 } 331 332 static int 333 nfp_bpf_map_delete_elem(struct bpf_offloaded_map *offmap, void *key) 334 { 335 if (offmap->map.map_type == BPF_MAP_TYPE_ARRAY) 336 return -EINVAL; 337 return nfp_bpf_ctrl_del_entry(offmap, key); 338 } 339 340 static const struct bpf_map_dev_ops nfp_bpf_map_ops = { 341 .map_get_next_key = nfp_bpf_map_get_next_key, 342 .map_lookup_elem = nfp_bpf_map_lookup_entry, 343 .map_update_elem = nfp_bpf_map_update_entry, 344 .map_delete_elem = nfp_bpf_map_delete_elem, 345 }; 346 347 static int 348 nfp_bpf_map_alloc(struct nfp_app_bpf *bpf, struct bpf_offloaded_map *offmap) 349 { 350 struct nfp_bpf_map *nfp_map; 351 unsigned int use_map_size; 352 long long int res; 353 354 if (!bpf->maps.types) 355 return -EOPNOTSUPP; 356 357 if (offmap->map.map_flags || 358 offmap->map.numa_node != NUMA_NO_NODE) { 359 pr_info("map flags are not supported\n"); 360 return -EINVAL; 361 } 362 363 if (!(bpf->maps.types & 1 << offmap->map.map_type)) { 364 pr_info("map type not supported\n"); 365 return -EOPNOTSUPP; 366 } 367 if (bpf->maps.max_maps == bpf->maps_in_use) { 368 pr_info("too many maps for a device\n"); 369 return -ENOMEM; 370 } 371 if (bpf->maps.max_elems - bpf->map_elems_in_use < 372 offmap->map.max_entries) { 373 pr_info("map with too many elements: %u, left: %u\n", 374 offmap->map.max_entries, 375 bpf->maps.max_elems - bpf->map_elems_in_use); 376 return -ENOMEM; 377 } 378 if (offmap->map.key_size > bpf->maps.max_key_sz || 379 offmap->map.value_size > bpf->maps.max_val_sz || 380 round_up(offmap->map.key_size, 8) + 381 round_up(offmap->map.value_size, 8) > bpf->maps.max_elem_sz) { 382 pr_info("elements don't fit in device constraints\n"); 383 return -ENOMEM; 384 } 385 386 use_map_size = DIV_ROUND_UP(offmap->map.value_size, 4) * 387 FIELD_SIZEOF(struct nfp_bpf_map, use_map[0]); 388 389 nfp_map = kzalloc(sizeof(*nfp_map) + use_map_size, GFP_USER); 390 if (!nfp_map) 391 return -ENOMEM; 392 393 offmap->dev_priv = nfp_map; 394 nfp_map->offmap = offmap; 395 nfp_map->bpf = bpf; 396 397 res = nfp_bpf_ctrl_alloc_map(bpf, &offmap->map); 398 if (res < 0) { 399 kfree(nfp_map); 400 return res; 401 } 402 403 nfp_map->tid = res; 404 offmap->dev_ops = &nfp_bpf_map_ops; 405 bpf->maps_in_use++; 406 bpf->map_elems_in_use += offmap->map.max_entries; 407 list_add_tail(&nfp_map->l, &bpf->map_list); 408 409 return 0; 410 } 411 412 static int 413 nfp_bpf_map_free(struct nfp_app_bpf *bpf, struct bpf_offloaded_map *offmap) 414 { 415 struct nfp_bpf_map *nfp_map = offmap->dev_priv; 416 417 nfp_bpf_ctrl_free_map(bpf, nfp_map); 418 list_del_init(&nfp_map->l); 419 bpf->map_elems_in_use -= offmap->map.max_entries; 420 bpf->maps_in_use--; 421 kfree(nfp_map); 422 423 return 0; 424 } 425 426 int nfp_ndo_bpf(struct nfp_app *app, struct nfp_net *nn, struct netdev_bpf *bpf) 427 { 428 switch (bpf->command) { 429 case BPF_OFFLOAD_VERIFIER_PREP: 430 return nfp_bpf_verifier_prep(app, nn, bpf); 431 case BPF_OFFLOAD_TRANSLATE: 432 return nfp_bpf_translate(nn, bpf->offload.prog); 433 case BPF_OFFLOAD_DESTROY: 434 return nfp_bpf_destroy(nn, bpf->offload.prog); 435 case BPF_OFFLOAD_MAP_ALLOC: 436 return nfp_bpf_map_alloc(app->priv, bpf->offmap); 437 case BPF_OFFLOAD_MAP_FREE: 438 return nfp_bpf_map_free(app->priv, bpf->offmap); 439 default: 440 return -EINVAL; 441 } 442 } 443 444 static unsigned long 445 nfp_bpf_perf_event_copy(void *dst, const void *src, 446 unsigned long off, unsigned long len) 447 { 448 memcpy(dst, src + off, len); 449 return 0; 450 } 451 452 int nfp_bpf_event_output(struct nfp_app_bpf *bpf, struct sk_buff *skb) 453 { 454 struct cmsg_bpf_event *cbe = (void *)skb->data; 455 u32 pkt_size, data_size; 456 struct bpf_map *map; 457 458 if (skb->len < sizeof(struct cmsg_bpf_event)) 459 goto err_drop; 460 461 pkt_size = be32_to_cpu(cbe->pkt_size); 462 data_size = be32_to_cpu(cbe->data_size); 463 map = (void *)(unsigned long)be64_to_cpu(cbe->map_ptr); 464 465 if (skb->len < sizeof(struct cmsg_bpf_event) + pkt_size + data_size) 466 goto err_drop; 467 if (cbe->hdr.ver != CMSG_MAP_ABI_VERSION) 468 goto err_drop; 469 470 rcu_read_lock(); 471 if (!rhashtable_lookup_fast(&bpf->maps_neutral, &map, 472 nfp_bpf_maps_neutral_params)) { 473 rcu_read_unlock(); 474 pr_warn("perf event: dest map pointer %px not recognized, dropping event\n", 475 map); 476 goto err_drop; 477 } 478 479 bpf_event_output(map, be32_to_cpu(cbe->cpu_id), 480 &cbe->data[round_up(pkt_size, 4)], data_size, 481 cbe->data, pkt_size, nfp_bpf_perf_event_copy); 482 rcu_read_unlock(); 483 484 dev_consume_skb_any(skb); 485 return 0; 486 err_drop: 487 dev_kfree_skb_any(skb); 488 return -EINVAL; 489 } 490 491 static int 492 nfp_net_bpf_load(struct nfp_net *nn, struct bpf_prog *prog, 493 struct netlink_ext_ack *extack) 494 { 495 struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv; 496 unsigned int max_mtu; 497 dma_addr_t dma_addr; 498 void *img; 499 int err; 500 501 max_mtu = nn_readb(nn, NFP_NET_CFG_BPF_INL_MTU) * 64 - 32; 502 if (max_mtu < nn->dp.netdev->mtu) { 503 NL_SET_ERR_MSG_MOD(extack, "BPF offload not supported with MTU larger than HW packet split boundary"); 504 return -EOPNOTSUPP; 505 } 506 507 img = nfp_bpf_relo_for_vnic(nfp_prog, nn->app_priv); 508 if (IS_ERR(img)) 509 return PTR_ERR(img); 510 511 dma_addr = dma_map_single(nn->dp.dev, img, 512 nfp_prog->prog_len * sizeof(u64), 513 DMA_TO_DEVICE); 514 if (dma_mapping_error(nn->dp.dev, dma_addr)) { 515 kfree(img); 516 return -ENOMEM; 517 } 518 519 nn_writew(nn, NFP_NET_CFG_BPF_SIZE, nfp_prog->prog_len); 520 nn_writeq(nn, NFP_NET_CFG_BPF_ADDR, dma_addr); 521 522 /* Load up the JITed code */ 523 err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_BPF); 524 if (err) 525 NL_SET_ERR_MSG_MOD(extack, 526 "FW command error while loading BPF"); 527 528 dma_unmap_single(nn->dp.dev, dma_addr, nfp_prog->prog_len * sizeof(u64), 529 DMA_TO_DEVICE); 530 kfree(img); 531 532 return err; 533 } 534 535 static void 536 nfp_net_bpf_start(struct nfp_net *nn, struct netlink_ext_ack *extack) 537 { 538 int err; 539 540 /* Enable passing packets through BPF function */ 541 nn->dp.ctrl |= NFP_NET_CFG_CTRL_BPF; 542 nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl); 543 err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN); 544 if (err) 545 NL_SET_ERR_MSG_MOD(extack, 546 "FW command error while enabling BPF"); 547 } 548 549 static int nfp_net_bpf_stop(struct nfp_net *nn) 550 { 551 if (!(nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF)) 552 return 0; 553 554 nn->dp.ctrl &= ~NFP_NET_CFG_CTRL_BPF; 555 nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl); 556 557 return nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN); 558 } 559 560 int nfp_net_bpf_offload(struct nfp_net *nn, struct bpf_prog *prog, 561 bool old_prog, struct netlink_ext_ack *extack) 562 { 563 int err; 564 565 if (prog) { 566 struct bpf_prog_offload *offload = prog->aux->offload; 567 568 if (!offload) 569 return -EINVAL; 570 if (offload->netdev != nn->dp.netdev) 571 return -EINVAL; 572 } 573 574 if (prog && old_prog) { 575 u8 cap; 576 577 cap = nn_readb(nn, NFP_NET_CFG_BPF_CAP); 578 if (!(cap & NFP_NET_BPF_CAP_RELO)) { 579 NL_SET_ERR_MSG_MOD(extack, 580 "FW does not support live reload"); 581 return -EBUSY; 582 } 583 } 584 585 /* Something else is loaded, different program type? */ 586 if (!old_prog && nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF) 587 return -EBUSY; 588 589 if (old_prog && !prog) 590 return nfp_net_bpf_stop(nn); 591 592 err = nfp_net_bpf_load(nn, prog, extack); 593 if (err) 594 return err; 595 596 if (!old_prog) 597 nfp_net_bpf_start(nn, extack); 598 599 return 0; 600 } 601