1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2016-2018 Netronome Systems, Inc. */ 3 4 /* 5 * nfp_net_offload.c 6 * Netronome network device driver: TC offload functions for PF and VF 7 */ 8 9 #define pr_fmt(fmt) "NFP net bpf: " fmt 10 11 #include <linux/bpf.h> 12 #include <linux/kernel.h> 13 #include <linux/netdevice.h> 14 #include <linux/pci.h> 15 #include <linux/jiffies.h> 16 #include <linux/timer.h> 17 #include <linux/list.h> 18 #include <linux/mm.h> 19 20 #include <net/pkt_cls.h> 21 #include <net/tc_act/tc_gact.h> 22 #include <net/tc_act/tc_mirred.h> 23 24 #include "main.h" 25 #include "../nfp_app.h" 26 #include "../nfp_net_ctrl.h" 27 #include "../nfp_net.h" 28 29 static int 30 nfp_map_ptr_record(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog, 31 struct bpf_map *map) 32 { 33 struct nfp_bpf_neutral_map *record; 34 int err; 35 36 /* Reuse path - other offloaded program is already tracking this map. */ 37 record = rhashtable_lookup_fast(&bpf->maps_neutral, &map->id, 38 nfp_bpf_maps_neutral_params); 39 if (record) { 40 nfp_prog->map_records[nfp_prog->map_records_cnt++] = record; 41 record->count++; 42 return 0; 43 } 44 45 /* Grab a single ref to the map for our record. The prog destroy ndo 46 * happens after free_used_maps(). 47 */ 48 map = bpf_map_inc(map, false); 49 if (IS_ERR(map)) 50 return PTR_ERR(map); 51 52 record = kmalloc(sizeof(*record), GFP_KERNEL); 53 if (!record) { 54 err = -ENOMEM; 55 goto err_map_put; 56 } 57 58 record->ptr = map; 59 record->map_id = map->id; 60 record->count = 1; 61 62 err = rhashtable_insert_fast(&bpf->maps_neutral, &record->l, 63 nfp_bpf_maps_neutral_params); 64 if (err) 65 goto err_free_rec; 66 67 nfp_prog->map_records[nfp_prog->map_records_cnt++] = record; 68 69 return 0; 70 71 err_free_rec: 72 kfree(record); 73 err_map_put: 74 bpf_map_put(map); 75 return err; 76 } 77 78 static void 79 nfp_map_ptrs_forget(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog) 80 { 81 bool freed = false; 82 int i; 83 84 for (i = 0; i < nfp_prog->map_records_cnt; i++) { 85 if (--nfp_prog->map_records[i]->count) { 86 nfp_prog->map_records[i] = NULL; 87 continue; 88 } 89 90 WARN_ON(rhashtable_remove_fast(&bpf->maps_neutral, 91 &nfp_prog->map_records[i]->l, 92 nfp_bpf_maps_neutral_params)); 93 freed = true; 94 } 95 96 if (freed) { 97 synchronize_rcu(); 98 99 for (i = 0; i < nfp_prog->map_records_cnt; i++) 100 if (nfp_prog->map_records[i]) { 101 bpf_map_put(nfp_prog->map_records[i]->ptr); 102 kfree(nfp_prog->map_records[i]); 103 } 104 } 105 106 kfree(nfp_prog->map_records); 107 nfp_prog->map_records = NULL; 108 nfp_prog->map_records_cnt = 0; 109 } 110 111 static int 112 nfp_map_ptrs_record(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog, 113 struct bpf_prog *prog) 114 { 115 int i, cnt, err; 116 117 /* Quickly count the maps we will have to remember */ 118 cnt = 0; 119 for (i = 0; i < prog->aux->used_map_cnt; i++) 120 if (bpf_map_offload_neutral(prog->aux->used_maps[i])) 121 cnt++; 122 if (!cnt) 123 return 0; 124 125 nfp_prog->map_records = kmalloc_array(cnt, 126 sizeof(nfp_prog->map_records[0]), 127 GFP_KERNEL); 128 if (!nfp_prog->map_records) 129 return -ENOMEM; 130 131 for (i = 0; i < prog->aux->used_map_cnt; i++) 132 if (bpf_map_offload_neutral(prog->aux->used_maps[i])) { 133 err = nfp_map_ptr_record(bpf, nfp_prog, 134 prog->aux->used_maps[i]); 135 if (err) { 136 nfp_map_ptrs_forget(bpf, nfp_prog); 137 return err; 138 } 139 } 140 WARN_ON(cnt != nfp_prog->map_records_cnt); 141 142 return 0; 143 } 144 145 static int 146 nfp_prog_prepare(struct nfp_prog *nfp_prog, const struct bpf_insn *prog, 147 unsigned int cnt) 148 { 149 struct nfp_insn_meta *meta; 150 unsigned int i; 151 152 for (i = 0; i < cnt; i++) { 153 meta = kzalloc(sizeof(*meta), GFP_KERNEL); 154 if (!meta) 155 return -ENOMEM; 156 157 meta->insn = prog[i]; 158 meta->n = i; 159 if (is_mbpf_alu(meta)) { 160 meta->umin_src = U64_MAX; 161 meta->umin_dst = U64_MAX; 162 } 163 164 list_add_tail(&meta->l, &nfp_prog->insns); 165 } 166 nfp_prog->n_insns = cnt; 167 168 nfp_bpf_jit_prepare(nfp_prog); 169 170 return 0; 171 } 172 173 static void nfp_prog_free(struct nfp_prog *nfp_prog) 174 { 175 struct nfp_insn_meta *meta, *tmp; 176 177 kfree(nfp_prog->subprog); 178 179 list_for_each_entry_safe(meta, tmp, &nfp_prog->insns, l) { 180 list_del(&meta->l); 181 kfree(meta); 182 } 183 kfree(nfp_prog); 184 } 185 186 static int nfp_bpf_verifier_prep(struct bpf_prog *prog) 187 { 188 struct nfp_prog *nfp_prog; 189 int ret; 190 191 nfp_prog = kzalloc(sizeof(*nfp_prog), GFP_KERNEL); 192 if (!nfp_prog) 193 return -ENOMEM; 194 prog->aux->offload->dev_priv = nfp_prog; 195 196 INIT_LIST_HEAD(&nfp_prog->insns); 197 nfp_prog->type = prog->type; 198 nfp_prog->bpf = bpf_offload_dev_priv(prog->aux->offload->offdev); 199 200 ret = nfp_prog_prepare(nfp_prog, prog->insnsi, prog->len); 201 if (ret) 202 goto err_free; 203 204 nfp_prog->verifier_meta = nfp_prog_first_meta(nfp_prog); 205 206 return 0; 207 208 err_free: 209 nfp_prog_free(nfp_prog); 210 211 return ret; 212 } 213 214 static int nfp_bpf_translate(struct bpf_prog *prog) 215 { 216 struct nfp_net *nn = netdev_priv(prog->aux->offload->netdev); 217 struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv; 218 unsigned int max_instr; 219 int err; 220 221 /* We depend on dead code elimination succeeding */ 222 if (prog->aux->offload->opt_failed) 223 return -EINVAL; 224 225 max_instr = nn_readw(nn, NFP_NET_CFG_BPF_MAX_LEN); 226 nfp_prog->__prog_alloc_len = max_instr * sizeof(u64); 227 228 nfp_prog->prog = kvmalloc(nfp_prog->__prog_alloc_len, GFP_KERNEL); 229 if (!nfp_prog->prog) 230 return -ENOMEM; 231 232 err = nfp_bpf_jit(nfp_prog); 233 if (err) 234 return err; 235 236 prog->aux->offload->jited_len = nfp_prog->prog_len * sizeof(u64); 237 prog->aux->offload->jited_image = nfp_prog->prog; 238 239 return nfp_map_ptrs_record(nfp_prog->bpf, nfp_prog, prog); 240 } 241 242 static void nfp_bpf_destroy(struct bpf_prog *prog) 243 { 244 struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv; 245 246 kvfree(nfp_prog->prog); 247 nfp_map_ptrs_forget(nfp_prog->bpf, nfp_prog); 248 nfp_prog_free(nfp_prog); 249 } 250 251 /* Atomic engine requires values to be in big endian, we need to byte swap 252 * the value words used with xadd. 253 */ 254 static void nfp_map_bpf_byte_swap(struct nfp_bpf_map *nfp_map, void *value) 255 { 256 u32 *word = value; 257 unsigned int i; 258 259 for (i = 0; i < DIV_ROUND_UP(nfp_map->offmap->map.value_size, 4); i++) 260 if (nfp_map->use_map[i].type == NFP_MAP_USE_ATOMIC_CNT) 261 word[i] = (__force u32)cpu_to_be32(word[i]); 262 } 263 264 /* Mark value as unsafely initialized in case it becomes atomic later 265 * and we didn't byte swap something non-byte swap neutral. 266 */ 267 static void 268 nfp_map_bpf_byte_swap_record(struct nfp_bpf_map *nfp_map, void *value) 269 { 270 u32 *word = value; 271 unsigned int i; 272 273 for (i = 0; i < DIV_ROUND_UP(nfp_map->offmap->map.value_size, 4); i++) 274 if (nfp_map->use_map[i].type == NFP_MAP_UNUSED && 275 word[i] != (__force u32)cpu_to_be32(word[i])) 276 nfp_map->use_map[i].non_zero_update = 1; 277 } 278 279 static int 280 nfp_bpf_map_lookup_entry(struct bpf_offloaded_map *offmap, 281 void *key, void *value) 282 { 283 int err; 284 285 err = nfp_bpf_ctrl_lookup_entry(offmap, key, value); 286 if (err) 287 return err; 288 289 nfp_map_bpf_byte_swap(offmap->dev_priv, value); 290 return 0; 291 } 292 293 static int 294 nfp_bpf_map_update_entry(struct bpf_offloaded_map *offmap, 295 void *key, void *value, u64 flags) 296 { 297 nfp_map_bpf_byte_swap(offmap->dev_priv, value); 298 nfp_map_bpf_byte_swap_record(offmap->dev_priv, value); 299 return nfp_bpf_ctrl_update_entry(offmap, key, value, flags); 300 } 301 302 static int 303 nfp_bpf_map_get_next_key(struct bpf_offloaded_map *offmap, 304 void *key, void *next_key) 305 { 306 if (!key) 307 return nfp_bpf_ctrl_getfirst_entry(offmap, next_key); 308 return nfp_bpf_ctrl_getnext_entry(offmap, key, next_key); 309 } 310 311 static int 312 nfp_bpf_map_delete_elem(struct bpf_offloaded_map *offmap, void *key) 313 { 314 if (offmap->map.map_type == BPF_MAP_TYPE_ARRAY) 315 return -EINVAL; 316 return nfp_bpf_ctrl_del_entry(offmap, key); 317 } 318 319 static const struct bpf_map_dev_ops nfp_bpf_map_ops = { 320 .map_get_next_key = nfp_bpf_map_get_next_key, 321 .map_lookup_elem = nfp_bpf_map_lookup_entry, 322 .map_update_elem = nfp_bpf_map_update_entry, 323 .map_delete_elem = nfp_bpf_map_delete_elem, 324 }; 325 326 static int 327 nfp_bpf_map_alloc(struct nfp_app_bpf *bpf, struct bpf_offloaded_map *offmap) 328 { 329 struct nfp_bpf_map *nfp_map; 330 unsigned int use_map_size; 331 long long int res; 332 333 if (!bpf->maps.types) 334 return -EOPNOTSUPP; 335 336 if (offmap->map.map_flags || 337 offmap->map.numa_node != NUMA_NO_NODE) { 338 pr_info("map flags are not supported\n"); 339 return -EINVAL; 340 } 341 342 if (!(bpf->maps.types & 1 << offmap->map.map_type)) { 343 pr_info("map type not supported\n"); 344 return -EOPNOTSUPP; 345 } 346 if (bpf->maps.max_maps == bpf->maps_in_use) { 347 pr_info("too many maps for a device\n"); 348 return -ENOMEM; 349 } 350 if (bpf->maps.max_elems - bpf->map_elems_in_use < 351 offmap->map.max_entries) { 352 pr_info("map with too many elements: %u, left: %u\n", 353 offmap->map.max_entries, 354 bpf->maps.max_elems - bpf->map_elems_in_use); 355 return -ENOMEM; 356 } 357 358 if (round_up(offmap->map.key_size, 8) + 359 round_up(offmap->map.value_size, 8) > bpf->maps.max_elem_sz) { 360 pr_info("map elements too large: %u, FW max element size (key+value): %u\n", 361 round_up(offmap->map.key_size, 8) + 362 round_up(offmap->map.value_size, 8), 363 bpf->maps.max_elem_sz); 364 return -ENOMEM; 365 } 366 if (offmap->map.key_size > bpf->maps.max_key_sz) { 367 pr_info("map key size %u, FW max is %u\n", 368 offmap->map.key_size, bpf->maps.max_key_sz); 369 return -ENOMEM; 370 } 371 if (offmap->map.value_size > bpf->maps.max_val_sz) { 372 pr_info("map value size %u, FW max is %u\n", 373 offmap->map.value_size, bpf->maps.max_val_sz); 374 return -ENOMEM; 375 } 376 377 use_map_size = DIV_ROUND_UP(offmap->map.value_size, 4) * 378 FIELD_SIZEOF(struct nfp_bpf_map, use_map[0]); 379 380 nfp_map = kzalloc(sizeof(*nfp_map) + use_map_size, GFP_USER); 381 if (!nfp_map) 382 return -ENOMEM; 383 384 offmap->dev_priv = nfp_map; 385 nfp_map->offmap = offmap; 386 nfp_map->bpf = bpf; 387 388 res = nfp_bpf_ctrl_alloc_map(bpf, &offmap->map); 389 if (res < 0) { 390 kfree(nfp_map); 391 return res; 392 } 393 394 nfp_map->tid = res; 395 offmap->dev_ops = &nfp_bpf_map_ops; 396 bpf->maps_in_use++; 397 bpf->map_elems_in_use += offmap->map.max_entries; 398 list_add_tail(&nfp_map->l, &bpf->map_list); 399 400 return 0; 401 } 402 403 static int 404 nfp_bpf_map_free(struct nfp_app_bpf *bpf, struct bpf_offloaded_map *offmap) 405 { 406 struct nfp_bpf_map *nfp_map = offmap->dev_priv; 407 408 nfp_bpf_ctrl_free_map(bpf, nfp_map); 409 list_del_init(&nfp_map->l); 410 bpf->map_elems_in_use -= offmap->map.max_entries; 411 bpf->maps_in_use--; 412 kfree(nfp_map); 413 414 return 0; 415 } 416 417 int nfp_ndo_bpf(struct nfp_app *app, struct nfp_net *nn, struct netdev_bpf *bpf) 418 { 419 switch (bpf->command) { 420 case BPF_OFFLOAD_MAP_ALLOC: 421 return nfp_bpf_map_alloc(app->priv, bpf->offmap); 422 case BPF_OFFLOAD_MAP_FREE: 423 return nfp_bpf_map_free(app->priv, bpf->offmap); 424 default: 425 return -EINVAL; 426 } 427 } 428 429 static unsigned long 430 nfp_bpf_perf_event_copy(void *dst, const void *src, 431 unsigned long off, unsigned long len) 432 { 433 memcpy(dst, src + off, len); 434 return 0; 435 } 436 437 int nfp_bpf_event_output(struct nfp_app_bpf *bpf, const void *data, 438 unsigned int len) 439 { 440 struct cmsg_bpf_event *cbe = (void *)data; 441 struct nfp_bpf_neutral_map *record; 442 u32 pkt_size, data_size, map_id; 443 u64 map_id_full; 444 445 if (len < sizeof(struct cmsg_bpf_event)) 446 return -EINVAL; 447 448 pkt_size = be32_to_cpu(cbe->pkt_size); 449 data_size = be32_to_cpu(cbe->data_size); 450 map_id_full = be64_to_cpu(cbe->map_ptr); 451 map_id = map_id_full; 452 453 if (len < sizeof(struct cmsg_bpf_event) + pkt_size + data_size) 454 return -EINVAL; 455 if (cbe->hdr.ver != CMSG_MAP_ABI_VERSION) 456 return -EINVAL; 457 458 rcu_read_lock(); 459 record = rhashtable_lookup_fast(&bpf->maps_neutral, &map_id, 460 nfp_bpf_maps_neutral_params); 461 if (!record || map_id_full > U32_MAX) { 462 rcu_read_unlock(); 463 cmsg_warn(bpf, "perf event: map id %lld (0x%llx) not recognized, dropping event\n", 464 map_id_full, map_id_full); 465 return -EINVAL; 466 } 467 468 bpf_event_output(record->ptr, be32_to_cpu(cbe->cpu_id), 469 &cbe->data[round_up(pkt_size, 4)], data_size, 470 cbe->data, pkt_size, nfp_bpf_perf_event_copy); 471 rcu_read_unlock(); 472 473 return 0; 474 } 475 476 static int 477 nfp_net_bpf_load(struct nfp_net *nn, struct bpf_prog *prog, 478 struct netlink_ext_ack *extack) 479 { 480 struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv; 481 unsigned int fw_mtu, pkt_off, max_stack, max_prog_len; 482 dma_addr_t dma_addr; 483 void *img; 484 int err; 485 486 fw_mtu = nn_readb(nn, NFP_NET_CFG_BPF_INL_MTU) * 64 - 32; 487 pkt_off = min(prog->aux->max_pkt_offset, nn->dp.netdev->mtu); 488 if (fw_mtu < pkt_off) { 489 NL_SET_ERR_MSG_MOD(extack, "BPF offload not supported with potential packet access beyond HW packet split boundary"); 490 return -EOPNOTSUPP; 491 } 492 493 max_stack = nn_readb(nn, NFP_NET_CFG_BPF_STACK_SZ) * 64; 494 if (nfp_prog->stack_size > max_stack) { 495 NL_SET_ERR_MSG_MOD(extack, "stack too large"); 496 return -EOPNOTSUPP; 497 } 498 499 max_prog_len = nn_readw(nn, NFP_NET_CFG_BPF_MAX_LEN); 500 if (nfp_prog->prog_len > max_prog_len) { 501 NL_SET_ERR_MSG_MOD(extack, "program too long"); 502 return -EOPNOTSUPP; 503 } 504 505 img = nfp_bpf_relo_for_vnic(nfp_prog, nn->app_priv); 506 if (IS_ERR(img)) 507 return PTR_ERR(img); 508 509 dma_addr = dma_map_single(nn->dp.dev, img, 510 nfp_prog->prog_len * sizeof(u64), 511 DMA_TO_DEVICE); 512 if (dma_mapping_error(nn->dp.dev, dma_addr)) { 513 kfree(img); 514 return -ENOMEM; 515 } 516 517 nn_writew(nn, NFP_NET_CFG_BPF_SIZE, nfp_prog->prog_len); 518 nn_writeq(nn, NFP_NET_CFG_BPF_ADDR, dma_addr); 519 520 /* Load up the JITed code */ 521 err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_BPF); 522 if (err) 523 NL_SET_ERR_MSG_MOD(extack, 524 "FW command error while loading BPF"); 525 526 dma_unmap_single(nn->dp.dev, dma_addr, nfp_prog->prog_len * sizeof(u64), 527 DMA_TO_DEVICE); 528 kfree(img); 529 530 return err; 531 } 532 533 static void 534 nfp_net_bpf_start(struct nfp_net *nn, struct netlink_ext_ack *extack) 535 { 536 int err; 537 538 /* Enable passing packets through BPF function */ 539 nn->dp.ctrl |= NFP_NET_CFG_CTRL_BPF; 540 nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl); 541 err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN); 542 if (err) 543 NL_SET_ERR_MSG_MOD(extack, 544 "FW command error while enabling BPF"); 545 } 546 547 static int nfp_net_bpf_stop(struct nfp_net *nn) 548 { 549 if (!(nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF)) 550 return 0; 551 552 nn->dp.ctrl &= ~NFP_NET_CFG_CTRL_BPF; 553 nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl); 554 555 return nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN); 556 } 557 558 int nfp_net_bpf_offload(struct nfp_net *nn, struct bpf_prog *prog, 559 bool old_prog, struct netlink_ext_ack *extack) 560 { 561 int err; 562 563 if (prog && !bpf_offload_dev_match(prog, nn->dp.netdev)) 564 return -EINVAL; 565 566 if (prog && old_prog) { 567 u8 cap; 568 569 cap = nn_readb(nn, NFP_NET_CFG_BPF_CAP); 570 if (!(cap & NFP_NET_BPF_CAP_RELO)) { 571 NL_SET_ERR_MSG_MOD(extack, 572 "FW does not support live reload"); 573 return -EBUSY; 574 } 575 } 576 577 /* Something else is loaded, different program type? */ 578 if (!old_prog && nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF) 579 return -EBUSY; 580 581 if (old_prog && !prog) 582 return nfp_net_bpf_stop(nn); 583 584 err = nfp_net_bpf_load(nn, prog, extack); 585 if (err) 586 return err; 587 588 if (!old_prog) 589 nfp_net_bpf_start(nn, extack); 590 591 return 0; 592 } 593 594 const struct bpf_prog_offload_ops nfp_bpf_dev_ops = { 595 .insn_hook = nfp_verify_insn, 596 .finalize = nfp_bpf_finalize, 597 .replace_insn = nfp_bpf_opt_replace_insn, 598 .remove_insns = nfp_bpf_opt_remove_insns, 599 .prepare = nfp_bpf_verifier_prep, 600 .translate = nfp_bpf_translate, 601 .destroy = nfp_bpf_destroy, 602 }; 603