1 /* 2 * Copyright (C) 2016-2018 Netronome Systems, Inc. 3 * 4 * This software is dual licensed under the GNU General License Version 2, 5 * June 1991 as shown in the file COPYING in the top-level directory of this 6 * source tree or the BSD 2-Clause License provided below. You have the 7 * option to license this software under the complete terms of either license. 8 * 9 * The BSD 2-Clause License: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * 1. Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * 2. Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 /* 35 * nfp_net_offload.c 36 * Netronome network device driver: TC offload functions for PF and VF 37 */ 38 39 #define pr_fmt(fmt) "NFP net bpf: " fmt 40 41 #include <linux/bpf.h> 42 #include <linux/kernel.h> 43 #include <linux/netdevice.h> 44 #include <linux/pci.h> 45 #include <linux/jiffies.h> 46 #include <linux/timer.h> 47 #include <linux/list.h> 48 #include <linux/mm.h> 49 50 #include <net/pkt_cls.h> 51 #include <net/tc_act/tc_gact.h> 52 #include <net/tc_act/tc_mirred.h> 53 54 #include "main.h" 55 #include "../nfp_app.h" 56 #include "../nfp_net_ctrl.h" 57 #include "../nfp_net.h" 58 59 static int 60 nfp_map_ptr_record(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog, 61 struct bpf_map *map) 62 { 63 struct nfp_bpf_neutral_map *record; 64 int err; 65 66 /* Map record paths are entered via ndo, update side is protected. */ 67 ASSERT_RTNL(); 68 69 /* Reuse path - other offloaded program is already tracking this map. */ 70 record = rhashtable_lookup_fast(&bpf->maps_neutral, &map, 71 nfp_bpf_maps_neutral_params); 72 if (record) { 73 nfp_prog->map_records[nfp_prog->map_records_cnt++] = record; 74 record->count++; 75 return 0; 76 } 77 78 /* Grab a single ref to the map for our record. The prog destroy ndo 79 * happens after free_used_maps(). 80 */ 81 map = bpf_map_inc(map, false); 82 if (IS_ERR(map)) 83 return PTR_ERR(map); 84 85 record = kmalloc(sizeof(*record), GFP_KERNEL); 86 if (!record) { 87 err = -ENOMEM; 88 goto err_map_put; 89 } 90 91 record->ptr = map; 92 record->count = 1; 93 94 err = rhashtable_insert_fast(&bpf->maps_neutral, &record->l, 95 nfp_bpf_maps_neutral_params); 96 if (err) 97 goto err_free_rec; 98 99 nfp_prog->map_records[nfp_prog->map_records_cnt++] = record; 100 101 return 0; 102 103 err_free_rec: 104 kfree(record); 105 err_map_put: 106 bpf_map_put(map); 107 return err; 108 } 109 110 static void 111 nfp_map_ptrs_forget(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog) 112 { 113 bool freed = false; 114 int i; 115 116 ASSERT_RTNL(); 117 118 for (i = 0; i < nfp_prog->map_records_cnt; i++) { 119 if (--nfp_prog->map_records[i]->count) { 120 nfp_prog->map_records[i] = NULL; 121 continue; 122 } 123 124 WARN_ON(rhashtable_remove_fast(&bpf->maps_neutral, 125 &nfp_prog->map_records[i]->l, 126 nfp_bpf_maps_neutral_params)); 127 freed = true; 128 } 129 130 if (freed) { 131 synchronize_rcu(); 132 133 for (i = 0; i < nfp_prog->map_records_cnt; i++) 134 if (nfp_prog->map_records[i]) { 135 bpf_map_put(nfp_prog->map_records[i]->ptr); 136 kfree(nfp_prog->map_records[i]); 137 } 138 } 139 140 kfree(nfp_prog->map_records); 141 nfp_prog->map_records = NULL; 142 nfp_prog->map_records_cnt = 0; 143 } 144 145 static int 146 nfp_map_ptrs_record(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog, 147 struct bpf_prog *prog) 148 { 149 int i, cnt, err; 150 151 /* Quickly count the maps we will have to remember */ 152 cnt = 0; 153 for (i = 0; i < prog->aux->used_map_cnt; i++) 154 if (bpf_map_offload_neutral(prog->aux->used_maps[i])) 155 cnt++; 156 if (!cnt) 157 return 0; 158 159 nfp_prog->map_records = kmalloc_array(cnt, 160 sizeof(nfp_prog->map_records[0]), 161 GFP_KERNEL); 162 if (!nfp_prog->map_records) 163 return -ENOMEM; 164 165 for (i = 0; i < prog->aux->used_map_cnt; i++) 166 if (bpf_map_offload_neutral(prog->aux->used_maps[i])) { 167 err = nfp_map_ptr_record(bpf, nfp_prog, 168 prog->aux->used_maps[i]); 169 if (err) { 170 nfp_map_ptrs_forget(bpf, nfp_prog); 171 return err; 172 } 173 } 174 WARN_ON(cnt != nfp_prog->map_records_cnt); 175 176 return 0; 177 } 178 179 static int 180 nfp_prog_prepare(struct nfp_prog *nfp_prog, const struct bpf_insn *prog, 181 unsigned int cnt) 182 { 183 struct nfp_insn_meta *meta; 184 unsigned int i; 185 186 for (i = 0; i < cnt; i++) { 187 meta = kzalloc(sizeof(*meta), GFP_KERNEL); 188 if (!meta) 189 return -ENOMEM; 190 191 meta->insn = prog[i]; 192 meta->n = i; 193 if (is_mbpf_indir_shift(meta)) 194 meta->umin = U64_MAX; 195 196 list_add_tail(&meta->l, &nfp_prog->insns); 197 } 198 199 nfp_bpf_jit_prepare(nfp_prog, cnt); 200 201 return 0; 202 } 203 204 static void nfp_prog_free(struct nfp_prog *nfp_prog) 205 { 206 struct nfp_insn_meta *meta, *tmp; 207 208 list_for_each_entry_safe(meta, tmp, &nfp_prog->insns, l) { 209 list_del(&meta->l); 210 kfree(meta); 211 } 212 kfree(nfp_prog); 213 } 214 215 static int 216 nfp_bpf_verifier_prep(struct nfp_app *app, struct nfp_net *nn, 217 struct netdev_bpf *bpf) 218 { 219 struct bpf_prog *prog = bpf->verifier.prog; 220 struct nfp_prog *nfp_prog; 221 int ret; 222 223 nfp_prog = kzalloc(sizeof(*nfp_prog), GFP_KERNEL); 224 if (!nfp_prog) 225 return -ENOMEM; 226 prog->aux->offload->dev_priv = nfp_prog; 227 228 INIT_LIST_HEAD(&nfp_prog->insns); 229 nfp_prog->type = prog->type; 230 nfp_prog->bpf = app->priv; 231 232 ret = nfp_prog_prepare(nfp_prog, prog->insnsi, prog->len); 233 if (ret) 234 goto err_free; 235 236 nfp_prog->verifier_meta = nfp_prog_first_meta(nfp_prog); 237 bpf->verifier.ops = &nfp_bpf_analyzer_ops; 238 239 return 0; 240 241 err_free: 242 nfp_prog_free(nfp_prog); 243 244 return ret; 245 } 246 247 static int nfp_bpf_translate(struct nfp_net *nn, struct bpf_prog *prog) 248 { 249 struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv; 250 unsigned int stack_size; 251 unsigned int max_instr; 252 int err; 253 254 stack_size = nn_readb(nn, NFP_NET_CFG_BPF_STACK_SZ) * 64; 255 if (prog->aux->stack_depth > stack_size) { 256 nn_info(nn, "stack too large: program %dB > FW stack %dB\n", 257 prog->aux->stack_depth, stack_size); 258 return -EOPNOTSUPP; 259 } 260 nfp_prog->stack_depth = round_up(prog->aux->stack_depth, 4); 261 262 max_instr = nn_readw(nn, NFP_NET_CFG_BPF_MAX_LEN); 263 nfp_prog->__prog_alloc_len = max_instr * sizeof(u64); 264 265 nfp_prog->prog = kvmalloc(nfp_prog->__prog_alloc_len, GFP_KERNEL); 266 if (!nfp_prog->prog) 267 return -ENOMEM; 268 269 err = nfp_bpf_jit(nfp_prog); 270 if (err) 271 return err; 272 273 prog->aux->offload->jited_len = nfp_prog->prog_len * sizeof(u64); 274 prog->aux->offload->jited_image = nfp_prog->prog; 275 276 return nfp_map_ptrs_record(nfp_prog->bpf, nfp_prog, prog); 277 } 278 279 static int nfp_bpf_destroy(struct nfp_net *nn, struct bpf_prog *prog) 280 { 281 struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv; 282 283 kvfree(nfp_prog->prog); 284 nfp_map_ptrs_forget(nfp_prog->bpf, nfp_prog); 285 nfp_prog_free(nfp_prog); 286 287 return 0; 288 } 289 290 /* Atomic engine requires values to be in big endian, we need to byte swap 291 * the value words used with xadd. 292 */ 293 static void nfp_map_bpf_byte_swap(struct nfp_bpf_map *nfp_map, void *value) 294 { 295 u32 *word = value; 296 unsigned int i; 297 298 for (i = 0; i < DIV_ROUND_UP(nfp_map->offmap->map.value_size, 4); i++) 299 if (nfp_map->use_map[i] == NFP_MAP_USE_ATOMIC_CNT) 300 word[i] = (__force u32)cpu_to_be32(word[i]); 301 } 302 303 static int 304 nfp_bpf_map_lookup_entry(struct bpf_offloaded_map *offmap, 305 void *key, void *value) 306 { 307 int err; 308 309 err = nfp_bpf_ctrl_lookup_entry(offmap, key, value); 310 if (err) 311 return err; 312 313 nfp_map_bpf_byte_swap(offmap->dev_priv, value); 314 return 0; 315 } 316 317 static int 318 nfp_bpf_map_update_entry(struct bpf_offloaded_map *offmap, 319 void *key, void *value, u64 flags) 320 { 321 nfp_map_bpf_byte_swap(offmap->dev_priv, value); 322 return nfp_bpf_ctrl_update_entry(offmap, key, value, flags); 323 } 324 325 static int 326 nfp_bpf_map_get_next_key(struct bpf_offloaded_map *offmap, 327 void *key, void *next_key) 328 { 329 if (!key) 330 return nfp_bpf_ctrl_getfirst_entry(offmap, next_key); 331 return nfp_bpf_ctrl_getnext_entry(offmap, key, next_key); 332 } 333 334 static int 335 nfp_bpf_map_delete_elem(struct bpf_offloaded_map *offmap, void *key) 336 { 337 if (offmap->map.map_type == BPF_MAP_TYPE_ARRAY) 338 return -EINVAL; 339 return nfp_bpf_ctrl_del_entry(offmap, key); 340 } 341 342 static const struct bpf_map_dev_ops nfp_bpf_map_ops = { 343 .map_get_next_key = nfp_bpf_map_get_next_key, 344 .map_lookup_elem = nfp_bpf_map_lookup_entry, 345 .map_update_elem = nfp_bpf_map_update_entry, 346 .map_delete_elem = nfp_bpf_map_delete_elem, 347 }; 348 349 static int 350 nfp_bpf_map_alloc(struct nfp_app_bpf *bpf, struct bpf_offloaded_map *offmap) 351 { 352 struct nfp_bpf_map *nfp_map; 353 unsigned int use_map_size; 354 long long int res; 355 356 if (!bpf->maps.types) 357 return -EOPNOTSUPP; 358 359 if (offmap->map.map_flags || 360 offmap->map.numa_node != NUMA_NO_NODE) { 361 pr_info("map flags are not supported\n"); 362 return -EINVAL; 363 } 364 365 if (!(bpf->maps.types & 1 << offmap->map.map_type)) { 366 pr_info("map type not supported\n"); 367 return -EOPNOTSUPP; 368 } 369 if (bpf->maps.max_maps == bpf->maps_in_use) { 370 pr_info("too many maps for a device\n"); 371 return -ENOMEM; 372 } 373 if (bpf->maps.max_elems - bpf->map_elems_in_use < 374 offmap->map.max_entries) { 375 pr_info("map with too many elements: %u, left: %u\n", 376 offmap->map.max_entries, 377 bpf->maps.max_elems - bpf->map_elems_in_use); 378 return -ENOMEM; 379 } 380 if (offmap->map.key_size > bpf->maps.max_key_sz || 381 offmap->map.value_size > bpf->maps.max_val_sz || 382 round_up(offmap->map.key_size, 8) + 383 round_up(offmap->map.value_size, 8) > bpf->maps.max_elem_sz) { 384 pr_info("elements don't fit in device constraints\n"); 385 return -ENOMEM; 386 } 387 388 use_map_size = DIV_ROUND_UP(offmap->map.value_size, 4) * 389 FIELD_SIZEOF(struct nfp_bpf_map, use_map[0]); 390 391 nfp_map = kzalloc(sizeof(*nfp_map) + use_map_size, GFP_USER); 392 if (!nfp_map) 393 return -ENOMEM; 394 395 offmap->dev_priv = nfp_map; 396 nfp_map->offmap = offmap; 397 nfp_map->bpf = bpf; 398 399 res = nfp_bpf_ctrl_alloc_map(bpf, &offmap->map); 400 if (res < 0) { 401 kfree(nfp_map); 402 return res; 403 } 404 405 nfp_map->tid = res; 406 offmap->dev_ops = &nfp_bpf_map_ops; 407 bpf->maps_in_use++; 408 bpf->map_elems_in_use += offmap->map.max_entries; 409 list_add_tail(&nfp_map->l, &bpf->map_list); 410 411 return 0; 412 } 413 414 static int 415 nfp_bpf_map_free(struct nfp_app_bpf *bpf, struct bpf_offloaded_map *offmap) 416 { 417 struct nfp_bpf_map *nfp_map = offmap->dev_priv; 418 419 nfp_bpf_ctrl_free_map(bpf, nfp_map); 420 list_del_init(&nfp_map->l); 421 bpf->map_elems_in_use -= offmap->map.max_entries; 422 bpf->maps_in_use--; 423 kfree(nfp_map); 424 425 return 0; 426 } 427 428 int nfp_ndo_bpf(struct nfp_app *app, struct nfp_net *nn, struct netdev_bpf *bpf) 429 { 430 switch (bpf->command) { 431 case BPF_OFFLOAD_VERIFIER_PREP: 432 return nfp_bpf_verifier_prep(app, nn, bpf); 433 case BPF_OFFLOAD_TRANSLATE: 434 return nfp_bpf_translate(nn, bpf->offload.prog); 435 case BPF_OFFLOAD_DESTROY: 436 return nfp_bpf_destroy(nn, bpf->offload.prog); 437 case BPF_OFFLOAD_MAP_ALLOC: 438 return nfp_bpf_map_alloc(app->priv, bpf->offmap); 439 case BPF_OFFLOAD_MAP_FREE: 440 return nfp_bpf_map_free(app->priv, bpf->offmap); 441 default: 442 return -EINVAL; 443 } 444 } 445 446 static unsigned long 447 nfp_bpf_perf_event_copy(void *dst, const void *src, 448 unsigned long off, unsigned long len) 449 { 450 memcpy(dst, src + off, len); 451 return 0; 452 } 453 454 int nfp_bpf_event_output(struct nfp_app_bpf *bpf, struct sk_buff *skb) 455 { 456 struct cmsg_bpf_event *cbe = (void *)skb->data; 457 u32 pkt_size, data_size; 458 struct bpf_map *map; 459 460 if (skb->len < sizeof(struct cmsg_bpf_event)) 461 goto err_drop; 462 463 pkt_size = be32_to_cpu(cbe->pkt_size); 464 data_size = be32_to_cpu(cbe->data_size); 465 map = (void *)(unsigned long)be64_to_cpu(cbe->map_ptr); 466 467 if (skb->len < sizeof(struct cmsg_bpf_event) + pkt_size + data_size) 468 goto err_drop; 469 if (cbe->hdr.ver != CMSG_MAP_ABI_VERSION) 470 goto err_drop; 471 472 rcu_read_lock(); 473 if (!rhashtable_lookup_fast(&bpf->maps_neutral, &map, 474 nfp_bpf_maps_neutral_params)) { 475 rcu_read_unlock(); 476 pr_warn("perf event: dest map pointer %px not recognized, dropping event\n", 477 map); 478 goto err_drop; 479 } 480 481 bpf_event_output(map, be32_to_cpu(cbe->cpu_id), 482 &cbe->data[round_up(pkt_size, 4)], data_size, 483 cbe->data, pkt_size, nfp_bpf_perf_event_copy); 484 rcu_read_unlock(); 485 486 dev_consume_skb_any(skb); 487 return 0; 488 err_drop: 489 dev_kfree_skb_any(skb); 490 return -EINVAL; 491 } 492 493 static int 494 nfp_net_bpf_load(struct nfp_net *nn, struct bpf_prog *prog, 495 struct netlink_ext_ack *extack) 496 { 497 struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv; 498 unsigned int max_mtu; 499 dma_addr_t dma_addr; 500 void *img; 501 int err; 502 503 max_mtu = nn_readb(nn, NFP_NET_CFG_BPF_INL_MTU) * 64 - 32; 504 if (max_mtu < nn->dp.netdev->mtu) { 505 NL_SET_ERR_MSG_MOD(extack, "BPF offload not supported with MTU larger than HW packet split boundary"); 506 return -EOPNOTSUPP; 507 } 508 509 img = nfp_bpf_relo_for_vnic(nfp_prog, nn->app_priv); 510 if (IS_ERR(img)) 511 return PTR_ERR(img); 512 513 dma_addr = dma_map_single(nn->dp.dev, img, 514 nfp_prog->prog_len * sizeof(u64), 515 DMA_TO_DEVICE); 516 if (dma_mapping_error(nn->dp.dev, dma_addr)) { 517 kfree(img); 518 return -ENOMEM; 519 } 520 521 nn_writew(nn, NFP_NET_CFG_BPF_SIZE, nfp_prog->prog_len); 522 nn_writeq(nn, NFP_NET_CFG_BPF_ADDR, dma_addr); 523 524 /* Load up the JITed code */ 525 err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_BPF); 526 if (err) 527 NL_SET_ERR_MSG_MOD(extack, 528 "FW command error while loading BPF"); 529 530 dma_unmap_single(nn->dp.dev, dma_addr, nfp_prog->prog_len * sizeof(u64), 531 DMA_TO_DEVICE); 532 kfree(img); 533 534 return err; 535 } 536 537 static void 538 nfp_net_bpf_start(struct nfp_net *nn, struct netlink_ext_ack *extack) 539 { 540 int err; 541 542 /* Enable passing packets through BPF function */ 543 nn->dp.ctrl |= NFP_NET_CFG_CTRL_BPF; 544 nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl); 545 err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN); 546 if (err) 547 NL_SET_ERR_MSG_MOD(extack, 548 "FW command error while enabling BPF"); 549 } 550 551 static int nfp_net_bpf_stop(struct nfp_net *nn) 552 { 553 if (!(nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF)) 554 return 0; 555 556 nn->dp.ctrl &= ~NFP_NET_CFG_CTRL_BPF; 557 nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl); 558 559 return nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN); 560 } 561 562 int nfp_net_bpf_offload(struct nfp_net *nn, struct bpf_prog *prog, 563 bool old_prog, struct netlink_ext_ack *extack) 564 { 565 int err; 566 567 if (prog) { 568 struct bpf_prog_offload *offload = prog->aux->offload; 569 570 if (!offload) 571 return -EINVAL; 572 if (offload->netdev != nn->dp.netdev) 573 return -EINVAL; 574 } 575 576 if (prog && old_prog) { 577 u8 cap; 578 579 cap = nn_readb(nn, NFP_NET_CFG_BPF_CAP); 580 if (!(cap & NFP_NET_BPF_CAP_RELO)) { 581 NL_SET_ERR_MSG_MOD(extack, 582 "FW does not support live reload"); 583 return -EBUSY; 584 } 585 } 586 587 /* Something else is loaded, different program type? */ 588 if (!old_prog && nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF) 589 return -EBUSY; 590 591 if (old_prog && !prog) 592 return nfp_net_bpf_stop(nn); 593 594 err = nfp_net_bpf_load(nn, prog, extack); 595 if (err) 596 return err; 597 598 if (!old_prog) 599 nfp_net_bpf_start(nn, extack); 600 601 return 0; 602 } 603