1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2017-2018 Netronome Systems, Inc. */ 3 4 #include <net/pkt_cls.h> 5 6 #include "../nfpcore/nfp_cpp.h" 7 #include "../nfpcore/nfp_nffw.h" 8 #include "../nfpcore/nfp_nsp.h" 9 #include "../nfp_app.h" 10 #include "../nfp_main.h" 11 #include "../nfp_net.h" 12 #include "../nfp_port.h" 13 #include "fw.h" 14 #include "main.h" 15 16 const struct rhashtable_params nfp_bpf_maps_neutral_params = { 17 .nelem_hint = 4, 18 .key_len = sizeof_field(struct bpf_map, id), 19 .key_offset = offsetof(struct nfp_bpf_neutral_map, map_id), 20 .head_offset = offsetof(struct nfp_bpf_neutral_map, l), 21 .automatic_shrinking = true, 22 }; 23 24 static bool nfp_net_ebpf_capable(struct nfp_net *nn) 25 { 26 #ifdef __LITTLE_ENDIAN 27 struct nfp_app_bpf *bpf = nn->app->priv; 28 29 return nn->cap & NFP_NET_CFG_CTRL_BPF && 30 bpf->abi_version && 31 nn_readb(nn, NFP_NET_CFG_BPF_ABI) == bpf->abi_version; 32 #else 33 return false; 34 #endif 35 } 36 37 static int 38 nfp_bpf_xdp_offload(struct nfp_app *app, struct nfp_net *nn, 39 struct bpf_prog *prog, struct netlink_ext_ack *extack) 40 { 41 bool running, xdp_running; 42 43 if (!nfp_net_ebpf_capable(nn)) 44 return -EINVAL; 45 46 running = nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF; 47 xdp_running = running && nn->xdp_hw.prog; 48 49 if (!prog && !xdp_running) 50 return 0; 51 if (prog && running && !xdp_running) 52 return -EBUSY; 53 54 return nfp_net_bpf_offload(nn, prog, running, extack); 55 } 56 57 static const char *nfp_bpf_extra_cap(struct nfp_app *app, struct nfp_net *nn) 58 { 59 return nfp_net_ebpf_capable(nn) ? "BPF" : ""; 60 } 61 62 static int 63 nfp_bpf_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, unsigned int id) 64 { 65 struct nfp_pf *pf = app->pf; 66 struct nfp_bpf_vnic *bv; 67 int err; 68 69 if (!pf->eth_tbl) { 70 nfp_err(pf->cpp, "No ETH table\n"); 71 return -EINVAL; 72 } 73 if (pf->max_data_vnics != pf->eth_tbl->count) { 74 nfp_err(pf->cpp, "ETH entries don't match vNICs (%d vs %d)\n", 75 pf->max_data_vnics, pf->eth_tbl->count); 76 return -EINVAL; 77 } 78 79 bv = kzalloc(sizeof(*bv), GFP_KERNEL); 80 if (!bv) 81 return -ENOMEM; 82 nn->app_priv = bv; 83 84 err = nfp_app_nic_vnic_alloc(app, nn, id); 85 if (err) 86 goto err_free_priv; 87 88 bv->start_off = nn_readw(nn, NFP_NET_CFG_BPF_START); 89 bv->tgt_done = nn_readw(nn, NFP_NET_CFG_BPF_DONE); 90 91 return 0; 92 err_free_priv: 93 kfree(nn->app_priv); 94 return err; 95 } 96 97 static void nfp_bpf_vnic_free(struct nfp_app *app, struct nfp_net *nn) 98 { 99 struct nfp_bpf_vnic *bv = nn->app_priv; 100 101 WARN_ON(bv->tc_prog); 102 kfree(bv); 103 } 104 105 static int nfp_bpf_setup_tc_block_cb(enum tc_setup_type type, 106 void *type_data, void *cb_priv) 107 { 108 struct tc_cls_bpf_offload *cls_bpf = type_data; 109 struct nfp_net *nn = cb_priv; 110 struct bpf_prog *oldprog; 111 struct nfp_bpf_vnic *bv; 112 int err; 113 114 if (type != TC_SETUP_CLSBPF) { 115 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack, 116 "only offload of BPF classifiers supported"); 117 return -EOPNOTSUPP; 118 } 119 if (!tc_cls_can_offload_and_chain0(nn->dp.netdev, &cls_bpf->common)) 120 return -EOPNOTSUPP; 121 if (!nfp_net_ebpf_capable(nn)) { 122 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack, 123 "NFP firmware does not support eBPF offload"); 124 return -EOPNOTSUPP; 125 } 126 if (cls_bpf->common.protocol != htons(ETH_P_ALL)) { 127 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack, 128 "only ETH_P_ALL supported as filter protocol"); 129 return -EOPNOTSUPP; 130 } 131 132 /* Only support TC direct action */ 133 if (!cls_bpf->exts_integrated || 134 tcf_exts_has_actions(cls_bpf->exts)) { 135 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack, 136 "only direct action with no legacy actions supported"); 137 return -EOPNOTSUPP; 138 } 139 140 if (cls_bpf->command != TC_CLSBPF_OFFLOAD) 141 return -EOPNOTSUPP; 142 143 bv = nn->app_priv; 144 oldprog = cls_bpf->oldprog; 145 146 /* Don't remove if oldprog doesn't match driver's state */ 147 if (bv->tc_prog != oldprog) { 148 oldprog = NULL; 149 if (!cls_bpf->prog) 150 return 0; 151 } 152 153 err = nfp_net_bpf_offload(nn, cls_bpf->prog, oldprog, 154 cls_bpf->common.extack); 155 if (err) 156 return err; 157 158 bv->tc_prog = cls_bpf->prog; 159 nn->port->tc_offload_cnt = !!bv->tc_prog; 160 return 0; 161 } 162 163 static LIST_HEAD(nfp_bpf_block_cb_list); 164 165 static int nfp_bpf_setup_tc(struct nfp_app *app, struct net_device *netdev, 166 enum tc_setup_type type, void *type_data) 167 { 168 struct nfp_net *nn = netdev_priv(netdev); 169 170 switch (type) { 171 case TC_SETUP_BLOCK: 172 return flow_block_cb_setup_simple(type_data, 173 &nfp_bpf_block_cb_list, 174 nfp_bpf_setup_tc_block_cb, 175 nn, nn, true); 176 default: 177 return -EOPNOTSUPP; 178 } 179 } 180 181 static int 182 nfp_bpf_check_mtu(struct nfp_app *app, struct net_device *netdev, int new_mtu) 183 { 184 struct nfp_net *nn = netdev_priv(netdev); 185 struct nfp_bpf_vnic *bv; 186 struct bpf_prog *prog; 187 188 if (~nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF) 189 return 0; 190 191 if (nn->xdp_hw.prog) { 192 prog = nn->xdp_hw.prog; 193 } else { 194 bv = nn->app_priv; 195 prog = bv->tc_prog; 196 } 197 198 if (nfp_bpf_offload_check_mtu(nn, prog, new_mtu)) { 199 nn_info(nn, "BPF offload active, potential packet access beyond hardware packet boundary"); 200 return -EBUSY; 201 } 202 return 0; 203 } 204 205 static int 206 nfp_bpf_parse_cap_adjust_head(struct nfp_app_bpf *bpf, void __iomem *value, 207 u32 length) 208 { 209 struct nfp_bpf_cap_tlv_adjust_head __iomem *cap = value; 210 struct nfp_cpp *cpp = bpf->app->pf->cpp; 211 212 if (length < sizeof(*cap)) { 213 nfp_err(cpp, "truncated adjust_head TLV: %d\n", length); 214 return -EINVAL; 215 } 216 217 bpf->adjust_head.flags = readl(&cap->flags); 218 bpf->adjust_head.off_min = readl(&cap->off_min); 219 bpf->adjust_head.off_max = readl(&cap->off_max); 220 bpf->adjust_head.guaranteed_sub = readl(&cap->guaranteed_sub); 221 bpf->adjust_head.guaranteed_add = readl(&cap->guaranteed_add); 222 223 if (bpf->adjust_head.off_min > bpf->adjust_head.off_max) { 224 nfp_err(cpp, "invalid adjust_head TLV: min > max\n"); 225 return -EINVAL; 226 } 227 if (!FIELD_FIT(UR_REG_IMM_MAX, bpf->adjust_head.off_min) || 228 !FIELD_FIT(UR_REG_IMM_MAX, bpf->adjust_head.off_max)) { 229 nfp_warn(cpp, "disabling adjust_head - driver expects min/max to fit in as immediates\n"); 230 memset(&bpf->adjust_head, 0, sizeof(bpf->adjust_head)); 231 return 0; 232 } 233 234 return 0; 235 } 236 237 static int 238 nfp_bpf_parse_cap_func(struct nfp_app_bpf *bpf, void __iomem *value, u32 length) 239 { 240 struct nfp_bpf_cap_tlv_func __iomem *cap = value; 241 242 if (length < sizeof(*cap)) { 243 nfp_err(bpf->app->cpp, "truncated function TLV: %d\n", length); 244 return -EINVAL; 245 } 246 247 switch (readl(&cap->func_id)) { 248 case BPF_FUNC_map_lookup_elem: 249 bpf->helpers.map_lookup = readl(&cap->func_addr); 250 break; 251 case BPF_FUNC_map_update_elem: 252 bpf->helpers.map_update = readl(&cap->func_addr); 253 break; 254 case BPF_FUNC_map_delete_elem: 255 bpf->helpers.map_delete = readl(&cap->func_addr); 256 break; 257 case BPF_FUNC_perf_event_output: 258 bpf->helpers.perf_event_output = readl(&cap->func_addr); 259 break; 260 } 261 262 return 0; 263 } 264 265 static int 266 nfp_bpf_parse_cap_maps(struct nfp_app_bpf *bpf, void __iomem *value, u32 length) 267 { 268 struct nfp_bpf_cap_tlv_maps __iomem *cap = value; 269 270 if (length < sizeof(*cap)) { 271 nfp_err(bpf->app->cpp, "truncated maps TLV: %d\n", length); 272 return -EINVAL; 273 } 274 275 bpf->maps.types = readl(&cap->types); 276 bpf->maps.max_maps = readl(&cap->max_maps); 277 bpf->maps.max_elems = readl(&cap->max_elems); 278 bpf->maps.max_key_sz = readl(&cap->max_key_sz); 279 bpf->maps.max_val_sz = readl(&cap->max_val_sz); 280 bpf->maps.max_elem_sz = readl(&cap->max_elem_sz); 281 282 return 0; 283 } 284 285 static int 286 nfp_bpf_parse_cap_random(struct nfp_app_bpf *bpf, void __iomem *value, 287 u32 length) 288 { 289 bpf->pseudo_random = true; 290 return 0; 291 } 292 293 static int 294 nfp_bpf_parse_cap_qsel(struct nfp_app_bpf *bpf, void __iomem *value, u32 length) 295 { 296 bpf->queue_select = true; 297 return 0; 298 } 299 300 static int 301 nfp_bpf_parse_cap_adjust_tail(struct nfp_app_bpf *bpf, void __iomem *value, 302 u32 length) 303 { 304 bpf->adjust_tail = true; 305 return 0; 306 } 307 308 static int 309 nfp_bpf_parse_cap_cmsg_multi_ent(struct nfp_app_bpf *bpf, void __iomem *value, 310 u32 length) 311 { 312 bpf->cmsg_multi_ent = true; 313 return 0; 314 } 315 316 static int 317 nfp_bpf_parse_cap_abi_version(struct nfp_app_bpf *bpf, void __iomem *value, 318 u32 length) 319 { 320 if (length < 4) { 321 nfp_err(bpf->app->cpp, "truncated ABI version TLV: %d\n", 322 length); 323 return -EINVAL; 324 } 325 326 bpf->abi_version = readl(value); 327 if (bpf->abi_version < 2 || bpf->abi_version > 3) { 328 nfp_warn(bpf->app->cpp, "unsupported BPF ABI version: %d\n", 329 bpf->abi_version); 330 bpf->abi_version = 0; 331 } 332 333 return 0; 334 } 335 336 static int nfp_bpf_parse_capabilities(struct nfp_app *app) 337 { 338 struct nfp_cpp *cpp = app->pf->cpp; 339 struct nfp_cpp_area *area; 340 u8 __iomem *mem, *start; 341 342 mem = nfp_rtsym_map(app->pf->rtbl, "_abi_bpf_capabilities", "bpf.cap", 343 8, &area); 344 if (IS_ERR(mem)) 345 return PTR_ERR(mem) == -ENOENT ? 0 : PTR_ERR(mem); 346 347 start = mem; 348 while (mem - start + 8 <= nfp_cpp_area_size(area)) { 349 u8 __iomem *value; 350 u32 type, length; 351 352 type = readl(mem); 353 length = readl(mem + 4); 354 value = mem + 8; 355 356 mem += 8 + length; 357 if (mem - start > nfp_cpp_area_size(area)) 358 goto err_release_free; 359 360 switch (type) { 361 case NFP_BPF_CAP_TYPE_FUNC: 362 if (nfp_bpf_parse_cap_func(app->priv, value, length)) 363 goto err_release_free; 364 break; 365 case NFP_BPF_CAP_TYPE_ADJUST_HEAD: 366 if (nfp_bpf_parse_cap_adjust_head(app->priv, value, 367 length)) 368 goto err_release_free; 369 break; 370 case NFP_BPF_CAP_TYPE_MAPS: 371 if (nfp_bpf_parse_cap_maps(app->priv, value, length)) 372 goto err_release_free; 373 break; 374 case NFP_BPF_CAP_TYPE_RANDOM: 375 if (nfp_bpf_parse_cap_random(app->priv, value, length)) 376 goto err_release_free; 377 break; 378 case NFP_BPF_CAP_TYPE_QUEUE_SELECT: 379 if (nfp_bpf_parse_cap_qsel(app->priv, value, length)) 380 goto err_release_free; 381 break; 382 case NFP_BPF_CAP_TYPE_ADJUST_TAIL: 383 if (nfp_bpf_parse_cap_adjust_tail(app->priv, value, 384 length)) 385 goto err_release_free; 386 break; 387 case NFP_BPF_CAP_TYPE_ABI_VERSION: 388 if (nfp_bpf_parse_cap_abi_version(app->priv, value, 389 length)) 390 goto err_release_free; 391 break; 392 case NFP_BPF_CAP_TYPE_CMSG_MULTI_ENT: 393 if (nfp_bpf_parse_cap_cmsg_multi_ent(app->priv, value, 394 length)) 395 goto err_release_free; 396 break; 397 default: 398 nfp_dbg(cpp, "unknown BPF capability: %d\n", type); 399 break; 400 } 401 } 402 if (mem - start != nfp_cpp_area_size(area)) { 403 nfp_err(cpp, "BPF capabilities left after parsing, parsed:%zd total length:%zu\n", 404 mem - start, nfp_cpp_area_size(area)); 405 goto err_release_free; 406 } 407 408 nfp_cpp_area_release_free(area); 409 410 return 0; 411 412 err_release_free: 413 nfp_err(cpp, "invalid BPF capabilities at offset:%zd\n", mem - start); 414 nfp_cpp_area_release_free(area); 415 return -EINVAL; 416 } 417 418 static void nfp_bpf_init_capabilities(struct nfp_app_bpf *bpf) 419 { 420 bpf->abi_version = 2; /* Original BPF ABI version */ 421 } 422 423 static int nfp_bpf_ndo_init(struct nfp_app *app, struct net_device *netdev) 424 { 425 struct nfp_app_bpf *bpf = app->priv; 426 427 return bpf_offload_dev_netdev_register(bpf->bpf_dev, netdev); 428 } 429 430 static void nfp_bpf_ndo_uninit(struct nfp_app *app, struct net_device *netdev) 431 { 432 struct nfp_app_bpf *bpf = app->priv; 433 434 bpf_offload_dev_netdev_unregister(bpf->bpf_dev, netdev); 435 } 436 437 static int nfp_bpf_start(struct nfp_app *app) 438 { 439 struct nfp_app_bpf *bpf = app->priv; 440 441 if (app->ctrl->dp.mtu < nfp_bpf_ctrl_cmsg_min_mtu(bpf)) { 442 nfp_err(bpf->app->cpp, 443 "ctrl channel MTU below min required %u < %u\n", 444 app->ctrl->dp.mtu, nfp_bpf_ctrl_cmsg_min_mtu(bpf)); 445 return -EINVAL; 446 } 447 448 if (bpf->cmsg_multi_ent) 449 bpf->cmsg_cache_cnt = nfp_bpf_ctrl_cmsg_cache_cnt(bpf); 450 else 451 bpf->cmsg_cache_cnt = 1; 452 453 return 0; 454 } 455 456 static int nfp_bpf_init(struct nfp_app *app) 457 { 458 struct nfp_app_bpf *bpf; 459 int err; 460 461 bpf = kzalloc(sizeof(*bpf), GFP_KERNEL); 462 if (!bpf) 463 return -ENOMEM; 464 bpf->app = app; 465 app->priv = bpf; 466 467 INIT_LIST_HEAD(&bpf->map_list); 468 469 err = nfp_ccm_init(&bpf->ccm, app); 470 if (err) 471 goto err_free_bpf; 472 473 err = rhashtable_init(&bpf->maps_neutral, &nfp_bpf_maps_neutral_params); 474 if (err) 475 goto err_clean_ccm; 476 477 nfp_bpf_init_capabilities(bpf); 478 479 err = nfp_bpf_parse_capabilities(app); 480 if (err) 481 goto err_free_neutral_maps; 482 483 if (bpf->abi_version < 3) { 484 bpf->cmsg_key_sz = CMSG_MAP_KEY_LW * 4; 485 bpf->cmsg_val_sz = CMSG_MAP_VALUE_LW * 4; 486 } else { 487 bpf->cmsg_key_sz = bpf->maps.max_key_sz; 488 bpf->cmsg_val_sz = bpf->maps.max_val_sz; 489 app->ctrl_mtu = nfp_bpf_ctrl_cmsg_mtu(bpf); 490 } 491 492 bpf->bpf_dev = bpf_offload_dev_create(&nfp_bpf_dev_ops, bpf); 493 err = PTR_ERR_OR_ZERO(bpf->bpf_dev); 494 if (err) 495 goto err_free_neutral_maps; 496 497 return 0; 498 499 err_free_neutral_maps: 500 rhashtable_destroy(&bpf->maps_neutral); 501 err_clean_ccm: 502 nfp_ccm_clean(&bpf->ccm); 503 err_free_bpf: 504 kfree(bpf); 505 return err; 506 } 507 508 static void nfp_bpf_clean(struct nfp_app *app) 509 { 510 struct nfp_app_bpf *bpf = app->priv; 511 512 bpf_offload_dev_destroy(bpf->bpf_dev); 513 nfp_ccm_clean(&bpf->ccm); 514 WARN_ON(!list_empty(&bpf->map_list)); 515 WARN_ON(bpf->maps_in_use || bpf->map_elems_in_use); 516 rhashtable_free_and_destroy(&bpf->maps_neutral, 517 nfp_check_rhashtable_empty, NULL); 518 kfree(bpf); 519 } 520 521 const struct nfp_app_type app_bpf = { 522 .id = NFP_APP_BPF_NIC, 523 .name = "ebpf", 524 525 .ctrl_cap_mask = 0, 526 527 .init = nfp_bpf_init, 528 .clean = nfp_bpf_clean, 529 .start = nfp_bpf_start, 530 531 .check_mtu = nfp_bpf_check_mtu, 532 533 .extra_cap = nfp_bpf_extra_cap, 534 535 .ndo_init = nfp_bpf_ndo_init, 536 .ndo_uninit = nfp_bpf_ndo_uninit, 537 538 .vnic_alloc = nfp_bpf_vnic_alloc, 539 .vnic_free = nfp_bpf_vnic_free, 540 541 .ctrl_msg_rx = nfp_bpf_ctrl_msg_rx, 542 .ctrl_msg_rx_raw = nfp_bpf_ctrl_msg_rx_raw, 543 544 .setup_tc = nfp_bpf_setup_tc, 545 .bpf = nfp_ndo_bpf, 546 .xdp_offload = nfp_bpf_xdp_offload, 547 }; 548