1 /* 2 * Copyright (C) 2017-2018 Netronome Systems, Inc. 3 * 4 * This software is dual licensed under the GNU General License Version 2, 5 * June 1991 as shown in the file COPYING in the top-level directory of this 6 * source tree or the BSD 2-Clause License provided below. You have the 7 * option to license this software under the complete terms of either license. 8 * 9 * The BSD 2-Clause License: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * 1. Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * 2. Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <net/pkt_cls.h> 35 36 #include "../nfpcore/nfp_cpp.h" 37 #include "../nfpcore/nfp_nffw.h" 38 #include "../nfpcore/nfp_nsp.h" 39 #include "../nfp_app.h" 40 #include "../nfp_main.h" 41 #include "../nfp_net.h" 42 #include "../nfp_port.h" 43 #include "fw.h" 44 #include "main.h" 45 46 const struct rhashtable_params nfp_bpf_maps_neutral_params = { 47 .nelem_hint = 4, 48 .key_len = FIELD_SIZEOF(struct nfp_bpf_neutral_map, ptr), 49 .key_offset = offsetof(struct nfp_bpf_neutral_map, ptr), 50 .head_offset = offsetof(struct nfp_bpf_neutral_map, l), 51 .automatic_shrinking = true, 52 }; 53 54 static bool nfp_net_ebpf_capable(struct nfp_net *nn) 55 { 56 #ifdef __LITTLE_ENDIAN 57 if (nn->cap & NFP_NET_CFG_CTRL_BPF && 58 nn_readb(nn, NFP_NET_CFG_BPF_ABI) == NFP_NET_BPF_ABI) 59 return true; 60 #endif 61 return false; 62 } 63 64 static int 65 nfp_bpf_xdp_offload(struct nfp_app *app, struct nfp_net *nn, 66 struct bpf_prog *prog, struct netlink_ext_ack *extack) 67 { 68 bool running, xdp_running; 69 70 if (!nfp_net_ebpf_capable(nn)) 71 return -EINVAL; 72 73 running = nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF; 74 xdp_running = running && nn->xdp_hw.prog; 75 76 if (!prog && !xdp_running) 77 return 0; 78 if (prog && running && !xdp_running) 79 return -EBUSY; 80 81 return nfp_net_bpf_offload(nn, prog, running, extack); 82 } 83 84 static const char *nfp_bpf_extra_cap(struct nfp_app *app, struct nfp_net *nn) 85 { 86 return nfp_net_ebpf_capable(nn) ? "BPF" : ""; 87 } 88 89 static int 90 nfp_bpf_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, unsigned int id) 91 { 92 struct nfp_pf *pf = app->pf; 93 struct nfp_bpf_vnic *bv; 94 int err; 95 96 if (!pf->eth_tbl) { 97 nfp_err(pf->cpp, "No ETH table\n"); 98 return -EINVAL; 99 } 100 if (pf->max_data_vnics != pf->eth_tbl->count) { 101 nfp_err(pf->cpp, "ETH entries don't match vNICs (%d vs %d)\n", 102 pf->max_data_vnics, pf->eth_tbl->count); 103 return -EINVAL; 104 } 105 106 bv = kzalloc(sizeof(*bv), GFP_KERNEL); 107 if (!bv) 108 return -ENOMEM; 109 nn->app_priv = bv; 110 111 err = nfp_app_nic_vnic_alloc(app, nn, id); 112 if (err) 113 goto err_free_priv; 114 115 bv->start_off = nn_readw(nn, NFP_NET_CFG_BPF_START); 116 bv->tgt_done = nn_readw(nn, NFP_NET_CFG_BPF_DONE); 117 118 return 0; 119 err_free_priv: 120 kfree(nn->app_priv); 121 return err; 122 } 123 124 static void nfp_bpf_vnic_free(struct nfp_app *app, struct nfp_net *nn) 125 { 126 struct nfp_bpf_vnic *bv = nn->app_priv; 127 128 WARN_ON(bv->tc_prog); 129 kfree(bv); 130 } 131 132 static int nfp_bpf_setup_tc_block_cb(enum tc_setup_type type, 133 void *type_data, void *cb_priv) 134 { 135 struct tc_cls_bpf_offload *cls_bpf = type_data; 136 struct nfp_net *nn = cb_priv; 137 struct bpf_prog *oldprog; 138 struct nfp_bpf_vnic *bv; 139 int err; 140 141 if (type != TC_SETUP_CLSBPF) { 142 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack, 143 "only offload of BPF classifiers supported"); 144 return -EOPNOTSUPP; 145 } 146 if (!tc_cls_can_offload_and_chain0(nn->dp.netdev, &cls_bpf->common)) 147 return -EOPNOTSUPP; 148 if (!nfp_net_ebpf_capable(nn)) { 149 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack, 150 "NFP firmware does not support eBPF offload"); 151 return -EOPNOTSUPP; 152 } 153 if (cls_bpf->common.protocol != htons(ETH_P_ALL)) { 154 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack, 155 "only ETH_P_ALL supported as filter protocol"); 156 return -EOPNOTSUPP; 157 } 158 159 /* Only support TC direct action */ 160 if (!cls_bpf->exts_integrated || 161 tcf_exts_has_actions(cls_bpf->exts)) { 162 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack, 163 "only direct action with no legacy actions supported"); 164 return -EOPNOTSUPP; 165 } 166 167 if (cls_bpf->command != TC_CLSBPF_OFFLOAD) 168 return -EOPNOTSUPP; 169 170 bv = nn->app_priv; 171 oldprog = cls_bpf->oldprog; 172 173 /* Don't remove if oldprog doesn't match driver's state */ 174 if (bv->tc_prog != oldprog) { 175 oldprog = NULL; 176 if (!cls_bpf->prog) 177 return 0; 178 } 179 180 err = nfp_net_bpf_offload(nn, cls_bpf->prog, oldprog, 181 cls_bpf->common.extack); 182 if (err) 183 return err; 184 185 bv->tc_prog = cls_bpf->prog; 186 nn->port->tc_offload_cnt = !!bv->tc_prog; 187 return 0; 188 } 189 190 static int nfp_bpf_setup_tc_block(struct net_device *netdev, 191 struct tc_block_offload *f) 192 { 193 struct nfp_net *nn = netdev_priv(netdev); 194 195 if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS) 196 return -EOPNOTSUPP; 197 198 switch (f->command) { 199 case TC_BLOCK_BIND: 200 return tcf_block_cb_register(f->block, 201 nfp_bpf_setup_tc_block_cb, 202 nn, nn, f->extack); 203 case TC_BLOCK_UNBIND: 204 tcf_block_cb_unregister(f->block, 205 nfp_bpf_setup_tc_block_cb, 206 nn); 207 return 0; 208 default: 209 return -EOPNOTSUPP; 210 } 211 } 212 213 static int nfp_bpf_setup_tc(struct nfp_app *app, struct net_device *netdev, 214 enum tc_setup_type type, void *type_data) 215 { 216 switch (type) { 217 case TC_SETUP_BLOCK: 218 return nfp_bpf_setup_tc_block(netdev, type_data); 219 default: 220 return -EOPNOTSUPP; 221 } 222 } 223 224 static int 225 nfp_bpf_check_mtu(struct nfp_app *app, struct net_device *netdev, int new_mtu) 226 { 227 struct nfp_net *nn = netdev_priv(netdev); 228 unsigned int max_mtu; 229 230 if (~nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF) 231 return 0; 232 233 max_mtu = nn_readb(nn, NFP_NET_CFG_BPF_INL_MTU) * 64 - 32; 234 if (new_mtu > max_mtu) { 235 nn_info(nn, "BPF offload active, MTU over %u not supported\n", 236 max_mtu); 237 return -EBUSY; 238 } 239 return 0; 240 } 241 242 static int 243 nfp_bpf_parse_cap_adjust_head(struct nfp_app_bpf *bpf, void __iomem *value, 244 u32 length) 245 { 246 struct nfp_bpf_cap_tlv_adjust_head __iomem *cap = value; 247 struct nfp_cpp *cpp = bpf->app->pf->cpp; 248 249 if (length < sizeof(*cap)) { 250 nfp_err(cpp, "truncated adjust_head TLV: %d\n", length); 251 return -EINVAL; 252 } 253 254 bpf->adjust_head.flags = readl(&cap->flags); 255 bpf->adjust_head.off_min = readl(&cap->off_min); 256 bpf->adjust_head.off_max = readl(&cap->off_max); 257 bpf->adjust_head.guaranteed_sub = readl(&cap->guaranteed_sub); 258 bpf->adjust_head.guaranteed_add = readl(&cap->guaranteed_add); 259 260 if (bpf->adjust_head.off_min > bpf->adjust_head.off_max) { 261 nfp_err(cpp, "invalid adjust_head TLV: min > max\n"); 262 return -EINVAL; 263 } 264 if (!FIELD_FIT(UR_REG_IMM_MAX, bpf->adjust_head.off_min) || 265 !FIELD_FIT(UR_REG_IMM_MAX, bpf->adjust_head.off_max)) { 266 nfp_warn(cpp, "disabling adjust_head - driver expects min/max to fit in as immediates\n"); 267 memset(&bpf->adjust_head, 0, sizeof(bpf->adjust_head)); 268 return 0; 269 } 270 271 return 0; 272 } 273 274 static int 275 nfp_bpf_parse_cap_func(struct nfp_app_bpf *bpf, void __iomem *value, u32 length) 276 { 277 struct nfp_bpf_cap_tlv_func __iomem *cap = value; 278 279 if (length < sizeof(*cap)) { 280 nfp_err(bpf->app->cpp, "truncated function TLV: %d\n", length); 281 return -EINVAL; 282 } 283 284 switch (readl(&cap->func_id)) { 285 case BPF_FUNC_map_lookup_elem: 286 bpf->helpers.map_lookup = readl(&cap->func_addr); 287 break; 288 case BPF_FUNC_map_update_elem: 289 bpf->helpers.map_update = readl(&cap->func_addr); 290 break; 291 case BPF_FUNC_map_delete_elem: 292 bpf->helpers.map_delete = readl(&cap->func_addr); 293 break; 294 case BPF_FUNC_perf_event_output: 295 bpf->helpers.perf_event_output = readl(&cap->func_addr); 296 break; 297 } 298 299 return 0; 300 } 301 302 static int 303 nfp_bpf_parse_cap_maps(struct nfp_app_bpf *bpf, void __iomem *value, u32 length) 304 { 305 struct nfp_bpf_cap_tlv_maps __iomem *cap = value; 306 307 if (length < sizeof(*cap)) { 308 nfp_err(bpf->app->cpp, "truncated maps TLV: %d\n", length); 309 return -EINVAL; 310 } 311 312 bpf->maps.types = readl(&cap->types); 313 bpf->maps.max_maps = readl(&cap->max_maps); 314 bpf->maps.max_elems = readl(&cap->max_elems); 315 bpf->maps.max_key_sz = readl(&cap->max_key_sz); 316 bpf->maps.max_val_sz = readl(&cap->max_val_sz); 317 bpf->maps.max_elem_sz = readl(&cap->max_elem_sz); 318 319 return 0; 320 } 321 322 static int 323 nfp_bpf_parse_cap_random(struct nfp_app_bpf *bpf, void __iomem *value, 324 u32 length) 325 { 326 bpf->pseudo_random = true; 327 return 0; 328 } 329 330 static int 331 nfp_bpf_parse_cap_qsel(struct nfp_app_bpf *bpf, void __iomem *value, u32 length) 332 { 333 bpf->queue_select = true; 334 return 0; 335 } 336 337 static int nfp_bpf_parse_capabilities(struct nfp_app *app) 338 { 339 struct nfp_cpp *cpp = app->pf->cpp; 340 struct nfp_cpp_area *area; 341 u8 __iomem *mem, *start; 342 343 mem = nfp_rtsym_map(app->pf->rtbl, "_abi_bpf_capabilities", "bpf.cap", 344 8, &area); 345 if (IS_ERR(mem)) 346 return PTR_ERR(mem) == -ENOENT ? 0 : PTR_ERR(mem); 347 348 start = mem; 349 while (mem - start + 8 <= nfp_cpp_area_size(area)) { 350 u8 __iomem *value; 351 u32 type, length; 352 353 type = readl(mem); 354 length = readl(mem + 4); 355 value = mem + 8; 356 357 mem += 8 + length; 358 if (mem - start > nfp_cpp_area_size(area)) 359 goto err_release_free; 360 361 switch (type) { 362 case NFP_BPF_CAP_TYPE_FUNC: 363 if (nfp_bpf_parse_cap_func(app->priv, value, length)) 364 goto err_release_free; 365 break; 366 case NFP_BPF_CAP_TYPE_ADJUST_HEAD: 367 if (nfp_bpf_parse_cap_adjust_head(app->priv, value, 368 length)) 369 goto err_release_free; 370 break; 371 case NFP_BPF_CAP_TYPE_MAPS: 372 if (nfp_bpf_parse_cap_maps(app->priv, value, length)) 373 goto err_release_free; 374 break; 375 case NFP_BPF_CAP_TYPE_RANDOM: 376 if (nfp_bpf_parse_cap_random(app->priv, value, length)) 377 goto err_release_free; 378 break; 379 case NFP_BPF_CAP_TYPE_QUEUE_SELECT: 380 if (nfp_bpf_parse_cap_qsel(app->priv, value, length)) 381 goto err_release_free; 382 break; 383 default: 384 nfp_dbg(cpp, "unknown BPF capability: %d\n", type); 385 break; 386 } 387 } 388 if (mem - start != nfp_cpp_area_size(area)) { 389 nfp_err(cpp, "BPF capabilities left after parsing, parsed:%zd total length:%zu\n", 390 mem - start, nfp_cpp_area_size(area)); 391 goto err_release_free; 392 } 393 394 nfp_cpp_area_release_free(area); 395 396 return 0; 397 398 err_release_free: 399 nfp_err(cpp, "invalid BPF capabilities at offset:%zd\n", mem - start); 400 nfp_cpp_area_release_free(area); 401 return -EINVAL; 402 } 403 404 static int nfp_bpf_ndo_init(struct nfp_app *app, struct net_device *netdev) 405 { 406 struct nfp_app_bpf *bpf = app->priv; 407 408 return bpf_offload_dev_netdev_register(bpf->bpf_dev, netdev); 409 } 410 411 static void nfp_bpf_ndo_uninit(struct nfp_app *app, struct net_device *netdev) 412 { 413 struct nfp_app_bpf *bpf = app->priv; 414 415 bpf_offload_dev_netdev_unregister(bpf->bpf_dev, netdev); 416 } 417 418 static int nfp_bpf_init(struct nfp_app *app) 419 { 420 struct nfp_app_bpf *bpf; 421 int err; 422 423 bpf = kzalloc(sizeof(*bpf), GFP_KERNEL); 424 if (!bpf) 425 return -ENOMEM; 426 bpf->app = app; 427 app->priv = bpf; 428 429 skb_queue_head_init(&bpf->cmsg_replies); 430 init_waitqueue_head(&bpf->cmsg_wq); 431 INIT_LIST_HEAD(&bpf->map_list); 432 433 err = rhashtable_init(&bpf->maps_neutral, &nfp_bpf_maps_neutral_params); 434 if (err) 435 goto err_free_bpf; 436 437 err = nfp_bpf_parse_capabilities(app); 438 if (err) 439 goto err_free_neutral_maps; 440 441 bpf->bpf_dev = bpf_offload_dev_create(); 442 err = PTR_ERR_OR_ZERO(bpf->bpf_dev); 443 if (err) 444 goto err_free_neutral_maps; 445 446 return 0; 447 448 err_free_neutral_maps: 449 rhashtable_destroy(&bpf->maps_neutral); 450 err_free_bpf: 451 kfree(bpf); 452 return err; 453 } 454 455 static void nfp_check_rhashtable_empty(void *ptr, void *arg) 456 { 457 WARN_ON_ONCE(1); 458 } 459 460 static void nfp_bpf_clean(struct nfp_app *app) 461 { 462 struct nfp_app_bpf *bpf = app->priv; 463 464 bpf_offload_dev_destroy(bpf->bpf_dev); 465 WARN_ON(!skb_queue_empty(&bpf->cmsg_replies)); 466 WARN_ON(!list_empty(&bpf->map_list)); 467 WARN_ON(bpf->maps_in_use || bpf->map_elems_in_use); 468 rhashtable_free_and_destroy(&bpf->maps_neutral, 469 nfp_check_rhashtable_empty, NULL); 470 kfree(bpf); 471 } 472 473 const struct nfp_app_type app_bpf = { 474 .id = NFP_APP_BPF_NIC, 475 .name = "ebpf", 476 477 .ctrl_cap_mask = 0, 478 479 .init = nfp_bpf_init, 480 .clean = nfp_bpf_clean, 481 482 .check_mtu = nfp_bpf_check_mtu, 483 484 .extra_cap = nfp_bpf_extra_cap, 485 486 .ndo_init = nfp_bpf_ndo_init, 487 .ndo_uninit = nfp_bpf_ndo_uninit, 488 489 .vnic_alloc = nfp_bpf_vnic_alloc, 490 .vnic_free = nfp_bpf_vnic_free, 491 492 .ctrl_msg_rx = nfp_bpf_ctrl_msg_rx, 493 494 .setup_tc = nfp_bpf_setup_tc, 495 .bpf = nfp_ndo_bpf, 496 .xdp_offload = nfp_bpf_xdp_offload, 497 }; 498