1 /* 2 * Copyright (C) 2017 Netronome Systems, Inc. 3 * 4 * This software is dual licensed under the GNU General License Version 2, 5 * June 1991 as shown in the file COPYING in the top-level directory of this 6 * source tree or the BSD 2-Clause License provided below. You have the 7 * option to license this software under the complete terms of either license. 8 * 9 * The BSD 2-Clause License: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * 1. Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * 2. Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <net/pkt_cls.h> 35 36 #include "../nfpcore/nfp_cpp.h" 37 #include "../nfpcore/nfp_nffw.h" 38 #include "../nfpcore/nfp_nsp.h" 39 #include "../nfp_app.h" 40 #include "../nfp_main.h" 41 #include "../nfp_net.h" 42 #include "../nfp_port.h" 43 #include "fw.h" 44 #include "main.h" 45 46 static bool nfp_net_ebpf_capable(struct nfp_net *nn) 47 { 48 #ifdef __LITTLE_ENDIAN 49 if (nn->cap & NFP_NET_CFG_CTRL_BPF && 50 nn_readb(nn, NFP_NET_CFG_BPF_ABI) == NFP_NET_BPF_ABI) 51 return true; 52 #endif 53 return false; 54 } 55 56 static int 57 nfp_bpf_xdp_offload(struct nfp_app *app, struct nfp_net *nn, 58 struct bpf_prog *prog, struct netlink_ext_ack *extack) 59 { 60 bool running, xdp_running; 61 int ret; 62 63 if (!nfp_net_ebpf_capable(nn)) 64 return -EINVAL; 65 66 running = nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF; 67 xdp_running = running && nn->dp.bpf_offload_xdp; 68 69 if (!prog && !xdp_running) 70 return 0; 71 if (prog && running && !xdp_running) 72 return -EBUSY; 73 74 ret = nfp_net_bpf_offload(nn, prog, running, extack); 75 /* Stop offload if replace not possible */ 76 if (ret && prog) 77 nfp_bpf_xdp_offload(app, nn, NULL, extack); 78 79 nn->dp.bpf_offload_xdp = prog && !ret; 80 return ret; 81 } 82 83 static const char *nfp_bpf_extra_cap(struct nfp_app *app, struct nfp_net *nn) 84 { 85 return nfp_net_ebpf_capable(nn) ? "BPF" : ""; 86 } 87 88 static int 89 nfp_bpf_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, unsigned int id) 90 { 91 struct nfp_pf *pf = app->pf; 92 struct nfp_bpf_vnic *bv; 93 int err; 94 95 if (!pf->eth_tbl) { 96 nfp_err(pf->cpp, "No ETH table\n"); 97 return -EINVAL; 98 } 99 if (pf->max_data_vnics != pf->eth_tbl->count) { 100 nfp_err(pf->cpp, "ETH entries don't match vNICs (%d vs %d)\n", 101 pf->max_data_vnics, pf->eth_tbl->count); 102 return -EINVAL; 103 } 104 105 bv = kzalloc(sizeof(*bv), GFP_KERNEL); 106 if (!bv) 107 return -ENOMEM; 108 nn->app_priv = bv; 109 110 err = nfp_app_nic_vnic_alloc(app, nn, id); 111 if (err) 112 goto err_free_priv; 113 114 bv->start_off = nn_readw(nn, NFP_NET_CFG_BPF_START); 115 bv->tgt_done = nn_readw(nn, NFP_NET_CFG_BPF_DONE); 116 117 return 0; 118 err_free_priv: 119 kfree(nn->app_priv); 120 return err; 121 } 122 123 static void nfp_bpf_vnic_free(struct nfp_app *app, struct nfp_net *nn) 124 { 125 struct nfp_bpf_vnic *bv = nn->app_priv; 126 127 WARN_ON(bv->tc_prog); 128 kfree(bv); 129 } 130 131 static int nfp_bpf_setup_tc_block_cb(enum tc_setup_type type, 132 void *type_data, void *cb_priv) 133 { 134 struct tc_cls_bpf_offload *cls_bpf = type_data; 135 struct nfp_net *nn = cb_priv; 136 struct bpf_prog *oldprog; 137 struct nfp_bpf_vnic *bv; 138 int err; 139 140 if (type != TC_SETUP_CLSBPF) { 141 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack, 142 "only offload of BPF classifiers supported"); 143 return -EOPNOTSUPP; 144 } 145 if (!tc_cls_can_offload_and_chain0(nn->dp.netdev, &cls_bpf->common)) 146 return -EOPNOTSUPP; 147 if (!nfp_net_ebpf_capable(nn)) { 148 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack, 149 "NFP firmware does not support eBPF offload"); 150 return -EOPNOTSUPP; 151 } 152 if (cls_bpf->common.protocol != htons(ETH_P_ALL)) { 153 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack, 154 "only ETH_P_ALL supported as filter protocol"); 155 return -EOPNOTSUPP; 156 } 157 158 /* Only support TC direct action */ 159 if (!cls_bpf->exts_integrated || 160 tcf_exts_has_actions(cls_bpf->exts)) { 161 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack, 162 "only direct action with no legacy actions supported"); 163 return -EOPNOTSUPP; 164 } 165 166 if (cls_bpf->command != TC_CLSBPF_OFFLOAD) 167 return -EOPNOTSUPP; 168 169 bv = nn->app_priv; 170 oldprog = cls_bpf->oldprog; 171 172 /* Don't remove if oldprog doesn't match driver's state */ 173 if (bv->tc_prog != oldprog) { 174 oldprog = NULL; 175 if (!cls_bpf->prog) 176 return 0; 177 } 178 179 err = nfp_net_bpf_offload(nn, cls_bpf->prog, oldprog, 180 cls_bpf->common.extack); 181 if (err) 182 return err; 183 184 bv->tc_prog = cls_bpf->prog; 185 nn->port->tc_offload_cnt = !!bv->tc_prog; 186 return 0; 187 } 188 189 static int nfp_bpf_setup_tc_block(struct net_device *netdev, 190 struct tc_block_offload *f) 191 { 192 struct nfp_net *nn = netdev_priv(netdev); 193 194 if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS) 195 return -EOPNOTSUPP; 196 197 switch (f->command) { 198 case TC_BLOCK_BIND: 199 return tcf_block_cb_register(f->block, 200 nfp_bpf_setup_tc_block_cb, 201 nn, nn); 202 case TC_BLOCK_UNBIND: 203 tcf_block_cb_unregister(f->block, 204 nfp_bpf_setup_tc_block_cb, 205 nn); 206 return 0; 207 default: 208 return -EOPNOTSUPP; 209 } 210 } 211 212 static int nfp_bpf_setup_tc(struct nfp_app *app, struct net_device *netdev, 213 enum tc_setup_type type, void *type_data) 214 { 215 switch (type) { 216 case TC_SETUP_BLOCK: 217 return nfp_bpf_setup_tc_block(netdev, type_data); 218 default: 219 return -EOPNOTSUPP; 220 } 221 } 222 223 static int 224 nfp_bpf_check_mtu(struct nfp_app *app, struct net_device *netdev, int new_mtu) 225 { 226 struct nfp_net *nn = netdev_priv(netdev); 227 unsigned int max_mtu; 228 229 if (~nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF) 230 return 0; 231 232 max_mtu = nn_readb(nn, NFP_NET_CFG_BPF_INL_MTU) * 64 - 32; 233 if (new_mtu > max_mtu) { 234 nn_info(nn, "BPF offload active, MTU over %u not supported\n", 235 max_mtu); 236 return -EBUSY; 237 } 238 return 0; 239 } 240 241 static int 242 nfp_bpf_parse_cap_adjust_head(struct nfp_app_bpf *bpf, void __iomem *value, 243 u32 length) 244 { 245 struct nfp_bpf_cap_tlv_adjust_head __iomem *cap = value; 246 struct nfp_cpp *cpp = bpf->app->pf->cpp; 247 248 if (length < sizeof(*cap)) { 249 nfp_err(cpp, "truncated adjust_head TLV: %d\n", length); 250 return -EINVAL; 251 } 252 253 bpf->adjust_head.flags = readl(&cap->flags); 254 bpf->adjust_head.off_min = readl(&cap->off_min); 255 bpf->adjust_head.off_max = readl(&cap->off_max); 256 bpf->adjust_head.guaranteed_sub = readl(&cap->guaranteed_sub); 257 bpf->adjust_head.guaranteed_add = readl(&cap->guaranteed_add); 258 259 if (bpf->adjust_head.off_min > bpf->adjust_head.off_max) { 260 nfp_err(cpp, "invalid adjust_head TLV: min > max\n"); 261 return -EINVAL; 262 } 263 if (!FIELD_FIT(UR_REG_IMM_MAX, bpf->adjust_head.off_min) || 264 !FIELD_FIT(UR_REG_IMM_MAX, bpf->adjust_head.off_max)) { 265 nfp_warn(cpp, "disabling adjust_head - driver expects min/max to fit in as immediates\n"); 266 memset(&bpf->adjust_head, 0, sizeof(bpf->adjust_head)); 267 return 0; 268 } 269 270 return 0; 271 } 272 273 static int 274 nfp_bpf_parse_cap_func(struct nfp_app_bpf *bpf, void __iomem *value, u32 length) 275 { 276 struct nfp_bpf_cap_tlv_func __iomem *cap = value; 277 278 if (length < sizeof(*cap)) { 279 nfp_err(bpf->app->cpp, "truncated function TLV: %d\n", length); 280 return -EINVAL; 281 } 282 283 switch (readl(&cap->func_id)) { 284 case BPF_FUNC_map_lookup_elem: 285 bpf->helpers.map_lookup = readl(&cap->func_addr); 286 break; 287 case BPF_FUNC_map_update_elem: 288 bpf->helpers.map_update = readl(&cap->func_addr); 289 break; 290 case BPF_FUNC_map_delete_elem: 291 bpf->helpers.map_delete = readl(&cap->func_addr); 292 break; 293 } 294 295 return 0; 296 } 297 298 static int 299 nfp_bpf_parse_cap_maps(struct nfp_app_bpf *bpf, void __iomem *value, u32 length) 300 { 301 struct nfp_bpf_cap_tlv_maps __iomem *cap = value; 302 303 if (length < sizeof(*cap)) { 304 nfp_err(bpf->app->cpp, "truncated maps TLV: %d\n", length); 305 return -EINVAL; 306 } 307 308 bpf->maps.types = readl(&cap->types); 309 bpf->maps.max_maps = readl(&cap->max_maps); 310 bpf->maps.max_elems = readl(&cap->max_elems); 311 bpf->maps.max_key_sz = readl(&cap->max_key_sz); 312 bpf->maps.max_val_sz = readl(&cap->max_val_sz); 313 bpf->maps.max_elem_sz = readl(&cap->max_elem_sz); 314 315 return 0; 316 } 317 318 static int 319 nfp_bpf_parse_cap_random(struct nfp_app_bpf *bpf, void __iomem *value, 320 u32 length) 321 { 322 bpf->pseudo_random = true; 323 return 0; 324 } 325 326 static int nfp_bpf_parse_capabilities(struct nfp_app *app) 327 { 328 struct nfp_cpp *cpp = app->pf->cpp; 329 struct nfp_cpp_area *area; 330 u8 __iomem *mem, *start; 331 332 mem = nfp_rtsym_map(app->pf->rtbl, "_abi_bpf_capabilities", "bpf.cap", 333 8, &area); 334 if (IS_ERR(mem)) 335 return PTR_ERR(mem) == -ENOENT ? 0 : PTR_ERR(mem); 336 337 start = mem; 338 while (mem - start + 8 < nfp_cpp_area_size(area)) { 339 u8 __iomem *value; 340 u32 type, length; 341 342 type = readl(mem); 343 length = readl(mem + 4); 344 value = mem + 8; 345 346 mem += 8 + length; 347 if (mem - start > nfp_cpp_area_size(area)) 348 goto err_release_free; 349 350 switch (type) { 351 case NFP_BPF_CAP_TYPE_FUNC: 352 if (nfp_bpf_parse_cap_func(app->priv, value, length)) 353 goto err_release_free; 354 break; 355 case NFP_BPF_CAP_TYPE_ADJUST_HEAD: 356 if (nfp_bpf_parse_cap_adjust_head(app->priv, value, 357 length)) 358 goto err_release_free; 359 break; 360 case NFP_BPF_CAP_TYPE_MAPS: 361 if (nfp_bpf_parse_cap_maps(app->priv, value, length)) 362 goto err_release_free; 363 break; 364 case NFP_BPF_CAP_TYPE_RANDOM: 365 if (nfp_bpf_parse_cap_random(app->priv, value, length)) 366 goto err_release_free; 367 break; 368 default: 369 nfp_dbg(cpp, "unknown BPF capability: %d\n", type); 370 break; 371 } 372 } 373 if (mem - start != nfp_cpp_area_size(area)) { 374 nfp_err(cpp, "BPF capabilities left after parsing, parsed:%zd total length:%zu\n", 375 mem - start, nfp_cpp_area_size(area)); 376 goto err_release_free; 377 } 378 379 nfp_cpp_area_release_free(area); 380 381 return 0; 382 383 err_release_free: 384 nfp_err(cpp, "invalid BPF capabilities at offset:%zd\n", mem - start); 385 nfp_cpp_area_release_free(area); 386 return -EINVAL; 387 } 388 389 static int nfp_bpf_init(struct nfp_app *app) 390 { 391 struct nfp_app_bpf *bpf; 392 int err; 393 394 bpf = kzalloc(sizeof(*bpf), GFP_KERNEL); 395 if (!bpf) 396 return -ENOMEM; 397 bpf->app = app; 398 app->priv = bpf; 399 400 skb_queue_head_init(&bpf->cmsg_replies); 401 init_waitqueue_head(&bpf->cmsg_wq); 402 INIT_LIST_HEAD(&bpf->map_list); 403 404 err = nfp_bpf_parse_capabilities(app); 405 if (err) 406 goto err_free_bpf; 407 408 return 0; 409 410 err_free_bpf: 411 kfree(bpf); 412 return err; 413 } 414 415 static void nfp_bpf_clean(struct nfp_app *app) 416 { 417 struct nfp_app_bpf *bpf = app->priv; 418 419 WARN_ON(!skb_queue_empty(&bpf->cmsg_replies)); 420 WARN_ON(!list_empty(&bpf->map_list)); 421 WARN_ON(bpf->maps_in_use || bpf->map_elems_in_use); 422 kfree(bpf); 423 } 424 425 const struct nfp_app_type app_bpf = { 426 .id = NFP_APP_BPF_NIC, 427 .name = "ebpf", 428 429 .ctrl_cap_mask = 0, 430 431 .init = nfp_bpf_init, 432 .clean = nfp_bpf_clean, 433 434 .check_mtu = nfp_bpf_check_mtu, 435 436 .extra_cap = nfp_bpf_extra_cap, 437 438 .vnic_alloc = nfp_bpf_vnic_alloc, 439 .vnic_free = nfp_bpf_vnic_free, 440 441 .ctrl_msg_rx = nfp_bpf_ctrl_msg_rx, 442 443 .setup_tc = nfp_bpf_setup_tc, 444 .bpf = nfp_ndo_bpf, 445 .xdp_offload = nfp_bpf_xdp_offload, 446 }; 447