1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2 /* Copyright (C) 2017-2018 Netronome Systems, Inc. */
3 
4 #include <net/pkt_cls.h>
5 
6 #include "../nfpcore/nfp_cpp.h"
7 #include "../nfpcore/nfp_nffw.h"
8 #include "../nfpcore/nfp_nsp.h"
9 #include "../nfp_app.h"
10 #include "../nfp_main.h"
11 #include "../nfp_net.h"
12 #include "../nfp_port.h"
13 #include "fw.h"
14 #include "main.h"
15 
16 const struct rhashtable_params nfp_bpf_maps_neutral_params = {
17 	.nelem_hint		= 4,
18 	.key_len		= sizeof_field(struct bpf_map, id),
19 	.key_offset		= offsetof(struct nfp_bpf_neutral_map, map_id),
20 	.head_offset		= offsetof(struct nfp_bpf_neutral_map, l),
21 	.automatic_shrinking	= true,
22 };
23 
24 static bool nfp_net_ebpf_capable(struct nfp_net *nn)
25 {
26 #ifdef __LITTLE_ENDIAN
27 	struct nfp_app_bpf *bpf = nn->app->priv;
28 
29 	return nn->cap & NFP_NET_CFG_CTRL_BPF &&
30 	       bpf->abi_version &&
31 	       nn_readb(nn, NFP_NET_CFG_BPF_ABI) == bpf->abi_version;
32 #else
33 	return false;
34 #endif
35 }
36 
37 static int
38 nfp_bpf_xdp_offload(struct nfp_app *app, struct nfp_net *nn,
39 		    struct bpf_prog *prog, struct netlink_ext_ack *extack)
40 {
41 	bool running, xdp_running;
42 
43 	if (!nfp_net_ebpf_capable(nn))
44 		return -EINVAL;
45 
46 	running = nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF;
47 	xdp_running = running && nn->xdp_hw.prog;
48 
49 	if (!prog && !xdp_running)
50 		return 0;
51 	if (prog && running && !xdp_running)
52 		return -EBUSY;
53 
54 	return nfp_net_bpf_offload(nn, prog, running, extack);
55 }
56 
57 static const char *nfp_bpf_extra_cap(struct nfp_app *app, struct nfp_net *nn)
58 {
59 	return nfp_net_ebpf_capable(nn) ? "BPF" : "";
60 }
61 
62 static int
63 nfp_bpf_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, unsigned int id)
64 {
65 	struct nfp_pf *pf = app->pf;
66 	struct nfp_bpf_vnic *bv;
67 	int err;
68 
69 	if (!pf->eth_tbl) {
70 		nfp_err(pf->cpp, "No ETH table\n");
71 		return -EINVAL;
72 	}
73 	if (pf->max_data_vnics != pf->eth_tbl->count) {
74 		nfp_err(pf->cpp, "ETH entries don't match vNICs (%d vs %d)\n",
75 			pf->max_data_vnics, pf->eth_tbl->count);
76 		return -EINVAL;
77 	}
78 
79 	bv = kzalloc(sizeof(*bv), GFP_KERNEL);
80 	if (!bv)
81 		return -ENOMEM;
82 	nn->app_priv = bv;
83 
84 	err = nfp_app_nic_vnic_alloc(app, nn, id);
85 	if (err)
86 		goto err_free_priv;
87 
88 	bv->start_off = nn_readw(nn, NFP_NET_CFG_BPF_START);
89 	bv->tgt_done = nn_readw(nn, NFP_NET_CFG_BPF_DONE);
90 
91 	return 0;
92 err_free_priv:
93 	kfree(nn->app_priv);
94 	return err;
95 }
96 
97 static void nfp_bpf_vnic_free(struct nfp_app *app, struct nfp_net *nn)
98 {
99 	struct nfp_bpf_vnic *bv = nn->app_priv;
100 
101 	WARN_ON(bv->tc_prog);
102 	kfree(bv);
103 }
104 
105 static int nfp_bpf_setup_tc_block_cb(enum tc_setup_type type,
106 				     void *type_data, void *cb_priv)
107 {
108 	struct tc_cls_bpf_offload *cls_bpf = type_data;
109 	struct nfp_net *nn = cb_priv;
110 	struct bpf_prog *oldprog;
111 	struct nfp_bpf_vnic *bv;
112 	int err;
113 
114 	if (type != TC_SETUP_CLSBPF) {
115 		NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
116 				   "only offload of BPF classifiers supported");
117 		return -EOPNOTSUPP;
118 	}
119 	if (!tc_cls_can_offload_and_chain0(nn->dp.netdev, &cls_bpf->common))
120 		return -EOPNOTSUPP;
121 	if (!nfp_net_ebpf_capable(nn)) {
122 		NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
123 				   "NFP firmware does not support eBPF offload");
124 		return -EOPNOTSUPP;
125 	}
126 	if (cls_bpf->common.protocol != htons(ETH_P_ALL)) {
127 		NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
128 				   "only ETH_P_ALL supported as filter protocol");
129 		return -EOPNOTSUPP;
130 	}
131 
132 	/* Only support TC direct action */
133 	if (!cls_bpf->exts_integrated ||
134 	    tcf_exts_has_actions(cls_bpf->exts)) {
135 		NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
136 				   "only direct action with no legacy actions supported");
137 		return -EOPNOTSUPP;
138 	}
139 
140 	if (cls_bpf->command != TC_CLSBPF_OFFLOAD)
141 		return -EOPNOTSUPP;
142 
143 	bv = nn->app_priv;
144 	oldprog = cls_bpf->oldprog;
145 
146 	/* Don't remove if oldprog doesn't match driver's state */
147 	if (bv->tc_prog != oldprog) {
148 		oldprog = NULL;
149 		if (!cls_bpf->prog)
150 			return 0;
151 	}
152 
153 	err = nfp_net_bpf_offload(nn, cls_bpf->prog, oldprog,
154 				  cls_bpf->common.extack);
155 	if (err)
156 		return err;
157 
158 	bv->tc_prog = cls_bpf->prog;
159 	nn->port->tc_offload_cnt = !!bv->tc_prog;
160 	return 0;
161 }
162 
163 static LIST_HEAD(nfp_bpf_block_cb_list);
164 
165 static int nfp_bpf_setup_tc(struct nfp_app *app, struct net_device *netdev,
166 			    enum tc_setup_type type, void *type_data)
167 {
168 	struct nfp_net *nn = netdev_priv(netdev);
169 
170 	switch (type) {
171 	case TC_SETUP_BLOCK:
172 		return flow_block_cb_setup_simple(type_data,
173 						  &nfp_bpf_block_cb_list,
174 						  nfp_bpf_setup_tc_block_cb,
175 						  nn, nn, true);
176 	default:
177 		return -EOPNOTSUPP;
178 	}
179 }
180 
181 static int
182 nfp_bpf_check_mtu(struct nfp_app *app, struct net_device *netdev, int new_mtu)
183 {
184 	struct nfp_net *nn = netdev_priv(netdev);
185 	struct nfp_bpf_vnic *bv;
186 	struct bpf_prog *prog;
187 
188 	if (~nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF)
189 		return 0;
190 
191 	if (nn->xdp_hw.prog) {
192 		prog = nn->xdp_hw.prog;
193 	} else {
194 		bv = nn->app_priv;
195 		prog = bv->tc_prog;
196 	}
197 
198 	if (nfp_bpf_offload_check_mtu(nn, prog, new_mtu)) {
199 		nn_info(nn, "BPF offload active, potential packet access beyond hardware packet boundary");
200 		return -EBUSY;
201 	}
202 	return 0;
203 }
204 
205 static int
206 nfp_bpf_parse_cap_adjust_head(struct nfp_app_bpf *bpf, void __iomem *value,
207 			      u32 length)
208 {
209 	struct nfp_bpf_cap_tlv_adjust_head __iomem *cap = value;
210 	struct nfp_cpp *cpp = bpf->app->pf->cpp;
211 
212 	if (length < sizeof(*cap)) {
213 		nfp_err(cpp, "truncated adjust_head TLV: %d\n", length);
214 		return -EINVAL;
215 	}
216 
217 	bpf->adjust_head.flags = readl(&cap->flags);
218 	bpf->adjust_head.off_min = readl(&cap->off_min);
219 	bpf->adjust_head.off_max = readl(&cap->off_max);
220 	bpf->adjust_head.guaranteed_sub = readl(&cap->guaranteed_sub);
221 	bpf->adjust_head.guaranteed_add = readl(&cap->guaranteed_add);
222 
223 	if (bpf->adjust_head.off_min > bpf->adjust_head.off_max) {
224 		nfp_err(cpp, "invalid adjust_head TLV: min > max\n");
225 		return -EINVAL;
226 	}
227 	if (!FIELD_FIT(UR_REG_IMM_MAX, bpf->adjust_head.off_min) ||
228 	    !FIELD_FIT(UR_REG_IMM_MAX, bpf->adjust_head.off_max)) {
229 		nfp_warn(cpp, "disabling adjust_head - driver expects min/max to fit in as immediates\n");
230 		memset(&bpf->adjust_head, 0, sizeof(bpf->adjust_head));
231 		return 0;
232 	}
233 
234 	return 0;
235 }
236 
237 static int
238 nfp_bpf_parse_cap_func(struct nfp_app_bpf *bpf, void __iomem *value, u32 length)
239 {
240 	struct nfp_bpf_cap_tlv_func __iomem *cap = value;
241 
242 	if (length < sizeof(*cap)) {
243 		nfp_err(bpf->app->cpp, "truncated function TLV: %d\n", length);
244 		return -EINVAL;
245 	}
246 
247 	switch (readl(&cap->func_id)) {
248 	case BPF_FUNC_map_lookup_elem:
249 		bpf->helpers.map_lookup = readl(&cap->func_addr);
250 		break;
251 	case BPF_FUNC_map_update_elem:
252 		bpf->helpers.map_update = readl(&cap->func_addr);
253 		break;
254 	case BPF_FUNC_map_delete_elem:
255 		bpf->helpers.map_delete = readl(&cap->func_addr);
256 		break;
257 	case BPF_FUNC_perf_event_output:
258 		bpf->helpers.perf_event_output = readl(&cap->func_addr);
259 		break;
260 	}
261 
262 	return 0;
263 }
264 
265 static int
266 nfp_bpf_parse_cap_maps(struct nfp_app_bpf *bpf, void __iomem *value, u32 length)
267 {
268 	struct nfp_bpf_cap_tlv_maps __iomem *cap = value;
269 
270 	if (length < sizeof(*cap)) {
271 		nfp_err(bpf->app->cpp, "truncated maps TLV: %d\n", length);
272 		return -EINVAL;
273 	}
274 
275 	bpf->maps.types = readl(&cap->types);
276 	bpf->maps.max_maps = readl(&cap->max_maps);
277 	bpf->maps.max_elems = readl(&cap->max_elems);
278 	bpf->maps.max_key_sz = readl(&cap->max_key_sz);
279 	bpf->maps.max_val_sz = readl(&cap->max_val_sz);
280 	bpf->maps.max_elem_sz = readl(&cap->max_elem_sz);
281 
282 	return 0;
283 }
284 
285 static int
286 nfp_bpf_parse_cap_random(struct nfp_app_bpf *bpf, void __iomem *value,
287 			 u32 length)
288 {
289 	bpf->pseudo_random = true;
290 	return 0;
291 }
292 
293 static int
294 nfp_bpf_parse_cap_qsel(struct nfp_app_bpf *bpf, void __iomem *value, u32 length)
295 {
296 	bpf->queue_select = true;
297 	return 0;
298 }
299 
300 static int
301 nfp_bpf_parse_cap_adjust_tail(struct nfp_app_bpf *bpf, void __iomem *value,
302 			      u32 length)
303 {
304 	bpf->adjust_tail = true;
305 	return 0;
306 }
307 
308 static int
309 nfp_bpf_parse_cap_cmsg_multi_ent(struct nfp_app_bpf *bpf, void __iomem *value,
310 				 u32 length)
311 {
312 	bpf->cmsg_multi_ent = true;
313 	return 0;
314 }
315 
316 static int
317 nfp_bpf_parse_cap_abi_version(struct nfp_app_bpf *bpf, void __iomem *value,
318 			      u32 length)
319 {
320 	if (length < 4) {
321 		nfp_err(bpf->app->cpp, "truncated ABI version TLV: %d\n",
322 			length);
323 		return -EINVAL;
324 	}
325 
326 	bpf->abi_version = readl(value);
327 	if (bpf->abi_version < 2 || bpf->abi_version > 3) {
328 		nfp_warn(bpf->app->cpp, "unsupported BPF ABI version: %d\n",
329 			 bpf->abi_version);
330 		bpf->abi_version = 0;
331 	}
332 
333 	return 0;
334 }
335 
336 static int nfp_bpf_parse_capabilities(struct nfp_app *app)
337 {
338 	struct nfp_cpp *cpp = app->pf->cpp;
339 	struct nfp_cpp_area *area;
340 	u8 __iomem *mem, *start;
341 
342 	mem = nfp_rtsym_map(app->pf->rtbl, "_abi_bpf_capabilities", "bpf.cap",
343 			    8, &area);
344 	if (IS_ERR(mem))
345 		return PTR_ERR(mem) == -ENOENT ? 0 : PTR_ERR(mem);
346 
347 	start = mem;
348 	while (mem - start + 8 <= nfp_cpp_area_size(area)) {
349 		u8 __iomem *value;
350 		u32 type, length;
351 
352 		type = readl(mem);
353 		length = readl(mem + 4);
354 		value = mem + 8;
355 
356 		mem += 8 + length;
357 		if (mem - start > nfp_cpp_area_size(area))
358 			goto err_release_free;
359 
360 		switch (type) {
361 		case NFP_BPF_CAP_TYPE_FUNC:
362 			if (nfp_bpf_parse_cap_func(app->priv, value, length))
363 				goto err_release_free;
364 			break;
365 		case NFP_BPF_CAP_TYPE_ADJUST_HEAD:
366 			if (nfp_bpf_parse_cap_adjust_head(app->priv, value,
367 							  length))
368 				goto err_release_free;
369 			break;
370 		case NFP_BPF_CAP_TYPE_MAPS:
371 			if (nfp_bpf_parse_cap_maps(app->priv, value, length))
372 				goto err_release_free;
373 			break;
374 		case NFP_BPF_CAP_TYPE_RANDOM:
375 			if (nfp_bpf_parse_cap_random(app->priv, value, length))
376 				goto err_release_free;
377 			break;
378 		case NFP_BPF_CAP_TYPE_QUEUE_SELECT:
379 			if (nfp_bpf_parse_cap_qsel(app->priv, value, length))
380 				goto err_release_free;
381 			break;
382 		case NFP_BPF_CAP_TYPE_ADJUST_TAIL:
383 			if (nfp_bpf_parse_cap_adjust_tail(app->priv, value,
384 							  length))
385 				goto err_release_free;
386 			break;
387 		case NFP_BPF_CAP_TYPE_ABI_VERSION:
388 			if (nfp_bpf_parse_cap_abi_version(app->priv, value,
389 							  length))
390 				goto err_release_free;
391 			break;
392 		case NFP_BPF_CAP_TYPE_CMSG_MULTI_ENT:
393 			if (nfp_bpf_parse_cap_cmsg_multi_ent(app->priv, value,
394 							     length))
395 				goto err_release_free;
396 			break;
397 		default:
398 			nfp_dbg(cpp, "unknown BPF capability: %d\n", type);
399 			break;
400 		}
401 	}
402 	if (mem - start != nfp_cpp_area_size(area)) {
403 		nfp_err(cpp, "BPF capabilities left after parsing, parsed:%zd total length:%zu\n",
404 			mem - start, nfp_cpp_area_size(area));
405 		goto err_release_free;
406 	}
407 
408 	nfp_cpp_area_release_free(area);
409 
410 	return 0;
411 
412 err_release_free:
413 	nfp_err(cpp, "invalid BPF capabilities at offset:%zd\n", mem - start);
414 	nfp_cpp_area_release_free(area);
415 	return -EINVAL;
416 }
417 
418 static void nfp_bpf_init_capabilities(struct nfp_app_bpf *bpf)
419 {
420 	bpf->abi_version = 2; /* Original BPF ABI version */
421 }
422 
423 static int nfp_bpf_ndo_init(struct nfp_app *app, struct net_device *netdev)
424 {
425 	struct nfp_app_bpf *bpf = app->priv;
426 
427 	return bpf_offload_dev_netdev_register(bpf->bpf_dev, netdev);
428 }
429 
430 static void nfp_bpf_ndo_uninit(struct nfp_app *app, struct net_device *netdev)
431 {
432 	struct nfp_app_bpf *bpf = app->priv;
433 
434 	bpf_offload_dev_netdev_unregister(bpf->bpf_dev, netdev);
435 }
436 
437 static int nfp_bpf_start(struct nfp_app *app)
438 {
439 	struct nfp_app_bpf *bpf = app->priv;
440 
441 	if (app->ctrl->dp.mtu < nfp_bpf_ctrl_cmsg_min_mtu(bpf)) {
442 		nfp_err(bpf->app->cpp,
443 			"ctrl channel MTU below min required %u < %u\n",
444 			app->ctrl->dp.mtu, nfp_bpf_ctrl_cmsg_min_mtu(bpf));
445 		return -EINVAL;
446 	}
447 
448 	if (bpf->cmsg_multi_ent)
449 		bpf->cmsg_cache_cnt = nfp_bpf_ctrl_cmsg_cache_cnt(bpf);
450 	else
451 		bpf->cmsg_cache_cnt = 1;
452 
453 	return 0;
454 }
455 
456 static int nfp_bpf_init(struct nfp_app *app)
457 {
458 	struct nfp_app_bpf *bpf;
459 	int err;
460 
461 	bpf = kzalloc(sizeof(*bpf), GFP_KERNEL);
462 	if (!bpf)
463 		return -ENOMEM;
464 	bpf->app = app;
465 	app->priv = bpf;
466 
467 	INIT_LIST_HEAD(&bpf->map_list);
468 
469 	err = nfp_ccm_init(&bpf->ccm, app);
470 	if (err)
471 		goto err_free_bpf;
472 
473 	err = rhashtable_init(&bpf->maps_neutral, &nfp_bpf_maps_neutral_params);
474 	if (err)
475 		goto err_clean_ccm;
476 
477 	nfp_bpf_init_capabilities(bpf);
478 
479 	err = nfp_bpf_parse_capabilities(app);
480 	if (err)
481 		goto err_free_neutral_maps;
482 
483 	if (bpf->abi_version < 3) {
484 		bpf->cmsg_key_sz = CMSG_MAP_KEY_LW * 4;
485 		bpf->cmsg_val_sz = CMSG_MAP_VALUE_LW * 4;
486 	} else {
487 		bpf->cmsg_key_sz = bpf->maps.max_key_sz;
488 		bpf->cmsg_val_sz = bpf->maps.max_val_sz;
489 		app->ctrl_mtu = nfp_bpf_ctrl_cmsg_mtu(bpf);
490 	}
491 
492 	bpf->bpf_dev = bpf_offload_dev_create(&nfp_bpf_dev_ops, bpf);
493 	err = PTR_ERR_OR_ZERO(bpf->bpf_dev);
494 	if (err)
495 		goto err_free_neutral_maps;
496 
497 	return 0;
498 
499 err_free_neutral_maps:
500 	rhashtable_destroy(&bpf->maps_neutral);
501 err_clean_ccm:
502 	nfp_ccm_clean(&bpf->ccm);
503 err_free_bpf:
504 	kfree(bpf);
505 	return err;
506 }
507 
508 static void nfp_bpf_clean(struct nfp_app *app)
509 {
510 	struct nfp_app_bpf *bpf = app->priv;
511 
512 	bpf_offload_dev_destroy(bpf->bpf_dev);
513 	nfp_ccm_clean(&bpf->ccm);
514 	WARN_ON(!list_empty(&bpf->map_list));
515 	WARN_ON(bpf->maps_in_use || bpf->map_elems_in_use);
516 	rhashtable_free_and_destroy(&bpf->maps_neutral,
517 				    nfp_check_rhashtable_empty, NULL);
518 	kfree(bpf);
519 }
520 
521 const struct nfp_app_type app_bpf = {
522 	.id		= NFP_APP_BPF_NIC,
523 	.name		= "ebpf",
524 
525 	.ctrl_cap_mask	= 0,
526 
527 	.init		= nfp_bpf_init,
528 	.clean		= nfp_bpf_clean,
529 	.start		= nfp_bpf_start,
530 
531 	.check_mtu	= nfp_bpf_check_mtu,
532 
533 	.extra_cap	= nfp_bpf_extra_cap,
534 
535 	.ndo_init	= nfp_bpf_ndo_init,
536 	.ndo_uninit	= nfp_bpf_ndo_uninit,
537 
538 	.vnic_alloc	= nfp_bpf_vnic_alloc,
539 	.vnic_free	= nfp_bpf_vnic_free,
540 
541 	.ctrl_msg_rx	= nfp_bpf_ctrl_msg_rx,
542 	.ctrl_msg_rx_raw	= nfp_bpf_ctrl_msg_rx_raw,
543 
544 	.setup_tc	= nfp_bpf_setup_tc,
545 	.bpf		= nfp_ndo_bpf,
546 	.xdp_offload	= nfp_bpf_xdp_offload,
547 };
548