1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2 /* Copyright (C) 2017-2018 Netronome Systems, Inc. */
3 
4 #include <net/pkt_cls.h>
5 
6 #include "../nfpcore/nfp_cpp.h"
7 #include "../nfpcore/nfp_nffw.h"
8 #include "../nfpcore/nfp_nsp.h"
9 #include "../nfp_app.h"
10 #include "../nfp_main.h"
11 #include "../nfp_net.h"
12 #include "../nfp_port.h"
13 #include "fw.h"
14 #include "main.h"
15 
16 const struct rhashtable_params nfp_bpf_maps_neutral_params = {
17 	.nelem_hint		= 4,
18 	.key_len		= FIELD_SIZEOF(struct bpf_map, id),
19 	.key_offset		= offsetof(struct nfp_bpf_neutral_map, map_id),
20 	.head_offset		= offsetof(struct nfp_bpf_neutral_map, l),
21 	.automatic_shrinking	= true,
22 };
23 
24 static bool nfp_net_ebpf_capable(struct nfp_net *nn)
25 {
26 #ifdef __LITTLE_ENDIAN
27 	struct nfp_app_bpf *bpf = nn->app->priv;
28 
29 	return nn->cap & NFP_NET_CFG_CTRL_BPF &&
30 	       bpf->abi_version &&
31 	       nn_readb(nn, NFP_NET_CFG_BPF_ABI) == bpf->abi_version;
32 #else
33 	return false;
34 #endif
35 }
36 
37 static int
38 nfp_bpf_xdp_offload(struct nfp_app *app, struct nfp_net *nn,
39 		    struct bpf_prog *prog, struct netlink_ext_ack *extack)
40 {
41 	bool running, xdp_running;
42 
43 	if (!nfp_net_ebpf_capable(nn))
44 		return -EINVAL;
45 
46 	running = nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF;
47 	xdp_running = running && nn->xdp_hw.prog;
48 
49 	if (!prog && !xdp_running)
50 		return 0;
51 	if (prog && running && !xdp_running)
52 		return -EBUSY;
53 
54 	return nfp_net_bpf_offload(nn, prog, running, extack);
55 }
56 
57 static const char *nfp_bpf_extra_cap(struct nfp_app *app, struct nfp_net *nn)
58 {
59 	return nfp_net_ebpf_capable(nn) ? "BPF" : "";
60 }
61 
62 static int
63 nfp_bpf_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, unsigned int id)
64 {
65 	struct nfp_pf *pf = app->pf;
66 	struct nfp_bpf_vnic *bv;
67 	int err;
68 
69 	if (!pf->eth_tbl) {
70 		nfp_err(pf->cpp, "No ETH table\n");
71 		return -EINVAL;
72 	}
73 	if (pf->max_data_vnics != pf->eth_tbl->count) {
74 		nfp_err(pf->cpp, "ETH entries don't match vNICs (%d vs %d)\n",
75 			pf->max_data_vnics, pf->eth_tbl->count);
76 		return -EINVAL;
77 	}
78 
79 	bv = kzalloc(sizeof(*bv), GFP_KERNEL);
80 	if (!bv)
81 		return -ENOMEM;
82 	nn->app_priv = bv;
83 
84 	err = nfp_app_nic_vnic_alloc(app, nn, id);
85 	if (err)
86 		goto err_free_priv;
87 
88 	bv->start_off = nn_readw(nn, NFP_NET_CFG_BPF_START);
89 	bv->tgt_done = nn_readw(nn, NFP_NET_CFG_BPF_DONE);
90 
91 	return 0;
92 err_free_priv:
93 	kfree(nn->app_priv);
94 	return err;
95 }
96 
97 static void nfp_bpf_vnic_free(struct nfp_app *app, struct nfp_net *nn)
98 {
99 	struct nfp_bpf_vnic *bv = nn->app_priv;
100 
101 	WARN_ON(bv->tc_prog);
102 	kfree(bv);
103 }
104 
105 static int nfp_bpf_setup_tc_block_cb(enum tc_setup_type type,
106 				     void *type_data, void *cb_priv)
107 {
108 	struct tc_cls_bpf_offload *cls_bpf = type_data;
109 	struct nfp_net *nn = cb_priv;
110 	struct bpf_prog *oldprog;
111 	struct nfp_bpf_vnic *bv;
112 	int err;
113 
114 	if (type != TC_SETUP_CLSBPF) {
115 		NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
116 				   "only offload of BPF classifiers supported");
117 		return -EOPNOTSUPP;
118 	}
119 	if (!tc_cls_can_offload_and_chain0(nn->dp.netdev, &cls_bpf->common))
120 		return -EOPNOTSUPP;
121 	if (!nfp_net_ebpf_capable(nn)) {
122 		NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
123 				   "NFP firmware does not support eBPF offload");
124 		return -EOPNOTSUPP;
125 	}
126 	if (cls_bpf->common.protocol != htons(ETH_P_ALL)) {
127 		NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
128 				   "only ETH_P_ALL supported as filter protocol");
129 		return -EOPNOTSUPP;
130 	}
131 
132 	/* Only support TC direct action */
133 	if (!cls_bpf->exts_integrated ||
134 	    tcf_exts_has_actions(cls_bpf->exts)) {
135 		NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
136 				   "only direct action with no legacy actions supported");
137 		return -EOPNOTSUPP;
138 	}
139 
140 	if (cls_bpf->command != TC_CLSBPF_OFFLOAD)
141 		return -EOPNOTSUPP;
142 
143 	bv = nn->app_priv;
144 	oldprog = cls_bpf->oldprog;
145 
146 	/* Don't remove if oldprog doesn't match driver's state */
147 	if (bv->tc_prog != oldprog) {
148 		oldprog = NULL;
149 		if (!cls_bpf->prog)
150 			return 0;
151 	}
152 
153 	err = nfp_net_bpf_offload(nn, cls_bpf->prog, oldprog,
154 				  cls_bpf->common.extack);
155 	if (err)
156 		return err;
157 
158 	bv->tc_prog = cls_bpf->prog;
159 	nn->port->tc_offload_cnt = !!bv->tc_prog;
160 	return 0;
161 }
162 
163 static int nfp_bpf_setup_tc_block(struct net_device *netdev,
164 				  struct tc_block_offload *f)
165 {
166 	struct nfp_net *nn = netdev_priv(netdev);
167 
168 	if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
169 		return -EOPNOTSUPP;
170 
171 	switch (f->command) {
172 	case TC_BLOCK_BIND:
173 		return tcf_block_cb_register(f->block,
174 					     nfp_bpf_setup_tc_block_cb,
175 					     nn, nn, f->extack);
176 	case TC_BLOCK_UNBIND:
177 		tcf_block_cb_unregister(f->block,
178 					nfp_bpf_setup_tc_block_cb,
179 					nn);
180 		return 0;
181 	default:
182 		return -EOPNOTSUPP;
183 	}
184 }
185 
186 static int nfp_bpf_setup_tc(struct nfp_app *app, struct net_device *netdev,
187 			    enum tc_setup_type type, void *type_data)
188 {
189 	switch (type) {
190 	case TC_SETUP_BLOCK:
191 		return nfp_bpf_setup_tc_block(netdev, type_data);
192 	default:
193 		return -EOPNOTSUPP;
194 	}
195 }
196 
197 static int
198 nfp_bpf_check_mtu(struct nfp_app *app, struct net_device *netdev, int new_mtu)
199 {
200 	struct nfp_net *nn = netdev_priv(netdev);
201 	unsigned int max_mtu;
202 
203 	if (~nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF)
204 		return 0;
205 
206 	max_mtu = nn_readb(nn, NFP_NET_CFG_BPF_INL_MTU) * 64 - 32;
207 	if (new_mtu > max_mtu) {
208 		nn_info(nn, "BPF offload active, MTU over %u not supported\n",
209 			max_mtu);
210 		return -EBUSY;
211 	}
212 	return 0;
213 }
214 
215 static int
216 nfp_bpf_parse_cap_adjust_head(struct nfp_app_bpf *bpf, void __iomem *value,
217 			      u32 length)
218 {
219 	struct nfp_bpf_cap_tlv_adjust_head __iomem *cap = value;
220 	struct nfp_cpp *cpp = bpf->app->pf->cpp;
221 
222 	if (length < sizeof(*cap)) {
223 		nfp_err(cpp, "truncated adjust_head TLV: %d\n", length);
224 		return -EINVAL;
225 	}
226 
227 	bpf->adjust_head.flags = readl(&cap->flags);
228 	bpf->adjust_head.off_min = readl(&cap->off_min);
229 	bpf->adjust_head.off_max = readl(&cap->off_max);
230 	bpf->adjust_head.guaranteed_sub = readl(&cap->guaranteed_sub);
231 	bpf->adjust_head.guaranteed_add = readl(&cap->guaranteed_add);
232 
233 	if (bpf->adjust_head.off_min > bpf->adjust_head.off_max) {
234 		nfp_err(cpp, "invalid adjust_head TLV: min > max\n");
235 		return -EINVAL;
236 	}
237 	if (!FIELD_FIT(UR_REG_IMM_MAX, bpf->adjust_head.off_min) ||
238 	    !FIELD_FIT(UR_REG_IMM_MAX, bpf->adjust_head.off_max)) {
239 		nfp_warn(cpp, "disabling adjust_head - driver expects min/max to fit in as immediates\n");
240 		memset(&bpf->adjust_head, 0, sizeof(bpf->adjust_head));
241 		return 0;
242 	}
243 
244 	return 0;
245 }
246 
247 static int
248 nfp_bpf_parse_cap_func(struct nfp_app_bpf *bpf, void __iomem *value, u32 length)
249 {
250 	struct nfp_bpf_cap_tlv_func __iomem *cap = value;
251 
252 	if (length < sizeof(*cap)) {
253 		nfp_err(bpf->app->cpp, "truncated function TLV: %d\n", length);
254 		return -EINVAL;
255 	}
256 
257 	switch (readl(&cap->func_id)) {
258 	case BPF_FUNC_map_lookup_elem:
259 		bpf->helpers.map_lookup = readl(&cap->func_addr);
260 		break;
261 	case BPF_FUNC_map_update_elem:
262 		bpf->helpers.map_update = readl(&cap->func_addr);
263 		break;
264 	case BPF_FUNC_map_delete_elem:
265 		bpf->helpers.map_delete = readl(&cap->func_addr);
266 		break;
267 	case BPF_FUNC_perf_event_output:
268 		bpf->helpers.perf_event_output = readl(&cap->func_addr);
269 		break;
270 	}
271 
272 	return 0;
273 }
274 
275 static int
276 nfp_bpf_parse_cap_maps(struct nfp_app_bpf *bpf, void __iomem *value, u32 length)
277 {
278 	struct nfp_bpf_cap_tlv_maps __iomem *cap = value;
279 
280 	if (length < sizeof(*cap)) {
281 		nfp_err(bpf->app->cpp, "truncated maps TLV: %d\n", length);
282 		return -EINVAL;
283 	}
284 
285 	bpf->maps.types = readl(&cap->types);
286 	bpf->maps.max_maps = readl(&cap->max_maps);
287 	bpf->maps.max_elems = readl(&cap->max_elems);
288 	bpf->maps.max_key_sz = readl(&cap->max_key_sz);
289 	bpf->maps.max_val_sz = readl(&cap->max_val_sz);
290 	bpf->maps.max_elem_sz = readl(&cap->max_elem_sz);
291 
292 	return 0;
293 }
294 
295 static int
296 nfp_bpf_parse_cap_random(struct nfp_app_bpf *bpf, void __iomem *value,
297 			 u32 length)
298 {
299 	bpf->pseudo_random = true;
300 	return 0;
301 }
302 
303 static int
304 nfp_bpf_parse_cap_qsel(struct nfp_app_bpf *bpf, void __iomem *value, u32 length)
305 {
306 	bpf->queue_select = true;
307 	return 0;
308 }
309 
310 static int
311 nfp_bpf_parse_cap_adjust_tail(struct nfp_app_bpf *bpf, void __iomem *value,
312 			      u32 length)
313 {
314 	bpf->adjust_tail = true;
315 	return 0;
316 }
317 
318 static int
319 nfp_bpf_parse_cap_abi_version(struct nfp_app_bpf *bpf, void __iomem *value,
320 			      u32 length)
321 {
322 	if (length < 4) {
323 		nfp_err(bpf->app->cpp, "truncated ABI version TLV: %d\n",
324 			length);
325 		return -EINVAL;
326 	}
327 
328 	bpf->abi_version = readl(value);
329 	if (bpf->abi_version < 2 || bpf->abi_version > 3) {
330 		nfp_warn(bpf->app->cpp, "unsupported BPF ABI version: %d\n",
331 			 bpf->abi_version);
332 		bpf->abi_version = 0;
333 	}
334 
335 	return 0;
336 }
337 
338 static int nfp_bpf_parse_capabilities(struct nfp_app *app)
339 {
340 	struct nfp_cpp *cpp = app->pf->cpp;
341 	struct nfp_cpp_area *area;
342 	u8 __iomem *mem, *start;
343 
344 	mem = nfp_rtsym_map(app->pf->rtbl, "_abi_bpf_capabilities", "bpf.cap",
345 			    8, &area);
346 	if (IS_ERR(mem))
347 		return PTR_ERR(mem) == -ENOENT ? 0 : PTR_ERR(mem);
348 
349 	start = mem;
350 	while (mem - start + 8 <= nfp_cpp_area_size(area)) {
351 		u8 __iomem *value;
352 		u32 type, length;
353 
354 		type = readl(mem);
355 		length = readl(mem + 4);
356 		value = mem + 8;
357 
358 		mem += 8 + length;
359 		if (mem - start > nfp_cpp_area_size(area))
360 			goto err_release_free;
361 
362 		switch (type) {
363 		case NFP_BPF_CAP_TYPE_FUNC:
364 			if (nfp_bpf_parse_cap_func(app->priv, value, length))
365 				goto err_release_free;
366 			break;
367 		case NFP_BPF_CAP_TYPE_ADJUST_HEAD:
368 			if (nfp_bpf_parse_cap_adjust_head(app->priv, value,
369 							  length))
370 				goto err_release_free;
371 			break;
372 		case NFP_BPF_CAP_TYPE_MAPS:
373 			if (nfp_bpf_parse_cap_maps(app->priv, value, length))
374 				goto err_release_free;
375 			break;
376 		case NFP_BPF_CAP_TYPE_RANDOM:
377 			if (nfp_bpf_parse_cap_random(app->priv, value, length))
378 				goto err_release_free;
379 			break;
380 		case NFP_BPF_CAP_TYPE_QUEUE_SELECT:
381 			if (nfp_bpf_parse_cap_qsel(app->priv, value, length))
382 				goto err_release_free;
383 			break;
384 		case NFP_BPF_CAP_TYPE_ADJUST_TAIL:
385 			if (nfp_bpf_parse_cap_adjust_tail(app->priv, value,
386 							  length))
387 				goto err_release_free;
388 			break;
389 		case NFP_BPF_CAP_TYPE_ABI_VERSION:
390 			if (nfp_bpf_parse_cap_abi_version(app->priv, value,
391 							  length))
392 				goto err_release_free;
393 			break;
394 		default:
395 			nfp_dbg(cpp, "unknown BPF capability: %d\n", type);
396 			break;
397 		}
398 	}
399 	if (mem - start != nfp_cpp_area_size(area)) {
400 		nfp_err(cpp, "BPF capabilities left after parsing, parsed:%zd total length:%zu\n",
401 			mem - start, nfp_cpp_area_size(area));
402 		goto err_release_free;
403 	}
404 
405 	nfp_cpp_area_release_free(area);
406 
407 	return 0;
408 
409 err_release_free:
410 	nfp_err(cpp, "invalid BPF capabilities at offset:%zd\n", mem - start);
411 	nfp_cpp_area_release_free(area);
412 	return -EINVAL;
413 }
414 
415 static void nfp_bpf_init_capabilities(struct nfp_app_bpf *bpf)
416 {
417 	bpf->abi_version = 2; /* Original BPF ABI version */
418 }
419 
420 static int nfp_bpf_ndo_init(struct nfp_app *app, struct net_device *netdev)
421 {
422 	struct nfp_app_bpf *bpf = app->priv;
423 
424 	return bpf_offload_dev_netdev_register(bpf->bpf_dev, netdev);
425 }
426 
427 static void nfp_bpf_ndo_uninit(struct nfp_app *app, struct net_device *netdev)
428 {
429 	struct nfp_app_bpf *bpf = app->priv;
430 
431 	bpf_offload_dev_netdev_unregister(bpf->bpf_dev, netdev);
432 }
433 
434 static int nfp_bpf_init(struct nfp_app *app)
435 {
436 	struct nfp_app_bpf *bpf;
437 	int err;
438 
439 	bpf = kzalloc(sizeof(*bpf), GFP_KERNEL);
440 	if (!bpf)
441 		return -ENOMEM;
442 	bpf->app = app;
443 	app->priv = bpf;
444 
445 	skb_queue_head_init(&bpf->cmsg_replies);
446 	init_waitqueue_head(&bpf->cmsg_wq);
447 	INIT_LIST_HEAD(&bpf->map_list);
448 
449 	err = rhashtable_init(&bpf->maps_neutral, &nfp_bpf_maps_neutral_params);
450 	if (err)
451 		goto err_free_bpf;
452 
453 	nfp_bpf_init_capabilities(bpf);
454 
455 	err = nfp_bpf_parse_capabilities(app);
456 	if (err)
457 		goto err_free_neutral_maps;
458 
459 	if (bpf->abi_version < 3) {
460 		bpf->cmsg_key_sz = CMSG_MAP_KEY_LW * 4;
461 		bpf->cmsg_val_sz = CMSG_MAP_VALUE_LW * 4;
462 	} else {
463 		bpf->cmsg_key_sz = bpf->maps.max_key_sz;
464 		bpf->cmsg_val_sz = bpf->maps.max_val_sz;
465 		app->ctrl_mtu = nfp_bpf_ctrl_cmsg_mtu(bpf);
466 	}
467 
468 	bpf->bpf_dev = bpf_offload_dev_create(&nfp_bpf_dev_ops, bpf);
469 	err = PTR_ERR_OR_ZERO(bpf->bpf_dev);
470 	if (err)
471 		goto err_free_neutral_maps;
472 
473 	return 0;
474 
475 err_free_neutral_maps:
476 	rhashtable_destroy(&bpf->maps_neutral);
477 err_free_bpf:
478 	kfree(bpf);
479 	return err;
480 }
481 
482 static void nfp_bpf_clean(struct nfp_app *app)
483 {
484 	struct nfp_app_bpf *bpf = app->priv;
485 
486 	bpf_offload_dev_destroy(bpf->bpf_dev);
487 	WARN_ON(!skb_queue_empty(&bpf->cmsg_replies));
488 	WARN_ON(!list_empty(&bpf->map_list));
489 	WARN_ON(bpf->maps_in_use || bpf->map_elems_in_use);
490 	rhashtable_free_and_destroy(&bpf->maps_neutral,
491 				    nfp_check_rhashtable_empty, NULL);
492 	kfree(bpf);
493 }
494 
495 const struct nfp_app_type app_bpf = {
496 	.id		= NFP_APP_BPF_NIC,
497 	.name		= "ebpf",
498 
499 	.ctrl_cap_mask	= 0,
500 
501 	.init		= nfp_bpf_init,
502 	.clean		= nfp_bpf_clean,
503 
504 	.check_mtu	= nfp_bpf_check_mtu,
505 
506 	.extra_cap	= nfp_bpf_extra_cap,
507 
508 	.ndo_init	= nfp_bpf_ndo_init,
509 	.ndo_uninit	= nfp_bpf_ndo_uninit,
510 
511 	.vnic_alloc	= nfp_bpf_vnic_alloc,
512 	.vnic_free	= nfp_bpf_vnic_free,
513 
514 	.ctrl_msg_rx	= nfp_bpf_ctrl_msg_rx,
515 	.ctrl_msg_rx_raw	= nfp_bpf_ctrl_msg_rx_raw,
516 
517 	.setup_tc	= nfp_bpf_setup_tc,
518 	.bpf		= nfp_ndo_bpf,
519 	.xdp_offload	= nfp_bpf_xdp_offload,
520 };
521