1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2 /* Copyright (C) 2016-2018 Netronome Systems, Inc. */
3 
4 /*
5  * nfp_net_offload.c
6  * Netronome network device driver: TC offload functions for PF and VF
7  */
8 
9 #define pr_fmt(fmt)	"NFP net bpf: " fmt
10 
11 #include <linux/bpf.h>
12 #include <linux/kernel.h>
13 #include <linux/netdevice.h>
14 #include <linux/pci.h>
15 #include <linux/jiffies.h>
16 #include <linux/timer.h>
17 #include <linux/list.h>
18 #include <linux/mm.h>
19 
20 #include <net/pkt_cls.h>
21 #include <net/tc_act/tc_gact.h>
22 #include <net/tc_act/tc_mirred.h>
23 
24 #include "main.h"
25 #include "../ccm.h"
26 #include "../nfp_app.h"
27 #include "../nfp_net_ctrl.h"
28 #include "../nfp_net.h"
29 
30 static int
31 nfp_map_ptr_record(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog,
32 		   struct bpf_map *map)
33 {
34 	struct nfp_bpf_neutral_map *record;
35 	int err;
36 
37 	/* Reuse path - other offloaded program is already tracking this map. */
38 	record = rhashtable_lookup_fast(&bpf->maps_neutral, &map->id,
39 					nfp_bpf_maps_neutral_params);
40 	if (record) {
41 		nfp_prog->map_records[nfp_prog->map_records_cnt++] = record;
42 		record->count++;
43 		return 0;
44 	}
45 
46 	/* Grab a single ref to the map for our record.  The prog destroy ndo
47 	 * happens after free_used_maps().
48 	 */
49 	bpf_map_inc(map);
50 
51 	record = kmalloc(sizeof(*record), GFP_KERNEL);
52 	if (!record) {
53 		err = -ENOMEM;
54 		goto err_map_put;
55 	}
56 
57 	record->ptr = map;
58 	record->map_id = map->id;
59 	record->count = 1;
60 
61 	err = rhashtable_insert_fast(&bpf->maps_neutral, &record->l,
62 				     nfp_bpf_maps_neutral_params);
63 	if (err)
64 		goto err_free_rec;
65 
66 	nfp_prog->map_records[nfp_prog->map_records_cnt++] = record;
67 
68 	return 0;
69 
70 err_free_rec:
71 	kfree(record);
72 err_map_put:
73 	bpf_map_put(map);
74 	return err;
75 }
76 
77 static void
78 nfp_map_ptrs_forget(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog)
79 {
80 	bool freed = false;
81 	int i;
82 
83 	for (i = 0; i < nfp_prog->map_records_cnt; i++) {
84 		if (--nfp_prog->map_records[i]->count) {
85 			nfp_prog->map_records[i] = NULL;
86 			continue;
87 		}
88 
89 		WARN_ON(rhashtable_remove_fast(&bpf->maps_neutral,
90 					       &nfp_prog->map_records[i]->l,
91 					       nfp_bpf_maps_neutral_params));
92 		freed = true;
93 	}
94 
95 	if (freed) {
96 		synchronize_rcu();
97 
98 		for (i = 0; i < nfp_prog->map_records_cnt; i++)
99 			if (nfp_prog->map_records[i]) {
100 				bpf_map_put(nfp_prog->map_records[i]->ptr);
101 				kfree(nfp_prog->map_records[i]);
102 			}
103 	}
104 
105 	kfree(nfp_prog->map_records);
106 	nfp_prog->map_records = NULL;
107 	nfp_prog->map_records_cnt = 0;
108 }
109 
110 static int
111 nfp_map_ptrs_record(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog,
112 		    struct bpf_prog *prog)
113 {
114 	int i, cnt, err = 0;
115 
116 	mutex_lock(&prog->aux->used_maps_mutex);
117 
118 	/* Quickly count the maps we will have to remember */
119 	cnt = 0;
120 	for (i = 0; i < prog->aux->used_map_cnt; i++)
121 		if (bpf_map_offload_neutral(prog->aux->used_maps[i]))
122 			cnt++;
123 	if (!cnt)
124 		goto out;
125 
126 	nfp_prog->map_records = kmalloc_array(cnt,
127 					      sizeof(nfp_prog->map_records[0]),
128 					      GFP_KERNEL);
129 	if (!nfp_prog->map_records) {
130 		err = -ENOMEM;
131 		goto out;
132 	}
133 
134 	for (i = 0; i < prog->aux->used_map_cnt; i++)
135 		if (bpf_map_offload_neutral(prog->aux->used_maps[i])) {
136 			err = nfp_map_ptr_record(bpf, nfp_prog,
137 						 prog->aux->used_maps[i]);
138 			if (err) {
139 				nfp_map_ptrs_forget(bpf, nfp_prog);
140 				goto out;
141 			}
142 		}
143 	WARN_ON(cnt != nfp_prog->map_records_cnt);
144 
145 out:
146 	mutex_unlock(&prog->aux->used_maps_mutex);
147 	return err;
148 }
149 
150 static int
151 nfp_prog_prepare(struct nfp_prog *nfp_prog, const struct bpf_insn *prog,
152 		 unsigned int cnt)
153 {
154 	struct nfp_insn_meta *meta;
155 	unsigned int i;
156 
157 	for (i = 0; i < cnt; i++) {
158 		meta = kzalloc(sizeof(*meta), GFP_KERNEL);
159 		if (!meta)
160 			return -ENOMEM;
161 
162 		meta->insn = prog[i];
163 		meta->n = i;
164 		if (is_mbpf_alu(meta)) {
165 			meta->umin_src = U64_MAX;
166 			meta->umin_dst = U64_MAX;
167 		}
168 
169 		list_add_tail(&meta->l, &nfp_prog->insns);
170 	}
171 	nfp_prog->n_insns = cnt;
172 
173 	nfp_bpf_jit_prepare(nfp_prog);
174 
175 	return 0;
176 }
177 
178 static void nfp_prog_free(struct nfp_prog *nfp_prog)
179 {
180 	struct nfp_insn_meta *meta, *tmp;
181 
182 	kfree(nfp_prog->subprog);
183 
184 	list_for_each_entry_safe(meta, tmp, &nfp_prog->insns, l) {
185 		list_del(&meta->l);
186 		kfree(meta);
187 	}
188 	kfree(nfp_prog);
189 }
190 
191 static int nfp_bpf_verifier_prep(struct bpf_prog *prog)
192 {
193 	struct nfp_prog *nfp_prog;
194 	int ret;
195 
196 	nfp_prog = kzalloc(sizeof(*nfp_prog), GFP_KERNEL);
197 	if (!nfp_prog)
198 		return -ENOMEM;
199 	prog->aux->offload->dev_priv = nfp_prog;
200 
201 	INIT_LIST_HEAD(&nfp_prog->insns);
202 	nfp_prog->type = prog->type;
203 	nfp_prog->bpf = bpf_offload_dev_priv(prog->aux->offload->offdev);
204 
205 	ret = nfp_prog_prepare(nfp_prog, prog->insnsi, prog->len);
206 	if (ret)
207 		goto err_free;
208 
209 	nfp_prog->verifier_meta = nfp_prog_first_meta(nfp_prog);
210 
211 	return 0;
212 
213 err_free:
214 	nfp_prog_free(nfp_prog);
215 
216 	return ret;
217 }
218 
219 static int nfp_bpf_translate(struct bpf_prog *prog)
220 {
221 	struct nfp_net *nn = netdev_priv(prog->aux->offload->netdev);
222 	struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
223 	unsigned int max_instr;
224 	int err;
225 
226 	/* We depend on dead code elimination succeeding */
227 	if (prog->aux->offload->opt_failed)
228 		return -EINVAL;
229 
230 	max_instr = nn_readw(nn, NFP_NET_CFG_BPF_MAX_LEN);
231 	nfp_prog->__prog_alloc_len = max_instr * sizeof(u64);
232 
233 	nfp_prog->prog = kvmalloc(nfp_prog->__prog_alloc_len, GFP_KERNEL);
234 	if (!nfp_prog->prog)
235 		return -ENOMEM;
236 
237 	err = nfp_bpf_jit(nfp_prog);
238 	if (err)
239 		return err;
240 
241 	prog->aux->offload->jited_len = nfp_prog->prog_len * sizeof(u64);
242 	prog->aux->offload->jited_image = nfp_prog->prog;
243 
244 	return nfp_map_ptrs_record(nfp_prog->bpf, nfp_prog, prog);
245 }
246 
247 static void nfp_bpf_destroy(struct bpf_prog *prog)
248 {
249 	struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
250 
251 	kvfree(nfp_prog->prog);
252 	nfp_map_ptrs_forget(nfp_prog->bpf, nfp_prog);
253 	nfp_prog_free(nfp_prog);
254 }
255 
256 /* Atomic engine requires values to be in big endian, we need to byte swap
257  * the value words used with xadd.
258  */
259 static void nfp_map_bpf_byte_swap(struct nfp_bpf_map *nfp_map, void *value)
260 {
261 	u32 *word = value;
262 	unsigned int i;
263 
264 	for (i = 0; i < DIV_ROUND_UP(nfp_map->offmap->map.value_size, 4); i++)
265 		if (nfp_map->use_map[i].type == NFP_MAP_USE_ATOMIC_CNT)
266 			word[i] = (__force u32)cpu_to_be32(word[i]);
267 }
268 
269 /* Mark value as unsafely initialized in case it becomes atomic later
270  * and we didn't byte swap something non-byte swap neutral.
271  */
272 static void
273 nfp_map_bpf_byte_swap_record(struct nfp_bpf_map *nfp_map, void *value)
274 {
275 	u32 *word = value;
276 	unsigned int i;
277 
278 	for (i = 0; i < DIV_ROUND_UP(nfp_map->offmap->map.value_size, 4); i++)
279 		if (nfp_map->use_map[i].type == NFP_MAP_UNUSED &&
280 		    word[i] != (__force u32)cpu_to_be32(word[i]))
281 			nfp_map->use_map[i].non_zero_update = 1;
282 }
283 
284 static int
285 nfp_bpf_map_lookup_entry(struct bpf_offloaded_map *offmap,
286 			 void *key, void *value)
287 {
288 	int err;
289 
290 	err = nfp_bpf_ctrl_lookup_entry(offmap, key, value);
291 	if (err)
292 		return err;
293 
294 	nfp_map_bpf_byte_swap(offmap->dev_priv, value);
295 	return 0;
296 }
297 
298 static int
299 nfp_bpf_map_update_entry(struct bpf_offloaded_map *offmap,
300 			 void *key, void *value, u64 flags)
301 {
302 	nfp_map_bpf_byte_swap(offmap->dev_priv, value);
303 	nfp_map_bpf_byte_swap_record(offmap->dev_priv, value);
304 	return nfp_bpf_ctrl_update_entry(offmap, key, value, flags);
305 }
306 
307 static int
308 nfp_bpf_map_get_next_key(struct bpf_offloaded_map *offmap,
309 			 void *key, void *next_key)
310 {
311 	if (!key)
312 		return nfp_bpf_ctrl_getfirst_entry(offmap, next_key);
313 	return nfp_bpf_ctrl_getnext_entry(offmap, key, next_key);
314 }
315 
316 static int
317 nfp_bpf_map_delete_elem(struct bpf_offloaded_map *offmap, void *key)
318 {
319 	if (offmap->map.map_type == BPF_MAP_TYPE_ARRAY)
320 		return -EINVAL;
321 	return nfp_bpf_ctrl_del_entry(offmap, key);
322 }
323 
324 static const struct bpf_map_dev_ops nfp_bpf_map_ops = {
325 	.map_get_next_key	= nfp_bpf_map_get_next_key,
326 	.map_lookup_elem	= nfp_bpf_map_lookup_entry,
327 	.map_update_elem	= nfp_bpf_map_update_entry,
328 	.map_delete_elem	= nfp_bpf_map_delete_elem,
329 };
330 
331 static int
332 nfp_bpf_map_alloc(struct nfp_app_bpf *bpf, struct bpf_offloaded_map *offmap)
333 {
334 	struct nfp_bpf_map *nfp_map;
335 	unsigned int use_map_size;
336 	long long int res;
337 
338 	if (!bpf->maps.types)
339 		return -EOPNOTSUPP;
340 
341 	if (offmap->map.map_flags ||
342 	    offmap->map.numa_node != NUMA_NO_NODE) {
343 		pr_info("map flags are not supported\n");
344 		return -EINVAL;
345 	}
346 
347 	if (!(bpf->maps.types & 1 << offmap->map.map_type)) {
348 		pr_info("map type not supported\n");
349 		return -EOPNOTSUPP;
350 	}
351 	if (bpf->maps.max_maps == bpf->maps_in_use) {
352 		pr_info("too many maps for a device\n");
353 		return -ENOMEM;
354 	}
355 	if (bpf->maps.max_elems - bpf->map_elems_in_use <
356 	    offmap->map.max_entries) {
357 		pr_info("map with too many elements: %u, left: %u\n",
358 			offmap->map.max_entries,
359 			bpf->maps.max_elems - bpf->map_elems_in_use);
360 		return -ENOMEM;
361 	}
362 
363 	if (round_up(offmap->map.key_size, 8) +
364 	    round_up(offmap->map.value_size, 8) > bpf->maps.max_elem_sz) {
365 		pr_info("map elements too large: %u, FW max element size (key+value): %u\n",
366 			round_up(offmap->map.key_size, 8) +
367 			round_up(offmap->map.value_size, 8),
368 			bpf->maps.max_elem_sz);
369 		return -ENOMEM;
370 	}
371 	if (offmap->map.key_size > bpf->maps.max_key_sz) {
372 		pr_info("map key size %u, FW max is %u\n",
373 			offmap->map.key_size, bpf->maps.max_key_sz);
374 		return -ENOMEM;
375 	}
376 	if (offmap->map.value_size > bpf->maps.max_val_sz) {
377 		pr_info("map value size %u, FW max is %u\n",
378 			offmap->map.value_size, bpf->maps.max_val_sz);
379 		return -ENOMEM;
380 	}
381 
382 	use_map_size = DIV_ROUND_UP(offmap->map.value_size, 4) *
383 		       sizeof_field(struct nfp_bpf_map, use_map[0]);
384 
385 	nfp_map = kzalloc(sizeof(*nfp_map) + use_map_size, GFP_USER);
386 	if (!nfp_map)
387 		return -ENOMEM;
388 
389 	offmap->dev_priv = nfp_map;
390 	nfp_map->offmap = offmap;
391 	nfp_map->bpf = bpf;
392 	spin_lock_init(&nfp_map->cache_lock);
393 
394 	res = nfp_bpf_ctrl_alloc_map(bpf, &offmap->map);
395 	if (res < 0) {
396 		kfree(nfp_map);
397 		return res;
398 	}
399 
400 	nfp_map->tid = res;
401 	offmap->dev_ops = &nfp_bpf_map_ops;
402 	bpf->maps_in_use++;
403 	bpf->map_elems_in_use += offmap->map.max_entries;
404 	list_add_tail(&nfp_map->l, &bpf->map_list);
405 
406 	return 0;
407 }
408 
409 static int
410 nfp_bpf_map_free(struct nfp_app_bpf *bpf, struct bpf_offloaded_map *offmap)
411 {
412 	struct nfp_bpf_map *nfp_map = offmap->dev_priv;
413 
414 	nfp_bpf_ctrl_free_map(bpf, nfp_map);
415 	dev_consume_skb_any(nfp_map->cache);
416 	WARN_ON_ONCE(nfp_map->cache_blockers);
417 	list_del_init(&nfp_map->l);
418 	bpf->map_elems_in_use -= offmap->map.max_entries;
419 	bpf->maps_in_use--;
420 	kfree(nfp_map);
421 
422 	return 0;
423 }
424 
425 int nfp_ndo_bpf(struct nfp_app *app, struct nfp_net *nn, struct netdev_bpf *bpf)
426 {
427 	switch (bpf->command) {
428 	case BPF_OFFLOAD_MAP_ALLOC:
429 		return nfp_bpf_map_alloc(app->priv, bpf->offmap);
430 	case BPF_OFFLOAD_MAP_FREE:
431 		return nfp_bpf_map_free(app->priv, bpf->offmap);
432 	default:
433 		return -EINVAL;
434 	}
435 }
436 
437 static unsigned long
438 nfp_bpf_perf_event_copy(void *dst, const void *src,
439 			unsigned long off, unsigned long len)
440 {
441 	memcpy(dst, src + off, len);
442 	return 0;
443 }
444 
445 int nfp_bpf_event_output(struct nfp_app_bpf *bpf, const void *data,
446 			 unsigned int len)
447 {
448 	struct cmsg_bpf_event *cbe = (void *)data;
449 	struct nfp_bpf_neutral_map *record;
450 	u32 pkt_size, data_size, map_id;
451 	u64 map_id_full;
452 
453 	if (len < sizeof(struct cmsg_bpf_event))
454 		return -EINVAL;
455 
456 	pkt_size = be32_to_cpu(cbe->pkt_size);
457 	data_size = be32_to_cpu(cbe->data_size);
458 	map_id_full = be64_to_cpu(cbe->map_ptr);
459 	map_id = map_id_full;
460 
461 	if (len < sizeof(struct cmsg_bpf_event) + pkt_size + data_size)
462 		return -EINVAL;
463 	if (cbe->hdr.ver != NFP_CCM_ABI_VERSION)
464 		return -EINVAL;
465 
466 	rcu_read_lock();
467 	record = rhashtable_lookup(&bpf->maps_neutral, &map_id,
468 				   nfp_bpf_maps_neutral_params);
469 	if (!record || map_id_full > U32_MAX) {
470 		rcu_read_unlock();
471 		cmsg_warn(bpf, "perf event: map id %lld (0x%llx) not recognized, dropping event\n",
472 			  map_id_full, map_id_full);
473 		return -EINVAL;
474 	}
475 
476 	bpf_event_output(record->ptr, be32_to_cpu(cbe->cpu_id),
477 			 &cbe->data[round_up(pkt_size, 4)], data_size,
478 			 cbe->data, pkt_size, nfp_bpf_perf_event_copy);
479 	rcu_read_unlock();
480 
481 	return 0;
482 }
483 
484 static int
485 nfp_net_bpf_load(struct nfp_net *nn, struct bpf_prog *prog,
486 		 struct netlink_ext_ack *extack)
487 {
488 	struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
489 	unsigned int fw_mtu, pkt_off, max_stack, max_prog_len;
490 	dma_addr_t dma_addr;
491 	void *img;
492 	int err;
493 
494 	fw_mtu = nn_readb(nn, NFP_NET_CFG_BPF_INL_MTU) * 64 - 32;
495 	pkt_off = min(prog->aux->max_pkt_offset, nn->dp.netdev->mtu);
496 	if (fw_mtu < pkt_off) {
497 		NL_SET_ERR_MSG_MOD(extack, "BPF offload not supported with potential packet access beyond HW packet split boundary");
498 		return -EOPNOTSUPP;
499 	}
500 
501 	max_stack = nn_readb(nn, NFP_NET_CFG_BPF_STACK_SZ) * 64;
502 	if (nfp_prog->stack_size > max_stack) {
503 		NL_SET_ERR_MSG_MOD(extack, "stack too large");
504 		return -EOPNOTSUPP;
505 	}
506 
507 	max_prog_len = nn_readw(nn, NFP_NET_CFG_BPF_MAX_LEN);
508 	if (nfp_prog->prog_len > max_prog_len) {
509 		NL_SET_ERR_MSG_MOD(extack, "program too long");
510 		return -EOPNOTSUPP;
511 	}
512 
513 	img = nfp_bpf_relo_for_vnic(nfp_prog, nn->app_priv);
514 	if (IS_ERR(img))
515 		return PTR_ERR(img);
516 
517 	dma_addr = dma_map_single(nn->dp.dev, img,
518 				  nfp_prog->prog_len * sizeof(u64),
519 				  DMA_TO_DEVICE);
520 	if (dma_mapping_error(nn->dp.dev, dma_addr)) {
521 		kfree(img);
522 		return -ENOMEM;
523 	}
524 
525 	nn_writew(nn, NFP_NET_CFG_BPF_SIZE, nfp_prog->prog_len);
526 	nn_writeq(nn, NFP_NET_CFG_BPF_ADDR, dma_addr);
527 
528 	/* Load up the JITed code */
529 	err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_BPF);
530 	if (err)
531 		NL_SET_ERR_MSG_MOD(extack,
532 				   "FW command error while loading BPF");
533 
534 	dma_unmap_single(nn->dp.dev, dma_addr, nfp_prog->prog_len * sizeof(u64),
535 			 DMA_TO_DEVICE);
536 	kfree(img);
537 
538 	return err;
539 }
540 
541 static void
542 nfp_net_bpf_start(struct nfp_net *nn, struct netlink_ext_ack *extack)
543 {
544 	int err;
545 
546 	/* Enable passing packets through BPF function */
547 	nn->dp.ctrl |= NFP_NET_CFG_CTRL_BPF;
548 	nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl);
549 	err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN);
550 	if (err)
551 		NL_SET_ERR_MSG_MOD(extack,
552 				   "FW command error while enabling BPF");
553 }
554 
555 static int nfp_net_bpf_stop(struct nfp_net *nn)
556 {
557 	if (!(nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF))
558 		return 0;
559 
560 	nn->dp.ctrl &= ~NFP_NET_CFG_CTRL_BPF;
561 	nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl);
562 
563 	return nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN);
564 }
565 
566 int nfp_net_bpf_offload(struct nfp_net *nn, struct bpf_prog *prog,
567 			bool old_prog, struct netlink_ext_ack *extack)
568 {
569 	int err;
570 
571 	if (prog && !bpf_offload_dev_match(prog, nn->dp.netdev))
572 		return -EINVAL;
573 
574 	if (prog && old_prog) {
575 		u8 cap;
576 
577 		cap = nn_readb(nn, NFP_NET_CFG_BPF_CAP);
578 		if (!(cap & NFP_NET_BPF_CAP_RELO)) {
579 			NL_SET_ERR_MSG_MOD(extack,
580 					   "FW does not support live reload");
581 			return -EBUSY;
582 		}
583 	}
584 
585 	/* Something else is loaded, different program type? */
586 	if (!old_prog && nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF)
587 		return -EBUSY;
588 
589 	if (old_prog && !prog)
590 		return nfp_net_bpf_stop(nn);
591 
592 	err = nfp_net_bpf_load(nn, prog, extack);
593 	if (err)
594 		return err;
595 
596 	if (!old_prog)
597 		nfp_net_bpf_start(nn, extack);
598 
599 	return 0;
600 }
601 
602 const struct bpf_prog_offload_ops nfp_bpf_dev_ops = {
603 	.insn_hook	= nfp_verify_insn,
604 	.finalize	= nfp_bpf_finalize,
605 	.replace_insn	= nfp_bpf_opt_replace_insn,
606 	.remove_insns	= nfp_bpf_opt_remove_insns,
607 	.prepare	= nfp_bpf_verifier_prep,
608 	.translate	= nfp_bpf_translate,
609 	.destroy	= nfp_bpf_destroy,
610 };
611