1 /*
2  * Copyright (C) 2016-2017 Netronome Systems, Inc.
3  *
4  * This software is dual licensed under the GNU General License Version 2,
5  * June 1991 as shown in the file COPYING in the top-level directory of this
6  * source tree or the BSD 2-Clause License provided below.  You have the
7  * option to license this software under the complete terms of either license.
8  *
9  * The BSD 2-Clause License:
10  *
11  *     Redistribution and use in source and binary forms, with or
12  *     without modification, are permitted provided that the following
13  *     conditions are met:
14  *
15  *      1. Redistributions of source code must retain the above
16  *         copyright notice, this list of conditions and the following
17  *         disclaimer.
18  *
19  *      2. Redistributions in binary form must reproduce the above
20  *         copyright notice, this list of conditions and the following
21  *         disclaimer in the documentation and/or other materials
22  *         provided with the distribution.
23  *
24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31  * SOFTWARE.
32  */
33 
34 /*
35  * nfp_net_offload.c
36  * Netronome network device driver: TC offload functions for PF and VF
37  */
38 
39 #define pr_fmt(fmt)	"NFP net bpf: " fmt
40 
41 #include <linux/bpf.h>
42 #include <linux/kernel.h>
43 #include <linux/netdevice.h>
44 #include <linux/pci.h>
45 #include <linux/jiffies.h>
46 #include <linux/timer.h>
47 #include <linux/list.h>
48 #include <linux/mm.h>
49 
50 #include <net/pkt_cls.h>
51 #include <net/tc_act/tc_gact.h>
52 #include <net/tc_act/tc_mirred.h>
53 
54 #include "main.h"
55 #include "../nfp_app.h"
56 #include "../nfp_net_ctrl.h"
57 #include "../nfp_net.h"
58 
59 static int
60 nfp_prog_prepare(struct nfp_prog *nfp_prog, const struct bpf_insn *prog,
61 		 unsigned int cnt)
62 {
63 	struct nfp_insn_meta *meta;
64 	unsigned int i;
65 
66 	for (i = 0; i < cnt; i++) {
67 		meta = kzalloc(sizeof(*meta), GFP_KERNEL);
68 		if (!meta)
69 			return -ENOMEM;
70 
71 		meta->insn = prog[i];
72 		meta->n = i;
73 
74 		list_add_tail(&meta->l, &nfp_prog->insns);
75 	}
76 
77 	nfp_bpf_jit_prepare(nfp_prog, cnt);
78 
79 	return 0;
80 }
81 
82 static void nfp_prog_free(struct nfp_prog *nfp_prog)
83 {
84 	struct nfp_insn_meta *meta, *tmp;
85 
86 	list_for_each_entry_safe(meta, tmp, &nfp_prog->insns, l) {
87 		list_del(&meta->l);
88 		kfree(meta);
89 	}
90 	kfree(nfp_prog);
91 }
92 
93 static int
94 nfp_bpf_verifier_prep(struct nfp_app *app, struct nfp_net *nn,
95 		      struct netdev_bpf *bpf)
96 {
97 	struct bpf_prog *prog = bpf->verifier.prog;
98 	struct nfp_prog *nfp_prog;
99 	int ret;
100 
101 	nfp_prog = kzalloc(sizeof(*nfp_prog), GFP_KERNEL);
102 	if (!nfp_prog)
103 		return -ENOMEM;
104 	prog->aux->offload->dev_priv = nfp_prog;
105 
106 	INIT_LIST_HEAD(&nfp_prog->insns);
107 	nfp_prog->type = prog->type;
108 	nfp_prog->bpf = app->priv;
109 
110 	ret = nfp_prog_prepare(nfp_prog, prog->insnsi, prog->len);
111 	if (ret)
112 		goto err_free;
113 
114 	nfp_prog->verifier_meta = nfp_prog_first_meta(nfp_prog);
115 	bpf->verifier.ops = &nfp_bpf_analyzer_ops;
116 
117 	return 0;
118 
119 err_free:
120 	nfp_prog_free(nfp_prog);
121 
122 	return ret;
123 }
124 
125 static int nfp_bpf_translate(struct nfp_net *nn, struct bpf_prog *prog)
126 {
127 	struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
128 	unsigned int stack_size;
129 	unsigned int max_instr;
130 	int err;
131 
132 	stack_size = nn_readb(nn, NFP_NET_CFG_BPF_STACK_SZ) * 64;
133 	if (prog->aux->stack_depth > stack_size) {
134 		nn_info(nn, "stack too large: program %dB > FW stack %dB\n",
135 			prog->aux->stack_depth, stack_size);
136 		return -EOPNOTSUPP;
137 	}
138 	nfp_prog->stack_depth = round_up(prog->aux->stack_depth, 4);
139 
140 	max_instr = nn_readw(nn, NFP_NET_CFG_BPF_MAX_LEN);
141 	nfp_prog->__prog_alloc_len = max_instr * sizeof(u64);
142 
143 	nfp_prog->prog = kvmalloc(nfp_prog->__prog_alloc_len, GFP_KERNEL);
144 	if (!nfp_prog->prog)
145 		return -ENOMEM;
146 
147 	err = nfp_bpf_jit(nfp_prog);
148 	if (err)
149 		return err;
150 
151 	prog->aux->offload->jited_len = nfp_prog->prog_len * sizeof(u64);
152 	prog->aux->offload->jited_image = nfp_prog->prog;
153 
154 	return 0;
155 }
156 
157 static int nfp_bpf_destroy(struct nfp_net *nn, struct bpf_prog *prog)
158 {
159 	struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
160 
161 	kvfree(nfp_prog->prog);
162 	nfp_prog_free(nfp_prog);
163 
164 	return 0;
165 }
166 
167 static int
168 nfp_bpf_map_get_next_key(struct bpf_offloaded_map *offmap,
169 			 void *key, void *next_key)
170 {
171 	if (!key)
172 		return nfp_bpf_ctrl_getfirst_entry(offmap, next_key);
173 	return nfp_bpf_ctrl_getnext_entry(offmap, key, next_key);
174 }
175 
176 static int
177 nfp_bpf_map_delete_elem(struct bpf_offloaded_map *offmap, void *key)
178 {
179 	if (offmap->map.map_type == BPF_MAP_TYPE_ARRAY)
180 		return -EINVAL;
181 	return nfp_bpf_ctrl_del_entry(offmap, key);
182 }
183 
184 static const struct bpf_map_dev_ops nfp_bpf_map_ops = {
185 	.map_get_next_key	= nfp_bpf_map_get_next_key,
186 	.map_lookup_elem	= nfp_bpf_ctrl_lookup_entry,
187 	.map_update_elem	= nfp_bpf_ctrl_update_entry,
188 	.map_delete_elem	= nfp_bpf_map_delete_elem,
189 };
190 
191 static int
192 nfp_bpf_map_alloc(struct nfp_app_bpf *bpf, struct bpf_offloaded_map *offmap)
193 {
194 	struct nfp_bpf_map *nfp_map;
195 	long long int res;
196 
197 	if (!bpf->maps.types)
198 		return -EOPNOTSUPP;
199 
200 	if (offmap->map.map_flags ||
201 	    offmap->map.numa_node != NUMA_NO_NODE) {
202 		pr_info("map flags are not supported\n");
203 		return -EINVAL;
204 	}
205 
206 	if (!(bpf->maps.types & 1 << offmap->map.map_type)) {
207 		pr_info("map type not supported\n");
208 		return -EOPNOTSUPP;
209 	}
210 	if (bpf->maps.max_maps == bpf->maps_in_use) {
211 		pr_info("too many maps for a device\n");
212 		return -ENOMEM;
213 	}
214 	if (bpf->maps.max_elems - bpf->map_elems_in_use <
215 	    offmap->map.max_entries) {
216 		pr_info("map with too many elements: %u, left: %u\n",
217 			offmap->map.max_entries,
218 			bpf->maps.max_elems - bpf->map_elems_in_use);
219 		return -ENOMEM;
220 	}
221 	if (offmap->map.key_size > bpf->maps.max_key_sz ||
222 	    offmap->map.value_size > bpf->maps.max_val_sz ||
223 	    round_up(offmap->map.key_size, 8) +
224 	    round_up(offmap->map.value_size, 8) > bpf->maps.max_elem_sz) {
225 		pr_info("elements don't fit in device constraints\n");
226 		return -ENOMEM;
227 	}
228 
229 	nfp_map = kzalloc(sizeof(*nfp_map), GFP_USER);
230 	if (!nfp_map)
231 		return -ENOMEM;
232 
233 	offmap->dev_priv = nfp_map;
234 	nfp_map->offmap = offmap;
235 	nfp_map->bpf = bpf;
236 
237 	res = nfp_bpf_ctrl_alloc_map(bpf, &offmap->map);
238 	if (res < 0) {
239 		kfree(nfp_map);
240 		return res;
241 	}
242 
243 	nfp_map->tid = res;
244 	offmap->dev_ops = &nfp_bpf_map_ops;
245 	bpf->maps_in_use++;
246 	bpf->map_elems_in_use += offmap->map.max_entries;
247 	list_add_tail(&nfp_map->l, &bpf->map_list);
248 
249 	return 0;
250 }
251 
252 static int
253 nfp_bpf_map_free(struct nfp_app_bpf *bpf, struct bpf_offloaded_map *offmap)
254 {
255 	struct nfp_bpf_map *nfp_map = offmap->dev_priv;
256 
257 	nfp_bpf_ctrl_free_map(bpf, nfp_map);
258 	list_del_init(&nfp_map->l);
259 	bpf->map_elems_in_use -= offmap->map.max_entries;
260 	bpf->maps_in_use--;
261 	kfree(nfp_map);
262 
263 	return 0;
264 }
265 
266 int nfp_ndo_bpf(struct nfp_app *app, struct nfp_net *nn, struct netdev_bpf *bpf)
267 {
268 	switch (bpf->command) {
269 	case BPF_OFFLOAD_VERIFIER_PREP:
270 		return nfp_bpf_verifier_prep(app, nn, bpf);
271 	case BPF_OFFLOAD_TRANSLATE:
272 		return nfp_bpf_translate(nn, bpf->offload.prog);
273 	case BPF_OFFLOAD_DESTROY:
274 		return nfp_bpf_destroy(nn, bpf->offload.prog);
275 	case BPF_OFFLOAD_MAP_ALLOC:
276 		return nfp_bpf_map_alloc(app->priv, bpf->offmap);
277 	case BPF_OFFLOAD_MAP_FREE:
278 		return nfp_bpf_map_free(app->priv, bpf->offmap);
279 	default:
280 		return -EINVAL;
281 	}
282 }
283 
284 static int
285 nfp_net_bpf_load(struct nfp_net *nn, struct bpf_prog *prog,
286 		 struct netlink_ext_ack *extack)
287 {
288 	struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
289 	unsigned int max_mtu;
290 	dma_addr_t dma_addr;
291 	void *img;
292 	int err;
293 
294 	max_mtu = nn_readb(nn, NFP_NET_CFG_BPF_INL_MTU) * 64 - 32;
295 	if (max_mtu < nn->dp.netdev->mtu) {
296 		NL_SET_ERR_MSG_MOD(extack, "BPF offload not supported with MTU larger than HW packet split boundary");
297 		return -EOPNOTSUPP;
298 	}
299 
300 	img = nfp_bpf_relo_for_vnic(nfp_prog, nn->app_priv);
301 	if (IS_ERR(img))
302 		return PTR_ERR(img);
303 
304 	dma_addr = dma_map_single(nn->dp.dev, img,
305 				  nfp_prog->prog_len * sizeof(u64),
306 				  DMA_TO_DEVICE);
307 	if (dma_mapping_error(nn->dp.dev, dma_addr)) {
308 		kfree(img);
309 		return -ENOMEM;
310 	}
311 
312 	nn_writew(nn, NFP_NET_CFG_BPF_SIZE, nfp_prog->prog_len);
313 	nn_writeq(nn, NFP_NET_CFG_BPF_ADDR, dma_addr);
314 
315 	/* Load up the JITed code */
316 	err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_BPF);
317 	if (err)
318 		NL_SET_ERR_MSG_MOD(extack,
319 				   "FW command error while loading BPF");
320 
321 	dma_unmap_single(nn->dp.dev, dma_addr, nfp_prog->prog_len * sizeof(u64),
322 			 DMA_TO_DEVICE);
323 	kfree(img);
324 
325 	return err;
326 }
327 
328 static void
329 nfp_net_bpf_start(struct nfp_net *nn, struct netlink_ext_ack *extack)
330 {
331 	int err;
332 
333 	/* Enable passing packets through BPF function */
334 	nn->dp.ctrl |= NFP_NET_CFG_CTRL_BPF;
335 	nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl);
336 	err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN);
337 	if (err)
338 		NL_SET_ERR_MSG_MOD(extack,
339 				   "FW command error while enabling BPF");
340 }
341 
342 static int nfp_net_bpf_stop(struct nfp_net *nn)
343 {
344 	if (!(nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF))
345 		return 0;
346 
347 	nn->dp.ctrl &= ~NFP_NET_CFG_CTRL_BPF;
348 	nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl);
349 
350 	return nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN);
351 }
352 
353 int nfp_net_bpf_offload(struct nfp_net *nn, struct bpf_prog *prog,
354 			bool old_prog, struct netlink_ext_ack *extack)
355 {
356 	int err;
357 
358 	if (prog) {
359 		struct bpf_prog_offload *offload = prog->aux->offload;
360 
361 		if (!offload)
362 			return -EINVAL;
363 		if (offload->netdev != nn->dp.netdev)
364 			return -EINVAL;
365 	}
366 
367 	if (prog && old_prog) {
368 		u8 cap;
369 
370 		cap = nn_readb(nn, NFP_NET_CFG_BPF_CAP);
371 		if (!(cap & NFP_NET_BPF_CAP_RELO)) {
372 			NL_SET_ERR_MSG_MOD(extack,
373 					   "FW does not support live reload");
374 			return -EBUSY;
375 		}
376 	}
377 
378 	/* Something else is loaded, different program type? */
379 	if (!old_prog && nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF)
380 		return -EBUSY;
381 
382 	if (old_prog && !prog)
383 		return nfp_net_bpf_stop(nn);
384 
385 	err = nfp_net_bpf_load(nn, prog, extack);
386 	if (err)
387 		return err;
388 
389 	if (!old_prog)
390 		nfp_net_bpf_start(nn, extack);
391 
392 	return 0;
393 }
394