1 /*
2  * Copyright (C) 2016-2017 Netronome Systems, Inc.
3  *
4  * This software is dual licensed under the GNU General License Version 2,
5  * June 1991 as shown in the file COPYING in the top-level directory of this
6  * source tree or the BSD 2-Clause License provided below.  You have the
7  * option to license this software under the complete terms of either license.
8  *
9  * The BSD 2-Clause License:
10  *
11  *     Redistribution and use in source and binary forms, with or
12  *     without modification, are permitted provided that the following
13  *     conditions are met:
14  *
15  *      1. Redistributions of source code must retain the above
16  *         copyright notice, this list of conditions and the following
17  *         disclaimer.
18  *
19  *      2. Redistributions in binary form must reproduce the above
20  *         copyright notice, this list of conditions and the following
21  *         disclaimer in the documentation and/or other materials
22  *         provided with the distribution.
23  *
24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31  * SOFTWARE.
32  */
33 
34 #ifndef __NFP_BPF_H__
35 #define __NFP_BPF_H__ 1
36 
37 #include <linux/bitfield.h>
38 #include <linux/bpf.h>
39 #include <linux/bpf_verifier.h>
40 #include <linux/kernel.h>
41 #include <linux/list.h>
42 #include <linux/skbuff.h>
43 #include <linux/types.h>
44 #include <linux/wait.h>
45 
46 #include "../nfp_asm.h"
47 #include "fw.h"
48 
49 /* For relocation logic use up-most byte of branch instruction as scratch
50  * area.  Remember to clear this before sending instructions to HW!
51  */
52 #define OP_RELO_TYPE	0xff00000000000000ULL
53 
54 enum nfp_relo_type {
55 	RELO_NONE = 0,
56 	/* standard internal jumps */
57 	RELO_BR_REL,
58 	/* internal jumps to parts of the outro */
59 	RELO_BR_GO_OUT,
60 	RELO_BR_GO_ABORT,
61 	/* external jumps to fixed addresses */
62 	RELO_BR_NEXT_PKT,
63 	RELO_BR_HELPER,
64 	/* immediate relocation against load address */
65 	RELO_IMMED_REL,
66 };
67 
68 /* To make absolute relocated branches (branches other than RELO_BR_REL)
69  * distinguishable in user space dumps from normal jumps, add a large offset
70  * to them.
71  */
72 #define BR_OFF_RELO		15000
73 
74 enum static_regs {
75 	STATIC_REG_IMM		= 21, /* Bank AB */
76 	STATIC_REG_STACK	= 22, /* Bank A */
77 	STATIC_REG_PKT_LEN	= 22, /* Bank B */
78 };
79 
80 enum pkt_vec {
81 	PKT_VEC_PKT_LEN		= 0,
82 	PKT_VEC_PKT_PTR		= 2,
83 };
84 
85 #define pv_len(np)	reg_lm(1, PKT_VEC_PKT_LEN)
86 #define pv_ctm_ptr(np)	reg_lm(1, PKT_VEC_PKT_PTR)
87 
88 #define stack_reg(np)	reg_a(STATIC_REG_STACK)
89 #define stack_imm(np)	imm_b(np)
90 #define plen_reg(np)	reg_b(STATIC_REG_PKT_LEN)
91 #define pptr_reg(np)	pv_ctm_ptr(np)
92 #define imm_a(np)	reg_a(STATIC_REG_IMM)
93 #define imm_b(np)	reg_b(STATIC_REG_IMM)
94 #define imm_both(np)	reg_both(STATIC_REG_IMM)
95 
96 #define NFP_BPF_ABI_FLAGS	reg_imm(0)
97 #define   NFP_BPF_ABI_FLAG_MARK	1
98 
99 /**
100  * struct nfp_app_bpf - bpf app priv structure
101  * @app:		backpointer to the app
102  *
103  * @tag_allocator:	bitmap of control message tags in use
104  * @tag_alloc_next:	next tag bit to allocate
105  * @tag_alloc_last:	next tag bit to be freed
106  *
107  * @cmsg_replies:	received cmsg replies waiting to be consumed
108  * @cmsg_wq:		work queue for waiting for cmsg replies
109  *
110  * @map_list:		list of offloaded maps
111  * @maps_in_use:	number of currently offloaded maps
112  * @map_elems_in_use:	number of elements allocated to offloaded maps
113  *
114  * @adjust_head:	adjust head capability
115  * @adjust_head.flags:		extra flags for adjust head
116  * @adjust_head.off_min:	minimal packet offset within buffer required
117  * @adjust_head.off_max:	maximum packet offset within buffer required
118  * @adjust_head.guaranteed_sub:	negative adjustment guaranteed possible
119  * @adjust_head.guaranteed_add:	positive adjustment guaranteed possible
120  *
121  * @maps:		map capability
122  * @maps.types:			supported map types
123  * @maps.max_maps:		max number of maps supported
124  * @maps.max_elems:		max number of entries in each map
125  * @maps.max_key_sz:		max size of map key
126  * @maps.max_val_sz:		max size of map value
127  * @maps.max_elem_sz:		max size of map entry (key + value)
128  *
129  * @helpers:		helper addressess for various calls
130  * @helpers.map_lookup:		map lookup helper address
131  */
132 struct nfp_app_bpf {
133 	struct nfp_app *app;
134 
135 	DECLARE_BITMAP(tag_allocator, U16_MAX + 1);
136 	u16 tag_alloc_next;
137 	u16 tag_alloc_last;
138 
139 	struct sk_buff_head cmsg_replies;
140 	struct wait_queue_head cmsg_wq;
141 
142 	struct list_head map_list;
143 	unsigned int maps_in_use;
144 	unsigned int map_elems_in_use;
145 
146 	struct nfp_bpf_cap_adjust_head {
147 		u32 flags;
148 		int off_min;
149 		int off_max;
150 		int guaranteed_sub;
151 		int guaranteed_add;
152 	} adjust_head;
153 
154 	struct {
155 		u32 types;
156 		u32 max_maps;
157 		u32 max_elems;
158 		u32 max_key_sz;
159 		u32 max_val_sz;
160 		u32 max_elem_sz;
161 	} maps;
162 
163 	struct {
164 		u32 map_lookup;
165 	} helpers;
166 };
167 
168 /**
169  * struct nfp_bpf_map - private per-map data attached to BPF maps for offload
170  * @offmap:	pointer to the offloaded BPF map
171  * @bpf:	back pointer to bpf app private structure
172  * @tid:	table id identifying map on datapath
173  * @l:		link on the nfp_app_bpf->map_list list
174  */
175 struct nfp_bpf_map {
176 	struct bpf_offloaded_map *offmap;
177 	struct nfp_app_bpf *bpf;
178 	u32 tid;
179 	struct list_head l;
180 };
181 
182 struct nfp_prog;
183 struct nfp_insn_meta;
184 typedef int (*instr_cb_t)(struct nfp_prog *, struct nfp_insn_meta *);
185 
186 #define nfp_prog_first_meta(nfp_prog)					\
187 	list_first_entry(&(nfp_prog)->insns, struct nfp_insn_meta, l)
188 #define nfp_prog_last_meta(nfp_prog)					\
189 	list_last_entry(&(nfp_prog)->insns, struct nfp_insn_meta, l)
190 #define nfp_meta_next(meta)	list_next_entry(meta, l)
191 #define nfp_meta_prev(meta)	list_prev_entry(meta, l)
192 
193 #define FLAG_INSN_IS_JUMP_DST	BIT(0)
194 
195 /**
196  * struct nfp_insn_meta - BPF instruction wrapper
197  * @insn: BPF instruction
198  * @ptr: pointer type for memory operations
199  * @ldst_gather_len: memcpy length gathered from load/store sequence
200  * @paired_st: the paired store insn at the head of the sequence
201  * @ptr_not_const: pointer is not always constant
202  * @jmp_dst: destination info for jump instructions
203  * @func_id: function id for call instructions
204  * @arg1: arg1 for call instructions
205  * @arg2: arg2 for call instructions
206  * @arg2_var_off: arg2 changes stack offset on different paths
207  * @off: index of first generated machine instruction (in nfp_prog.prog)
208  * @n: eBPF instruction number
209  * @flags: eBPF instruction extra optimization flags
210  * @skip: skip this instruction (optimized out)
211  * @double_cb: callback for second part of the instruction
212  * @l: link on nfp_prog->insns list
213  */
214 struct nfp_insn_meta {
215 	struct bpf_insn insn;
216 	union {
217 		struct {
218 			struct bpf_reg_state ptr;
219 			struct bpf_insn *paired_st;
220 			s16 ldst_gather_len;
221 			bool ptr_not_const;
222 		};
223 		struct nfp_insn_meta *jmp_dst;
224 		struct {
225 			u32 func_id;
226 			struct bpf_reg_state arg1;
227 			struct bpf_reg_state arg2;
228 			bool arg2_var_off;
229 		};
230 	};
231 	unsigned int off;
232 	unsigned short n;
233 	unsigned short flags;
234 	bool skip;
235 	instr_cb_t double_cb;
236 
237 	struct list_head l;
238 };
239 
240 #define BPF_SIZE_MASK	0x18
241 
242 static inline u8 mbpf_class(const struct nfp_insn_meta *meta)
243 {
244 	return BPF_CLASS(meta->insn.code);
245 }
246 
247 static inline u8 mbpf_src(const struct nfp_insn_meta *meta)
248 {
249 	return BPF_SRC(meta->insn.code);
250 }
251 
252 static inline u8 mbpf_op(const struct nfp_insn_meta *meta)
253 {
254 	return BPF_OP(meta->insn.code);
255 }
256 
257 static inline u8 mbpf_mode(const struct nfp_insn_meta *meta)
258 {
259 	return BPF_MODE(meta->insn.code);
260 }
261 
262 static inline bool is_mbpf_load(const struct nfp_insn_meta *meta)
263 {
264 	return (meta->insn.code & ~BPF_SIZE_MASK) == (BPF_LDX | BPF_MEM);
265 }
266 
267 static inline bool is_mbpf_store(const struct nfp_insn_meta *meta)
268 {
269 	return (meta->insn.code & ~BPF_SIZE_MASK) == (BPF_STX | BPF_MEM);
270 }
271 
272 /**
273  * struct nfp_prog - nfp BPF program
274  * @bpf: backpointer to the bpf app priv structure
275  * @prog: machine code
276  * @prog_len: number of valid instructions in @prog array
277  * @__prog_alloc_len: alloc size of @prog array
278  * @verifier_meta: temporary storage for verifier's insn meta
279  * @type: BPF program type
280  * @last_bpf_off: address of the last instruction translated from BPF
281  * @tgt_out: jump target for normal exit
282  * @tgt_abort: jump target for abort (e.g. access outside of packet buffer)
283  * @n_translated: number of successfully translated instructions (for errors)
284  * @error: error code if something went wrong
285  * @stack_depth: max stack depth from the verifier
286  * @adjust_head_location: if program has single adjust head call - the insn no.
287  * @insns: list of BPF instruction wrappers (struct nfp_insn_meta)
288  */
289 struct nfp_prog {
290 	struct nfp_app_bpf *bpf;
291 
292 	u64 *prog;
293 	unsigned int prog_len;
294 	unsigned int __prog_alloc_len;
295 
296 	struct nfp_insn_meta *verifier_meta;
297 
298 	enum bpf_prog_type type;
299 
300 	unsigned int last_bpf_off;
301 	unsigned int tgt_out;
302 	unsigned int tgt_abort;
303 
304 	unsigned int n_translated;
305 	int error;
306 
307 	unsigned int stack_depth;
308 	unsigned int adjust_head_location;
309 
310 	struct list_head insns;
311 };
312 
313 /**
314  * struct nfp_bpf_vnic - per-vNIC BPF priv structure
315  * @tc_prog:	currently loaded cls_bpf program
316  * @start_off:	address of the first instruction in the memory
317  * @tgt_done:	jump target to get the next packet
318  */
319 struct nfp_bpf_vnic {
320 	struct bpf_prog *tc_prog;
321 	unsigned int start_off;
322 	unsigned int tgt_done;
323 };
324 
325 void nfp_bpf_jit_prepare(struct nfp_prog *nfp_prog, unsigned int cnt);
326 int nfp_bpf_jit(struct nfp_prog *prog);
327 bool nfp_bpf_supported_opcode(u8 code);
328 
329 extern const struct bpf_prog_offload_ops nfp_bpf_analyzer_ops;
330 
331 struct netdev_bpf;
332 struct nfp_app;
333 struct nfp_net;
334 
335 int nfp_ndo_bpf(struct nfp_app *app, struct nfp_net *nn,
336 		struct netdev_bpf *bpf);
337 int nfp_net_bpf_offload(struct nfp_net *nn, struct bpf_prog *prog,
338 			bool old_prog, struct netlink_ext_ack *extack);
339 
340 struct nfp_insn_meta *
341 nfp_bpf_goto_meta(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
342 		  unsigned int insn_idx, unsigned int n_insns);
343 
344 void *nfp_bpf_relo_for_vnic(struct nfp_prog *nfp_prog, struct nfp_bpf_vnic *bv);
345 
346 long long int
347 nfp_bpf_ctrl_alloc_map(struct nfp_app_bpf *bpf, struct bpf_map *map);
348 void
349 nfp_bpf_ctrl_free_map(struct nfp_app_bpf *bpf, struct nfp_bpf_map *nfp_map);
350 int nfp_bpf_ctrl_getfirst_entry(struct bpf_offloaded_map *offmap,
351 				void *next_key);
352 int nfp_bpf_ctrl_update_entry(struct bpf_offloaded_map *offmap,
353 			      void *key, void *value, u64 flags);
354 int nfp_bpf_ctrl_del_entry(struct bpf_offloaded_map *offmap, void *key);
355 int nfp_bpf_ctrl_lookup_entry(struct bpf_offloaded_map *offmap,
356 			      void *key, void *value);
357 int nfp_bpf_ctrl_getnext_entry(struct bpf_offloaded_map *offmap,
358 			       void *key, void *next_key);
359 
360 void nfp_bpf_ctrl_msg_rx(struct nfp_app *app, struct sk_buff *skb);
361 #endif
362