1d9ae7f2bSJakub Kicinski /*
2d9ae7f2bSJakub Kicinski  * Copyright (C) 2016 Netronome Systems, Inc.
3d9ae7f2bSJakub Kicinski  *
4d9ae7f2bSJakub Kicinski  * This software is dual licensed under the GNU General License Version 2,
5d9ae7f2bSJakub Kicinski  * June 1991 as shown in the file COPYING in the top-level directory of this
6d9ae7f2bSJakub Kicinski  * source tree or the BSD 2-Clause License provided below.  You have the
7d9ae7f2bSJakub Kicinski  * option to license this software under the complete terms of either license.
8d9ae7f2bSJakub Kicinski  *
9d9ae7f2bSJakub Kicinski  * The BSD 2-Clause License:
10d9ae7f2bSJakub Kicinski  *
11d9ae7f2bSJakub Kicinski  *     Redistribution and use in source and binary forms, with or
12d9ae7f2bSJakub Kicinski  *     without modification, are permitted provided that the following
13d9ae7f2bSJakub Kicinski  *     conditions are met:
14d9ae7f2bSJakub Kicinski  *
15d9ae7f2bSJakub Kicinski  *      1. Redistributions of source code must retain the above
16d9ae7f2bSJakub Kicinski  *         copyright notice, this list of conditions and the following
17d9ae7f2bSJakub Kicinski  *         disclaimer.
18d9ae7f2bSJakub Kicinski  *
19d9ae7f2bSJakub Kicinski  *      2. Redistributions in binary form must reproduce the above
20d9ae7f2bSJakub Kicinski  *         copyright notice, this list of conditions and the following
21d9ae7f2bSJakub Kicinski  *         disclaimer in the documentation and/or other materials
22d9ae7f2bSJakub Kicinski  *         provided with the distribution.
23d9ae7f2bSJakub Kicinski  *
24d9ae7f2bSJakub Kicinski  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25d9ae7f2bSJakub Kicinski  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26d9ae7f2bSJakub Kicinski  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27d9ae7f2bSJakub Kicinski  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28d9ae7f2bSJakub Kicinski  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29d9ae7f2bSJakub Kicinski  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30d9ae7f2bSJakub Kicinski  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31d9ae7f2bSJakub Kicinski  * SOFTWARE.
32d9ae7f2bSJakub Kicinski  */
33d9ae7f2bSJakub Kicinski 
34d9ae7f2bSJakub Kicinski #define pr_fmt(fmt)	"NFP net bpf: " fmt
35d9ae7f2bSJakub Kicinski 
36d9ae7f2bSJakub Kicinski #include <linux/kernel.h>
37d9ae7f2bSJakub Kicinski #include <linux/bpf.h>
38d9ae7f2bSJakub Kicinski #include <linux/filter.h>
39d9ae7f2bSJakub Kicinski #include <linux/pkt_cls.h>
40d9ae7f2bSJakub Kicinski #include <linux/unistd.h>
41d9ae7f2bSJakub Kicinski 
42d9ae7f2bSJakub Kicinski #include "main.h"
43d9ae7f2bSJakub Kicinski #include "../nfp_asm.h"
44d9ae7f2bSJakub Kicinski 
45d9ae7f2bSJakub Kicinski /* --- NFP prog --- */
46d9ae7f2bSJakub Kicinski /* Foreach "multiple" entries macros provide pos and next<n> pointers.
47d9ae7f2bSJakub Kicinski  * It's safe to modify the next pointers (but not pos).
48d9ae7f2bSJakub Kicinski  */
49d9ae7f2bSJakub Kicinski #define nfp_for_each_insn_walk2(nfp_prog, pos, next)			\
50d9ae7f2bSJakub Kicinski 	for (pos = list_first_entry(&(nfp_prog)->insns, typeof(*pos), l), \
51d9ae7f2bSJakub Kicinski 	     next = list_next_entry(pos, l);			\
52d9ae7f2bSJakub Kicinski 	     &(nfp_prog)->insns != &pos->l &&			\
53d9ae7f2bSJakub Kicinski 	     &(nfp_prog)->insns != &next->l;			\
54d9ae7f2bSJakub Kicinski 	     pos = nfp_meta_next(pos),				\
55d9ae7f2bSJakub Kicinski 	     next = nfp_meta_next(pos))
56d9ae7f2bSJakub Kicinski 
57d9ae7f2bSJakub Kicinski #define nfp_for_each_insn_walk3(nfp_prog, pos, next, next2)		\
58d9ae7f2bSJakub Kicinski 	for (pos = list_first_entry(&(nfp_prog)->insns, typeof(*pos), l), \
59d9ae7f2bSJakub Kicinski 	     next = list_next_entry(pos, l),			\
60d9ae7f2bSJakub Kicinski 	     next2 = list_next_entry(next, l);			\
61d9ae7f2bSJakub Kicinski 	     &(nfp_prog)->insns != &pos->l &&			\
62d9ae7f2bSJakub Kicinski 	     &(nfp_prog)->insns != &next->l &&			\
63d9ae7f2bSJakub Kicinski 	     &(nfp_prog)->insns != &next2->l;			\
64d9ae7f2bSJakub Kicinski 	     pos = nfp_meta_next(pos),				\
65d9ae7f2bSJakub Kicinski 	     next = nfp_meta_next(pos),				\
66d9ae7f2bSJakub Kicinski 	     next2 = nfp_meta_next(next))
67d9ae7f2bSJakub Kicinski 
68d9ae7f2bSJakub Kicinski static bool
69d9ae7f2bSJakub Kicinski nfp_meta_has_next(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
70d9ae7f2bSJakub Kicinski {
71d9ae7f2bSJakub Kicinski 	return meta->l.next != &nfp_prog->insns;
72d9ae7f2bSJakub Kicinski }
73d9ae7f2bSJakub Kicinski 
74d9ae7f2bSJakub Kicinski static bool
75d9ae7f2bSJakub Kicinski nfp_meta_has_prev(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
76d9ae7f2bSJakub Kicinski {
77d9ae7f2bSJakub Kicinski 	return meta->l.prev != &nfp_prog->insns;
78d9ae7f2bSJakub Kicinski }
79d9ae7f2bSJakub Kicinski 
80d9ae7f2bSJakub Kicinski static void nfp_prog_free(struct nfp_prog *nfp_prog)
81d9ae7f2bSJakub Kicinski {
82d9ae7f2bSJakub Kicinski 	struct nfp_insn_meta *meta, *tmp;
83d9ae7f2bSJakub Kicinski 
84d9ae7f2bSJakub Kicinski 	list_for_each_entry_safe(meta, tmp, &nfp_prog->insns, l) {
85d9ae7f2bSJakub Kicinski 		list_del(&meta->l);
86d9ae7f2bSJakub Kicinski 		kfree(meta);
87d9ae7f2bSJakub Kicinski 	}
88d9ae7f2bSJakub Kicinski 	kfree(nfp_prog);
89d9ae7f2bSJakub Kicinski }
90d9ae7f2bSJakub Kicinski 
91d9ae7f2bSJakub Kicinski static void nfp_prog_push(struct nfp_prog *nfp_prog, u64 insn)
92d9ae7f2bSJakub Kicinski {
93d9ae7f2bSJakub Kicinski 	if (nfp_prog->__prog_alloc_len == nfp_prog->prog_len) {
94d9ae7f2bSJakub Kicinski 		nfp_prog->error = -ENOSPC;
95d9ae7f2bSJakub Kicinski 		return;
96d9ae7f2bSJakub Kicinski 	}
97d9ae7f2bSJakub Kicinski 
98d9ae7f2bSJakub Kicinski 	nfp_prog->prog[nfp_prog->prog_len] = insn;
99d9ae7f2bSJakub Kicinski 	nfp_prog->prog_len++;
100d9ae7f2bSJakub Kicinski }
101d9ae7f2bSJakub Kicinski 
102d9ae7f2bSJakub Kicinski static unsigned int nfp_prog_current_offset(struct nfp_prog *nfp_prog)
103d9ae7f2bSJakub Kicinski {
104d9ae7f2bSJakub Kicinski 	return nfp_prog->start_off + nfp_prog->prog_len;
105d9ae7f2bSJakub Kicinski }
106d9ae7f2bSJakub Kicinski 
107d9ae7f2bSJakub Kicinski static unsigned int
108d9ae7f2bSJakub Kicinski nfp_prog_offset_to_index(struct nfp_prog *nfp_prog, unsigned int offset)
109d9ae7f2bSJakub Kicinski {
110d9ae7f2bSJakub Kicinski 	return offset - nfp_prog->start_off;
111d9ae7f2bSJakub Kicinski }
112d9ae7f2bSJakub Kicinski 
113d9ae7f2bSJakub Kicinski /* --- Emitters --- */
114d9ae7f2bSJakub Kicinski static void
115d9ae7f2bSJakub Kicinski __emit_cmd(struct nfp_prog *nfp_prog, enum cmd_tgt_map op,
116d9ae7f2bSJakub Kicinski 	   u8 mode, u8 xfer, u8 areg, u8 breg, u8 size, bool sync)
117d9ae7f2bSJakub Kicinski {
118d9ae7f2bSJakub Kicinski 	enum cmd_ctx_swap ctx;
119d9ae7f2bSJakub Kicinski 	u64 insn;
120d9ae7f2bSJakub Kicinski 
121d9ae7f2bSJakub Kicinski 	if (sync)
122d9ae7f2bSJakub Kicinski 		ctx = CMD_CTX_SWAP;
123d9ae7f2bSJakub Kicinski 	else
124d9ae7f2bSJakub Kicinski 		ctx = CMD_CTX_NO_SWAP;
125d9ae7f2bSJakub Kicinski 
126d9ae7f2bSJakub Kicinski 	insn =	FIELD_PREP(OP_CMD_A_SRC, areg) |
127d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_CMD_CTX, ctx) |
128d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_CMD_B_SRC, breg) |
129d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_CMD_TOKEN, cmd_tgt_act[op].token) |
130d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_CMD_XFER, xfer) |
131d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_CMD_CNT, size) |
132d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_CMD_SIG, sync) |
133d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_CMD_TGT_CMD, cmd_tgt_act[op].tgt_cmd) |
134d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_CMD_MODE, mode);
135d9ae7f2bSJakub Kicinski 
136d9ae7f2bSJakub Kicinski 	nfp_prog_push(nfp_prog, insn);
137d9ae7f2bSJakub Kicinski }
138d9ae7f2bSJakub Kicinski 
139d9ae7f2bSJakub Kicinski static void
140d9ae7f2bSJakub Kicinski emit_cmd(struct nfp_prog *nfp_prog, enum cmd_tgt_map op,
141b3f868dfSJakub Kicinski 	 u8 mode, u8 xfer, swreg lreg, swreg rreg, u8 size, bool sync)
142d9ae7f2bSJakub Kicinski {
143d9ae7f2bSJakub Kicinski 	struct nfp_insn_re_regs reg;
144d9ae7f2bSJakub Kicinski 	int err;
145d9ae7f2bSJakub Kicinski 
146d9ae7f2bSJakub Kicinski 	err = swreg_to_restricted(reg_none(), lreg, rreg, &reg, false);
147d9ae7f2bSJakub Kicinski 	if (err) {
148d9ae7f2bSJakub Kicinski 		nfp_prog->error = err;
149d9ae7f2bSJakub Kicinski 		return;
150d9ae7f2bSJakub Kicinski 	}
151d9ae7f2bSJakub Kicinski 	if (reg.swap) {
152d9ae7f2bSJakub Kicinski 		pr_err("cmd can't swap arguments\n");
153d9ae7f2bSJakub Kicinski 		nfp_prog->error = -EFAULT;
154d9ae7f2bSJakub Kicinski 		return;
155d9ae7f2bSJakub Kicinski 	}
156995e101fSJakub Kicinski 	if (reg.dst_lmextn || reg.src_lmextn) {
157995e101fSJakub Kicinski 		pr_err("cmd can't use LMextn\n");
158995e101fSJakub Kicinski 		nfp_prog->error = -EFAULT;
159995e101fSJakub Kicinski 		return;
160995e101fSJakub Kicinski 	}
161d9ae7f2bSJakub Kicinski 
162d9ae7f2bSJakub Kicinski 	__emit_cmd(nfp_prog, op, mode, xfer, reg.areg, reg.breg, size, sync);
163d9ae7f2bSJakub Kicinski }
164d9ae7f2bSJakub Kicinski 
165d9ae7f2bSJakub Kicinski static void
166d9ae7f2bSJakub Kicinski __emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, enum br_ev_pip ev_pip,
167d9ae7f2bSJakub Kicinski 	  enum br_ctx_signal_state css, u16 addr, u8 defer)
168d9ae7f2bSJakub Kicinski {
169d9ae7f2bSJakub Kicinski 	u16 addr_lo, addr_hi;
170d9ae7f2bSJakub Kicinski 	u64 insn;
171d9ae7f2bSJakub Kicinski 
172d9ae7f2bSJakub Kicinski 	addr_lo = addr & (OP_BR_ADDR_LO >> __bf_shf(OP_BR_ADDR_LO));
173d9ae7f2bSJakub Kicinski 	addr_hi = addr != addr_lo;
174d9ae7f2bSJakub Kicinski 
175d9ae7f2bSJakub Kicinski 	insn = OP_BR_BASE |
176d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_BR_MASK, mask) |
177d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_BR_EV_PIP, ev_pip) |
178d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_BR_CSS, css) |
179d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_BR_DEFBR, defer) |
180d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_BR_ADDR_LO, addr_lo) |
181d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_BR_ADDR_HI, addr_hi);
182d9ae7f2bSJakub Kicinski 
183d9ae7f2bSJakub Kicinski 	nfp_prog_push(nfp_prog, insn);
184d9ae7f2bSJakub Kicinski }
185d9ae7f2bSJakub Kicinski 
186d9ae7f2bSJakub Kicinski static void emit_br_def(struct nfp_prog *nfp_prog, u16 addr, u8 defer)
187d9ae7f2bSJakub Kicinski {
188d9ae7f2bSJakub Kicinski 	if (defer > 2) {
189d9ae7f2bSJakub Kicinski 		pr_err("BUG: branch defer out of bounds %d\n", defer);
190d9ae7f2bSJakub Kicinski 		nfp_prog->error = -EFAULT;
191d9ae7f2bSJakub Kicinski 		return;
192d9ae7f2bSJakub Kicinski 	}
193d9ae7f2bSJakub Kicinski 	__emit_br(nfp_prog, BR_UNC, BR_EV_PIP_UNCOND, BR_CSS_NONE, addr, defer);
194d9ae7f2bSJakub Kicinski }
195d9ae7f2bSJakub Kicinski 
196d9ae7f2bSJakub Kicinski static void
197d9ae7f2bSJakub Kicinski emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, u16 addr, u8 defer)
198d9ae7f2bSJakub Kicinski {
199d9ae7f2bSJakub Kicinski 	__emit_br(nfp_prog, mask,
200d9ae7f2bSJakub Kicinski 		  mask != BR_UNC ? BR_EV_PIP_COND : BR_EV_PIP_UNCOND,
201d9ae7f2bSJakub Kicinski 		  BR_CSS_NONE, addr, defer);
202d9ae7f2bSJakub Kicinski }
203d9ae7f2bSJakub Kicinski 
204d9ae7f2bSJakub Kicinski static void
205d9ae7f2bSJakub Kicinski __emit_br_byte(struct nfp_prog *nfp_prog, u8 areg, u8 breg, bool imm8,
206995e101fSJakub Kicinski 	       u8 byte, bool equal, u16 addr, u8 defer, bool src_lmextn)
207d9ae7f2bSJakub Kicinski {
208d9ae7f2bSJakub Kicinski 	u16 addr_lo, addr_hi;
209d9ae7f2bSJakub Kicinski 	u64 insn;
210d9ae7f2bSJakub Kicinski 
211d9ae7f2bSJakub Kicinski 	addr_lo = addr & (OP_BB_ADDR_LO >> __bf_shf(OP_BB_ADDR_LO));
212d9ae7f2bSJakub Kicinski 	addr_hi = addr != addr_lo;
213d9ae7f2bSJakub Kicinski 
214d9ae7f2bSJakub Kicinski 	insn = OP_BBYTE_BASE |
215d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_BB_A_SRC, areg) |
216d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_BB_BYTE, byte) |
217d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_BB_B_SRC, breg) |
218d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_BB_I8, imm8) |
219d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_BB_EQ, equal) |
220d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_BB_DEFBR, defer) |
221d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_BB_ADDR_LO, addr_lo) |
222995e101fSJakub Kicinski 		FIELD_PREP(OP_BB_ADDR_HI, addr_hi) |
223995e101fSJakub Kicinski 		FIELD_PREP(OP_BB_SRC_LMEXTN, src_lmextn);
224d9ae7f2bSJakub Kicinski 
225d9ae7f2bSJakub Kicinski 	nfp_prog_push(nfp_prog, insn);
226d9ae7f2bSJakub Kicinski }
227d9ae7f2bSJakub Kicinski 
228d9ae7f2bSJakub Kicinski static void
229d9ae7f2bSJakub Kicinski emit_br_byte_neq(struct nfp_prog *nfp_prog,
230995e101fSJakub Kicinski 		 swreg src, u8 imm, u8 byte, u16 addr, u8 defer)
231d9ae7f2bSJakub Kicinski {
232d9ae7f2bSJakub Kicinski 	struct nfp_insn_re_regs reg;
233d9ae7f2bSJakub Kicinski 	int err;
234d9ae7f2bSJakub Kicinski 
235995e101fSJakub Kicinski 	err = swreg_to_restricted(reg_none(), src, reg_imm(imm), &reg, true);
236d9ae7f2bSJakub Kicinski 	if (err) {
237d9ae7f2bSJakub Kicinski 		nfp_prog->error = err;
238d9ae7f2bSJakub Kicinski 		return;
239d9ae7f2bSJakub Kicinski 	}
240d9ae7f2bSJakub Kicinski 
241d9ae7f2bSJakub Kicinski 	__emit_br_byte(nfp_prog, reg.areg, reg.breg, reg.i8, byte, false, addr,
242995e101fSJakub Kicinski 		       defer, reg.src_lmextn);
243d9ae7f2bSJakub Kicinski }
244d9ae7f2bSJakub Kicinski 
245d9ae7f2bSJakub Kicinski static void
246d9ae7f2bSJakub Kicinski __emit_immed(struct nfp_prog *nfp_prog, u16 areg, u16 breg, u16 imm_hi,
247d9ae7f2bSJakub Kicinski 	     enum immed_width width, bool invert,
248995e101fSJakub Kicinski 	     enum immed_shift shift, bool wr_both,
249995e101fSJakub Kicinski 	     bool dst_lmextn, bool src_lmextn)
250d9ae7f2bSJakub Kicinski {
251d9ae7f2bSJakub Kicinski 	u64 insn;
252d9ae7f2bSJakub Kicinski 
253d9ae7f2bSJakub Kicinski 	insn = OP_IMMED_BASE |
254d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_IMMED_A_SRC, areg) |
255d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_IMMED_B_SRC, breg) |
256d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_IMMED_IMM, imm_hi) |
257d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_IMMED_WIDTH, width) |
258d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_IMMED_INV, invert) |
259d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_IMMED_SHIFT, shift) |
260995e101fSJakub Kicinski 		FIELD_PREP(OP_IMMED_WR_AB, wr_both) |
261995e101fSJakub Kicinski 		FIELD_PREP(OP_IMMED_SRC_LMEXTN, src_lmextn) |
262995e101fSJakub Kicinski 		FIELD_PREP(OP_IMMED_DST_LMEXTN, dst_lmextn);
263d9ae7f2bSJakub Kicinski 
264d9ae7f2bSJakub Kicinski 	nfp_prog_push(nfp_prog, insn);
265d9ae7f2bSJakub Kicinski }
266d9ae7f2bSJakub Kicinski 
267d9ae7f2bSJakub Kicinski static void
268b3f868dfSJakub Kicinski emit_immed(struct nfp_prog *nfp_prog, swreg dst, u16 imm,
269d9ae7f2bSJakub Kicinski 	   enum immed_width width, bool invert, enum immed_shift shift)
270d9ae7f2bSJakub Kicinski {
271d9ae7f2bSJakub Kicinski 	struct nfp_insn_ur_regs reg;
272d9ae7f2bSJakub Kicinski 	int err;
273d9ae7f2bSJakub Kicinski 
274b3f868dfSJakub Kicinski 	if (swreg_type(dst) == NN_REG_IMM) {
275d9ae7f2bSJakub Kicinski 		nfp_prog->error = -EFAULT;
276d9ae7f2bSJakub Kicinski 		return;
277d9ae7f2bSJakub Kicinski 	}
278d9ae7f2bSJakub Kicinski 
279d9ae7f2bSJakub Kicinski 	err = swreg_to_unrestricted(dst, dst, reg_imm(imm & 0xff), &reg);
280d9ae7f2bSJakub Kicinski 	if (err) {
281d9ae7f2bSJakub Kicinski 		nfp_prog->error = err;
282d9ae7f2bSJakub Kicinski 		return;
283d9ae7f2bSJakub Kicinski 	}
284d9ae7f2bSJakub Kicinski 
285d9ae7f2bSJakub Kicinski 	__emit_immed(nfp_prog, reg.areg, reg.breg, imm >> 8, width,
286995e101fSJakub Kicinski 		     invert, shift, reg.wr_both,
287995e101fSJakub Kicinski 		     reg.dst_lmextn, reg.src_lmextn);
288d9ae7f2bSJakub Kicinski }
289d9ae7f2bSJakub Kicinski 
290d9ae7f2bSJakub Kicinski static void
291d9ae7f2bSJakub Kicinski __emit_shf(struct nfp_prog *nfp_prog, u16 dst, enum alu_dst_ab dst_ab,
292d9ae7f2bSJakub Kicinski 	   enum shf_sc sc, u8 shift,
293995e101fSJakub Kicinski 	   u16 areg, enum shf_op op, u16 breg, bool i8, bool sw, bool wr_both,
294995e101fSJakub Kicinski 	   bool dst_lmextn, bool src_lmextn)
295d9ae7f2bSJakub Kicinski {
296d9ae7f2bSJakub Kicinski 	u64 insn;
297d9ae7f2bSJakub Kicinski 
298d9ae7f2bSJakub Kicinski 	if (!FIELD_FIT(OP_SHF_SHIFT, shift)) {
299d9ae7f2bSJakub Kicinski 		nfp_prog->error = -EFAULT;
300d9ae7f2bSJakub Kicinski 		return;
301d9ae7f2bSJakub Kicinski 	}
302d9ae7f2bSJakub Kicinski 
303d9ae7f2bSJakub Kicinski 	if (sc == SHF_SC_L_SHF)
304d9ae7f2bSJakub Kicinski 		shift = 32 - shift;
305d9ae7f2bSJakub Kicinski 
306d9ae7f2bSJakub Kicinski 	insn = OP_SHF_BASE |
307d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_SHF_A_SRC, areg) |
308d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_SHF_SC, sc) |
309d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_SHF_B_SRC, breg) |
310d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_SHF_I8, i8) |
311d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_SHF_SW, sw) |
312d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_SHF_DST, dst) |
313d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_SHF_SHIFT, shift) |
314d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_SHF_OP, op) |
315d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_SHF_DST_AB, dst_ab) |
316995e101fSJakub Kicinski 		FIELD_PREP(OP_SHF_WR_AB, wr_both) |
317995e101fSJakub Kicinski 		FIELD_PREP(OP_SHF_SRC_LMEXTN, src_lmextn) |
318995e101fSJakub Kicinski 		FIELD_PREP(OP_SHF_DST_LMEXTN, dst_lmextn);
319d9ae7f2bSJakub Kicinski 
320d9ae7f2bSJakub Kicinski 	nfp_prog_push(nfp_prog, insn);
321d9ae7f2bSJakub Kicinski }
322d9ae7f2bSJakub Kicinski 
323d9ae7f2bSJakub Kicinski static void
324b3f868dfSJakub Kicinski emit_shf(struct nfp_prog *nfp_prog, swreg dst,
325b3f868dfSJakub Kicinski 	 swreg lreg, enum shf_op op, swreg rreg, enum shf_sc sc, u8 shift)
326d9ae7f2bSJakub Kicinski {
327d9ae7f2bSJakub Kicinski 	struct nfp_insn_re_regs reg;
328d9ae7f2bSJakub Kicinski 	int err;
329d9ae7f2bSJakub Kicinski 
330d9ae7f2bSJakub Kicinski 	err = swreg_to_restricted(dst, lreg, rreg, &reg, true);
331d9ae7f2bSJakub Kicinski 	if (err) {
332d9ae7f2bSJakub Kicinski 		nfp_prog->error = err;
333d9ae7f2bSJakub Kicinski 		return;
334d9ae7f2bSJakub Kicinski 	}
335d9ae7f2bSJakub Kicinski 
336d9ae7f2bSJakub Kicinski 	__emit_shf(nfp_prog, reg.dst, reg.dst_ab, sc, shift,
337995e101fSJakub Kicinski 		   reg.areg, op, reg.breg, reg.i8, reg.swap, reg.wr_both,
338995e101fSJakub Kicinski 		   reg.dst_lmextn, reg.src_lmextn);
339d9ae7f2bSJakub Kicinski }
340d9ae7f2bSJakub Kicinski 
341d9ae7f2bSJakub Kicinski static void
342d9ae7f2bSJakub Kicinski __emit_alu(struct nfp_prog *nfp_prog, u16 dst, enum alu_dst_ab dst_ab,
343995e101fSJakub Kicinski 	   u16 areg, enum alu_op op, u16 breg, bool swap, bool wr_both,
344995e101fSJakub Kicinski 	   bool dst_lmextn, bool src_lmextn)
345d9ae7f2bSJakub Kicinski {
346d9ae7f2bSJakub Kicinski 	u64 insn;
347d9ae7f2bSJakub Kicinski 
348d9ae7f2bSJakub Kicinski 	insn = OP_ALU_BASE |
349d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_ALU_A_SRC, areg) |
350d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_ALU_B_SRC, breg) |
351d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_ALU_DST, dst) |
352d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_ALU_SW, swap) |
353d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_ALU_OP, op) |
354d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_ALU_DST_AB, dst_ab) |
355995e101fSJakub Kicinski 		FIELD_PREP(OP_ALU_WR_AB, wr_both) |
356995e101fSJakub Kicinski 		FIELD_PREP(OP_ALU_SRC_LMEXTN, src_lmextn) |
357995e101fSJakub Kicinski 		FIELD_PREP(OP_ALU_DST_LMEXTN, dst_lmextn);
358d9ae7f2bSJakub Kicinski 
359d9ae7f2bSJakub Kicinski 	nfp_prog_push(nfp_prog, insn);
360d9ae7f2bSJakub Kicinski }
361d9ae7f2bSJakub Kicinski 
362d9ae7f2bSJakub Kicinski static void
363b3f868dfSJakub Kicinski emit_alu(struct nfp_prog *nfp_prog, swreg dst,
364b3f868dfSJakub Kicinski 	 swreg lreg, enum alu_op op, swreg rreg)
365d9ae7f2bSJakub Kicinski {
366d9ae7f2bSJakub Kicinski 	struct nfp_insn_ur_regs reg;
367d9ae7f2bSJakub Kicinski 	int err;
368d9ae7f2bSJakub Kicinski 
369d9ae7f2bSJakub Kicinski 	err = swreg_to_unrestricted(dst, lreg, rreg, &reg);
370d9ae7f2bSJakub Kicinski 	if (err) {
371d9ae7f2bSJakub Kicinski 		nfp_prog->error = err;
372d9ae7f2bSJakub Kicinski 		return;
373d9ae7f2bSJakub Kicinski 	}
374d9ae7f2bSJakub Kicinski 
375d9ae7f2bSJakub Kicinski 	__emit_alu(nfp_prog, reg.dst, reg.dst_ab,
376995e101fSJakub Kicinski 		   reg.areg, op, reg.breg, reg.swap, reg.wr_both,
377995e101fSJakub Kicinski 		   reg.dst_lmextn, reg.src_lmextn);
378d9ae7f2bSJakub Kicinski }
379d9ae7f2bSJakub Kicinski 
380d9ae7f2bSJakub Kicinski static void
381d9ae7f2bSJakub Kicinski __emit_ld_field(struct nfp_prog *nfp_prog, enum shf_sc sc,
382d9ae7f2bSJakub Kicinski 		u8 areg, u8 bmask, u8 breg, u8 shift, bool imm8,
383995e101fSJakub Kicinski 		bool zero, bool swap, bool wr_both,
384995e101fSJakub Kicinski 		bool dst_lmextn, bool src_lmextn)
385d9ae7f2bSJakub Kicinski {
386d9ae7f2bSJakub Kicinski 	u64 insn;
387d9ae7f2bSJakub Kicinski 
388d9ae7f2bSJakub Kicinski 	insn = OP_LDF_BASE |
389d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_LDF_A_SRC, areg) |
390d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_LDF_SC, sc) |
391d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_LDF_B_SRC, breg) |
392d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_LDF_I8, imm8) |
393d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_LDF_SW, swap) |
394d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_LDF_ZF, zero) |
395d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_LDF_BMASK, bmask) |
396d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_LDF_SHF, shift) |
397995e101fSJakub Kicinski 		FIELD_PREP(OP_LDF_WR_AB, wr_both) |
398995e101fSJakub Kicinski 		FIELD_PREP(OP_LDF_SRC_LMEXTN, src_lmextn) |
399995e101fSJakub Kicinski 		FIELD_PREP(OP_LDF_DST_LMEXTN, dst_lmextn);
400d9ae7f2bSJakub Kicinski 
401d9ae7f2bSJakub Kicinski 	nfp_prog_push(nfp_prog, insn);
402d9ae7f2bSJakub Kicinski }
403d9ae7f2bSJakub Kicinski 
404d9ae7f2bSJakub Kicinski static void
405bc8c80a8SJakub Kicinski emit_ld_field_any(struct nfp_prog *nfp_prog, swreg dst, u8 bmask, swreg src,
406bc8c80a8SJakub Kicinski 		  enum shf_sc sc, u8 shift, bool zero)
407d9ae7f2bSJakub Kicinski {
408d9ae7f2bSJakub Kicinski 	struct nfp_insn_re_regs reg;
409d9ae7f2bSJakub Kicinski 	int err;
410d9ae7f2bSJakub Kicinski 
4112de1be1dSJakub Kicinski 	/* Note: ld_field is special as it uses one of the src regs as dst */
4122de1be1dSJakub Kicinski 	err = swreg_to_restricted(dst, dst, src, &reg, true);
413d9ae7f2bSJakub Kicinski 	if (err) {
414d9ae7f2bSJakub Kicinski 		nfp_prog->error = err;
415d9ae7f2bSJakub Kicinski 		return;
416d9ae7f2bSJakub Kicinski 	}
417d9ae7f2bSJakub Kicinski 
418d9ae7f2bSJakub Kicinski 	__emit_ld_field(nfp_prog, sc, reg.areg, bmask, reg.breg, shift,
419995e101fSJakub Kicinski 			reg.i8, zero, reg.swap, reg.wr_both,
420995e101fSJakub Kicinski 			reg.dst_lmextn, reg.src_lmextn);
421d9ae7f2bSJakub Kicinski }
422d9ae7f2bSJakub Kicinski 
423d9ae7f2bSJakub Kicinski static void
424b3f868dfSJakub Kicinski emit_ld_field(struct nfp_prog *nfp_prog, swreg dst, u8 bmask, swreg src,
425d9ae7f2bSJakub Kicinski 	      enum shf_sc sc, u8 shift)
426d9ae7f2bSJakub Kicinski {
427bc8c80a8SJakub Kicinski 	emit_ld_field_any(nfp_prog, dst, bmask, src, sc, shift, false);
428d9ae7f2bSJakub Kicinski }
429d9ae7f2bSJakub Kicinski 
4301c03e03fSJakub Kicinski static void emit_nop(struct nfp_prog *nfp_prog)
4311c03e03fSJakub Kicinski {
4321c03e03fSJakub Kicinski 	__emit_immed(nfp_prog, UR_REG_IMM, UR_REG_IMM, 0, 0, 0, 0, 0, 0, 0);
4331c03e03fSJakub Kicinski }
4341c03e03fSJakub Kicinski 
435d9ae7f2bSJakub Kicinski /* --- Wrappers --- */
436d9ae7f2bSJakub Kicinski static bool pack_immed(u32 imm, u16 *val, enum immed_shift *shift)
437d9ae7f2bSJakub Kicinski {
438d9ae7f2bSJakub Kicinski 	if (!(imm & 0xffff0000)) {
439d9ae7f2bSJakub Kicinski 		*val = imm;
440d9ae7f2bSJakub Kicinski 		*shift = IMMED_SHIFT_0B;
441d9ae7f2bSJakub Kicinski 	} else if (!(imm & 0xff0000ff)) {
442d9ae7f2bSJakub Kicinski 		*val = imm >> 8;
443d9ae7f2bSJakub Kicinski 		*shift = IMMED_SHIFT_1B;
444d9ae7f2bSJakub Kicinski 	} else if (!(imm & 0x0000ffff)) {
445d9ae7f2bSJakub Kicinski 		*val = imm >> 16;
446d9ae7f2bSJakub Kicinski 		*shift = IMMED_SHIFT_2B;
447d9ae7f2bSJakub Kicinski 	} else {
448d9ae7f2bSJakub Kicinski 		return false;
449d9ae7f2bSJakub Kicinski 	}
450d9ae7f2bSJakub Kicinski 
451d9ae7f2bSJakub Kicinski 	return true;
452d9ae7f2bSJakub Kicinski }
453d9ae7f2bSJakub Kicinski 
454b3f868dfSJakub Kicinski static void wrp_immed(struct nfp_prog *nfp_prog, swreg dst, u32 imm)
455d9ae7f2bSJakub Kicinski {
456d9ae7f2bSJakub Kicinski 	enum immed_shift shift;
457d9ae7f2bSJakub Kicinski 	u16 val;
458d9ae7f2bSJakub Kicinski 
459d9ae7f2bSJakub Kicinski 	if (pack_immed(imm, &val, &shift)) {
460d9ae7f2bSJakub Kicinski 		emit_immed(nfp_prog, dst, val, IMMED_WIDTH_ALL, false, shift);
461d9ae7f2bSJakub Kicinski 	} else if (pack_immed(~imm, &val, &shift)) {
462d9ae7f2bSJakub Kicinski 		emit_immed(nfp_prog, dst, val, IMMED_WIDTH_ALL, true, shift);
463d9ae7f2bSJakub Kicinski 	} else {
464d9ae7f2bSJakub Kicinski 		emit_immed(nfp_prog, dst, imm & 0xffff, IMMED_WIDTH_ALL,
465d9ae7f2bSJakub Kicinski 			   false, IMMED_SHIFT_0B);
466d9ae7f2bSJakub Kicinski 		emit_immed(nfp_prog, dst, imm >> 16, IMMED_WIDTH_WORD,
467d9ae7f2bSJakub Kicinski 			   false, IMMED_SHIFT_2B);
468d9ae7f2bSJakub Kicinski 	}
469d9ae7f2bSJakub Kicinski }
470d9ae7f2bSJakub Kicinski 
471d9ae7f2bSJakub Kicinski /* ur_load_imm_any() - encode immediate or use tmp register (unrestricted)
472d9ae7f2bSJakub Kicinski  * If the @imm is small enough encode it directly in operand and return
473d9ae7f2bSJakub Kicinski  * otherwise load @imm to a spare register and return its encoding.
474d9ae7f2bSJakub Kicinski  */
475b3f868dfSJakub Kicinski static swreg ur_load_imm_any(struct nfp_prog *nfp_prog, u32 imm, swreg tmp_reg)
476d9ae7f2bSJakub Kicinski {
477d9ae7f2bSJakub Kicinski 	if (FIELD_FIT(UR_REG_IMM_MAX, imm))
478d9ae7f2bSJakub Kicinski 		return reg_imm(imm);
479d9ae7f2bSJakub Kicinski 
480d9ae7f2bSJakub Kicinski 	wrp_immed(nfp_prog, tmp_reg, imm);
481d9ae7f2bSJakub Kicinski 	return tmp_reg;
482d9ae7f2bSJakub Kicinski }
483d9ae7f2bSJakub Kicinski 
484d9ae7f2bSJakub Kicinski /* re_load_imm_any() - encode immediate or use tmp register (restricted)
485d9ae7f2bSJakub Kicinski  * If the @imm is small enough encode it directly in operand and return
486d9ae7f2bSJakub Kicinski  * otherwise load @imm to a spare register and return its encoding.
487d9ae7f2bSJakub Kicinski  */
488b3f868dfSJakub Kicinski static swreg re_load_imm_any(struct nfp_prog *nfp_prog, u32 imm, swreg tmp_reg)
489d9ae7f2bSJakub Kicinski {
490d9ae7f2bSJakub Kicinski 	if (FIELD_FIT(RE_REG_IMM_MAX, imm))
491d9ae7f2bSJakub Kicinski 		return reg_imm(imm);
492d9ae7f2bSJakub Kicinski 
493d9ae7f2bSJakub Kicinski 	wrp_immed(nfp_prog, tmp_reg, imm);
494d9ae7f2bSJakub Kicinski 	return tmp_reg;
495d9ae7f2bSJakub Kicinski }
496d9ae7f2bSJakub Kicinski 
497d9ae7f2bSJakub Kicinski static void
498d9ae7f2bSJakub Kicinski wrp_br_special(struct nfp_prog *nfp_prog, enum br_mask mask,
499d9ae7f2bSJakub Kicinski 	       enum br_special special)
500d9ae7f2bSJakub Kicinski {
501d9ae7f2bSJakub Kicinski 	emit_br(nfp_prog, mask, 0, 0);
502d9ae7f2bSJakub Kicinski 
503d9ae7f2bSJakub Kicinski 	nfp_prog->prog[nfp_prog->prog_len - 1] |=
504d9ae7f2bSJakub Kicinski 		FIELD_PREP(OP_BR_SPECIAL, special);
505d9ae7f2bSJakub Kicinski }
506d9ae7f2bSJakub Kicinski 
507c000dfb5SJakub Kicinski static void wrp_mov(struct nfp_prog *nfp_prog, swreg dst, swreg src)
508c000dfb5SJakub Kicinski {
509c000dfb5SJakub Kicinski 	emit_alu(nfp_prog, dst, reg_none(), ALU_OP_NONE, src);
510c000dfb5SJakub Kicinski }
511c000dfb5SJakub Kicinski 
512d9ae7f2bSJakub Kicinski static void wrp_reg_mov(struct nfp_prog *nfp_prog, u16 dst, u16 src)
513d9ae7f2bSJakub Kicinski {
514c000dfb5SJakub Kicinski 	wrp_mov(nfp_prog, reg_both(dst), reg_b(src));
515d9ae7f2bSJakub Kicinski }
516d9ae7f2bSJakub Kicinski 
517d9ae7f2bSJakub Kicinski static int
5180a793977SJakub Kicinski data_ld(struct nfp_prog *nfp_prog, swreg offset, u8 dst_gpr, int size)
519d9ae7f2bSJakub Kicinski {
520d9ae7f2bSJakub Kicinski 	unsigned int i;
521d9ae7f2bSJakub Kicinski 	u16 shift, sz;
522d9ae7f2bSJakub Kicinski 
523d9ae7f2bSJakub Kicinski 	/* We load the value from the address indicated in @offset and then
524d9ae7f2bSJakub Kicinski 	 * shift out the data we don't need.  Note: this is big endian!
525d9ae7f2bSJakub Kicinski 	 */
5260a793977SJakub Kicinski 	sz = max(size, 4);
527d9ae7f2bSJakub Kicinski 	shift = size < 4 ? 4 - size : 0;
528d9ae7f2bSJakub Kicinski 
5290a793977SJakub Kicinski 	emit_cmd(nfp_prog, CMD_TGT_READ8, CMD_MODE_32b, 0,
5300a793977SJakub Kicinski 		 pptr_reg(nfp_prog), offset, sz - 1, true);
5310a793977SJakub Kicinski 
5320a793977SJakub Kicinski 	i = 0;
5330a793977SJakub Kicinski 	if (shift)
5340a793977SJakub Kicinski 		emit_shf(nfp_prog, reg_both(dst_gpr), reg_none(), SHF_OP_NONE,
5350a793977SJakub Kicinski 			 reg_xfer(0), SHF_SC_R_SHF, shift * 8);
5360a793977SJakub Kicinski 	else
5370a793977SJakub Kicinski 		for (; i * 4 < size; i++)
5380a793977SJakub Kicinski 			wrp_mov(nfp_prog, reg_both(dst_gpr + i), reg_xfer(i));
5390a793977SJakub Kicinski 
5400a793977SJakub Kicinski 	if (i < 2)
5410a793977SJakub Kicinski 		wrp_immed(nfp_prog, reg_both(dst_gpr + 1), 0);
5420a793977SJakub Kicinski 
5430a793977SJakub Kicinski 	return 0;
5440a793977SJakub Kicinski }
5450a793977SJakub Kicinski 
5460a793977SJakub Kicinski static int
5470a793977SJakub Kicinski construct_data_ind_ld(struct nfp_prog *nfp_prog, u16 offset, u16 src, u8 size)
5480a793977SJakub Kicinski {
5490a793977SJakub Kicinski 	swreg tmp_reg;
5500a793977SJakub Kicinski 
551d9ae7f2bSJakub Kicinski 	/* Calculate the true offset (src_reg + imm) */
552d9ae7f2bSJakub Kicinski 	tmp_reg = ur_load_imm_any(nfp_prog, offset, imm_b(nfp_prog));
5530a793977SJakub Kicinski 	emit_alu(nfp_prog, imm_both(nfp_prog), reg_a(src), ALU_OP_ADD, tmp_reg);
5540a793977SJakub Kicinski 
555d9ae7f2bSJakub Kicinski 	/* Check packet length (size guaranteed to fit b/c it's u8) */
556d9ae7f2bSJakub Kicinski 	emit_alu(nfp_prog, imm_a(nfp_prog),
557d9ae7f2bSJakub Kicinski 		 imm_a(nfp_prog), ALU_OP_ADD, reg_imm(size));
558d9ae7f2bSJakub Kicinski 	emit_alu(nfp_prog, reg_none(),
55918e53b6cSJakub Kicinski 		 plen_reg(nfp_prog), ALU_OP_SUB, imm_a(nfp_prog));
560d9ae7f2bSJakub Kicinski 	wrp_br_special(nfp_prog, BR_BLO, OP_BR_GO_ABORT);
5610a793977SJakub Kicinski 
562d9ae7f2bSJakub Kicinski 	/* Load data */
5630a793977SJakub Kicinski 	return data_ld(nfp_prog, imm_b(nfp_prog), 0, size);
564d9ae7f2bSJakub Kicinski }
565d9ae7f2bSJakub Kicinski 
566d9ae7f2bSJakub Kicinski static int construct_data_ld(struct nfp_prog *nfp_prog, u16 offset, u8 size)
567d9ae7f2bSJakub Kicinski {
5680a793977SJakub Kicinski 	swreg tmp_reg;
5690a793977SJakub Kicinski 
5700a793977SJakub Kicinski 	/* Check packet length */
5710a793977SJakub Kicinski 	tmp_reg = ur_load_imm_any(nfp_prog, offset + size, imm_a(nfp_prog));
5720a793977SJakub Kicinski 	emit_alu(nfp_prog, reg_none(), plen_reg(nfp_prog), ALU_OP_SUB, tmp_reg);
5730a793977SJakub Kicinski 	wrp_br_special(nfp_prog, BR_BLO, OP_BR_GO_ABORT);
5740a793977SJakub Kicinski 
5750a793977SJakub Kicinski 	/* Load data */
5760a793977SJakub Kicinski 	tmp_reg = re_load_imm_any(nfp_prog, offset, imm_b(nfp_prog));
5770a793977SJakub Kicinski 	return data_ld(nfp_prog, tmp_reg, 0, size);
578d9ae7f2bSJakub Kicinski }
579d9ae7f2bSJakub Kicinski 
580d9ae7f2bSJakub Kicinski static void
581d9ae7f2bSJakub Kicinski wrp_alu_imm(struct nfp_prog *nfp_prog, u8 dst, enum alu_op alu_op, u32 imm)
582d9ae7f2bSJakub Kicinski {
583b3f868dfSJakub Kicinski 	swreg tmp_reg;
584d9ae7f2bSJakub Kicinski 
585d9ae7f2bSJakub Kicinski 	if (alu_op == ALU_OP_AND) {
586d9ae7f2bSJakub Kicinski 		if (!imm)
587d9ae7f2bSJakub Kicinski 			wrp_immed(nfp_prog, reg_both(dst), 0);
588d9ae7f2bSJakub Kicinski 		if (!imm || !~imm)
589d9ae7f2bSJakub Kicinski 			return;
590d9ae7f2bSJakub Kicinski 	}
591d9ae7f2bSJakub Kicinski 	if (alu_op == ALU_OP_OR) {
592d9ae7f2bSJakub Kicinski 		if (!~imm)
593d9ae7f2bSJakub Kicinski 			wrp_immed(nfp_prog, reg_both(dst), ~0U);
594d9ae7f2bSJakub Kicinski 		if (!imm || !~imm)
595d9ae7f2bSJakub Kicinski 			return;
596d9ae7f2bSJakub Kicinski 	}
597d9ae7f2bSJakub Kicinski 	if (alu_op == ALU_OP_XOR) {
598d9ae7f2bSJakub Kicinski 		if (!~imm)
599d9ae7f2bSJakub Kicinski 			emit_alu(nfp_prog, reg_both(dst), reg_none(),
600d9ae7f2bSJakub Kicinski 				 ALU_OP_NEG, reg_b(dst));
601d9ae7f2bSJakub Kicinski 		if (!imm || !~imm)
602d9ae7f2bSJakub Kicinski 			return;
603d9ae7f2bSJakub Kicinski 	}
604d9ae7f2bSJakub Kicinski 
605d9ae7f2bSJakub Kicinski 	tmp_reg = ur_load_imm_any(nfp_prog, imm, imm_b(nfp_prog));
606d9ae7f2bSJakub Kicinski 	emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, tmp_reg);
607d9ae7f2bSJakub Kicinski }
608d9ae7f2bSJakub Kicinski 
609d9ae7f2bSJakub Kicinski static int
610d9ae7f2bSJakub Kicinski wrp_alu64_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
611d9ae7f2bSJakub Kicinski 	      enum alu_op alu_op, bool skip)
612d9ae7f2bSJakub Kicinski {
613d9ae7f2bSJakub Kicinski 	const struct bpf_insn *insn = &meta->insn;
614d9ae7f2bSJakub Kicinski 	u64 imm = insn->imm; /* sign extend */
615d9ae7f2bSJakub Kicinski 
616d9ae7f2bSJakub Kicinski 	if (skip) {
617d9ae7f2bSJakub Kicinski 		meta->skip = true;
618d9ae7f2bSJakub Kicinski 		return 0;
619d9ae7f2bSJakub Kicinski 	}
620d9ae7f2bSJakub Kicinski 
621d9ae7f2bSJakub Kicinski 	wrp_alu_imm(nfp_prog, insn->dst_reg * 2, alu_op, imm & ~0U);
622d9ae7f2bSJakub Kicinski 	wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, alu_op, imm >> 32);
623d9ae7f2bSJakub Kicinski 
624d9ae7f2bSJakub Kicinski 	return 0;
625d9ae7f2bSJakub Kicinski }
626d9ae7f2bSJakub Kicinski 
627d9ae7f2bSJakub Kicinski static int
628d9ae7f2bSJakub Kicinski wrp_alu64_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
629d9ae7f2bSJakub Kicinski 	      enum alu_op alu_op)
630d9ae7f2bSJakub Kicinski {
631d9ae7f2bSJakub Kicinski 	u8 dst = meta->insn.dst_reg * 2, src = meta->insn.src_reg * 2;
632d9ae7f2bSJakub Kicinski 
633d9ae7f2bSJakub Kicinski 	emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, reg_b(src));
634d9ae7f2bSJakub Kicinski 	emit_alu(nfp_prog, reg_both(dst + 1),
635d9ae7f2bSJakub Kicinski 		 reg_a(dst + 1), alu_op, reg_b(src + 1));
636d9ae7f2bSJakub Kicinski 
637d9ae7f2bSJakub Kicinski 	return 0;
638d9ae7f2bSJakub Kicinski }
639d9ae7f2bSJakub Kicinski 
640d9ae7f2bSJakub Kicinski static int
641d9ae7f2bSJakub Kicinski wrp_alu32_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
642d9ae7f2bSJakub Kicinski 	      enum alu_op alu_op, bool skip)
643d9ae7f2bSJakub Kicinski {
644d9ae7f2bSJakub Kicinski 	const struct bpf_insn *insn = &meta->insn;
645d9ae7f2bSJakub Kicinski 
646d9ae7f2bSJakub Kicinski 	if (skip) {
647d9ae7f2bSJakub Kicinski 		meta->skip = true;
648d9ae7f2bSJakub Kicinski 		return 0;
649d9ae7f2bSJakub Kicinski 	}
650d9ae7f2bSJakub Kicinski 
651d9ae7f2bSJakub Kicinski 	wrp_alu_imm(nfp_prog, insn->dst_reg * 2, alu_op, insn->imm);
652d9ae7f2bSJakub Kicinski 	wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0);
653d9ae7f2bSJakub Kicinski 
654d9ae7f2bSJakub Kicinski 	return 0;
655d9ae7f2bSJakub Kicinski }
656d9ae7f2bSJakub Kicinski 
657d9ae7f2bSJakub Kicinski static int
658d9ae7f2bSJakub Kicinski wrp_alu32_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
659d9ae7f2bSJakub Kicinski 	      enum alu_op alu_op)
660d9ae7f2bSJakub Kicinski {
661d9ae7f2bSJakub Kicinski 	u8 dst = meta->insn.dst_reg * 2, src = meta->insn.src_reg * 2;
662d9ae7f2bSJakub Kicinski 
663d9ae7f2bSJakub Kicinski 	emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, reg_b(src));
664d9ae7f2bSJakub Kicinski 	wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0);
665d9ae7f2bSJakub Kicinski 
666d9ae7f2bSJakub Kicinski 	return 0;
667d9ae7f2bSJakub Kicinski }
668d9ae7f2bSJakub Kicinski 
669d9ae7f2bSJakub Kicinski static void
670d9ae7f2bSJakub Kicinski wrp_test_reg_one(struct nfp_prog *nfp_prog, u8 dst, enum alu_op alu_op, u8 src,
671d9ae7f2bSJakub Kicinski 		 enum br_mask br_mask, u16 off)
672d9ae7f2bSJakub Kicinski {
673d9ae7f2bSJakub Kicinski 	emit_alu(nfp_prog, reg_none(), reg_a(dst), alu_op, reg_b(src));
674d9ae7f2bSJakub Kicinski 	emit_br(nfp_prog, br_mask, off, 0);
675d9ae7f2bSJakub Kicinski }
676d9ae7f2bSJakub Kicinski 
677d9ae7f2bSJakub Kicinski static int
678d9ae7f2bSJakub Kicinski wrp_test_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
679d9ae7f2bSJakub Kicinski 	     enum alu_op alu_op, enum br_mask br_mask)
680d9ae7f2bSJakub Kicinski {
681d9ae7f2bSJakub Kicinski 	const struct bpf_insn *insn = &meta->insn;
682d9ae7f2bSJakub Kicinski 
683d9ae7f2bSJakub Kicinski 	if (insn->off < 0) /* TODO */
684d9ae7f2bSJakub Kicinski 		return -EOPNOTSUPP;
685d9ae7f2bSJakub Kicinski 
686d9ae7f2bSJakub Kicinski 	wrp_test_reg_one(nfp_prog, insn->dst_reg * 2, alu_op,
687d9ae7f2bSJakub Kicinski 			 insn->src_reg * 2, br_mask, insn->off);
688d9ae7f2bSJakub Kicinski 	wrp_test_reg_one(nfp_prog, insn->dst_reg * 2 + 1, alu_op,
689d9ae7f2bSJakub Kicinski 			 insn->src_reg * 2 + 1, br_mask, insn->off);
690d9ae7f2bSJakub Kicinski 
691d9ae7f2bSJakub Kicinski 	return 0;
692d9ae7f2bSJakub Kicinski }
693d9ae7f2bSJakub Kicinski 
694d9ae7f2bSJakub Kicinski static int
695d9ae7f2bSJakub Kicinski wrp_cmp_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
696d9ae7f2bSJakub Kicinski 	    enum br_mask br_mask, bool swap)
697d9ae7f2bSJakub Kicinski {
698d9ae7f2bSJakub Kicinski 	const struct bpf_insn *insn = &meta->insn;
699d9ae7f2bSJakub Kicinski 	u64 imm = insn->imm; /* sign extend */
700d9ae7f2bSJakub Kicinski 	u8 reg = insn->dst_reg * 2;
701b3f868dfSJakub Kicinski 	swreg tmp_reg;
702d9ae7f2bSJakub Kicinski 
703d9ae7f2bSJakub Kicinski 	if (insn->off < 0) /* TODO */
704d9ae7f2bSJakub Kicinski 		return -EOPNOTSUPP;
705d9ae7f2bSJakub Kicinski 
706d9ae7f2bSJakub Kicinski 	tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog));
707d9ae7f2bSJakub Kicinski 	if (!swap)
708d9ae7f2bSJakub Kicinski 		emit_alu(nfp_prog, reg_none(), reg_a(reg), ALU_OP_SUB, tmp_reg);
709d9ae7f2bSJakub Kicinski 	else
710d9ae7f2bSJakub Kicinski 		emit_alu(nfp_prog, reg_none(), tmp_reg, ALU_OP_SUB, reg_a(reg));
711d9ae7f2bSJakub Kicinski 
712d9ae7f2bSJakub Kicinski 	tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog));
713d9ae7f2bSJakub Kicinski 	if (!swap)
714d9ae7f2bSJakub Kicinski 		emit_alu(nfp_prog, reg_none(),
715d9ae7f2bSJakub Kicinski 			 reg_a(reg + 1), ALU_OP_SUB_C, tmp_reg);
716d9ae7f2bSJakub Kicinski 	else
717d9ae7f2bSJakub Kicinski 		emit_alu(nfp_prog, reg_none(),
718d9ae7f2bSJakub Kicinski 			 tmp_reg, ALU_OP_SUB_C, reg_a(reg + 1));
719d9ae7f2bSJakub Kicinski 
720d9ae7f2bSJakub Kicinski 	emit_br(nfp_prog, br_mask, insn->off, 0);
721d9ae7f2bSJakub Kicinski 
722d9ae7f2bSJakub Kicinski 	return 0;
723d9ae7f2bSJakub Kicinski }
724d9ae7f2bSJakub Kicinski 
725d9ae7f2bSJakub Kicinski static int
726d9ae7f2bSJakub Kicinski wrp_cmp_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
727d9ae7f2bSJakub Kicinski 	    enum br_mask br_mask, bool swap)
728d9ae7f2bSJakub Kicinski {
729d9ae7f2bSJakub Kicinski 	const struct bpf_insn *insn = &meta->insn;
73026fa818dSJakub Kicinski 	u8 areg, breg;
73126fa818dSJakub Kicinski 
73226fa818dSJakub Kicinski 	areg = insn->dst_reg * 2;
73326fa818dSJakub Kicinski 	breg = insn->src_reg * 2;
734d9ae7f2bSJakub Kicinski 
735d9ae7f2bSJakub Kicinski 	if (insn->off < 0) /* TODO */
736d9ae7f2bSJakub Kicinski 		return -EOPNOTSUPP;
737d9ae7f2bSJakub Kicinski 
738d9ae7f2bSJakub Kicinski 	if (swap) {
739d9ae7f2bSJakub Kicinski 		areg ^= breg;
740d9ae7f2bSJakub Kicinski 		breg ^= areg;
741d9ae7f2bSJakub Kicinski 		areg ^= breg;
742d9ae7f2bSJakub Kicinski 	}
743d9ae7f2bSJakub Kicinski 
744d9ae7f2bSJakub Kicinski 	emit_alu(nfp_prog, reg_none(), reg_a(areg), ALU_OP_SUB, reg_b(breg));
745d9ae7f2bSJakub Kicinski 	emit_alu(nfp_prog, reg_none(),
746d9ae7f2bSJakub Kicinski 		 reg_a(areg + 1), ALU_OP_SUB_C, reg_b(breg + 1));
747d9ae7f2bSJakub Kicinski 	emit_br(nfp_prog, br_mask, insn->off, 0);
748d9ae7f2bSJakub Kicinski 
749d9ae7f2bSJakub Kicinski 	return 0;
750d9ae7f2bSJakub Kicinski }
751d9ae7f2bSJakub Kicinski 
7523119d1fdSJakub Kicinski static void wrp_end32(struct nfp_prog *nfp_prog, swreg reg_in, u8 gpr_out)
7533119d1fdSJakub Kicinski {
7543119d1fdSJakub Kicinski 	emit_ld_field(nfp_prog, reg_both(gpr_out), 0xf, reg_in,
7553119d1fdSJakub Kicinski 		      SHF_SC_R_ROT, 8);
7563119d1fdSJakub Kicinski 	emit_ld_field(nfp_prog, reg_both(gpr_out), 0x5, reg_a(gpr_out),
7573119d1fdSJakub Kicinski 		      SHF_SC_R_ROT, 16);
7583119d1fdSJakub Kicinski }
7593119d1fdSJakub Kicinski 
760d9ae7f2bSJakub Kicinski /* --- Callbacks --- */
761d9ae7f2bSJakub Kicinski static int mov_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
762d9ae7f2bSJakub Kicinski {
763d9ae7f2bSJakub Kicinski 	const struct bpf_insn *insn = &meta->insn;
764d9ae7f2bSJakub Kicinski 
765d9ae7f2bSJakub Kicinski 	wrp_reg_mov(nfp_prog, insn->dst_reg * 2, insn->src_reg * 2);
766d9ae7f2bSJakub Kicinski 	wrp_reg_mov(nfp_prog, insn->dst_reg * 2 + 1, insn->src_reg * 2 + 1);
767d9ae7f2bSJakub Kicinski 
768d9ae7f2bSJakub Kicinski 	return 0;
769d9ae7f2bSJakub Kicinski }
770d9ae7f2bSJakub Kicinski 
771d9ae7f2bSJakub Kicinski static int mov_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
772d9ae7f2bSJakub Kicinski {
773d9ae7f2bSJakub Kicinski 	u64 imm = meta->insn.imm; /* sign extend */
774d9ae7f2bSJakub Kicinski 
775d9ae7f2bSJakub Kicinski 	wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2), imm & ~0U);
776d9ae7f2bSJakub Kicinski 	wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), imm >> 32);
777d9ae7f2bSJakub Kicinski 
778d9ae7f2bSJakub Kicinski 	return 0;
779d9ae7f2bSJakub Kicinski }
780d9ae7f2bSJakub Kicinski 
781d9ae7f2bSJakub Kicinski static int xor_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
782d9ae7f2bSJakub Kicinski {
783d9ae7f2bSJakub Kicinski 	return wrp_alu64_reg(nfp_prog, meta, ALU_OP_XOR);
784d9ae7f2bSJakub Kicinski }
785d9ae7f2bSJakub Kicinski 
786d9ae7f2bSJakub Kicinski static int xor_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
787d9ae7f2bSJakub Kicinski {
788d9ae7f2bSJakub Kicinski 	return wrp_alu64_imm(nfp_prog, meta, ALU_OP_XOR, !meta->insn.imm);
789d9ae7f2bSJakub Kicinski }
790d9ae7f2bSJakub Kicinski 
791d9ae7f2bSJakub Kicinski static int and_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
792d9ae7f2bSJakub Kicinski {
793d9ae7f2bSJakub Kicinski 	return wrp_alu64_reg(nfp_prog, meta, ALU_OP_AND);
794d9ae7f2bSJakub Kicinski }
795d9ae7f2bSJakub Kicinski 
796d9ae7f2bSJakub Kicinski static int and_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
797d9ae7f2bSJakub Kicinski {
798d9ae7f2bSJakub Kicinski 	return wrp_alu64_imm(nfp_prog, meta, ALU_OP_AND, !~meta->insn.imm);
799d9ae7f2bSJakub Kicinski }
800d9ae7f2bSJakub Kicinski 
801d9ae7f2bSJakub Kicinski static int or_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
802d9ae7f2bSJakub Kicinski {
803d9ae7f2bSJakub Kicinski 	return wrp_alu64_reg(nfp_prog, meta, ALU_OP_OR);
804d9ae7f2bSJakub Kicinski }
805d9ae7f2bSJakub Kicinski 
806d9ae7f2bSJakub Kicinski static int or_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
807d9ae7f2bSJakub Kicinski {
808d9ae7f2bSJakub Kicinski 	return wrp_alu64_imm(nfp_prog, meta, ALU_OP_OR, !meta->insn.imm);
809d9ae7f2bSJakub Kicinski }
810d9ae7f2bSJakub Kicinski 
811d9ae7f2bSJakub Kicinski static int add_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
812d9ae7f2bSJakub Kicinski {
813d9ae7f2bSJakub Kicinski 	const struct bpf_insn *insn = &meta->insn;
814d9ae7f2bSJakub Kicinski 
815d9ae7f2bSJakub Kicinski 	emit_alu(nfp_prog, reg_both(insn->dst_reg * 2),
816d9ae7f2bSJakub Kicinski 		 reg_a(insn->dst_reg * 2), ALU_OP_ADD,
817d9ae7f2bSJakub Kicinski 		 reg_b(insn->src_reg * 2));
818d9ae7f2bSJakub Kicinski 	emit_alu(nfp_prog, reg_both(insn->dst_reg * 2 + 1),
819d9ae7f2bSJakub Kicinski 		 reg_a(insn->dst_reg * 2 + 1), ALU_OP_ADD_C,
820d9ae7f2bSJakub Kicinski 		 reg_b(insn->src_reg * 2 + 1));
821d9ae7f2bSJakub Kicinski 
822d9ae7f2bSJakub Kicinski 	return 0;
823d9ae7f2bSJakub Kicinski }
824d9ae7f2bSJakub Kicinski 
825d9ae7f2bSJakub Kicinski static int add_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
826d9ae7f2bSJakub Kicinski {
827d9ae7f2bSJakub Kicinski 	const struct bpf_insn *insn = &meta->insn;
828d9ae7f2bSJakub Kicinski 	u64 imm = insn->imm; /* sign extend */
829d9ae7f2bSJakub Kicinski 
830d9ae7f2bSJakub Kicinski 	wrp_alu_imm(nfp_prog, insn->dst_reg * 2, ALU_OP_ADD, imm & ~0U);
831d9ae7f2bSJakub Kicinski 	wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, ALU_OP_ADD_C, imm >> 32);
832d9ae7f2bSJakub Kicinski 
833d9ae7f2bSJakub Kicinski 	return 0;
834d9ae7f2bSJakub Kicinski }
835d9ae7f2bSJakub Kicinski 
836d9ae7f2bSJakub Kicinski static int sub_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
837d9ae7f2bSJakub Kicinski {
838d9ae7f2bSJakub Kicinski 	const struct bpf_insn *insn = &meta->insn;
839d9ae7f2bSJakub Kicinski 
840d9ae7f2bSJakub Kicinski 	emit_alu(nfp_prog, reg_both(insn->dst_reg * 2),
841d9ae7f2bSJakub Kicinski 		 reg_a(insn->dst_reg * 2), ALU_OP_SUB,
842d9ae7f2bSJakub Kicinski 		 reg_b(insn->src_reg * 2));
843d9ae7f2bSJakub Kicinski 	emit_alu(nfp_prog, reg_both(insn->dst_reg * 2 + 1),
844d9ae7f2bSJakub Kicinski 		 reg_a(insn->dst_reg * 2 + 1), ALU_OP_SUB_C,
845d9ae7f2bSJakub Kicinski 		 reg_b(insn->src_reg * 2 + 1));
846d9ae7f2bSJakub Kicinski 
847d9ae7f2bSJakub Kicinski 	return 0;
848d9ae7f2bSJakub Kicinski }
849d9ae7f2bSJakub Kicinski 
850d9ae7f2bSJakub Kicinski static int sub_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
851d9ae7f2bSJakub Kicinski {
852d9ae7f2bSJakub Kicinski 	const struct bpf_insn *insn = &meta->insn;
853d9ae7f2bSJakub Kicinski 	u64 imm = insn->imm; /* sign extend */
854d9ae7f2bSJakub Kicinski 
855d9ae7f2bSJakub Kicinski 	wrp_alu_imm(nfp_prog, insn->dst_reg * 2, ALU_OP_SUB, imm & ~0U);
856d9ae7f2bSJakub Kicinski 	wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, ALU_OP_SUB_C, imm >> 32);
857d9ae7f2bSJakub Kicinski 
858d9ae7f2bSJakub Kicinski 	return 0;
859d9ae7f2bSJakub Kicinski }
860d9ae7f2bSJakub Kicinski 
861d9ae7f2bSJakub Kicinski static int shl_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
862d9ae7f2bSJakub Kicinski {
863d9ae7f2bSJakub Kicinski 	const struct bpf_insn *insn = &meta->insn;
8643cae1319SJakub Kicinski 	u8 dst = insn->dst_reg * 2;
865d9ae7f2bSJakub Kicinski 
8663cae1319SJakub Kicinski 	if (insn->imm < 32) {
8673cae1319SJakub Kicinski 		emit_shf(nfp_prog, reg_both(dst + 1),
8683cae1319SJakub Kicinski 			 reg_a(dst + 1), SHF_OP_NONE, reg_b(dst),
8693cae1319SJakub Kicinski 			 SHF_SC_R_DSHF, 32 - insn->imm);
8703cae1319SJakub Kicinski 		emit_shf(nfp_prog, reg_both(dst),
8713cae1319SJakub Kicinski 			 reg_none(), SHF_OP_NONE, reg_b(dst),
8723cae1319SJakub Kicinski 			 SHF_SC_L_SHF, insn->imm);
8733cae1319SJakub Kicinski 	} else if (insn->imm == 32) {
8743cae1319SJakub Kicinski 		wrp_reg_mov(nfp_prog, dst + 1, dst);
8753cae1319SJakub Kicinski 		wrp_immed(nfp_prog, reg_both(dst), 0);
8763cae1319SJakub Kicinski 	} else if (insn->imm > 32) {
8773cae1319SJakub Kicinski 		emit_shf(nfp_prog, reg_both(dst + 1),
8783cae1319SJakub Kicinski 			 reg_none(), SHF_OP_NONE, reg_b(dst),
8793cae1319SJakub Kicinski 			 SHF_SC_L_SHF, insn->imm - 32);
8803cae1319SJakub Kicinski 		wrp_immed(nfp_prog, reg_both(dst), 0);
8813cae1319SJakub Kicinski 	}
882d9ae7f2bSJakub Kicinski 
883d9ae7f2bSJakub Kicinski 	return 0;
884d9ae7f2bSJakub Kicinski }
885d9ae7f2bSJakub Kicinski 
886d9ae7f2bSJakub Kicinski static int shr_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
887d9ae7f2bSJakub Kicinski {
888d9ae7f2bSJakub Kicinski 	const struct bpf_insn *insn = &meta->insn;
8893cae1319SJakub Kicinski 	u8 dst = insn->dst_reg * 2;
890d9ae7f2bSJakub Kicinski 
8913cae1319SJakub Kicinski 	if (insn->imm < 32) {
8923cae1319SJakub Kicinski 		emit_shf(nfp_prog, reg_both(dst),
8933cae1319SJakub Kicinski 			 reg_a(dst + 1), SHF_OP_NONE, reg_b(dst),
8943cae1319SJakub Kicinski 			 SHF_SC_R_DSHF, insn->imm);
8953cae1319SJakub Kicinski 		emit_shf(nfp_prog, reg_both(dst + 1),
8963cae1319SJakub Kicinski 			 reg_none(), SHF_OP_NONE, reg_b(dst + 1),
8973cae1319SJakub Kicinski 			 SHF_SC_R_SHF, insn->imm);
8983cae1319SJakub Kicinski 	} else if (insn->imm == 32) {
8993cae1319SJakub Kicinski 		wrp_reg_mov(nfp_prog, dst, dst + 1);
9003cae1319SJakub Kicinski 		wrp_immed(nfp_prog, reg_both(dst + 1), 0);
9013cae1319SJakub Kicinski 	} else if (insn->imm > 32) {
9023cae1319SJakub Kicinski 		emit_shf(nfp_prog, reg_both(dst),
9033cae1319SJakub Kicinski 			 reg_none(), SHF_OP_NONE, reg_b(dst + 1),
9043cae1319SJakub Kicinski 			 SHF_SC_R_SHF, insn->imm - 32);
9053cae1319SJakub Kicinski 		wrp_immed(nfp_prog, reg_both(dst + 1), 0);
9063cae1319SJakub Kicinski 	}
907d9ae7f2bSJakub Kicinski 
908d9ae7f2bSJakub Kicinski 	return 0;
909d9ae7f2bSJakub Kicinski }
910d9ae7f2bSJakub Kicinski 
911d9ae7f2bSJakub Kicinski static int mov_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
912d9ae7f2bSJakub Kicinski {
913d9ae7f2bSJakub Kicinski 	const struct bpf_insn *insn = &meta->insn;
914d9ae7f2bSJakub Kicinski 
915d9ae7f2bSJakub Kicinski 	wrp_reg_mov(nfp_prog, insn->dst_reg * 2,  insn->src_reg * 2);
916d9ae7f2bSJakub Kicinski 	wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0);
917d9ae7f2bSJakub Kicinski 
918d9ae7f2bSJakub Kicinski 	return 0;
919d9ae7f2bSJakub Kicinski }
920d9ae7f2bSJakub Kicinski 
921d9ae7f2bSJakub Kicinski static int mov_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
922d9ae7f2bSJakub Kicinski {
923d9ae7f2bSJakub Kicinski 	const struct bpf_insn *insn = &meta->insn;
924d9ae7f2bSJakub Kicinski 
925d9ae7f2bSJakub Kicinski 	wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2), insn->imm);
926d9ae7f2bSJakub Kicinski 	wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0);
927d9ae7f2bSJakub Kicinski 
928d9ae7f2bSJakub Kicinski 	return 0;
929d9ae7f2bSJakub Kicinski }
930d9ae7f2bSJakub Kicinski 
931d9ae7f2bSJakub Kicinski static int xor_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
932d9ae7f2bSJakub Kicinski {
933d9ae7f2bSJakub Kicinski 	return wrp_alu32_reg(nfp_prog, meta, ALU_OP_XOR);
934d9ae7f2bSJakub Kicinski }
935d9ae7f2bSJakub Kicinski 
936d9ae7f2bSJakub Kicinski static int xor_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
937d9ae7f2bSJakub Kicinski {
938d9ae7f2bSJakub Kicinski 	return wrp_alu32_imm(nfp_prog, meta, ALU_OP_XOR, !~meta->insn.imm);
939d9ae7f2bSJakub Kicinski }
940d9ae7f2bSJakub Kicinski 
941d9ae7f2bSJakub Kicinski static int and_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
942d9ae7f2bSJakub Kicinski {
943d9ae7f2bSJakub Kicinski 	return wrp_alu32_reg(nfp_prog, meta, ALU_OP_AND);
944d9ae7f2bSJakub Kicinski }
945d9ae7f2bSJakub Kicinski 
946d9ae7f2bSJakub Kicinski static int and_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
947d9ae7f2bSJakub Kicinski {
948d9ae7f2bSJakub Kicinski 	return wrp_alu32_imm(nfp_prog, meta, ALU_OP_AND, !~meta->insn.imm);
949d9ae7f2bSJakub Kicinski }
950d9ae7f2bSJakub Kicinski 
951d9ae7f2bSJakub Kicinski static int or_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
952d9ae7f2bSJakub Kicinski {
953d9ae7f2bSJakub Kicinski 	return wrp_alu32_reg(nfp_prog, meta, ALU_OP_OR);
954d9ae7f2bSJakub Kicinski }
955d9ae7f2bSJakub Kicinski 
956d9ae7f2bSJakub Kicinski static int or_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
957d9ae7f2bSJakub Kicinski {
958d9ae7f2bSJakub Kicinski 	return wrp_alu32_imm(nfp_prog, meta, ALU_OP_OR, !meta->insn.imm);
959d9ae7f2bSJakub Kicinski }
960d9ae7f2bSJakub Kicinski 
961d9ae7f2bSJakub Kicinski static int add_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
962d9ae7f2bSJakub Kicinski {
963d9ae7f2bSJakub Kicinski 	return wrp_alu32_reg(nfp_prog, meta, ALU_OP_ADD);
964d9ae7f2bSJakub Kicinski }
965d9ae7f2bSJakub Kicinski 
966d9ae7f2bSJakub Kicinski static int add_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
967d9ae7f2bSJakub Kicinski {
968d9ae7f2bSJakub Kicinski 	return wrp_alu32_imm(nfp_prog, meta, ALU_OP_ADD, !meta->insn.imm);
969d9ae7f2bSJakub Kicinski }
970d9ae7f2bSJakub Kicinski 
971d9ae7f2bSJakub Kicinski static int sub_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
972d9ae7f2bSJakub Kicinski {
973d9ae7f2bSJakub Kicinski 	return wrp_alu32_reg(nfp_prog, meta, ALU_OP_SUB);
974d9ae7f2bSJakub Kicinski }
975d9ae7f2bSJakub Kicinski 
976d9ae7f2bSJakub Kicinski static int sub_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
977d9ae7f2bSJakub Kicinski {
978d9ae7f2bSJakub Kicinski 	return wrp_alu32_imm(nfp_prog, meta, ALU_OP_SUB, !meta->insn.imm);
979d9ae7f2bSJakub Kicinski }
980d9ae7f2bSJakub Kicinski 
981d9ae7f2bSJakub Kicinski static int shl_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
982d9ae7f2bSJakub Kicinski {
983d9ae7f2bSJakub Kicinski 	const struct bpf_insn *insn = &meta->insn;
984d9ae7f2bSJakub Kicinski 
985d9ae7f2bSJakub Kicinski 	if (!insn->imm)
986d9ae7f2bSJakub Kicinski 		return 1; /* TODO: zero shift means indirect */
987d9ae7f2bSJakub Kicinski 
988d9ae7f2bSJakub Kicinski 	emit_shf(nfp_prog, reg_both(insn->dst_reg * 2),
989d9ae7f2bSJakub Kicinski 		 reg_none(), SHF_OP_NONE, reg_b(insn->dst_reg * 2),
990d9ae7f2bSJakub Kicinski 		 SHF_SC_L_SHF, insn->imm);
991d9ae7f2bSJakub Kicinski 	wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0);
992d9ae7f2bSJakub Kicinski 
993d9ae7f2bSJakub Kicinski 	return 0;
994d9ae7f2bSJakub Kicinski }
995d9ae7f2bSJakub Kicinski 
9963119d1fdSJakub Kicinski static int end_reg32(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
9973119d1fdSJakub Kicinski {
9983119d1fdSJakub Kicinski 	const struct bpf_insn *insn = &meta->insn;
9993119d1fdSJakub Kicinski 	u8 gpr = insn->dst_reg * 2;
10003119d1fdSJakub Kicinski 
10013119d1fdSJakub Kicinski 	switch (insn->imm) {
10023119d1fdSJakub Kicinski 	case 16:
10033119d1fdSJakub Kicinski 		emit_ld_field(nfp_prog, reg_both(gpr), 0x9, reg_b(gpr),
10043119d1fdSJakub Kicinski 			      SHF_SC_R_ROT, 8);
10053119d1fdSJakub Kicinski 		emit_ld_field(nfp_prog, reg_both(gpr), 0xe, reg_a(gpr),
10063119d1fdSJakub Kicinski 			      SHF_SC_R_SHF, 16);
10073119d1fdSJakub Kicinski 
10083119d1fdSJakub Kicinski 		wrp_immed(nfp_prog, reg_both(gpr + 1), 0);
10093119d1fdSJakub Kicinski 		break;
10103119d1fdSJakub Kicinski 	case 32:
10113119d1fdSJakub Kicinski 		wrp_end32(nfp_prog, reg_a(gpr), gpr);
10123119d1fdSJakub Kicinski 		wrp_immed(nfp_prog, reg_both(gpr + 1), 0);
10133119d1fdSJakub Kicinski 		break;
10143119d1fdSJakub Kicinski 	case 64:
10153119d1fdSJakub Kicinski 		wrp_mov(nfp_prog, imm_a(nfp_prog), reg_b(gpr + 1));
10163119d1fdSJakub Kicinski 
10173119d1fdSJakub Kicinski 		wrp_end32(nfp_prog, reg_a(gpr), gpr + 1);
10183119d1fdSJakub Kicinski 		wrp_end32(nfp_prog, imm_a(nfp_prog), gpr);
10193119d1fdSJakub Kicinski 		break;
10203119d1fdSJakub Kicinski 	}
10213119d1fdSJakub Kicinski 
10223119d1fdSJakub Kicinski 	return 0;
10233119d1fdSJakub Kicinski }
10243119d1fdSJakub Kicinski 
1025d9ae7f2bSJakub Kicinski static int imm_ld8_part2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1026d9ae7f2bSJakub Kicinski {
1027d9ae7f2bSJakub Kicinski 	wrp_immed(nfp_prog, reg_both(nfp_meta_prev(meta)->insn.dst_reg * 2 + 1),
1028d9ae7f2bSJakub Kicinski 		  meta->insn.imm);
1029d9ae7f2bSJakub Kicinski 
1030d9ae7f2bSJakub Kicinski 	return 0;
1031d9ae7f2bSJakub Kicinski }
1032d9ae7f2bSJakub Kicinski 
1033d9ae7f2bSJakub Kicinski static int imm_ld8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1034d9ae7f2bSJakub Kicinski {
1035d9ae7f2bSJakub Kicinski 	const struct bpf_insn *insn = &meta->insn;
1036d9ae7f2bSJakub Kicinski 
1037d9ae7f2bSJakub Kicinski 	meta->double_cb = imm_ld8_part2;
1038d9ae7f2bSJakub Kicinski 	wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2), insn->imm);
1039d9ae7f2bSJakub Kicinski 
1040d9ae7f2bSJakub Kicinski 	return 0;
1041d9ae7f2bSJakub Kicinski }
1042d9ae7f2bSJakub Kicinski 
1043d9ae7f2bSJakub Kicinski static int data_ld1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1044d9ae7f2bSJakub Kicinski {
1045d9ae7f2bSJakub Kicinski 	return construct_data_ld(nfp_prog, meta->insn.imm, 1);
1046d9ae7f2bSJakub Kicinski }
1047d9ae7f2bSJakub Kicinski 
1048d9ae7f2bSJakub Kicinski static int data_ld2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1049d9ae7f2bSJakub Kicinski {
1050d9ae7f2bSJakub Kicinski 	return construct_data_ld(nfp_prog, meta->insn.imm, 2);
1051d9ae7f2bSJakub Kicinski }
1052d9ae7f2bSJakub Kicinski 
1053d9ae7f2bSJakub Kicinski static int data_ld4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1054d9ae7f2bSJakub Kicinski {
1055d9ae7f2bSJakub Kicinski 	return construct_data_ld(nfp_prog, meta->insn.imm, 4);
1056d9ae7f2bSJakub Kicinski }
1057d9ae7f2bSJakub Kicinski 
1058d9ae7f2bSJakub Kicinski static int data_ind_ld1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1059d9ae7f2bSJakub Kicinski {
1060d9ae7f2bSJakub Kicinski 	return construct_data_ind_ld(nfp_prog, meta->insn.imm,
10610a793977SJakub Kicinski 				     meta->insn.src_reg * 2, 1);
1062d9ae7f2bSJakub Kicinski }
1063d9ae7f2bSJakub Kicinski 
1064d9ae7f2bSJakub Kicinski static int data_ind_ld2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1065d9ae7f2bSJakub Kicinski {
1066d9ae7f2bSJakub Kicinski 	return construct_data_ind_ld(nfp_prog, meta->insn.imm,
10670a793977SJakub Kicinski 				     meta->insn.src_reg * 2, 2);
1068d9ae7f2bSJakub Kicinski }
1069d9ae7f2bSJakub Kicinski 
1070d9ae7f2bSJakub Kicinski static int data_ind_ld4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1071d9ae7f2bSJakub Kicinski {
1072d9ae7f2bSJakub Kicinski 	return construct_data_ind_ld(nfp_prog, meta->insn.imm,
10730a793977SJakub Kicinski 				     meta->insn.src_reg * 2, 4);
1074d9ae7f2bSJakub Kicinski }
1075d9ae7f2bSJakub Kicinski 
1076943c57b9SJakub Kicinski static int mem_ldx_skb(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
1077943c57b9SJakub Kicinski 		       u8 size)
1078d9ae7f2bSJakub Kicinski {
1079943c57b9SJakub Kicinski 	switch (meta->insn.off) {
1080943c57b9SJakub Kicinski 	case offsetof(struct sk_buff, len):
1081943c57b9SJakub Kicinski 		if (size != FIELD_SIZEOF(struct sk_buff, len))
1082943c57b9SJakub Kicinski 			return -EOPNOTSUPP;
1083c000dfb5SJakub Kicinski 		wrp_mov(nfp_prog,
1084c000dfb5SJakub Kicinski 			reg_both(meta->insn.dst_reg * 2), plen_reg(nfp_prog));
1085943c57b9SJakub Kicinski 		break;
1086943c57b9SJakub Kicinski 	default:
1087d9ae7f2bSJakub Kicinski 		return -EOPNOTSUPP;
1088943c57b9SJakub Kicinski 	}
1089943c57b9SJakub Kicinski 
1090943c57b9SJakub Kicinski 	wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0);
1091d9ae7f2bSJakub Kicinski 
1092d9ae7f2bSJakub Kicinski 	return 0;
1093d9ae7f2bSJakub Kicinski }
1094d9ae7f2bSJakub Kicinski 
1095943c57b9SJakub Kicinski static int mem_ldx_xdp(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
1096943c57b9SJakub Kicinski 		       u8 size)
1097d9ae7f2bSJakub Kicinski {
1098b3f868dfSJakub Kicinski 	swreg dst = reg_both(meta->insn.dst_reg * 2);
1099d9ae7f2bSJakub Kicinski 
1100943c57b9SJakub Kicinski 	if (size != sizeof(void *))
1101943c57b9SJakub Kicinski 		return -EINVAL;
1102d9ae7f2bSJakub Kicinski 
1103943c57b9SJakub Kicinski 	switch (meta->insn.off) {
1104943c57b9SJakub Kicinski 	case offsetof(struct xdp_buff, data):
1105c000dfb5SJakub Kicinski 		wrp_mov(nfp_prog, dst, pptr_reg(nfp_prog));
1106943c57b9SJakub Kicinski 		break;
1107943c57b9SJakub Kicinski 	case offsetof(struct xdp_buff, data_end):
1108943c57b9SJakub Kicinski 		emit_alu(nfp_prog, dst,
1109943c57b9SJakub Kicinski 			 plen_reg(nfp_prog), ALU_OP_ADD, pptr_reg(nfp_prog));
1110943c57b9SJakub Kicinski 		break;
1111943c57b9SJakub Kicinski 	default:
1112943c57b9SJakub Kicinski 		return -EOPNOTSUPP;
1113943c57b9SJakub Kicinski 	}
1114d9ae7f2bSJakub Kicinski 
1115943c57b9SJakub Kicinski 	wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0);
1116d9ae7f2bSJakub Kicinski 
1117d9ae7f2bSJakub Kicinski 	return 0;
1118d9ae7f2bSJakub Kicinski }
1119d9ae7f2bSJakub Kicinski 
1120d9ae7f2bSJakub Kicinski static int mem_ldx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1121d9ae7f2bSJakub Kicinski {
1122d9ae7f2bSJakub Kicinski 	if (nfp_prog->act == NN_ACT_XDP)
1123943c57b9SJakub Kicinski 		return mem_ldx_xdp(nfp_prog, meta, 4);
1124d9ae7f2bSJakub Kicinski 	else
1125943c57b9SJakub Kicinski 		return mem_ldx_skb(nfp_prog, meta, 4);
1126d9ae7f2bSJakub Kicinski }
1127d9ae7f2bSJakub Kicinski 
1128d9ae7f2bSJakub Kicinski static int mem_stx4_skb(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1129d9ae7f2bSJakub Kicinski {
1130d9ae7f2bSJakub Kicinski 	return -EOPNOTSUPP;
1131d9ae7f2bSJakub Kicinski }
1132d9ae7f2bSJakub Kicinski 
1133d9ae7f2bSJakub Kicinski static int mem_stx4_xdp(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1134d9ae7f2bSJakub Kicinski {
1135d9ae7f2bSJakub Kicinski 	return -EOPNOTSUPP;
1136d9ae7f2bSJakub Kicinski }
1137d9ae7f2bSJakub Kicinski 
1138d9ae7f2bSJakub Kicinski static int mem_stx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1139d9ae7f2bSJakub Kicinski {
1140d9ae7f2bSJakub Kicinski 	if (nfp_prog->act == NN_ACT_XDP)
1141d9ae7f2bSJakub Kicinski 		return mem_stx4_xdp(nfp_prog, meta);
1142d9ae7f2bSJakub Kicinski 	return mem_stx4_skb(nfp_prog, meta);
1143d9ae7f2bSJakub Kicinski }
1144d9ae7f2bSJakub Kicinski 
1145d9ae7f2bSJakub Kicinski static int jump(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1146d9ae7f2bSJakub Kicinski {
1147d9ae7f2bSJakub Kicinski 	if (meta->insn.off < 0) /* TODO */
1148d9ae7f2bSJakub Kicinski 		return -EOPNOTSUPP;
1149d9ae7f2bSJakub Kicinski 	emit_br(nfp_prog, BR_UNC, meta->insn.off, 0);
1150d9ae7f2bSJakub Kicinski 
1151d9ae7f2bSJakub Kicinski 	return 0;
1152d9ae7f2bSJakub Kicinski }
1153d9ae7f2bSJakub Kicinski 
1154d9ae7f2bSJakub Kicinski static int jeq_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1155d9ae7f2bSJakub Kicinski {
1156d9ae7f2bSJakub Kicinski 	const struct bpf_insn *insn = &meta->insn;
1157d9ae7f2bSJakub Kicinski 	u64 imm = insn->imm; /* sign extend */
1158b3f868dfSJakub Kicinski 	swreg or1, or2, tmp_reg;
1159b3f868dfSJakub Kicinski 
1160b3f868dfSJakub Kicinski 	or1 = reg_a(insn->dst_reg * 2);
1161b3f868dfSJakub Kicinski 	or2 = reg_b(insn->dst_reg * 2 + 1);
1162d9ae7f2bSJakub Kicinski 
1163d9ae7f2bSJakub Kicinski 	if (insn->off < 0) /* TODO */
1164d9ae7f2bSJakub Kicinski 		return -EOPNOTSUPP;
1165d9ae7f2bSJakub Kicinski 
1166d9ae7f2bSJakub Kicinski 	if (imm & ~0U) {
1167d9ae7f2bSJakub Kicinski 		tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog));
1168d9ae7f2bSJakub Kicinski 		emit_alu(nfp_prog, imm_a(nfp_prog),
1169d9ae7f2bSJakub Kicinski 			 reg_a(insn->dst_reg * 2), ALU_OP_XOR, tmp_reg);
1170d9ae7f2bSJakub Kicinski 		or1 = imm_a(nfp_prog);
1171d9ae7f2bSJakub Kicinski 	}
1172d9ae7f2bSJakub Kicinski 
1173d9ae7f2bSJakub Kicinski 	if (imm >> 32) {
1174d9ae7f2bSJakub Kicinski 		tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog));
1175d9ae7f2bSJakub Kicinski 		emit_alu(nfp_prog, imm_b(nfp_prog),
1176d9ae7f2bSJakub Kicinski 			 reg_a(insn->dst_reg * 2 + 1), ALU_OP_XOR, tmp_reg);
1177d9ae7f2bSJakub Kicinski 		or2 = imm_b(nfp_prog);
1178d9ae7f2bSJakub Kicinski 	}
1179d9ae7f2bSJakub Kicinski 
1180d9ae7f2bSJakub Kicinski 	emit_alu(nfp_prog, reg_none(), or1, ALU_OP_OR, or2);
1181d9ae7f2bSJakub Kicinski 	emit_br(nfp_prog, BR_BEQ, insn->off, 0);
1182d9ae7f2bSJakub Kicinski 
1183d9ae7f2bSJakub Kicinski 	return 0;
1184d9ae7f2bSJakub Kicinski }
1185d9ae7f2bSJakub Kicinski 
1186d9ae7f2bSJakub Kicinski static int jgt_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1187d9ae7f2bSJakub Kicinski {
118826fa818dSJakub Kicinski 	return wrp_cmp_imm(nfp_prog, meta, BR_BLO, true);
1189d9ae7f2bSJakub Kicinski }
1190d9ae7f2bSJakub Kicinski 
1191d9ae7f2bSJakub Kicinski static int jge_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1192d9ae7f2bSJakub Kicinski {
119326fa818dSJakub Kicinski 	return wrp_cmp_imm(nfp_prog, meta, BR_BHS, false);
1194d9ae7f2bSJakub Kicinski }
1195d9ae7f2bSJakub Kicinski 
11965dd294d4SDaniel Borkmann static int jlt_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
11975dd294d4SDaniel Borkmann {
119826fa818dSJakub Kicinski 	return wrp_cmp_imm(nfp_prog, meta, BR_BLO, false);
11995dd294d4SDaniel Borkmann }
12005dd294d4SDaniel Borkmann 
12015dd294d4SDaniel Borkmann static int jle_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
12025dd294d4SDaniel Borkmann {
120326fa818dSJakub Kicinski 	return wrp_cmp_imm(nfp_prog, meta, BR_BHS, true);
12045dd294d4SDaniel Borkmann }
12055dd294d4SDaniel Borkmann 
1206d9ae7f2bSJakub Kicinski static int jset_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1207d9ae7f2bSJakub Kicinski {
1208d9ae7f2bSJakub Kicinski 	const struct bpf_insn *insn = &meta->insn;
1209d9ae7f2bSJakub Kicinski 	u64 imm = insn->imm; /* sign extend */
1210b3f868dfSJakub Kicinski 	swreg tmp_reg;
1211d9ae7f2bSJakub Kicinski 
1212d9ae7f2bSJakub Kicinski 	if (insn->off < 0) /* TODO */
1213d9ae7f2bSJakub Kicinski 		return -EOPNOTSUPP;
1214d9ae7f2bSJakub Kicinski 
1215d9ae7f2bSJakub Kicinski 	if (!imm) {
1216d9ae7f2bSJakub Kicinski 		meta->skip = true;
1217d9ae7f2bSJakub Kicinski 		return 0;
1218d9ae7f2bSJakub Kicinski 	}
1219d9ae7f2bSJakub Kicinski 
1220d9ae7f2bSJakub Kicinski 	if (imm & ~0U) {
1221d9ae7f2bSJakub Kicinski 		tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog));
1222d9ae7f2bSJakub Kicinski 		emit_alu(nfp_prog, reg_none(),
1223d9ae7f2bSJakub Kicinski 			 reg_a(insn->dst_reg * 2), ALU_OP_AND, tmp_reg);
1224d9ae7f2bSJakub Kicinski 		emit_br(nfp_prog, BR_BNE, insn->off, 0);
1225d9ae7f2bSJakub Kicinski 	}
1226d9ae7f2bSJakub Kicinski 
1227d9ae7f2bSJakub Kicinski 	if (imm >> 32) {
1228d9ae7f2bSJakub Kicinski 		tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog));
1229d9ae7f2bSJakub Kicinski 		emit_alu(nfp_prog, reg_none(),
1230d9ae7f2bSJakub Kicinski 			 reg_a(insn->dst_reg * 2 + 1), ALU_OP_AND, tmp_reg);
1231d9ae7f2bSJakub Kicinski 		emit_br(nfp_prog, BR_BNE, insn->off, 0);
1232d9ae7f2bSJakub Kicinski 	}
1233d9ae7f2bSJakub Kicinski 
1234d9ae7f2bSJakub Kicinski 	return 0;
1235d9ae7f2bSJakub Kicinski }
1236d9ae7f2bSJakub Kicinski 
1237d9ae7f2bSJakub Kicinski static int jne_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1238d9ae7f2bSJakub Kicinski {
1239d9ae7f2bSJakub Kicinski 	const struct bpf_insn *insn = &meta->insn;
1240d9ae7f2bSJakub Kicinski 	u64 imm = insn->imm; /* sign extend */
1241b3f868dfSJakub Kicinski 	swreg tmp_reg;
1242d9ae7f2bSJakub Kicinski 
1243d9ae7f2bSJakub Kicinski 	if (insn->off < 0) /* TODO */
1244d9ae7f2bSJakub Kicinski 		return -EOPNOTSUPP;
1245d9ae7f2bSJakub Kicinski 
1246d9ae7f2bSJakub Kicinski 	if (!imm) {
1247d9ae7f2bSJakub Kicinski 		emit_alu(nfp_prog, reg_none(), reg_a(insn->dst_reg * 2),
1248d9ae7f2bSJakub Kicinski 			 ALU_OP_OR, reg_b(insn->dst_reg * 2 + 1));
1249d9ae7f2bSJakub Kicinski 		emit_br(nfp_prog, BR_BNE, insn->off, 0);
125082837370SJakub Kicinski 		return 0;
1251d9ae7f2bSJakub Kicinski 	}
1252d9ae7f2bSJakub Kicinski 
1253d9ae7f2bSJakub Kicinski 	tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog));
1254d9ae7f2bSJakub Kicinski 	emit_alu(nfp_prog, reg_none(),
1255d9ae7f2bSJakub Kicinski 		 reg_a(insn->dst_reg * 2), ALU_OP_XOR, tmp_reg);
1256d9ae7f2bSJakub Kicinski 	emit_br(nfp_prog, BR_BNE, insn->off, 0);
1257d9ae7f2bSJakub Kicinski 
1258d9ae7f2bSJakub Kicinski 	tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog));
1259d9ae7f2bSJakub Kicinski 	emit_alu(nfp_prog, reg_none(),
1260d9ae7f2bSJakub Kicinski 		 reg_a(insn->dst_reg * 2 + 1), ALU_OP_XOR, tmp_reg);
1261d9ae7f2bSJakub Kicinski 	emit_br(nfp_prog, BR_BNE, insn->off, 0);
1262d9ae7f2bSJakub Kicinski 
1263d9ae7f2bSJakub Kicinski 	return 0;
1264d9ae7f2bSJakub Kicinski }
1265d9ae7f2bSJakub Kicinski 
1266d9ae7f2bSJakub Kicinski static int jeq_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1267d9ae7f2bSJakub Kicinski {
1268d9ae7f2bSJakub Kicinski 	const struct bpf_insn *insn = &meta->insn;
1269d9ae7f2bSJakub Kicinski 
1270d9ae7f2bSJakub Kicinski 	if (insn->off < 0) /* TODO */
1271d9ae7f2bSJakub Kicinski 		return -EOPNOTSUPP;
1272d9ae7f2bSJakub Kicinski 
1273d9ae7f2bSJakub Kicinski 	emit_alu(nfp_prog, imm_a(nfp_prog), reg_a(insn->dst_reg * 2),
1274d9ae7f2bSJakub Kicinski 		 ALU_OP_XOR, reg_b(insn->src_reg * 2));
1275d9ae7f2bSJakub Kicinski 	emit_alu(nfp_prog, imm_b(nfp_prog), reg_a(insn->dst_reg * 2 + 1),
1276d9ae7f2bSJakub Kicinski 		 ALU_OP_XOR, reg_b(insn->src_reg * 2 + 1));
1277d9ae7f2bSJakub Kicinski 	emit_alu(nfp_prog, reg_none(),
1278d9ae7f2bSJakub Kicinski 		 imm_a(nfp_prog), ALU_OP_OR, imm_b(nfp_prog));
1279d9ae7f2bSJakub Kicinski 	emit_br(nfp_prog, BR_BEQ, insn->off, 0);
1280d9ae7f2bSJakub Kicinski 
1281d9ae7f2bSJakub Kicinski 	return 0;
1282d9ae7f2bSJakub Kicinski }
1283d9ae7f2bSJakub Kicinski 
1284d9ae7f2bSJakub Kicinski static int jgt_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1285d9ae7f2bSJakub Kicinski {
128626fa818dSJakub Kicinski 	return wrp_cmp_reg(nfp_prog, meta, BR_BLO, true);
1287d9ae7f2bSJakub Kicinski }
1288d9ae7f2bSJakub Kicinski 
1289d9ae7f2bSJakub Kicinski static int jge_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1290d9ae7f2bSJakub Kicinski {
129126fa818dSJakub Kicinski 	return wrp_cmp_reg(nfp_prog, meta, BR_BHS, false);
1292d9ae7f2bSJakub Kicinski }
1293d9ae7f2bSJakub Kicinski 
12945dd294d4SDaniel Borkmann static int jlt_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
12955dd294d4SDaniel Borkmann {
129626fa818dSJakub Kicinski 	return wrp_cmp_reg(nfp_prog, meta, BR_BLO, false);
12975dd294d4SDaniel Borkmann }
12985dd294d4SDaniel Borkmann 
12995dd294d4SDaniel Borkmann static int jle_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
13005dd294d4SDaniel Borkmann {
130126fa818dSJakub Kicinski 	return wrp_cmp_reg(nfp_prog, meta, BR_BHS, true);
13025dd294d4SDaniel Borkmann }
13035dd294d4SDaniel Borkmann 
1304d9ae7f2bSJakub Kicinski static int jset_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1305d9ae7f2bSJakub Kicinski {
1306d9ae7f2bSJakub Kicinski 	return wrp_test_reg(nfp_prog, meta, ALU_OP_AND, BR_BNE);
1307d9ae7f2bSJakub Kicinski }
1308d9ae7f2bSJakub Kicinski 
1309d9ae7f2bSJakub Kicinski static int jne_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1310d9ae7f2bSJakub Kicinski {
1311d9ae7f2bSJakub Kicinski 	return wrp_test_reg(nfp_prog, meta, ALU_OP_XOR, BR_BNE);
1312d9ae7f2bSJakub Kicinski }
1313d9ae7f2bSJakub Kicinski 
1314d9ae7f2bSJakub Kicinski static int goto_out(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1315d9ae7f2bSJakub Kicinski {
1316d9ae7f2bSJakub Kicinski 	wrp_br_special(nfp_prog, BR_UNC, OP_BR_GO_OUT);
1317d9ae7f2bSJakub Kicinski 
1318d9ae7f2bSJakub Kicinski 	return 0;
1319d9ae7f2bSJakub Kicinski }
1320d9ae7f2bSJakub Kicinski 
1321d9ae7f2bSJakub Kicinski static const instr_cb_t instr_cb[256] = {
1322d9ae7f2bSJakub Kicinski 	[BPF_ALU64 | BPF_MOV | BPF_X] =	mov_reg64,
1323d9ae7f2bSJakub Kicinski 	[BPF_ALU64 | BPF_MOV | BPF_K] =	mov_imm64,
1324d9ae7f2bSJakub Kicinski 	[BPF_ALU64 | BPF_XOR | BPF_X] =	xor_reg64,
1325d9ae7f2bSJakub Kicinski 	[BPF_ALU64 | BPF_XOR | BPF_K] =	xor_imm64,
1326d9ae7f2bSJakub Kicinski 	[BPF_ALU64 | BPF_AND | BPF_X] =	and_reg64,
1327d9ae7f2bSJakub Kicinski 	[BPF_ALU64 | BPF_AND | BPF_K] =	and_imm64,
1328d9ae7f2bSJakub Kicinski 	[BPF_ALU64 | BPF_OR | BPF_X] =	or_reg64,
1329d9ae7f2bSJakub Kicinski 	[BPF_ALU64 | BPF_OR | BPF_K] =	or_imm64,
1330d9ae7f2bSJakub Kicinski 	[BPF_ALU64 | BPF_ADD | BPF_X] =	add_reg64,
1331d9ae7f2bSJakub Kicinski 	[BPF_ALU64 | BPF_ADD | BPF_K] =	add_imm64,
1332d9ae7f2bSJakub Kicinski 	[BPF_ALU64 | BPF_SUB | BPF_X] =	sub_reg64,
1333d9ae7f2bSJakub Kicinski 	[BPF_ALU64 | BPF_SUB | BPF_K] =	sub_imm64,
1334d9ae7f2bSJakub Kicinski 	[BPF_ALU64 | BPF_LSH | BPF_K] =	shl_imm64,
1335d9ae7f2bSJakub Kicinski 	[BPF_ALU64 | BPF_RSH | BPF_K] =	shr_imm64,
1336d9ae7f2bSJakub Kicinski 	[BPF_ALU | BPF_MOV | BPF_X] =	mov_reg,
1337d9ae7f2bSJakub Kicinski 	[BPF_ALU | BPF_MOV | BPF_K] =	mov_imm,
1338d9ae7f2bSJakub Kicinski 	[BPF_ALU | BPF_XOR | BPF_X] =	xor_reg,
1339d9ae7f2bSJakub Kicinski 	[BPF_ALU | BPF_XOR | BPF_K] =	xor_imm,
1340d9ae7f2bSJakub Kicinski 	[BPF_ALU | BPF_AND | BPF_X] =	and_reg,
1341d9ae7f2bSJakub Kicinski 	[BPF_ALU | BPF_AND | BPF_K] =	and_imm,
1342d9ae7f2bSJakub Kicinski 	[BPF_ALU | BPF_OR | BPF_X] =	or_reg,
1343d9ae7f2bSJakub Kicinski 	[BPF_ALU | BPF_OR | BPF_K] =	or_imm,
1344d9ae7f2bSJakub Kicinski 	[BPF_ALU | BPF_ADD | BPF_X] =	add_reg,
1345d9ae7f2bSJakub Kicinski 	[BPF_ALU | BPF_ADD | BPF_K] =	add_imm,
1346d9ae7f2bSJakub Kicinski 	[BPF_ALU | BPF_SUB | BPF_X] =	sub_reg,
1347d9ae7f2bSJakub Kicinski 	[BPF_ALU | BPF_SUB | BPF_K] =	sub_imm,
1348d9ae7f2bSJakub Kicinski 	[BPF_ALU | BPF_LSH | BPF_K] =	shl_imm,
13493119d1fdSJakub Kicinski 	[BPF_ALU | BPF_END | BPF_X] =	end_reg32,
1350d9ae7f2bSJakub Kicinski 	[BPF_LD | BPF_IMM | BPF_DW] =	imm_ld8,
1351d9ae7f2bSJakub Kicinski 	[BPF_LD | BPF_ABS | BPF_B] =	data_ld1,
1352d9ae7f2bSJakub Kicinski 	[BPF_LD | BPF_ABS | BPF_H] =	data_ld2,
1353d9ae7f2bSJakub Kicinski 	[BPF_LD | BPF_ABS | BPF_W] =	data_ld4,
1354d9ae7f2bSJakub Kicinski 	[BPF_LD | BPF_IND | BPF_B] =	data_ind_ld1,
1355d9ae7f2bSJakub Kicinski 	[BPF_LD | BPF_IND | BPF_H] =	data_ind_ld2,
1356d9ae7f2bSJakub Kicinski 	[BPF_LD | BPF_IND | BPF_W] =	data_ind_ld4,
1357d9ae7f2bSJakub Kicinski 	[BPF_LDX | BPF_MEM | BPF_W] =	mem_ldx4,
1358d9ae7f2bSJakub Kicinski 	[BPF_STX | BPF_MEM | BPF_W] =	mem_stx4,
1359d9ae7f2bSJakub Kicinski 	[BPF_JMP | BPF_JA | BPF_K] =	jump,
1360d9ae7f2bSJakub Kicinski 	[BPF_JMP | BPF_JEQ | BPF_K] =	jeq_imm,
1361d9ae7f2bSJakub Kicinski 	[BPF_JMP | BPF_JGT | BPF_K] =	jgt_imm,
1362d9ae7f2bSJakub Kicinski 	[BPF_JMP | BPF_JGE | BPF_K] =	jge_imm,
13635dd294d4SDaniel Borkmann 	[BPF_JMP | BPF_JLT | BPF_K] =	jlt_imm,
13645dd294d4SDaniel Borkmann 	[BPF_JMP | BPF_JLE | BPF_K] =	jle_imm,
1365d9ae7f2bSJakub Kicinski 	[BPF_JMP | BPF_JSET | BPF_K] =	jset_imm,
1366d9ae7f2bSJakub Kicinski 	[BPF_JMP | BPF_JNE | BPF_K] =	jne_imm,
1367d9ae7f2bSJakub Kicinski 	[BPF_JMP | BPF_JEQ | BPF_X] =	jeq_reg,
1368d9ae7f2bSJakub Kicinski 	[BPF_JMP | BPF_JGT | BPF_X] =	jgt_reg,
1369d9ae7f2bSJakub Kicinski 	[BPF_JMP | BPF_JGE | BPF_X] =	jge_reg,
13705dd294d4SDaniel Borkmann 	[BPF_JMP | BPF_JLT | BPF_X] =	jlt_reg,
13715dd294d4SDaniel Borkmann 	[BPF_JMP | BPF_JLE | BPF_X] =	jle_reg,
1372d9ae7f2bSJakub Kicinski 	[BPF_JMP | BPF_JSET | BPF_X] =	jset_reg,
1373d9ae7f2bSJakub Kicinski 	[BPF_JMP | BPF_JNE | BPF_X] =	jne_reg,
1374d9ae7f2bSJakub Kicinski 	[BPF_JMP | BPF_EXIT] =		goto_out,
1375d9ae7f2bSJakub Kicinski };
1376d9ae7f2bSJakub Kicinski 
1377d9ae7f2bSJakub Kicinski /* --- Misc code --- */
1378d9ae7f2bSJakub Kicinski static void br_set_offset(u64 *instr, u16 offset)
1379d9ae7f2bSJakub Kicinski {
1380d9ae7f2bSJakub Kicinski 	u16 addr_lo, addr_hi;
1381d9ae7f2bSJakub Kicinski 
1382d9ae7f2bSJakub Kicinski 	addr_lo = offset & (OP_BR_ADDR_LO >> __bf_shf(OP_BR_ADDR_LO));
1383d9ae7f2bSJakub Kicinski 	addr_hi = offset != addr_lo;
1384d9ae7f2bSJakub Kicinski 	*instr &= ~(OP_BR_ADDR_HI | OP_BR_ADDR_LO);
1385d9ae7f2bSJakub Kicinski 	*instr |= FIELD_PREP(OP_BR_ADDR_HI, addr_hi);
1386d9ae7f2bSJakub Kicinski 	*instr |= FIELD_PREP(OP_BR_ADDR_LO, addr_lo);
1387d9ae7f2bSJakub Kicinski }
1388d9ae7f2bSJakub Kicinski 
1389d9ae7f2bSJakub Kicinski /* --- Assembler logic --- */
1390d9ae7f2bSJakub Kicinski static int nfp_fixup_branches(struct nfp_prog *nfp_prog)
1391d9ae7f2bSJakub Kicinski {
1392d9ae7f2bSJakub Kicinski 	struct nfp_insn_meta *meta, *next;
1393d9ae7f2bSJakub Kicinski 	u32 off, br_idx;
1394d9ae7f2bSJakub Kicinski 	u32 idx;
1395d9ae7f2bSJakub Kicinski 
1396d9ae7f2bSJakub Kicinski 	nfp_for_each_insn_walk2(nfp_prog, meta, next) {
1397d9ae7f2bSJakub Kicinski 		if (meta->skip)
1398d9ae7f2bSJakub Kicinski 			continue;
1399d9ae7f2bSJakub Kicinski 		if (BPF_CLASS(meta->insn.code) != BPF_JMP)
1400d9ae7f2bSJakub Kicinski 			continue;
1401d9ae7f2bSJakub Kicinski 
1402d9ae7f2bSJakub Kicinski 		br_idx = nfp_prog_offset_to_index(nfp_prog, next->off) - 1;
1403d9ae7f2bSJakub Kicinski 		if (!nfp_is_br(nfp_prog->prog[br_idx])) {
1404d9ae7f2bSJakub Kicinski 			pr_err("Fixup found block not ending in branch %d %02x %016llx!!\n",
1405d9ae7f2bSJakub Kicinski 			       br_idx, meta->insn.code, nfp_prog->prog[br_idx]);
1406d9ae7f2bSJakub Kicinski 			return -ELOOP;
1407d9ae7f2bSJakub Kicinski 		}
1408d9ae7f2bSJakub Kicinski 		/* Leave special branches for later */
1409d9ae7f2bSJakub Kicinski 		if (FIELD_GET(OP_BR_SPECIAL, nfp_prog->prog[br_idx]))
1410d9ae7f2bSJakub Kicinski 			continue;
1411d9ae7f2bSJakub Kicinski 
1412d9ae7f2bSJakub Kicinski 		/* Find the target offset in assembler realm */
1413d9ae7f2bSJakub Kicinski 		off = meta->insn.off;
1414d9ae7f2bSJakub Kicinski 		if (!off) {
1415d9ae7f2bSJakub Kicinski 			pr_err("Fixup found zero offset!!\n");
1416d9ae7f2bSJakub Kicinski 			return -ELOOP;
1417d9ae7f2bSJakub Kicinski 		}
1418d9ae7f2bSJakub Kicinski 
1419d9ae7f2bSJakub Kicinski 		while (off && nfp_meta_has_next(nfp_prog, next)) {
1420d9ae7f2bSJakub Kicinski 			next = nfp_meta_next(next);
1421d9ae7f2bSJakub Kicinski 			off--;
1422d9ae7f2bSJakub Kicinski 		}
1423d9ae7f2bSJakub Kicinski 		if (off) {
1424d9ae7f2bSJakub Kicinski 			pr_err("Fixup found too large jump!! %d\n", off);
1425d9ae7f2bSJakub Kicinski 			return -ELOOP;
1426d9ae7f2bSJakub Kicinski 		}
1427d9ae7f2bSJakub Kicinski 
1428d9ae7f2bSJakub Kicinski 		if (next->skip) {
1429d9ae7f2bSJakub Kicinski 			pr_err("Branch landing on removed instruction!!\n");
1430d9ae7f2bSJakub Kicinski 			return -ELOOP;
1431d9ae7f2bSJakub Kicinski 		}
1432d9ae7f2bSJakub Kicinski 
1433d9ae7f2bSJakub Kicinski 		for (idx = nfp_prog_offset_to_index(nfp_prog, meta->off);
1434d9ae7f2bSJakub Kicinski 		     idx <= br_idx; idx++) {
1435d9ae7f2bSJakub Kicinski 			if (!nfp_is_br(nfp_prog->prog[idx]))
1436d9ae7f2bSJakub Kicinski 				continue;
1437d9ae7f2bSJakub Kicinski 			br_set_offset(&nfp_prog->prog[idx], next->off);
1438d9ae7f2bSJakub Kicinski 		}
1439d9ae7f2bSJakub Kicinski 	}
1440d9ae7f2bSJakub Kicinski 
1441d9ae7f2bSJakub Kicinski 	/* Fixup 'goto out's separately, they can be scattered around */
1442d9ae7f2bSJakub Kicinski 	for (br_idx = 0; br_idx < nfp_prog->prog_len; br_idx++) {
1443d9ae7f2bSJakub Kicinski 		enum br_special special;
1444d9ae7f2bSJakub Kicinski 
1445d9ae7f2bSJakub Kicinski 		if ((nfp_prog->prog[br_idx] & OP_BR_BASE_MASK) != OP_BR_BASE)
1446d9ae7f2bSJakub Kicinski 			continue;
1447d9ae7f2bSJakub Kicinski 
1448d9ae7f2bSJakub Kicinski 		special = FIELD_GET(OP_BR_SPECIAL, nfp_prog->prog[br_idx]);
1449d9ae7f2bSJakub Kicinski 		switch (special) {
1450d9ae7f2bSJakub Kicinski 		case OP_BR_NORMAL:
1451d9ae7f2bSJakub Kicinski 			break;
1452d9ae7f2bSJakub Kicinski 		case OP_BR_GO_OUT:
1453d9ae7f2bSJakub Kicinski 			br_set_offset(&nfp_prog->prog[br_idx],
1454d9ae7f2bSJakub Kicinski 				      nfp_prog->tgt_out);
1455d9ae7f2bSJakub Kicinski 			break;
1456d9ae7f2bSJakub Kicinski 		case OP_BR_GO_ABORT:
1457d9ae7f2bSJakub Kicinski 			br_set_offset(&nfp_prog->prog[br_idx],
1458d9ae7f2bSJakub Kicinski 				      nfp_prog->tgt_abort);
1459d9ae7f2bSJakub Kicinski 			break;
1460d9ae7f2bSJakub Kicinski 		}
1461d9ae7f2bSJakub Kicinski 
1462d9ae7f2bSJakub Kicinski 		nfp_prog->prog[br_idx] &= ~OP_BR_SPECIAL;
1463d9ae7f2bSJakub Kicinski 	}
1464d9ae7f2bSJakub Kicinski 
1465d9ae7f2bSJakub Kicinski 	return 0;
1466d9ae7f2bSJakub Kicinski }
1467d9ae7f2bSJakub Kicinski 
1468d9ae7f2bSJakub Kicinski static void nfp_intro(struct nfp_prog *nfp_prog)
1469d9ae7f2bSJakub Kicinski {
147018e53b6cSJakub Kicinski 	wrp_immed(nfp_prog, plen_reg(nfp_prog), GENMASK(13, 0));
147118e53b6cSJakub Kicinski 	emit_alu(nfp_prog, plen_reg(nfp_prog),
147218e53b6cSJakub Kicinski 		 plen_reg(nfp_prog), ALU_OP_AND, pv_len(nfp_prog));
1473d9ae7f2bSJakub Kicinski }
1474d9ae7f2bSJakub Kicinski 
1475d9ae7f2bSJakub Kicinski static void nfp_outro_tc_legacy(struct nfp_prog *nfp_prog)
1476d9ae7f2bSJakub Kicinski {
1477d9ae7f2bSJakub Kicinski 	const u8 act2code[] = {
1478d9ae7f2bSJakub Kicinski 		[NN_ACT_TC_DROP]  = 0x22,
1479d9ae7f2bSJakub Kicinski 		[NN_ACT_TC_REDIR] = 0x24
1480d9ae7f2bSJakub Kicinski 	};
1481d9ae7f2bSJakub Kicinski 	/* Target for aborts */
1482d9ae7f2bSJakub Kicinski 	nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog);
1483d9ae7f2bSJakub Kicinski 	wrp_immed(nfp_prog, reg_both(0), 0);
1484d9ae7f2bSJakub Kicinski 
1485d9ae7f2bSJakub Kicinski 	/* Target for normal exits */
1486d9ae7f2bSJakub Kicinski 	nfp_prog->tgt_out = nfp_prog_current_offset(nfp_prog);
1487d9ae7f2bSJakub Kicinski 	/* Legacy TC mode:
1488d9ae7f2bSJakub Kicinski 	 *   0        0x11 -> pass,  count as stat0
1489d9ae7f2bSJakub Kicinski 	 *  -1  drop  0x22 -> drop,  count as stat1
1490d9ae7f2bSJakub Kicinski 	 *     redir  0x24 -> redir, count as stat1
1491d9ae7f2bSJakub Kicinski 	 *  ife mark  0x21 -> pass,  count as stat1
1492d9ae7f2bSJakub Kicinski 	 *  ife + tx  0x24 -> redir, count as stat1
1493d9ae7f2bSJakub Kicinski 	 */
1494d9ae7f2bSJakub Kicinski 	emit_br_byte_neq(nfp_prog, reg_b(0), 0xff, 0, nfp_prog->tgt_done, 2);
1495c000dfb5SJakub Kicinski 	wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS);
1496d9ae7f2bSJakub Kicinski 	emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x11), SHF_SC_L_SHF, 16);
1497d9ae7f2bSJakub Kicinski 
1498d9ae7f2bSJakub Kicinski 	emit_br(nfp_prog, BR_UNC, nfp_prog->tgt_done, 1);
1499d9ae7f2bSJakub Kicinski 	emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(act2code[nfp_prog->act]),
1500d9ae7f2bSJakub Kicinski 		      SHF_SC_L_SHF, 16);
1501d9ae7f2bSJakub Kicinski }
1502d9ae7f2bSJakub Kicinski 
1503d9ae7f2bSJakub Kicinski static void nfp_outro_tc_da(struct nfp_prog *nfp_prog)
1504d9ae7f2bSJakub Kicinski {
1505d9ae7f2bSJakub Kicinski 	/* TC direct-action mode:
1506d9ae7f2bSJakub Kicinski 	 *   0,1   ok        NOT SUPPORTED[1]
1507d9ae7f2bSJakub Kicinski 	 *   2   drop  0x22 -> drop,  count as stat1
1508d9ae7f2bSJakub Kicinski 	 *   4,5 nuke  0x02 -> drop
1509d9ae7f2bSJakub Kicinski 	 *   7  redir  0x44 -> redir, count as stat2
1510d9ae7f2bSJakub Kicinski 	 *   * unspec  0x11 -> pass,  count as stat0
1511d9ae7f2bSJakub Kicinski 	 *
1512d9ae7f2bSJakub Kicinski 	 * [1] We can't support OK and RECLASSIFY because we can't tell TC
1513d9ae7f2bSJakub Kicinski 	 *     the exact decision made.  We are forced to support UNSPEC
1514d9ae7f2bSJakub Kicinski 	 *     to handle aborts so that's the only one we handle for passing
1515d9ae7f2bSJakub Kicinski 	 *     packets up the stack.
1516d9ae7f2bSJakub Kicinski 	 */
1517d9ae7f2bSJakub Kicinski 	/* Target for aborts */
1518d9ae7f2bSJakub Kicinski 	nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog);
1519d9ae7f2bSJakub Kicinski 
1520d9ae7f2bSJakub Kicinski 	emit_br_def(nfp_prog, nfp_prog->tgt_done, 2);
1521d9ae7f2bSJakub Kicinski 
1522c000dfb5SJakub Kicinski 	wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS);
1523d9ae7f2bSJakub Kicinski 	emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x11), SHF_SC_L_SHF, 16);
1524d9ae7f2bSJakub Kicinski 
1525d9ae7f2bSJakub Kicinski 	/* Target for normal exits */
1526d9ae7f2bSJakub Kicinski 	nfp_prog->tgt_out = nfp_prog_current_offset(nfp_prog);
1527d9ae7f2bSJakub Kicinski 
1528d9ae7f2bSJakub Kicinski 	/* if R0 > 7 jump to abort */
1529d9ae7f2bSJakub Kicinski 	emit_alu(nfp_prog, reg_none(), reg_imm(7), ALU_OP_SUB, reg_b(0));
1530d9ae7f2bSJakub Kicinski 	emit_br(nfp_prog, BR_BLO, nfp_prog->tgt_abort, 0);
1531c000dfb5SJakub Kicinski 	wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS);
1532d9ae7f2bSJakub Kicinski 
1533d9ae7f2bSJakub Kicinski 	wrp_immed(nfp_prog, reg_b(2), 0x41221211);
1534d9ae7f2bSJakub Kicinski 	wrp_immed(nfp_prog, reg_b(3), 0x41001211);
1535d9ae7f2bSJakub Kicinski 
1536d9ae7f2bSJakub Kicinski 	emit_shf(nfp_prog, reg_a(1),
1537d9ae7f2bSJakub Kicinski 		 reg_none(), SHF_OP_NONE, reg_b(0), SHF_SC_L_SHF, 2);
1538d9ae7f2bSJakub Kicinski 
1539d9ae7f2bSJakub Kicinski 	emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0));
1540d9ae7f2bSJakub Kicinski 	emit_shf(nfp_prog, reg_a(2),
1541d9ae7f2bSJakub Kicinski 		 reg_imm(0xf), SHF_OP_AND, reg_b(2), SHF_SC_R_SHF, 0);
1542d9ae7f2bSJakub Kicinski 
1543d9ae7f2bSJakub Kicinski 	emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0));
1544d9ae7f2bSJakub Kicinski 	emit_shf(nfp_prog, reg_b(2),
1545d9ae7f2bSJakub Kicinski 		 reg_imm(0xf), SHF_OP_AND, reg_b(3), SHF_SC_R_SHF, 0);
1546d9ae7f2bSJakub Kicinski 
1547d9ae7f2bSJakub Kicinski 	emit_br_def(nfp_prog, nfp_prog->tgt_done, 2);
1548d9ae7f2bSJakub Kicinski 
1549d9ae7f2bSJakub Kicinski 	emit_shf(nfp_prog, reg_b(2),
1550d9ae7f2bSJakub Kicinski 		 reg_a(2), SHF_OP_OR, reg_b(2), SHF_SC_L_SHF, 4);
1551d9ae7f2bSJakub Kicinski 	emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_b(2), SHF_SC_L_SHF, 16);
1552d9ae7f2bSJakub Kicinski }
1553d9ae7f2bSJakub Kicinski 
1554d9ae7f2bSJakub Kicinski static void nfp_outro_xdp(struct nfp_prog *nfp_prog)
1555d9ae7f2bSJakub Kicinski {
1556d9ae7f2bSJakub Kicinski 	/* XDP return codes:
1557d9ae7f2bSJakub Kicinski 	 *   0 aborted  0x82 -> drop,  count as stat3
1558d9ae7f2bSJakub Kicinski 	 *   1    drop  0x22 -> drop,  count as stat1
1559d9ae7f2bSJakub Kicinski 	 *   2    pass  0x11 -> pass,  count as stat0
1560d9ae7f2bSJakub Kicinski 	 *   3      tx  0x44 -> redir, count as stat2
1561d9ae7f2bSJakub Kicinski 	 *   * unknown  0x82 -> drop,  count as stat3
1562d9ae7f2bSJakub Kicinski 	 */
1563d9ae7f2bSJakub Kicinski 	/* Target for aborts */
1564d9ae7f2bSJakub Kicinski 	nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog);
1565d9ae7f2bSJakub Kicinski 
1566d9ae7f2bSJakub Kicinski 	emit_br_def(nfp_prog, nfp_prog->tgt_done, 2);
1567d9ae7f2bSJakub Kicinski 
1568c000dfb5SJakub Kicinski 	wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS);
1569d9ae7f2bSJakub Kicinski 	emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x82), SHF_SC_L_SHF, 16);
1570d9ae7f2bSJakub Kicinski 
1571d9ae7f2bSJakub Kicinski 	/* Target for normal exits */
1572d9ae7f2bSJakub Kicinski 	nfp_prog->tgt_out = nfp_prog_current_offset(nfp_prog);
1573d9ae7f2bSJakub Kicinski 
1574d9ae7f2bSJakub Kicinski 	/* if R0 > 3 jump to abort */
1575d9ae7f2bSJakub Kicinski 	emit_alu(nfp_prog, reg_none(), reg_imm(3), ALU_OP_SUB, reg_b(0));
1576d9ae7f2bSJakub Kicinski 	emit_br(nfp_prog, BR_BLO, nfp_prog->tgt_abort, 0);
1577d9ae7f2bSJakub Kicinski 
1578d9ae7f2bSJakub Kicinski 	wrp_immed(nfp_prog, reg_b(2), 0x44112282);
1579d9ae7f2bSJakub Kicinski 
1580d9ae7f2bSJakub Kicinski 	emit_shf(nfp_prog, reg_a(1),
1581d9ae7f2bSJakub Kicinski 		 reg_none(), SHF_OP_NONE, reg_b(0), SHF_SC_L_SHF, 3);
1582d9ae7f2bSJakub Kicinski 
1583d9ae7f2bSJakub Kicinski 	emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0));
1584d9ae7f2bSJakub Kicinski 	emit_shf(nfp_prog, reg_b(2),
1585d9ae7f2bSJakub Kicinski 		 reg_imm(0xff), SHF_OP_AND, reg_b(2), SHF_SC_R_SHF, 0);
1586d9ae7f2bSJakub Kicinski 
1587d9ae7f2bSJakub Kicinski 	emit_br_def(nfp_prog, nfp_prog->tgt_done, 2);
1588d9ae7f2bSJakub Kicinski 
1589c000dfb5SJakub Kicinski 	wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS);
1590d9ae7f2bSJakub Kicinski 	emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_b(2), SHF_SC_L_SHF, 16);
1591d9ae7f2bSJakub Kicinski }
1592d9ae7f2bSJakub Kicinski 
1593d9ae7f2bSJakub Kicinski static void nfp_outro(struct nfp_prog *nfp_prog)
1594d9ae7f2bSJakub Kicinski {
1595d9ae7f2bSJakub Kicinski 	switch (nfp_prog->act) {
1596d9ae7f2bSJakub Kicinski 	case NN_ACT_DIRECT:
1597d9ae7f2bSJakub Kicinski 		nfp_outro_tc_da(nfp_prog);
1598d9ae7f2bSJakub Kicinski 		break;
1599d9ae7f2bSJakub Kicinski 	case NN_ACT_TC_DROP:
1600d9ae7f2bSJakub Kicinski 	case NN_ACT_TC_REDIR:
1601d9ae7f2bSJakub Kicinski 		nfp_outro_tc_legacy(nfp_prog);
1602d9ae7f2bSJakub Kicinski 		break;
1603d9ae7f2bSJakub Kicinski 	case NN_ACT_XDP:
1604d9ae7f2bSJakub Kicinski 		nfp_outro_xdp(nfp_prog);
1605d9ae7f2bSJakub Kicinski 		break;
1606d9ae7f2bSJakub Kicinski 	}
1607d9ae7f2bSJakub Kicinski }
1608d9ae7f2bSJakub Kicinski 
1609d9ae7f2bSJakub Kicinski static int nfp_translate(struct nfp_prog *nfp_prog)
1610d9ae7f2bSJakub Kicinski {
1611d9ae7f2bSJakub Kicinski 	struct nfp_insn_meta *meta;
16121c03e03fSJakub Kicinski 	int i, err;
1613d9ae7f2bSJakub Kicinski 
1614d9ae7f2bSJakub Kicinski 	nfp_intro(nfp_prog);
1615d9ae7f2bSJakub Kicinski 	if (nfp_prog->error)
1616d9ae7f2bSJakub Kicinski 		return nfp_prog->error;
1617d9ae7f2bSJakub Kicinski 
1618d9ae7f2bSJakub Kicinski 	list_for_each_entry(meta, &nfp_prog->insns, l) {
1619d9ae7f2bSJakub Kicinski 		instr_cb_t cb = instr_cb[meta->insn.code];
1620d9ae7f2bSJakub Kicinski 
1621d9ae7f2bSJakub Kicinski 		meta->off = nfp_prog_current_offset(nfp_prog);
1622d9ae7f2bSJakub Kicinski 
1623d9ae7f2bSJakub Kicinski 		if (meta->skip) {
1624d9ae7f2bSJakub Kicinski 			nfp_prog->n_translated++;
1625d9ae7f2bSJakub Kicinski 			continue;
1626d9ae7f2bSJakub Kicinski 		}
1627d9ae7f2bSJakub Kicinski 
1628d9ae7f2bSJakub Kicinski 		if (nfp_meta_has_prev(nfp_prog, meta) &&
1629d9ae7f2bSJakub Kicinski 		    nfp_meta_prev(meta)->double_cb)
1630d9ae7f2bSJakub Kicinski 			cb = nfp_meta_prev(meta)->double_cb;
1631d9ae7f2bSJakub Kicinski 		if (!cb)
1632d9ae7f2bSJakub Kicinski 			return -ENOENT;
1633d9ae7f2bSJakub Kicinski 		err = cb(nfp_prog, meta);
1634d9ae7f2bSJakub Kicinski 		if (err)
1635d9ae7f2bSJakub Kicinski 			return err;
1636d9ae7f2bSJakub Kicinski 
1637d9ae7f2bSJakub Kicinski 		nfp_prog->n_translated++;
1638d9ae7f2bSJakub Kicinski 	}
1639d9ae7f2bSJakub Kicinski 
1640d9ae7f2bSJakub Kicinski 	nfp_outro(nfp_prog);
1641d9ae7f2bSJakub Kicinski 	if (nfp_prog->error)
1642d9ae7f2bSJakub Kicinski 		return nfp_prog->error;
1643d9ae7f2bSJakub Kicinski 
16441c03e03fSJakub Kicinski 	for (i = 0; i < NFP_USTORE_PREFETCH_WINDOW; i++)
16451c03e03fSJakub Kicinski 		emit_nop(nfp_prog);
16461c03e03fSJakub Kicinski 	if (nfp_prog->error)
16471c03e03fSJakub Kicinski 		return nfp_prog->error;
16481c03e03fSJakub Kicinski 
1649d9ae7f2bSJakub Kicinski 	return nfp_fixup_branches(nfp_prog);
1650d9ae7f2bSJakub Kicinski }
1651d9ae7f2bSJakub Kicinski 
1652d9ae7f2bSJakub Kicinski static int
1653d9ae7f2bSJakub Kicinski nfp_prog_prepare(struct nfp_prog *nfp_prog, const struct bpf_insn *prog,
1654d9ae7f2bSJakub Kicinski 		 unsigned int cnt)
1655d9ae7f2bSJakub Kicinski {
1656d9ae7f2bSJakub Kicinski 	unsigned int i;
1657d9ae7f2bSJakub Kicinski 
1658d9ae7f2bSJakub Kicinski 	for (i = 0; i < cnt; i++) {
1659d9ae7f2bSJakub Kicinski 		struct nfp_insn_meta *meta;
1660d9ae7f2bSJakub Kicinski 
1661d9ae7f2bSJakub Kicinski 		meta = kzalloc(sizeof(*meta), GFP_KERNEL);
1662d9ae7f2bSJakub Kicinski 		if (!meta)
1663d9ae7f2bSJakub Kicinski 			return -ENOMEM;
1664d9ae7f2bSJakub Kicinski 
1665d9ae7f2bSJakub Kicinski 		meta->insn = prog[i];
1666d9ae7f2bSJakub Kicinski 		meta->n = i;
1667d9ae7f2bSJakub Kicinski 
1668d9ae7f2bSJakub Kicinski 		list_add_tail(&meta->l, &nfp_prog->insns);
1669d9ae7f2bSJakub Kicinski 	}
1670d9ae7f2bSJakub Kicinski 
1671d9ae7f2bSJakub Kicinski 	return 0;
1672d9ae7f2bSJakub Kicinski }
1673d9ae7f2bSJakub Kicinski 
1674d9ae7f2bSJakub Kicinski /* --- Optimizations --- */
1675d9ae7f2bSJakub Kicinski static void nfp_bpf_opt_reg_init(struct nfp_prog *nfp_prog)
1676d9ae7f2bSJakub Kicinski {
1677d9ae7f2bSJakub Kicinski 	struct nfp_insn_meta *meta;
1678d9ae7f2bSJakub Kicinski 
1679d9ae7f2bSJakub Kicinski 	list_for_each_entry(meta, &nfp_prog->insns, l) {
1680d9ae7f2bSJakub Kicinski 		struct bpf_insn insn = meta->insn;
1681d9ae7f2bSJakub Kicinski 
1682d9ae7f2bSJakub Kicinski 		/* Programs converted from cBPF start with register xoring */
1683d9ae7f2bSJakub Kicinski 		if (insn.code == (BPF_ALU64 | BPF_XOR | BPF_X) &&
1684d9ae7f2bSJakub Kicinski 		    insn.src_reg == insn.dst_reg)
1685d9ae7f2bSJakub Kicinski 			continue;
1686d9ae7f2bSJakub Kicinski 
1687d9ae7f2bSJakub Kicinski 		/* Programs start with R6 = R1 but we ignore the skb pointer */
1688d9ae7f2bSJakub Kicinski 		if (insn.code == (BPF_ALU64 | BPF_MOV | BPF_X) &&
1689d9ae7f2bSJakub Kicinski 		    insn.src_reg == 1 && insn.dst_reg == 6)
1690d9ae7f2bSJakub Kicinski 			meta->skip = true;
1691d9ae7f2bSJakub Kicinski 
1692d9ae7f2bSJakub Kicinski 		/* Return as soon as something doesn't match */
1693d9ae7f2bSJakub Kicinski 		if (!meta->skip)
1694d9ae7f2bSJakub Kicinski 			return;
1695d9ae7f2bSJakub Kicinski 	}
1696d9ae7f2bSJakub Kicinski }
1697d9ae7f2bSJakub Kicinski 
1698d9ae7f2bSJakub Kicinski /* Remove masking after load since our load guarantees this is not needed */
1699d9ae7f2bSJakub Kicinski static void nfp_bpf_opt_ld_mask(struct nfp_prog *nfp_prog)
1700d9ae7f2bSJakub Kicinski {
1701d9ae7f2bSJakub Kicinski 	struct nfp_insn_meta *meta1, *meta2;
1702d9ae7f2bSJakub Kicinski 	const s32 exp_mask[] = {
1703d9ae7f2bSJakub Kicinski 		[BPF_B] = 0x000000ffU,
1704d9ae7f2bSJakub Kicinski 		[BPF_H] = 0x0000ffffU,
1705d9ae7f2bSJakub Kicinski 		[BPF_W] = 0xffffffffU,
1706d9ae7f2bSJakub Kicinski 	};
1707d9ae7f2bSJakub Kicinski 
1708d9ae7f2bSJakub Kicinski 	nfp_for_each_insn_walk2(nfp_prog, meta1, meta2) {
1709d9ae7f2bSJakub Kicinski 		struct bpf_insn insn, next;
1710d9ae7f2bSJakub Kicinski 
1711d9ae7f2bSJakub Kicinski 		insn = meta1->insn;
1712d9ae7f2bSJakub Kicinski 		next = meta2->insn;
1713d9ae7f2bSJakub Kicinski 
1714d9ae7f2bSJakub Kicinski 		if (BPF_CLASS(insn.code) != BPF_LD)
1715d9ae7f2bSJakub Kicinski 			continue;
1716d9ae7f2bSJakub Kicinski 		if (BPF_MODE(insn.code) != BPF_ABS &&
1717d9ae7f2bSJakub Kicinski 		    BPF_MODE(insn.code) != BPF_IND)
1718d9ae7f2bSJakub Kicinski 			continue;
1719d9ae7f2bSJakub Kicinski 
1720d9ae7f2bSJakub Kicinski 		if (next.code != (BPF_ALU64 | BPF_AND | BPF_K))
1721d9ae7f2bSJakub Kicinski 			continue;
1722d9ae7f2bSJakub Kicinski 
1723d9ae7f2bSJakub Kicinski 		if (!exp_mask[BPF_SIZE(insn.code)])
1724d9ae7f2bSJakub Kicinski 			continue;
1725d9ae7f2bSJakub Kicinski 		if (exp_mask[BPF_SIZE(insn.code)] != next.imm)
1726d9ae7f2bSJakub Kicinski 			continue;
1727d9ae7f2bSJakub Kicinski 
1728d9ae7f2bSJakub Kicinski 		if (next.src_reg || next.dst_reg)
1729d9ae7f2bSJakub Kicinski 			continue;
1730d9ae7f2bSJakub Kicinski 
1731d9ae7f2bSJakub Kicinski 		meta2->skip = true;
1732d9ae7f2bSJakub Kicinski 	}
1733d9ae7f2bSJakub Kicinski }
1734d9ae7f2bSJakub Kicinski 
1735d9ae7f2bSJakub Kicinski static void nfp_bpf_opt_ld_shift(struct nfp_prog *nfp_prog)
1736d9ae7f2bSJakub Kicinski {
1737d9ae7f2bSJakub Kicinski 	struct nfp_insn_meta *meta1, *meta2, *meta3;
1738d9ae7f2bSJakub Kicinski 
1739d9ae7f2bSJakub Kicinski 	nfp_for_each_insn_walk3(nfp_prog, meta1, meta2, meta3) {
1740d9ae7f2bSJakub Kicinski 		struct bpf_insn insn, next1, next2;
1741d9ae7f2bSJakub Kicinski 
1742d9ae7f2bSJakub Kicinski 		insn = meta1->insn;
1743d9ae7f2bSJakub Kicinski 		next1 = meta2->insn;
1744d9ae7f2bSJakub Kicinski 		next2 = meta3->insn;
1745d9ae7f2bSJakub Kicinski 
1746d9ae7f2bSJakub Kicinski 		if (BPF_CLASS(insn.code) != BPF_LD)
1747d9ae7f2bSJakub Kicinski 			continue;
1748d9ae7f2bSJakub Kicinski 		if (BPF_MODE(insn.code) != BPF_ABS &&
1749d9ae7f2bSJakub Kicinski 		    BPF_MODE(insn.code) != BPF_IND)
1750d9ae7f2bSJakub Kicinski 			continue;
1751d9ae7f2bSJakub Kicinski 		if (BPF_SIZE(insn.code) != BPF_W)
1752d9ae7f2bSJakub Kicinski 			continue;
1753d9ae7f2bSJakub Kicinski 
1754d9ae7f2bSJakub Kicinski 		if (!(next1.code == (BPF_LSH | BPF_K | BPF_ALU64) &&
1755d9ae7f2bSJakub Kicinski 		      next2.code == (BPF_RSH | BPF_K | BPF_ALU64)) &&
1756d9ae7f2bSJakub Kicinski 		    !(next1.code == (BPF_RSH | BPF_K | BPF_ALU64) &&
1757d9ae7f2bSJakub Kicinski 		      next2.code == (BPF_LSH | BPF_K | BPF_ALU64)))
1758d9ae7f2bSJakub Kicinski 			continue;
1759d9ae7f2bSJakub Kicinski 
1760d9ae7f2bSJakub Kicinski 		if (next1.src_reg || next1.dst_reg ||
1761d9ae7f2bSJakub Kicinski 		    next2.src_reg || next2.dst_reg)
1762d9ae7f2bSJakub Kicinski 			continue;
1763d9ae7f2bSJakub Kicinski 
1764d9ae7f2bSJakub Kicinski 		if (next1.imm != 0x20 || next2.imm != 0x20)
1765d9ae7f2bSJakub Kicinski 			continue;
1766d9ae7f2bSJakub Kicinski 
1767d9ae7f2bSJakub Kicinski 		meta2->skip = true;
1768d9ae7f2bSJakub Kicinski 		meta3->skip = true;
1769d9ae7f2bSJakub Kicinski 	}
1770d9ae7f2bSJakub Kicinski }
1771d9ae7f2bSJakub Kicinski 
1772d9ae7f2bSJakub Kicinski static int nfp_bpf_optimize(struct nfp_prog *nfp_prog)
1773d9ae7f2bSJakub Kicinski {
1774d9ae7f2bSJakub Kicinski 	nfp_bpf_opt_reg_init(nfp_prog);
1775d9ae7f2bSJakub Kicinski 
1776d9ae7f2bSJakub Kicinski 	nfp_bpf_opt_ld_mask(nfp_prog);
1777d9ae7f2bSJakub Kicinski 	nfp_bpf_opt_ld_shift(nfp_prog);
1778d9ae7f2bSJakub Kicinski 
1779d9ae7f2bSJakub Kicinski 	return 0;
1780d9ae7f2bSJakub Kicinski }
1781d9ae7f2bSJakub Kicinski 
17822e85d388SJakub Kicinski static int nfp_bpf_ustore_calc(struct nfp_prog *nfp_prog, __le64 *ustore)
1783fd068ddcSJakub Kicinski {
1784fd068ddcSJakub Kicinski 	int i;
1785fd068ddcSJakub Kicinski 
1786fd068ddcSJakub Kicinski 	for (i = 0; i < nfp_prog->prog_len; i++) {
1787fd068ddcSJakub Kicinski 		int err;
1788fd068ddcSJakub Kicinski 
1789fd068ddcSJakub Kicinski 		err = nfp_ustore_check_valid_no_ecc(nfp_prog->prog[i]);
1790fd068ddcSJakub Kicinski 		if (err)
1791fd068ddcSJakub Kicinski 			return err;
1792fd068ddcSJakub Kicinski 
1793fd068ddcSJakub Kicinski 		nfp_prog->prog[i] = nfp_ustore_calc_ecc_insn(nfp_prog->prog[i]);
17942e85d388SJakub Kicinski 
17952e85d388SJakub Kicinski 		ustore[i] = cpu_to_le64(nfp_prog->prog[i]);
1796fd068ddcSJakub Kicinski 	}
1797fd068ddcSJakub Kicinski 
1798fd068ddcSJakub Kicinski 	return 0;
1799fd068ddcSJakub Kicinski }
1800fd068ddcSJakub Kicinski 
1801d9ae7f2bSJakub Kicinski /**
1802d9ae7f2bSJakub Kicinski  * nfp_bpf_jit() - translate BPF code into NFP assembly
1803d9ae7f2bSJakub Kicinski  * @filter:	kernel BPF filter struct
1804d9ae7f2bSJakub Kicinski  * @prog_mem:	memory to store assembler instructions
1805d9ae7f2bSJakub Kicinski  * @act:	action attached to this eBPF program
1806d9ae7f2bSJakub Kicinski  * @prog_start:	offset of the first instruction when loaded
1807d9ae7f2bSJakub Kicinski  * @prog_done:	where to jump on exit
1808d9ae7f2bSJakub Kicinski  * @prog_sz:	size of @prog_mem in instructions
1809d9ae7f2bSJakub Kicinski  * @res:	achieved parameters of translation results
1810d9ae7f2bSJakub Kicinski  */
1811d9ae7f2bSJakub Kicinski int
1812d9ae7f2bSJakub Kicinski nfp_bpf_jit(struct bpf_prog *filter, void *prog_mem,
1813d9ae7f2bSJakub Kicinski 	    enum nfp_bpf_action_type act,
1814d9ae7f2bSJakub Kicinski 	    unsigned int prog_start, unsigned int prog_done,
1815d9ae7f2bSJakub Kicinski 	    unsigned int prog_sz, struct nfp_bpf_result *res)
1816d9ae7f2bSJakub Kicinski {
1817d9ae7f2bSJakub Kicinski 	struct nfp_prog *nfp_prog;
1818d9ae7f2bSJakub Kicinski 	int ret;
1819d9ae7f2bSJakub Kicinski 
1820d9ae7f2bSJakub Kicinski 	nfp_prog = kzalloc(sizeof(*nfp_prog), GFP_KERNEL);
1821d9ae7f2bSJakub Kicinski 	if (!nfp_prog)
1822d9ae7f2bSJakub Kicinski 		return -ENOMEM;
1823d9ae7f2bSJakub Kicinski 
1824d9ae7f2bSJakub Kicinski 	INIT_LIST_HEAD(&nfp_prog->insns);
1825d9ae7f2bSJakub Kicinski 	nfp_prog->act = act;
1826d9ae7f2bSJakub Kicinski 	nfp_prog->start_off = prog_start;
1827d9ae7f2bSJakub Kicinski 	nfp_prog->tgt_done = prog_done;
1828d9ae7f2bSJakub Kicinski 
1829d9ae7f2bSJakub Kicinski 	ret = nfp_prog_prepare(nfp_prog, filter->insnsi, filter->len);
1830d9ae7f2bSJakub Kicinski 	if (ret)
1831d9ae7f2bSJakub Kicinski 		goto out;
1832d9ae7f2bSJakub Kicinski 
1833d9ae7f2bSJakub Kicinski 	ret = nfp_prog_verify(nfp_prog, filter);
1834d9ae7f2bSJakub Kicinski 	if (ret)
1835d9ae7f2bSJakub Kicinski 		goto out;
1836d9ae7f2bSJakub Kicinski 
1837d9ae7f2bSJakub Kicinski 	ret = nfp_bpf_optimize(nfp_prog);
1838d9ae7f2bSJakub Kicinski 	if (ret)
1839d9ae7f2bSJakub Kicinski 		goto out;
1840d9ae7f2bSJakub Kicinski 
1841226e0e94SJakub Kicinski 	nfp_prog->num_regs = MAX_BPF_REG;
1842d9ae7f2bSJakub Kicinski 	nfp_prog->regs_per_thread = 32;
1843d9ae7f2bSJakub Kicinski 
1844d9ae7f2bSJakub Kicinski 	nfp_prog->prog = prog_mem;
1845d9ae7f2bSJakub Kicinski 	nfp_prog->__prog_alloc_len = prog_sz;
1846d9ae7f2bSJakub Kicinski 
1847d9ae7f2bSJakub Kicinski 	ret = nfp_translate(nfp_prog);
1848d9ae7f2bSJakub Kicinski 	if (ret) {
1849d9ae7f2bSJakub Kicinski 		pr_err("Translation failed with error %d (translated: %u)\n",
1850d9ae7f2bSJakub Kicinski 		       ret, nfp_prog->n_translated);
1851d9ae7f2bSJakub Kicinski 		ret = -EINVAL;
1852fd068ddcSJakub Kicinski 		goto out;
1853d9ae7f2bSJakub Kicinski 	}
1854d9ae7f2bSJakub Kicinski 
18552e85d388SJakub Kicinski 	ret = nfp_bpf_ustore_calc(nfp_prog, (__force __le64 *)prog_mem);
1856fd068ddcSJakub Kicinski 
1857d9ae7f2bSJakub Kicinski 	res->n_instr = nfp_prog->prog_len;
1858226e0e94SJakub Kicinski 	res->dense_mode = false;
1859d9ae7f2bSJakub Kicinski out:
1860d9ae7f2bSJakub Kicinski 	nfp_prog_free(nfp_prog);
1861d9ae7f2bSJakub Kicinski 
1862d9ae7f2bSJakub Kicinski 	return ret;
1863d9ae7f2bSJakub Kicinski }
1864