xref: /openbmc/linux/arch/s390/net/bpf_jit_comp.c (revision a2cce7a9)
1 /*
2  * BPF Jit compiler for s390.
3  *
4  * Minimum build requirements:
5  *
6  *  - HAVE_MARCH_Z196_FEATURES: laal, laalg
7  *  - HAVE_MARCH_Z10_FEATURES: msfi, cgrj, clgrj
8  *  - HAVE_MARCH_Z9_109_FEATURES: alfi, llilf, clfi, oilf, nilf
9  *  - PACK_STACK
10  *  - 64BIT
11  *
12  * Copyright IBM Corp. 2012,2015
13  *
14  * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
15  *	      Michael Holzheu <holzheu@linux.vnet.ibm.com>
16  */
17 
18 #define KMSG_COMPONENT "bpf_jit"
19 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
20 
21 #include <linux/netdevice.h>
22 #include <linux/filter.h>
23 #include <linux/init.h>
24 #include <linux/bpf.h>
25 #include <asm/cacheflush.h>
26 #include <asm/dis.h>
27 #include "bpf_jit.h"
28 
29 int bpf_jit_enable __read_mostly;
30 
31 struct bpf_jit {
32 	u32 seen;		/* Flags to remember seen eBPF instructions */
33 	u32 seen_reg[16];	/* Array to remember which registers are used */
34 	u32 *addrs;		/* Array with relative instruction addresses */
35 	u8 *prg_buf;		/* Start of program */
36 	int size;		/* Size of program and literal pool */
37 	int size_prg;		/* Size of program */
38 	int prg;		/* Current position in program */
39 	int lit_start;		/* Start of literal pool */
40 	int lit;		/* Current position in literal pool */
41 	int base_ip;		/* Base address for literal pool */
42 	int ret0_ip;		/* Address of return 0 */
43 	int exit_ip;		/* Address of exit */
44 	int tail_call_start;	/* Tail call start offset */
45 	int labels[1];		/* Labels for local jumps */
46 };
47 
48 #define BPF_SIZE_MAX	0x7ffff	/* Max size for program (20 bit signed displ) */
49 
50 #define SEEN_SKB	1	/* skb access */
51 #define SEEN_MEM	2	/* use mem[] for temporary storage */
52 #define SEEN_RET0	4	/* ret0_ip points to a valid return 0 */
53 #define SEEN_LITERAL	8	/* code uses literals */
54 #define SEEN_FUNC	16	/* calls C functions */
55 #define SEEN_TAIL_CALL	32	/* code uses tail calls */
56 #define SEEN_SKB_CHANGE	64	/* code changes skb data */
57 #define SEEN_STACK	(SEEN_FUNC | SEEN_MEM | SEEN_SKB)
58 
59 /*
60  * s390 registers
61  */
62 #define REG_W0		(__MAX_BPF_REG+0)	/* Work register 1 (even) */
63 #define REG_W1		(__MAX_BPF_REG+1)	/* Work register 2 (odd) */
64 #define REG_SKB_DATA	(__MAX_BPF_REG+2)	/* SKB data register */
65 #define REG_L		(__MAX_BPF_REG+3)	/* Literal pool register */
66 #define REG_15		(__MAX_BPF_REG+4)	/* Register 15 */
67 #define REG_0		REG_W0			/* Register 0 */
68 #define REG_1		REG_W1			/* Register 1 */
69 #define REG_2		BPF_REG_1		/* Register 2 */
70 #define REG_14		BPF_REG_0		/* Register 14 */
71 
72 /*
73  * Mapping of BPF registers to s390 registers
74  */
75 static const int reg2hex[] = {
76 	/* Return code */
77 	[BPF_REG_0]	= 14,
78 	/* Function parameters */
79 	[BPF_REG_1]	= 2,
80 	[BPF_REG_2]	= 3,
81 	[BPF_REG_3]	= 4,
82 	[BPF_REG_4]	= 5,
83 	[BPF_REG_5]	= 6,
84 	/* Call saved registers */
85 	[BPF_REG_6]	= 7,
86 	[BPF_REG_7]	= 8,
87 	[BPF_REG_8]	= 9,
88 	[BPF_REG_9]	= 10,
89 	/* BPF stack pointer */
90 	[BPF_REG_FP]	= 13,
91 	/* SKB data pointer */
92 	[REG_SKB_DATA]	= 12,
93 	/* Work registers for s390x backend */
94 	[REG_W0]	= 0,
95 	[REG_W1]	= 1,
96 	[REG_L]		= 11,
97 	[REG_15]	= 15,
98 };
99 
100 static inline u32 reg(u32 dst_reg, u32 src_reg)
101 {
102 	return reg2hex[dst_reg] << 4 | reg2hex[src_reg];
103 }
104 
105 static inline u32 reg_high(u32 reg)
106 {
107 	return reg2hex[reg] << 4;
108 }
109 
110 static inline void reg_set_seen(struct bpf_jit *jit, u32 b1)
111 {
112 	u32 r1 = reg2hex[b1];
113 
114 	if (!jit->seen_reg[r1] && r1 >= 6 && r1 <= 15)
115 		jit->seen_reg[r1] = 1;
116 }
117 
118 #define REG_SET_SEEN(b1)					\
119 ({								\
120 	reg_set_seen(jit, b1);					\
121 })
122 
123 #define REG_SEEN(b1) jit->seen_reg[reg2hex[(b1)]]
124 
125 /*
126  * EMIT macros for code generation
127  */
128 
129 #define _EMIT2(op)						\
130 ({								\
131 	if (jit->prg_buf)					\
132 		*(u16 *) (jit->prg_buf + jit->prg) = op;	\
133 	jit->prg += 2;						\
134 })
135 
136 #define EMIT2(op, b1, b2)					\
137 ({								\
138 	_EMIT2(op | reg(b1, b2));				\
139 	REG_SET_SEEN(b1);					\
140 	REG_SET_SEEN(b2);					\
141 })
142 
143 #define _EMIT4(op)						\
144 ({								\
145 	if (jit->prg_buf)					\
146 		*(u32 *) (jit->prg_buf + jit->prg) = op;	\
147 	jit->prg += 4;						\
148 })
149 
150 #define EMIT4(op, b1, b2)					\
151 ({								\
152 	_EMIT4(op | reg(b1, b2));				\
153 	REG_SET_SEEN(b1);					\
154 	REG_SET_SEEN(b2);					\
155 })
156 
157 #define EMIT4_RRF(op, b1, b2, b3)				\
158 ({								\
159 	_EMIT4(op | reg_high(b3) << 8 | reg(b1, b2));		\
160 	REG_SET_SEEN(b1);					\
161 	REG_SET_SEEN(b2);					\
162 	REG_SET_SEEN(b3);					\
163 })
164 
165 #define _EMIT4_DISP(op, disp)					\
166 ({								\
167 	unsigned int __disp = (disp) & 0xfff;			\
168 	_EMIT4(op | __disp);					\
169 })
170 
171 #define EMIT4_DISP(op, b1, b2, disp)				\
172 ({								\
173 	_EMIT4_DISP(op | reg_high(b1) << 16 |			\
174 		    reg_high(b2) << 8, disp);			\
175 	REG_SET_SEEN(b1);					\
176 	REG_SET_SEEN(b2);					\
177 })
178 
179 #define EMIT4_IMM(op, b1, imm)					\
180 ({								\
181 	unsigned int __imm = (imm) & 0xffff;			\
182 	_EMIT4(op | reg_high(b1) << 16 | __imm);		\
183 	REG_SET_SEEN(b1);					\
184 })
185 
186 #define EMIT4_PCREL(op, pcrel)					\
187 ({								\
188 	long __pcrel = ((pcrel) >> 1) & 0xffff;			\
189 	_EMIT4(op | __pcrel);					\
190 })
191 
192 #define _EMIT6(op1, op2)					\
193 ({								\
194 	if (jit->prg_buf) {					\
195 		*(u32 *) (jit->prg_buf + jit->prg) = op1;	\
196 		*(u16 *) (jit->prg_buf + jit->prg + 4) = op2;	\
197 	}							\
198 	jit->prg += 6;						\
199 })
200 
201 #define _EMIT6_DISP(op1, op2, disp)				\
202 ({								\
203 	unsigned int __disp = (disp) & 0xfff;			\
204 	_EMIT6(op1 | __disp, op2);				\
205 })
206 
207 #define _EMIT6_DISP_LH(op1, op2, disp)				\
208 ({								\
209 	u32 _disp = (u32) disp;					\
210 	unsigned int __disp_h = _disp & 0xff000;		\
211 	unsigned int __disp_l = _disp & 0x00fff;		\
212 	_EMIT6(op1 | __disp_l, op2 | __disp_h >> 4);		\
213 })
214 
215 #define EMIT6_DISP_LH(op1, op2, b1, b2, b3, disp)		\
216 ({								\
217 	_EMIT6_DISP_LH(op1 | reg(b1, b2) << 16 |		\
218 		       reg_high(b3) << 8, op2, disp);		\
219 	REG_SET_SEEN(b1);					\
220 	REG_SET_SEEN(b2);					\
221 	REG_SET_SEEN(b3);					\
222 })
223 
224 #define EMIT6_PCREL_LABEL(op1, op2, b1, b2, label, mask)	\
225 ({								\
226 	int rel = (jit->labels[label] - jit->prg) >> 1;		\
227 	_EMIT6(op1 | reg(b1, b2) << 16 | (rel & 0xffff),	\
228 	       op2 | mask << 12);				\
229 	REG_SET_SEEN(b1);					\
230 	REG_SET_SEEN(b2);					\
231 })
232 
233 #define EMIT6_PCREL_IMM_LABEL(op1, op2, b1, imm, label, mask)	\
234 ({								\
235 	int rel = (jit->labels[label] - jit->prg) >> 1;		\
236 	_EMIT6(op1 | (reg_high(b1) | mask) << 16 |		\
237 		(rel & 0xffff), op2 | (imm & 0xff) << 8);	\
238 	REG_SET_SEEN(b1);					\
239 	BUILD_BUG_ON(((unsigned long) imm) > 0xff);		\
240 })
241 
242 #define EMIT6_PCREL(op1, op2, b1, b2, i, off, mask)		\
243 ({								\
244 	/* Branch instruction needs 6 bytes */			\
245 	int rel = (addrs[i + off + 1] - (addrs[i + 1] - 6)) / 2;\
246 	_EMIT6(op1 | reg(b1, b2) << 16 | (rel & 0xffff), op2 | mask);	\
247 	REG_SET_SEEN(b1);					\
248 	REG_SET_SEEN(b2);					\
249 })
250 
251 #define _EMIT6_IMM(op, imm)					\
252 ({								\
253 	unsigned int __imm = (imm);				\
254 	_EMIT6(op | (__imm >> 16), __imm & 0xffff);		\
255 })
256 
257 #define EMIT6_IMM(op, b1, imm)					\
258 ({								\
259 	_EMIT6_IMM(op | reg_high(b1) << 16, imm);		\
260 	REG_SET_SEEN(b1);					\
261 })
262 
263 #define EMIT_CONST_U32(val)					\
264 ({								\
265 	unsigned int ret;					\
266 	ret = jit->lit - jit->base_ip;				\
267 	jit->seen |= SEEN_LITERAL;				\
268 	if (jit->prg_buf)					\
269 		*(u32 *) (jit->prg_buf + jit->lit) = (u32) val;	\
270 	jit->lit += 4;						\
271 	ret;							\
272 })
273 
274 #define EMIT_CONST_U64(val)					\
275 ({								\
276 	unsigned int ret;					\
277 	ret = jit->lit - jit->base_ip;				\
278 	jit->seen |= SEEN_LITERAL;				\
279 	if (jit->prg_buf)					\
280 		*(u64 *) (jit->prg_buf + jit->lit) = (u64) val;	\
281 	jit->lit += 8;						\
282 	ret;							\
283 })
284 
285 #define EMIT_ZERO(b1)						\
286 ({								\
287 	/* llgfr %dst,%dst (zero extend to 64 bit) */		\
288 	EMIT4(0xb9160000, b1, b1);				\
289 	REG_SET_SEEN(b1);					\
290 })
291 
292 /*
293  * Fill whole space with illegal instructions
294  */
295 static void jit_fill_hole(void *area, unsigned int size)
296 {
297 	memset(area, 0, size);
298 }
299 
300 /*
301  * Save registers from "rs" (register start) to "re" (register end) on stack
302  */
303 static void save_regs(struct bpf_jit *jit, u32 rs, u32 re)
304 {
305 	u32 off = STK_OFF_R6 + (rs - 6) * 8;
306 
307 	if (rs == re)
308 		/* stg %rs,off(%r15) */
309 		_EMIT6(0xe300f000 | rs << 20 | off, 0x0024);
310 	else
311 		/* stmg %rs,%re,off(%r15) */
312 		_EMIT6_DISP(0xeb00f000 | rs << 20 | re << 16, 0x0024, off);
313 }
314 
315 /*
316  * Restore registers from "rs" (register start) to "re" (register end) on stack
317  */
318 static void restore_regs(struct bpf_jit *jit, u32 rs, u32 re)
319 {
320 	u32 off = STK_OFF_R6 + (rs - 6) * 8;
321 
322 	if (jit->seen & SEEN_STACK)
323 		off += STK_OFF;
324 
325 	if (rs == re)
326 		/* lg %rs,off(%r15) */
327 		_EMIT6(0xe300f000 | rs << 20 | off, 0x0004);
328 	else
329 		/* lmg %rs,%re,off(%r15) */
330 		_EMIT6_DISP(0xeb00f000 | rs << 20 | re << 16, 0x0004, off);
331 }
332 
333 /*
334  * Return first seen register (from start)
335  */
336 static int get_start(struct bpf_jit *jit, int start)
337 {
338 	int i;
339 
340 	for (i = start; i <= 15; i++) {
341 		if (jit->seen_reg[i])
342 			return i;
343 	}
344 	return 0;
345 }
346 
347 /*
348  * Return last seen register (from start) (gap >= 2)
349  */
350 static int get_end(struct bpf_jit *jit, int start)
351 {
352 	int i;
353 
354 	for (i = start; i < 15; i++) {
355 		if (!jit->seen_reg[i] && !jit->seen_reg[i + 1])
356 			return i - 1;
357 	}
358 	return jit->seen_reg[15] ? 15 : 14;
359 }
360 
361 #define REGS_SAVE	1
362 #define REGS_RESTORE	0
363 /*
364  * Save and restore clobbered registers (6-15) on stack.
365  * We save/restore registers in chunks with gap >= 2 registers.
366  */
367 static void save_restore_regs(struct bpf_jit *jit, int op)
368 {
369 
370 	int re = 6, rs;
371 
372 	do {
373 		rs = get_start(jit, re);
374 		if (!rs)
375 			break;
376 		re = get_end(jit, rs + 1);
377 		if (op == REGS_SAVE)
378 			save_regs(jit, rs, re);
379 		else
380 			restore_regs(jit, rs, re);
381 		re++;
382 	} while (re <= 15);
383 }
384 
385 /*
386  * For SKB access %b1 contains the SKB pointer. For "bpf_jit.S"
387  * we store the SKB header length on the stack and the SKB data
388  * pointer in REG_SKB_DATA.
389  */
390 static void emit_load_skb_data_hlen(struct bpf_jit *jit)
391 {
392 	/* Header length: llgf %w1,<len>(%b1) */
393 	EMIT6_DISP_LH(0xe3000000, 0x0016, REG_W1, REG_0, BPF_REG_1,
394 		      offsetof(struct sk_buff, len));
395 	/* s %w1,<data_len>(%b1) */
396 	EMIT4_DISP(0x5b000000, REG_W1, BPF_REG_1,
397 		   offsetof(struct sk_buff, data_len));
398 	/* stg %w1,ST_OFF_HLEN(%r0,%r15) */
399 	EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0, REG_15, STK_OFF_HLEN);
400 	/* lg %skb_data,data_off(%b1) */
401 	EMIT6_DISP_LH(0xe3000000, 0x0004, REG_SKB_DATA, REG_0,
402 		      BPF_REG_1, offsetof(struct sk_buff, data));
403 }
404 
405 /*
406  * Emit function prologue
407  *
408  * Save registers and create stack frame if necessary.
409  * See stack frame layout desription in "bpf_jit.h"!
410  */
411 static void bpf_jit_prologue(struct bpf_jit *jit, bool is_classic)
412 {
413 	if (jit->seen & SEEN_TAIL_CALL) {
414 		/* xc STK_OFF_TCCNT(4,%r15),STK_OFF_TCCNT(%r15) */
415 		_EMIT6(0xd703f000 | STK_OFF_TCCNT, 0xf000 | STK_OFF_TCCNT);
416 	} else {
417 		/* j tail_call_start: NOP if no tail calls are used */
418 		EMIT4_PCREL(0xa7f40000, 6);
419 		_EMIT2(0);
420 	}
421 	/* Tail calls have to skip above initialization */
422 	jit->tail_call_start = jit->prg;
423 	/* Save registers */
424 	save_restore_regs(jit, REGS_SAVE);
425 	/* Setup literal pool */
426 	if (jit->seen & SEEN_LITERAL) {
427 		/* basr %r13,0 */
428 		EMIT2(0x0d00, REG_L, REG_0);
429 		jit->base_ip = jit->prg;
430 	}
431 	/* Setup stack and backchain */
432 	if (jit->seen & SEEN_STACK) {
433 		if (jit->seen & SEEN_FUNC)
434 			/* lgr %w1,%r15 (backchain) */
435 			EMIT4(0xb9040000, REG_W1, REG_15);
436 		/* la %bfp,STK_160_UNUSED(%r15) (BPF frame pointer) */
437 		EMIT4_DISP(0x41000000, BPF_REG_FP, REG_15, STK_160_UNUSED);
438 		/* aghi %r15,-STK_OFF */
439 		EMIT4_IMM(0xa70b0000, REG_15, -STK_OFF);
440 		if (jit->seen & SEEN_FUNC)
441 			/* stg %w1,152(%r15) (backchain) */
442 			EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0,
443 				      REG_15, 152);
444 	}
445 	if (jit->seen & SEEN_SKB)
446 		emit_load_skb_data_hlen(jit);
447 	if (jit->seen & SEEN_SKB_CHANGE)
448 		/* stg %b1,ST_OFF_SKBP(%r0,%r15) */
449 		EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0, REG_15,
450 			      STK_OFF_SKBP);
451 	/* Clear A (%b0) and X (%b7) registers for converted BPF programs */
452 	if (is_classic) {
453 		if (REG_SEEN(BPF_REG_A))
454 			/* lghi %ba,0 */
455 			EMIT4_IMM(0xa7090000, BPF_REG_A, 0);
456 		if (REG_SEEN(BPF_REG_X))
457 			/* lghi %bx,0 */
458 			EMIT4_IMM(0xa7090000, BPF_REG_X, 0);
459 	}
460 }
461 
462 /*
463  * Function epilogue
464  */
465 static void bpf_jit_epilogue(struct bpf_jit *jit)
466 {
467 	/* Return 0 */
468 	if (jit->seen & SEEN_RET0) {
469 		jit->ret0_ip = jit->prg;
470 		/* lghi %b0,0 */
471 		EMIT4_IMM(0xa7090000, BPF_REG_0, 0);
472 	}
473 	jit->exit_ip = jit->prg;
474 	/* Load exit code: lgr %r2,%b0 */
475 	EMIT4(0xb9040000, REG_2, BPF_REG_0);
476 	/* Restore registers */
477 	save_restore_regs(jit, REGS_RESTORE);
478 	/* br %r14 */
479 	_EMIT2(0x07fe);
480 }
481 
482 /*
483  * Compile one eBPF instruction into s390x code
484  *
485  * NOTE: Use noinline because for gcov (-fprofile-arcs) gcc allocates a lot of
486  * stack space for the large switch statement.
487  */
488 static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i)
489 {
490 	struct bpf_insn *insn = &fp->insnsi[i];
491 	int jmp_off, last, insn_count = 1;
492 	unsigned int func_addr, mask;
493 	u32 dst_reg = insn->dst_reg;
494 	u32 src_reg = insn->src_reg;
495 	u32 *addrs = jit->addrs;
496 	s32 imm = insn->imm;
497 	s16 off = insn->off;
498 
499 	switch (insn->code) {
500 	/*
501 	 * BPF_MOV
502 	 */
503 	case BPF_ALU | BPF_MOV | BPF_X: /* dst = (u32) src */
504 		/* llgfr %dst,%src */
505 		EMIT4(0xb9160000, dst_reg, src_reg);
506 		break;
507 	case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */
508 		/* lgr %dst,%src */
509 		EMIT4(0xb9040000, dst_reg, src_reg);
510 		break;
511 	case BPF_ALU | BPF_MOV | BPF_K: /* dst = (u32) imm */
512 		/* llilf %dst,imm */
513 		EMIT6_IMM(0xc00f0000, dst_reg, imm);
514 		break;
515 	case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = imm */
516 		/* lgfi %dst,imm */
517 		EMIT6_IMM(0xc0010000, dst_reg, imm);
518 		break;
519 	/*
520 	 * BPF_LD 64
521 	 */
522 	case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */
523 	{
524 		/* 16 byte instruction that uses two 'struct bpf_insn' */
525 		u64 imm64;
526 
527 		imm64 = (u64)(u32) insn[0].imm | ((u64)(u32) insn[1].imm) << 32;
528 		/* lg %dst,<d(imm)>(%l) */
529 		EMIT6_DISP_LH(0xe3000000, 0x0004, dst_reg, REG_0, REG_L,
530 			      EMIT_CONST_U64(imm64));
531 		insn_count = 2;
532 		break;
533 	}
534 	/*
535 	 * BPF_ADD
536 	 */
537 	case BPF_ALU | BPF_ADD | BPF_X: /* dst = (u32) dst + (u32) src */
538 		/* ar %dst,%src */
539 		EMIT2(0x1a00, dst_reg, src_reg);
540 		EMIT_ZERO(dst_reg);
541 		break;
542 	case BPF_ALU64 | BPF_ADD | BPF_X: /* dst = dst + src */
543 		/* agr %dst,%src */
544 		EMIT4(0xb9080000, dst_reg, src_reg);
545 		break;
546 	case BPF_ALU | BPF_ADD | BPF_K: /* dst = (u32) dst + (u32) imm */
547 		if (!imm)
548 			break;
549 		/* alfi %dst,imm */
550 		EMIT6_IMM(0xc20b0000, dst_reg, imm);
551 		EMIT_ZERO(dst_reg);
552 		break;
553 	case BPF_ALU64 | BPF_ADD | BPF_K: /* dst = dst + imm */
554 		if (!imm)
555 			break;
556 		/* agfi %dst,imm */
557 		EMIT6_IMM(0xc2080000, dst_reg, imm);
558 		break;
559 	/*
560 	 * BPF_SUB
561 	 */
562 	case BPF_ALU | BPF_SUB | BPF_X: /* dst = (u32) dst - (u32) src */
563 		/* sr %dst,%src */
564 		EMIT2(0x1b00, dst_reg, src_reg);
565 		EMIT_ZERO(dst_reg);
566 		break;
567 	case BPF_ALU64 | BPF_SUB | BPF_X: /* dst = dst - src */
568 		/* sgr %dst,%src */
569 		EMIT4(0xb9090000, dst_reg, src_reg);
570 		break;
571 	case BPF_ALU | BPF_SUB | BPF_K: /* dst = (u32) dst - (u32) imm */
572 		if (!imm)
573 			break;
574 		/* alfi %dst,-imm */
575 		EMIT6_IMM(0xc20b0000, dst_reg, -imm);
576 		EMIT_ZERO(dst_reg);
577 		break;
578 	case BPF_ALU64 | BPF_SUB | BPF_K: /* dst = dst - imm */
579 		if (!imm)
580 			break;
581 		/* agfi %dst,-imm */
582 		EMIT6_IMM(0xc2080000, dst_reg, -imm);
583 		break;
584 	/*
585 	 * BPF_MUL
586 	 */
587 	case BPF_ALU | BPF_MUL | BPF_X: /* dst = (u32) dst * (u32) src */
588 		/* msr %dst,%src */
589 		EMIT4(0xb2520000, dst_reg, src_reg);
590 		EMIT_ZERO(dst_reg);
591 		break;
592 	case BPF_ALU64 | BPF_MUL | BPF_X: /* dst = dst * src */
593 		/* msgr %dst,%src */
594 		EMIT4(0xb90c0000, dst_reg, src_reg);
595 		break;
596 	case BPF_ALU | BPF_MUL | BPF_K: /* dst = (u32) dst * (u32) imm */
597 		if (imm == 1)
598 			break;
599 		/* msfi %r5,imm */
600 		EMIT6_IMM(0xc2010000, dst_reg, imm);
601 		EMIT_ZERO(dst_reg);
602 		break;
603 	case BPF_ALU64 | BPF_MUL | BPF_K: /* dst = dst * imm */
604 		if (imm == 1)
605 			break;
606 		/* msgfi %dst,imm */
607 		EMIT6_IMM(0xc2000000, dst_reg, imm);
608 		break;
609 	/*
610 	 * BPF_DIV / BPF_MOD
611 	 */
612 	case BPF_ALU | BPF_DIV | BPF_X: /* dst = (u32) dst / (u32) src */
613 	case BPF_ALU | BPF_MOD | BPF_X: /* dst = (u32) dst % (u32) src */
614 	{
615 		int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0;
616 
617 		jit->seen |= SEEN_RET0;
618 		/* ltr %src,%src (if src == 0 goto fail) */
619 		EMIT2(0x1200, src_reg, src_reg);
620 		/* jz <ret0> */
621 		EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg);
622 		/* lhi %w0,0 */
623 		EMIT4_IMM(0xa7080000, REG_W0, 0);
624 		/* lr %w1,%dst */
625 		EMIT2(0x1800, REG_W1, dst_reg);
626 		/* dlr %w0,%src */
627 		EMIT4(0xb9970000, REG_W0, src_reg);
628 		/* llgfr %dst,%rc */
629 		EMIT4(0xb9160000, dst_reg, rc_reg);
630 		break;
631 	}
632 	case BPF_ALU64 | BPF_DIV | BPF_X: /* dst = dst / src */
633 	case BPF_ALU64 | BPF_MOD | BPF_X: /* dst = dst % src */
634 	{
635 		int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0;
636 
637 		jit->seen |= SEEN_RET0;
638 		/* ltgr %src,%src (if src == 0 goto fail) */
639 		EMIT4(0xb9020000, src_reg, src_reg);
640 		/* jz <ret0> */
641 		EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg);
642 		/* lghi %w0,0 */
643 		EMIT4_IMM(0xa7090000, REG_W0, 0);
644 		/* lgr %w1,%dst */
645 		EMIT4(0xb9040000, REG_W1, dst_reg);
646 		/* dlgr %w0,%dst */
647 		EMIT4(0xb9870000, REG_W0, src_reg);
648 		/* lgr %dst,%rc */
649 		EMIT4(0xb9040000, dst_reg, rc_reg);
650 		break;
651 	}
652 	case BPF_ALU | BPF_DIV | BPF_K: /* dst = (u32) dst / (u32) imm */
653 	case BPF_ALU | BPF_MOD | BPF_K: /* dst = (u32) dst % (u32) imm */
654 	{
655 		int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0;
656 
657 		if (imm == 1) {
658 			if (BPF_OP(insn->code) == BPF_MOD)
659 				/* lhgi %dst,0 */
660 				EMIT4_IMM(0xa7090000, dst_reg, 0);
661 			break;
662 		}
663 		/* lhi %w0,0 */
664 		EMIT4_IMM(0xa7080000, REG_W0, 0);
665 		/* lr %w1,%dst */
666 		EMIT2(0x1800, REG_W1, dst_reg);
667 		/* dl %w0,<d(imm)>(%l) */
668 		EMIT6_DISP_LH(0xe3000000, 0x0097, REG_W0, REG_0, REG_L,
669 			      EMIT_CONST_U32(imm));
670 		/* llgfr %dst,%rc */
671 		EMIT4(0xb9160000, dst_reg, rc_reg);
672 		break;
673 	}
674 	case BPF_ALU64 | BPF_DIV | BPF_K: /* dst = dst / imm */
675 	case BPF_ALU64 | BPF_MOD | BPF_K: /* dst = dst % imm */
676 	{
677 		int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0;
678 
679 		if (imm == 1) {
680 			if (BPF_OP(insn->code) == BPF_MOD)
681 				/* lhgi %dst,0 */
682 				EMIT4_IMM(0xa7090000, dst_reg, 0);
683 			break;
684 		}
685 		/* lghi %w0,0 */
686 		EMIT4_IMM(0xa7090000, REG_W0, 0);
687 		/* lgr %w1,%dst */
688 		EMIT4(0xb9040000, REG_W1, dst_reg);
689 		/* dlg %w0,<d(imm)>(%l) */
690 		EMIT6_DISP_LH(0xe3000000, 0x0087, REG_W0, REG_0, REG_L,
691 			      EMIT_CONST_U64(imm));
692 		/* lgr %dst,%rc */
693 		EMIT4(0xb9040000, dst_reg, rc_reg);
694 		break;
695 	}
696 	/*
697 	 * BPF_AND
698 	 */
699 	case BPF_ALU | BPF_AND | BPF_X: /* dst = (u32) dst & (u32) src */
700 		/* nr %dst,%src */
701 		EMIT2(0x1400, dst_reg, src_reg);
702 		EMIT_ZERO(dst_reg);
703 		break;
704 	case BPF_ALU64 | BPF_AND | BPF_X: /* dst = dst & src */
705 		/* ngr %dst,%src */
706 		EMIT4(0xb9800000, dst_reg, src_reg);
707 		break;
708 	case BPF_ALU | BPF_AND | BPF_K: /* dst = (u32) dst & (u32) imm */
709 		/* nilf %dst,imm */
710 		EMIT6_IMM(0xc00b0000, dst_reg, imm);
711 		EMIT_ZERO(dst_reg);
712 		break;
713 	case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */
714 		/* ng %dst,<d(imm)>(%l) */
715 		EMIT6_DISP_LH(0xe3000000, 0x0080, dst_reg, REG_0, REG_L,
716 			      EMIT_CONST_U64(imm));
717 		break;
718 	/*
719 	 * BPF_OR
720 	 */
721 	case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */
722 		/* or %dst,%src */
723 		EMIT2(0x1600, dst_reg, src_reg);
724 		EMIT_ZERO(dst_reg);
725 		break;
726 	case BPF_ALU64 | BPF_OR | BPF_X: /* dst = dst | src */
727 		/* ogr %dst,%src */
728 		EMIT4(0xb9810000, dst_reg, src_reg);
729 		break;
730 	case BPF_ALU | BPF_OR | BPF_K: /* dst = (u32) dst | (u32) imm */
731 		/* oilf %dst,imm */
732 		EMIT6_IMM(0xc00d0000, dst_reg, imm);
733 		EMIT_ZERO(dst_reg);
734 		break;
735 	case BPF_ALU64 | BPF_OR | BPF_K: /* dst = dst | imm */
736 		/* og %dst,<d(imm)>(%l) */
737 		EMIT6_DISP_LH(0xe3000000, 0x0081, dst_reg, REG_0, REG_L,
738 			      EMIT_CONST_U64(imm));
739 		break;
740 	/*
741 	 * BPF_XOR
742 	 */
743 	case BPF_ALU | BPF_XOR | BPF_X: /* dst = (u32) dst ^ (u32) src */
744 		/* xr %dst,%src */
745 		EMIT2(0x1700, dst_reg, src_reg);
746 		EMIT_ZERO(dst_reg);
747 		break;
748 	case BPF_ALU64 | BPF_XOR | BPF_X: /* dst = dst ^ src */
749 		/* xgr %dst,%src */
750 		EMIT4(0xb9820000, dst_reg, src_reg);
751 		break;
752 	case BPF_ALU | BPF_XOR | BPF_K: /* dst = (u32) dst ^ (u32) imm */
753 		if (!imm)
754 			break;
755 		/* xilf %dst,imm */
756 		EMIT6_IMM(0xc0070000, dst_reg, imm);
757 		EMIT_ZERO(dst_reg);
758 		break;
759 	case BPF_ALU64 | BPF_XOR | BPF_K: /* dst = dst ^ imm */
760 		/* xg %dst,<d(imm)>(%l) */
761 		EMIT6_DISP_LH(0xe3000000, 0x0082, dst_reg, REG_0, REG_L,
762 			      EMIT_CONST_U64(imm));
763 		break;
764 	/*
765 	 * BPF_LSH
766 	 */
767 	case BPF_ALU | BPF_LSH | BPF_X: /* dst = (u32) dst << (u32) src */
768 		/* sll %dst,0(%src) */
769 		EMIT4_DISP(0x89000000, dst_reg, src_reg, 0);
770 		EMIT_ZERO(dst_reg);
771 		break;
772 	case BPF_ALU64 | BPF_LSH | BPF_X: /* dst = dst << src */
773 		/* sllg %dst,%dst,0(%src) */
774 		EMIT6_DISP_LH(0xeb000000, 0x000d, dst_reg, dst_reg, src_reg, 0);
775 		break;
776 	case BPF_ALU | BPF_LSH | BPF_K: /* dst = (u32) dst << (u32) imm */
777 		if (imm == 0)
778 			break;
779 		/* sll %dst,imm(%r0) */
780 		EMIT4_DISP(0x89000000, dst_reg, REG_0, imm);
781 		EMIT_ZERO(dst_reg);
782 		break;
783 	case BPF_ALU64 | BPF_LSH | BPF_K: /* dst = dst << imm */
784 		if (imm == 0)
785 			break;
786 		/* sllg %dst,%dst,imm(%r0) */
787 		EMIT6_DISP_LH(0xeb000000, 0x000d, dst_reg, dst_reg, REG_0, imm);
788 		break;
789 	/*
790 	 * BPF_RSH
791 	 */
792 	case BPF_ALU | BPF_RSH | BPF_X: /* dst = (u32) dst >> (u32) src */
793 		/* srl %dst,0(%src) */
794 		EMIT4_DISP(0x88000000, dst_reg, src_reg, 0);
795 		EMIT_ZERO(dst_reg);
796 		break;
797 	case BPF_ALU64 | BPF_RSH | BPF_X: /* dst = dst >> src */
798 		/* srlg %dst,%dst,0(%src) */
799 		EMIT6_DISP_LH(0xeb000000, 0x000c, dst_reg, dst_reg, src_reg, 0);
800 		break;
801 	case BPF_ALU | BPF_RSH | BPF_K: /* dst = (u32) dst >> (u32) imm */
802 		if (imm == 0)
803 			break;
804 		/* srl %dst,imm(%r0) */
805 		EMIT4_DISP(0x88000000, dst_reg, REG_0, imm);
806 		EMIT_ZERO(dst_reg);
807 		break;
808 	case BPF_ALU64 | BPF_RSH | BPF_K: /* dst = dst >> imm */
809 		if (imm == 0)
810 			break;
811 		/* srlg %dst,%dst,imm(%r0) */
812 		EMIT6_DISP_LH(0xeb000000, 0x000c, dst_reg, dst_reg, REG_0, imm);
813 		break;
814 	/*
815 	 * BPF_ARSH
816 	 */
817 	case BPF_ALU64 | BPF_ARSH | BPF_X: /* ((s64) dst) >>= src */
818 		/* srag %dst,%dst,0(%src) */
819 		EMIT6_DISP_LH(0xeb000000, 0x000a, dst_reg, dst_reg, src_reg, 0);
820 		break;
821 	case BPF_ALU64 | BPF_ARSH | BPF_K: /* ((s64) dst) >>= imm */
822 		if (imm == 0)
823 			break;
824 		/* srag %dst,%dst,imm(%r0) */
825 		EMIT6_DISP_LH(0xeb000000, 0x000a, dst_reg, dst_reg, REG_0, imm);
826 		break;
827 	/*
828 	 * BPF_NEG
829 	 */
830 	case BPF_ALU | BPF_NEG: /* dst = (u32) -dst */
831 		/* lcr %dst,%dst */
832 		EMIT2(0x1300, dst_reg, dst_reg);
833 		EMIT_ZERO(dst_reg);
834 		break;
835 	case BPF_ALU64 | BPF_NEG: /* dst = -dst */
836 		/* lcgr %dst,%dst */
837 		EMIT4(0xb9130000, dst_reg, dst_reg);
838 		break;
839 	/*
840 	 * BPF_FROM_BE/LE
841 	 */
842 	case BPF_ALU | BPF_END | BPF_FROM_BE:
843 		/* s390 is big endian, therefore only clear high order bytes */
844 		switch (imm) {
845 		case 16: /* dst = (u16) cpu_to_be16(dst) */
846 			/* llghr %dst,%dst */
847 			EMIT4(0xb9850000, dst_reg, dst_reg);
848 			break;
849 		case 32: /* dst = (u32) cpu_to_be32(dst) */
850 			/* llgfr %dst,%dst */
851 			EMIT4(0xb9160000, dst_reg, dst_reg);
852 			break;
853 		case 64: /* dst = (u64) cpu_to_be64(dst) */
854 			break;
855 		}
856 		break;
857 	case BPF_ALU | BPF_END | BPF_FROM_LE:
858 		switch (imm) {
859 		case 16: /* dst = (u16) cpu_to_le16(dst) */
860 			/* lrvr %dst,%dst */
861 			EMIT4(0xb91f0000, dst_reg, dst_reg);
862 			/* srl %dst,16(%r0) */
863 			EMIT4_DISP(0x88000000, dst_reg, REG_0, 16);
864 			/* llghr %dst,%dst */
865 			EMIT4(0xb9850000, dst_reg, dst_reg);
866 			break;
867 		case 32: /* dst = (u32) cpu_to_le32(dst) */
868 			/* lrvr %dst,%dst */
869 			EMIT4(0xb91f0000, dst_reg, dst_reg);
870 			/* llgfr %dst,%dst */
871 			EMIT4(0xb9160000, dst_reg, dst_reg);
872 			break;
873 		case 64: /* dst = (u64) cpu_to_le64(dst) */
874 			/* lrvgr %dst,%dst */
875 			EMIT4(0xb90f0000, dst_reg, dst_reg);
876 			break;
877 		}
878 		break;
879 	/*
880 	 * BPF_ST(X)
881 	 */
882 	case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src_reg */
883 		/* stcy %src,off(%dst) */
884 		EMIT6_DISP_LH(0xe3000000, 0x0072, src_reg, dst_reg, REG_0, off);
885 		jit->seen |= SEEN_MEM;
886 		break;
887 	case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */
888 		/* sthy %src,off(%dst) */
889 		EMIT6_DISP_LH(0xe3000000, 0x0070, src_reg, dst_reg, REG_0, off);
890 		jit->seen |= SEEN_MEM;
891 		break;
892 	case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */
893 		/* sty %src,off(%dst) */
894 		EMIT6_DISP_LH(0xe3000000, 0x0050, src_reg, dst_reg, REG_0, off);
895 		jit->seen |= SEEN_MEM;
896 		break;
897 	case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */
898 		/* stg %src,off(%dst) */
899 		EMIT6_DISP_LH(0xe3000000, 0x0024, src_reg, dst_reg, REG_0, off);
900 		jit->seen |= SEEN_MEM;
901 		break;
902 	case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */
903 		/* lhi %w0,imm */
904 		EMIT4_IMM(0xa7080000, REG_W0, (u8) imm);
905 		/* stcy %w0,off(dst) */
906 		EMIT6_DISP_LH(0xe3000000, 0x0072, REG_W0, dst_reg, REG_0, off);
907 		jit->seen |= SEEN_MEM;
908 		break;
909 	case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */
910 		/* lhi %w0,imm */
911 		EMIT4_IMM(0xa7080000, REG_W0, (u16) imm);
912 		/* sthy %w0,off(dst) */
913 		EMIT6_DISP_LH(0xe3000000, 0x0070, REG_W0, dst_reg, REG_0, off);
914 		jit->seen |= SEEN_MEM;
915 		break;
916 	case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */
917 		/* llilf %w0,imm  */
918 		EMIT6_IMM(0xc00f0000, REG_W0, (u32) imm);
919 		/* sty %w0,off(%dst) */
920 		EMIT6_DISP_LH(0xe3000000, 0x0050, REG_W0, dst_reg, REG_0, off);
921 		jit->seen |= SEEN_MEM;
922 		break;
923 	case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */
924 		/* lgfi %w0,imm */
925 		EMIT6_IMM(0xc0010000, REG_W0, imm);
926 		/* stg %w0,off(%dst) */
927 		EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W0, dst_reg, REG_0, off);
928 		jit->seen |= SEEN_MEM;
929 		break;
930 	/*
931 	 * BPF_STX XADD (atomic_add)
932 	 */
933 	case BPF_STX | BPF_XADD | BPF_W: /* *(u32 *)(dst + off) += src */
934 		/* laal %w0,%src,off(%dst) */
935 		EMIT6_DISP_LH(0xeb000000, 0x00fa, REG_W0, src_reg,
936 			      dst_reg, off);
937 		jit->seen |= SEEN_MEM;
938 		break;
939 	case BPF_STX | BPF_XADD | BPF_DW: /* *(u64 *)(dst + off) += src */
940 		/* laalg %w0,%src,off(%dst) */
941 		EMIT6_DISP_LH(0xeb000000, 0x00ea, REG_W0, src_reg,
942 			      dst_reg, off);
943 		jit->seen |= SEEN_MEM;
944 		break;
945 	/*
946 	 * BPF_LDX
947 	 */
948 	case BPF_LDX | BPF_MEM | BPF_B: /* dst = *(u8 *)(ul) (src + off) */
949 		/* llgc %dst,0(off,%src) */
950 		EMIT6_DISP_LH(0xe3000000, 0x0090, dst_reg, src_reg, REG_0, off);
951 		jit->seen |= SEEN_MEM;
952 		break;
953 	case BPF_LDX | BPF_MEM | BPF_H: /* dst = *(u16 *)(ul) (src + off) */
954 		/* llgh %dst,0(off,%src) */
955 		EMIT6_DISP_LH(0xe3000000, 0x0091, dst_reg, src_reg, REG_0, off);
956 		jit->seen |= SEEN_MEM;
957 		break;
958 	case BPF_LDX | BPF_MEM | BPF_W: /* dst = *(u32 *)(ul) (src + off) */
959 		/* llgf %dst,off(%src) */
960 		jit->seen |= SEEN_MEM;
961 		EMIT6_DISP_LH(0xe3000000, 0x0016, dst_reg, src_reg, REG_0, off);
962 		break;
963 	case BPF_LDX | BPF_MEM | BPF_DW: /* dst = *(u64 *)(ul) (src + off) */
964 		/* lg %dst,0(off,%src) */
965 		jit->seen |= SEEN_MEM;
966 		EMIT6_DISP_LH(0xe3000000, 0x0004, dst_reg, src_reg, REG_0, off);
967 		break;
968 	/*
969 	 * BPF_JMP / CALL
970 	 */
971 	case BPF_JMP | BPF_CALL:
972 	{
973 		/*
974 		 * b0 = (__bpf_call_base + imm)(b1, b2, b3, b4, b5)
975 		 */
976 		const u64 func = (u64)__bpf_call_base + imm;
977 
978 		REG_SET_SEEN(BPF_REG_5);
979 		jit->seen |= SEEN_FUNC;
980 		/* lg %w1,<d(imm)>(%l) */
981 		EMIT6_DISP_LH(0xe3000000, 0x0004, REG_W1, REG_0, REG_L,
982 			      EMIT_CONST_U64(func));
983 		/* basr %r14,%w1 */
984 		EMIT2(0x0d00, REG_14, REG_W1);
985 		/* lgr %b0,%r2: load return value into %b0 */
986 		EMIT4(0xb9040000, BPF_REG_0, REG_2);
987 		if (bpf_helper_changes_skb_data((void *)func)) {
988 			jit->seen |= SEEN_SKB_CHANGE;
989 			/* lg %b1,ST_OFF_SKBP(%r15) */
990 			EMIT6_DISP_LH(0xe3000000, 0x0004, BPF_REG_1, REG_0,
991 				      REG_15, STK_OFF_SKBP);
992 			emit_load_skb_data_hlen(jit);
993 		}
994 		break;
995 	}
996 	case BPF_JMP | BPF_CALL | BPF_X:
997 		/*
998 		 * Implicit input:
999 		 *  B1: pointer to ctx
1000 		 *  B2: pointer to bpf_array
1001 		 *  B3: index in bpf_array
1002 		 */
1003 		jit->seen |= SEEN_TAIL_CALL;
1004 
1005 		/*
1006 		 * if (index >= array->map.max_entries)
1007 		 *         goto out;
1008 		 */
1009 
1010 		/* llgf %w1,map.max_entries(%b2) */
1011 		EMIT6_DISP_LH(0xe3000000, 0x0016, REG_W1, REG_0, BPF_REG_2,
1012 			      offsetof(struct bpf_array, map.max_entries));
1013 		/* clgrj %b3,%w1,0xa,label0: if %b3 >= %w1 goto out */
1014 		EMIT6_PCREL_LABEL(0xec000000, 0x0065, BPF_REG_3,
1015 				  REG_W1, 0, 0xa);
1016 
1017 		/*
1018 		 * if (tail_call_cnt++ > MAX_TAIL_CALL_CNT)
1019 		 *         goto out;
1020 		 */
1021 
1022 		if (jit->seen & SEEN_STACK)
1023 			off = STK_OFF_TCCNT + STK_OFF;
1024 		else
1025 			off = STK_OFF_TCCNT;
1026 		/* lhi %w0,1 */
1027 		EMIT4_IMM(0xa7080000, REG_W0, 1);
1028 		/* laal %w1,%w0,off(%r15) */
1029 		EMIT6_DISP_LH(0xeb000000, 0x00fa, REG_W1, REG_W0, REG_15, off);
1030 		/* clij %w1,MAX_TAIL_CALL_CNT,0x2,label0 */
1031 		EMIT6_PCREL_IMM_LABEL(0xec000000, 0x007f, REG_W1,
1032 				      MAX_TAIL_CALL_CNT, 0, 0x2);
1033 
1034 		/*
1035 		 * prog = array->ptrs[index];
1036 		 * if (prog == NULL)
1037 		 *         goto out;
1038 		 */
1039 
1040 		/* sllg %r1,%b3,3: %r1 = index * 8 */
1041 		EMIT6_DISP_LH(0xeb000000, 0x000d, REG_1, BPF_REG_3, REG_0, 3);
1042 		/* lg %r1,prog(%b2,%r1) */
1043 		EMIT6_DISP_LH(0xe3000000, 0x0004, REG_1, BPF_REG_2,
1044 			      REG_1, offsetof(struct bpf_array, ptrs));
1045 		/* clgij %r1,0,0x8,label0 */
1046 		EMIT6_PCREL_IMM_LABEL(0xec000000, 0x007d, REG_1, 0, 0, 0x8);
1047 
1048 		/*
1049 		 * Restore registers before calling function
1050 		 */
1051 		save_restore_regs(jit, REGS_RESTORE);
1052 
1053 		/*
1054 		 * goto *(prog->bpf_func + tail_call_start);
1055 		 */
1056 
1057 		/* lg %r1,bpf_func(%r1) */
1058 		EMIT6_DISP_LH(0xe3000000, 0x0004, REG_1, REG_1, REG_0,
1059 			      offsetof(struct bpf_prog, bpf_func));
1060 		/* bc 0xf,tail_call_start(%r1) */
1061 		_EMIT4(0x47f01000 + jit->tail_call_start);
1062 		/* out: */
1063 		jit->labels[0] = jit->prg;
1064 		break;
1065 	case BPF_JMP | BPF_EXIT: /* return b0 */
1066 		last = (i == fp->len - 1) ? 1 : 0;
1067 		if (last && !(jit->seen & SEEN_RET0))
1068 			break;
1069 		/* j <exit> */
1070 		EMIT4_PCREL(0xa7f40000, jit->exit_ip - jit->prg);
1071 		break;
1072 	/*
1073 	 * Branch relative (number of skipped instructions) to offset on
1074 	 * condition.
1075 	 *
1076 	 * Condition code to mask mapping:
1077 	 *
1078 	 * CC | Description	   | Mask
1079 	 * ------------------------------
1080 	 * 0  | Operands equal	   |	8
1081 	 * 1  | First operand low  |	4
1082 	 * 2  | First operand high |	2
1083 	 * 3  | Unused		   |	1
1084 	 *
1085 	 * For s390x relative branches: ip = ip + off_bytes
1086 	 * For BPF relative branches:	insn = insn + off_insns + 1
1087 	 *
1088 	 * For example for s390x with offset 0 we jump to the branch
1089 	 * instruction itself (loop) and for BPF with offset 0 we
1090 	 * branch to the instruction behind the branch.
1091 	 */
1092 	case BPF_JMP | BPF_JA: /* if (true) */
1093 		mask = 0xf000; /* j */
1094 		goto branch_oc;
1095 	case BPF_JMP | BPF_JSGT | BPF_K: /* ((s64) dst > (s64) imm) */
1096 		mask = 0x2000; /* jh */
1097 		goto branch_ks;
1098 	case BPF_JMP | BPF_JSGE | BPF_K: /* ((s64) dst >= (s64) imm) */
1099 		mask = 0xa000; /* jhe */
1100 		goto branch_ks;
1101 	case BPF_JMP | BPF_JGT | BPF_K: /* (dst_reg > imm) */
1102 		mask = 0x2000; /* jh */
1103 		goto branch_ku;
1104 	case BPF_JMP | BPF_JGE | BPF_K: /* (dst_reg >= imm) */
1105 		mask = 0xa000; /* jhe */
1106 		goto branch_ku;
1107 	case BPF_JMP | BPF_JNE | BPF_K: /* (dst_reg != imm) */
1108 		mask = 0x7000; /* jne */
1109 		goto branch_ku;
1110 	case BPF_JMP | BPF_JEQ | BPF_K: /* (dst_reg == imm) */
1111 		mask = 0x8000; /* je */
1112 		goto branch_ku;
1113 	case BPF_JMP | BPF_JSET | BPF_K: /* (dst_reg & imm) */
1114 		mask = 0x7000; /* jnz */
1115 		/* lgfi %w1,imm (load sign extend imm) */
1116 		EMIT6_IMM(0xc0010000, REG_W1, imm);
1117 		/* ngr %w1,%dst */
1118 		EMIT4(0xb9800000, REG_W1, dst_reg);
1119 		goto branch_oc;
1120 
1121 	case BPF_JMP | BPF_JSGT | BPF_X: /* ((s64) dst > (s64) src) */
1122 		mask = 0x2000; /* jh */
1123 		goto branch_xs;
1124 	case BPF_JMP | BPF_JSGE | BPF_X: /* ((s64) dst >= (s64) src) */
1125 		mask = 0xa000; /* jhe */
1126 		goto branch_xs;
1127 	case BPF_JMP | BPF_JGT | BPF_X: /* (dst > src) */
1128 		mask = 0x2000; /* jh */
1129 		goto branch_xu;
1130 	case BPF_JMP | BPF_JGE | BPF_X: /* (dst >= src) */
1131 		mask = 0xa000; /* jhe */
1132 		goto branch_xu;
1133 	case BPF_JMP | BPF_JNE | BPF_X: /* (dst != src) */
1134 		mask = 0x7000; /* jne */
1135 		goto branch_xu;
1136 	case BPF_JMP | BPF_JEQ | BPF_X: /* (dst == src) */
1137 		mask = 0x8000; /* je */
1138 		goto branch_xu;
1139 	case BPF_JMP | BPF_JSET | BPF_X: /* (dst & src) */
1140 		mask = 0x7000; /* jnz */
1141 		/* ngrk %w1,%dst,%src */
1142 		EMIT4_RRF(0xb9e40000, REG_W1, dst_reg, src_reg);
1143 		goto branch_oc;
1144 branch_ks:
1145 		/* lgfi %w1,imm (load sign extend imm) */
1146 		EMIT6_IMM(0xc0010000, REG_W1, imm);
1147 		/* cgrj %dst,%w1,mask,off */
1148 		EMIT6_PCREL(0xec000000, 0x0064, dst_reg, REG_W1, i, off, mask);
1149 		break;
1150 branch_ku:
1151 		/* lgfi %w1,imm (load sign extend imm) */
1152 		EMIT6_IMM(0xc0010000, REG_W1, imm);
1153 		/* clgrj %dst,%w1,mask,off */
1154 		EMIT6_PCREL(0xec000000, 0x0065, dst_reg, REG_W1, i, off, mask);
1155 		break;
1156 branch_xs:
1157 		/* cgrj %dst,%src,mask,off */
1158 		EMIT6_PCREL(0xec000000, 0x0064, dst_reg, src_reg, i, off, mask);
1159 		break;
1160 branch_xu:
1161 		/* clgrj %dst,%src,mask,off */
1162 		EMIT6_PCREL(0xec000000, 0x0065, dst_reg, src_reg, i, off, mask);
1163 		break;
1164 branch_oc:
1165 		/* brc mask,jmp_off (branch instruction needs 4 bytes) */
1166 		jmp_off = addrs[i + off + 1] - (addrs[i + 1] - 4);
1167 		EMIT4_PCREL(0xa7040000 | mask << 8, jmp_off);
1168 		break;
1169 	/*
1170 	 * BPF_LD
1171 	 */
1172 	case BPF_LD | BPF_ABS | BPF_B: /* b0 = *(u8 *) (skb->data+imm) */
1173 	case BPF_LD | BPF_IND | BPF_B: /* b0 = *(u8 *) (skb->data+imm+src) */
1174 		if ((BPF_MODE(insn->code) == BPF_ABS) && (imm >= 0))
1175 			func_addr = __pa(sk_load_byte_pos);
1176 		else
1177 			func_addr = __pa(sk_load_byte);
1178 		goto call_fn;
1179 	case BPF_LD | BPF_ABS | BPF_H: /* b0 = *(u16 *) (skb->data+imm) */
1180 	case BPF_LD | BPF_IND | BPF_H: /* b0 = *(u16 *) (skb->data+imm+src) */
1181 		if ((BPF_MODE(insn->code) == BPF_ABS) && (imm >= 0))
1182 			func_addr = __pa(sk_load_half_pos);
1183 		else
1184 			func_addr = __pa(sk_load_half);
1185 		goto call_fn;
1186 	case BPF_LD | BPF_ABS | BPF_W: /* b0 = *(u32 *) (skb->data+imm) */
1187 	case BPF_LD | BPF_IND | BPF_W: /* b0 = *(u32 *) (skb->data+imm+src) */
1188 		if ((BPF_MODE(insn->code) == BPF_ABS) && (imm >= 0))
1189 			func_addr = __pa(sk_load_word_pos);
1190 		else
1191 			func_addr = __pa(sk_load_word);
1192 		goto call_fn;
1193 call_fn:
1194 		jit->seen |= SEEN_SKB | SEEN_RET0 | SEEN_FUNC;
1195 		REG_SET_SEEN(REG_14); /* Return address of possible func call */
1196 
1197 		/*
1198 		 * Implicit input:
1199 		 *  BPF_REG_6	 (R7) : skb pointer
1200 		 *  REG_SKB_DATA (R12): skb data pointer
1201 		 *
1202 		 * Calculated input:
1203 		 *  BPF_REG_2	 (R3) : offset of byte(s) to fetch in skb
1204 		 *  BPF_REG_5	 (R6) : return address
1205 		 *
1206 		 * Output:
1207 		 *  BPF_REG_0	 (R14): data read from skb
1208 		 *
1209 		 * Scratch registers (BPF_REG_1-5)
1210 		 */
1211 
1212 		/* Call function: llilf %w1,func_addr  */
1213 		EMIT6_IMM(0xc00f0000, REG_W1, func_addr);
1214 
1215 		/* Offset: lgfi %b2,imm */
1216 		EMIT6_IMM(0xc0010000, BPF_REG_2, imm);
1217 		if (BPF_MODE(insn->code) == BPF_IND)
1218 			/* agfr %b2,%src (%src is s32 here) */
1219 			EMIT4(0xb9180000, BPF_REG_2, src_reg);
1220 
1221 		/* basr %b5,%w1 (%b5 is call saved) */
1222 		EMIT2(0x0d00, BPF_REG_5, REG_W1);
1223 
1224 		/*
1225 		 * Note: For fast access we jump directly after the
1226 		 * jnz instruction from bpf_jit.S
1227 		 */
1228 		/* jnz <ret0> */
1229 		EMIT4_PCREL(0xa7740000, jit->ret0_ip - jit->prg);
1230 		break;
1231 	default: /* too complex, give up */
1232 		pr_err("Unknown opcode %02x\n", insn->code);
1233 		return -1;
1234 	}
1235 	return insn_count;
1236 }
1237 
1238 /*
1239  * Compile eBPF program into s390x code
1240  */
1241 static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp)
1242 {
1243 	int i, insn_count;
1244 
1245 	jit->lit = jit->lit_start;
1246 	jit->prg = 0;
1247 
1248 	bpf_jit_prologue(jit, bpf_prog_was_classic(fp));
1249 	for (i = 0; i < fp->len; i += insn_count) {
1250 		insn_count = bpf_jit_insn(jit, fp, i);
1251 		if (insn_count < 0)
1252 			return -1;
1253 		jit->addrs[i + 1] = jit->prg; /* Next instruction address */
1254 	}
1255 	bpf_jit_epilogue(jit);
1256 
1257 	jit->lit_start = jit->prg;
1258 	jit->size = jit->lit;
1259 	jit->size_prg = jit->prg;
1260 	return 0;
1261 }
1262 
1263 /*
1264  * Classic BPF function stub. BPF programs will be converted into
1265  * eBPF and then bpf_int_jit_compile() will be called.
1266  */
1267 void bpf_jit_compile(struct bpf_prog *fp)
1268 {
1269 }
1270 
1271 /*
1272  * Compile eBPF program "fp"
1273  */
1274 void bpf_int_jit_compile(struct bpf_prog *fp)
1275 {
1276 	struct bpf_binary_header *header;
1277 	struct bpf_jit jit;
1278 	int pass;
1279 
1280 	if (!bpf_jit_enable)
1281 		return;
1282 	memset(&jit, 0, sizeof(jit));
1283 	jit.addrs = kcalloc(fp->len + 1, sizeof(*jit.addrs), GFP_KERNEL);
1284 	if (jit.addrs == NULL)
1285 		return;
1286 	/*
1287 	 * Three initial passes:
1288 	 *   - 1/2: Determine clobbered registers
1289 	 *   - 3:   Calculate program size and addrs arrray
1290 	 */
1291 	for (pass = 1; pass <= 3; pass++) {
1292 		if (bpf_jit_prog(&jit, fp))
1293 			goto free_addrs;
1294 	}
1295 	/*
1296 	 * Final pass: Allocate and generate program
1297 	 */
1298 	if (jit.size >= BPF_SIZE_MAX)
1299 		goto free_addrs;
1300 	header = bpf_jit_binary_alloc(jit.size, &jit.prg_buf, 2, jit_fill_hole);
1301 	if (!header)
1302 		goto free_addrs;
1303 	if (bpf_jit_prog(&jit, fp))
1304 		goto free_addrs;
1305 	if (bpf_jit_enable > 1) {
1306 		bpf_jit_dump(fp->len, jit.size, pass, jit.prg_buf);
1307 		if (jit.prg_buf)
1308 			print_fn_code(jit.prg_buf, jit.size_prg);
1309 	}
1310 	if (jit.prg_buf) {
1311 		set_memory_ro((unsigned long)header, header->pages);
1312 		fp->bpf_func = (void *) jit.prg_buf;
1313 		fp->jited = true;
1314 	}
1315 free_addrs:
1316 	kfree(jit.addrs);
1317 }
1318 
1319 /*
1320  * Free eBPF program
1321  */
1322 void bpf_jit_free(struct bpf_prog *fp)
1323 {
1324 	unsigned long addr = (unsigned long)fp->bpf_func & PAGE_MASK;
1325 	struct bpf_binary_header *header = (void *)addr;
1326 
1327 	if (!fp->jited)
1328 		goto free_filter;
1329 
1330 	set_memory_rw(addr, header->pages);
1331 	bpf_jit_binary_free(header);
1332 
1333 free_filter:
1334 	bpf_prog_unlock_free(fp);
1335 }
1336