1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * eBPF JIT compiler for PPC32
4 *
5 * Copyright 2020 Christophe Leroy <christophe.leroy@csgroup.eu>
6 * CS GROUP France
7 *
8 * Based on PPC64 eBPF JIT compiler by Naveen N. Rao
9 */
10 #include <linux/moduleloader.h>
11 #include <asm/cacheflush.h>
12 #include <asm/asm-compat.h>
13 #include <linux/netdevice.h>
14 #include <linux/filter.h>
15 #include <linux/if_vlan.h>
16 #include <asm/kprobes.h>
17 #include <linux/bpf.h>
18
19 #include "bpf_jit.h"
20
21 /*
22 * Stack layout:
23 *
24 * [ prev sp ] <-------------
25 * [ nv gpr save area ] 16 * 4 |
26 * fp (r31) --> [ ebpf stack space ] upto 512 |
27 * [ frame header ] 16 |
28 * sp (r1) ---> [ stack pointer ] --------------
29 */
30
31 /* for gpr non volatile registers r17 to r31 (14) + tail call */
32 #define BPF_PPC_STACK_SAVE (15 * 4 + 4)
33 /* stack frame, ensure this is quadword aligned */
34 #define BPF_PPC_STACKFRAME(ctx) (STACK_FRAME_MIN_SIZE + BPF_PPC_STACK_SAVE + (ctx)->stack_size)
35
36 #define PPC_EX32(r, i) EMIT(PPC_RAW_LI((r), (i) < 0 ? -1 : 0))
37
38 /* PPC NVR range -- update this if we ever use NVRs below r17 */
39 #define BPF_PPC_NVR_MIN _R17
40 #define BPF_PPC_TC _R16
41
42 /* BPF register usage */
43 #define TMP_REG (MAX_BPF_JIT_REG + 0)
44
45 /* BPF to ppc register mappings */
bpf_jit_init_reg_mapping(struct codegen_context * ctx)46 void bpf_jit_init_reg_mapping(struct codegen_context *ctx)
47 {
48 /* function return value */
49 ctx->b2p[BPF_REG_0] = _R12;
50 /* function arguments */
51 ctx->b2p[BPF_REG_1] = _R4;
52 ctx->b2p[BPF_REG_2] = _R6;
53 ctx->b2p[BPF_REG_3] = _R8;
54 ctx->b2p[BPF_REG_4] = _R10;
55 ctx->b2p[BPF_REG_5] = _R22;
56 /* non volatile registers */
57 ctx->b2p[BPF_REG_6] = _R24;
58 ctx->b2p[BPF_REG_7] = _R26;
59 ctx->b2p[BPF_REG_8] = _R28;
60 ctx->b2p[BPF_REG_9] = _R30;
61 /* frame pointer aka BPF_REG_10 */
62 ctx->b2p[BPF_REG_FP] = _R18;
63 /* eBPF jit internal registers */
64 ctx->b2p[BPF_REG_AX] = _R20;
65 ctx->b2p[TMP_REG] = _R31; /* 32 bits */
66 }
67
bpf_jit_stack_offsetof(struct codegen_context * ctx,int reg)68 static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg)
69 {
70 if ((reg >= BPF_PPC_NVR_MIN && reg < 32) || reg == BPF_PPC_TC)
71 return BPF_PPC_STACKFRAME(ctx) - 4 * (32 - reg);
72
73 WARN(true, "BPF JIT is asking about unknown registers, will crash the stack");
74 /* Use the hole we have left for alignment */
75 return BPF_PPC_STACKFRAME(ctx) - 4;
76 }
77
78 #define SEEN_VREG_MASK 0x1ff80000 /* Volatile registers r3-r12 */
79 #define SEEN_NVREG_FULL_MASK 0x0003ffff /* Non volatile registers r14-r31 */
80 #define SEEN_NVREG_TEMP_MASK 0x00001e01 /* BPF_REG_5, BPF_REG_AX, TMP_REG */
81
bpf_has_stack_frame(struct codegen_context * ctx)82 static inline bool bpf_has_stack_frame(struct codegen_context *ctx)
83 {
84 /*
85 * We only need a stack frame if:
86 * - we call other functions (kernel helpers), or
87 * - we use non volatile registers, or
88 * - we use tail call counter
89 * - the bpf program uses its stack area
90 * The latter condition is deduced from the usage of BPF_REG_FP
91 */
92 return ctx->seen & (SEEN_FUNC | SEEN_TAILCALL | SEEN_NVREG_FULL_MASK) ||
93 bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_FP));
94 }
95
bpf_jit_realloc_regs(struct codegen_context * ctx)96 void bpf_jit_realloc_regs(struct codegen_context *ctx)
97 {
98 unsigned int nvreg_mask;
99
100 if (ctx->seen & SEEN_FUNC)
101 nvreg_mask = SEEN_NVREG_TEMP_MASK;
102 else
103 nvreg_mask = SEEN_NVREG_FULL_MASK;
104
105 while (ctx->seen & nvreg_mask &&
106 (ctx->seen & SEEN_VREG_MASK) != SEEN_VREG_MASK) {
107 int old = 32 - fls(ctx->seen & (nvreg_mask & 0xaaaaaaab));
108 int new = 32 - fls(~ctx->seen & (SEEN_VREG_MASK & 0xaaaaaaaa));
109 int i;
110
111 for (i = BPF_REG_0; i <= TMP_REG; i++) {
112 if (ctx->b2p[i] != old)
113 continue;
114 ctx->b2p[i] = new;
115 bpf_set_seen_register(ctx, new);
116 bpf_clear_seen_register(ctx, old);
117 if (i != TMP_REG) {
118 bpf_set_seen_register(ctx, new - 1);
119 bpf_clear_seen_register(ctx, old - 1);
120 }
121 break;
122 }
123 }
124 }
125
bpf_jit_build_prologue(u32 * image,struct codegen_context * ctx)126 void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx)
127 {
128 int i;
129
130 /* Initialize tail_call_cnt, to be skipped if we do tail calls. */
131 if (ctx->seen & SEEN_TAILCALL)
132 EMIT(PPC_RAW_LI(_R4, 0));
133 else
134 EMIT(PPC_RAW_NOP());
135
136 #define BPF_TAILCALL_PROLOGUE_SIZE 4
137
138 if (bpf_has_stack_frame(ctx))
139 EMIT(PPC_RAW_STWU(_R1, _R1, -BPF_PPC_STACKFRAME(ctx)));
140
141 if (ctx->seen & SEEN_TAILCALL)
142 EMIT(PPC_RAW_STW(_R4, _R1, bpf_jit_stack_offsetof(ctx, BPF_PPC_TC)));
143
144 /* First arg comes in as a 32 bits pointer. */
145 EMIT(PPC_RAW_MR(bpf_to_ppc(BPF_REG_1), _R3));
146 EMIT(PPC_RAW_LI(bpf_to_ppc(BPF_REG_1) - 1, 0));
147
148 /*
149 * We need a stack frame, but we don't necessarily need to
150 * save/restore LR unless we call other functions
151 */
152 if (ctx->seen & SEEN_FUNC)
153 EMIT(PPC_RAW_MFLR(_R0));
154
155 /*
156 * Back up non-volatile regs -- registers r18-r31
157 */
158 for (i = BPF_PPC_NVR_MIN; i <= 31; i++)
159 if (bpf_is_seen_register(ctx, i))
160 EMIT(PPC_RAW_STW(i, _R1, bpf_jit_stack_offsetof(ctx, i)));
161
162 /* Setup frame pointer to point to the bpf stack area */
163 if (bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_FP))) {
164 EMIT(PPC_RAW_LI(bpf_to_ppc(BPF_REG_FP) - 1, 0));
165 EMIT(PPC_RAW_ADDI(bpf_to_ppc(BPF_REG_FP), _R1,
166 STACK_FRAME_MIN_SIZE + ctx->stack_size));
167 }
168
169 if (ctx->seen & SEEN_FUNC)
170 EMIT(PPC_RAW_STW(_R0, _R1, BPF_PPC_STACKFRAME(ctx) + PPC_LR_STKOFF));
171 }
172
bpf_jit_emit_common_epilogue(u32 * image,struct codegen_context * ctx)173 static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context *ctx)
174 {
175 int i;
176
177 /* Restore NVRs */
178 for (i = BPF_PPC_NVR_MIN; i <= 31; i++)
179 if (bpf_is_seen_register(ctx, i))
180 EMIT(PPC_RAW_LWZ(i, _R1, bpf_jit_stack_offsetof(ctx, i)));
181
182 if (ctx->seen & SEEN_FUNC)
183 EMIT(PPC_RAW_LWZ(_R0, _R1, BPF_PPC_STACKFRAME(ctx) + PPC_LR_STKOFF));
184
185 /* Tear down our stack frame */
186 if (bpf_has_stack_frame(ctx))
187 EMIT(PPC_RAW_ADDI(_R1, _R1, BPF_PPC_STACKFRAME(ctx)));
188
189 if (ctx->seen & SEEN_FUNC)
190 EMIT(PPC_RAW_MTLR(_R0));
191
192 }
193
bpf_jit_build_epilogue(u32 * image,struct codegen_context * ctx)194 void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx)
195 {
196 EMIT(PPC_RAW_MR(_R3, bpf_to_ppc(BPF_REG_0)));
197
198 bpf_jit_emit_common_epilogue(image, ctx);
199
200 EMIT(PPC_RAW_BLR());
201 }
202
bpf_jit_emit_func_call_rel(u32 * image,struct codegen_context * ctx,u64 func)203 int bpf_jit_emit_func_call_rel(u32 *image, struct codegen_context *ctx, u64 func)
204 {
205 s32 rel = (s32)func - (s32)(image + ctx->idx);
206
207 if (image && rel < 0x2000000 && rel >= -0x2000000) {
208 PPC_BL(func);
209 } else {
210 /* Load function address into r0 */
211 EMIT(PPC_RAW_LIS(_R0, IMM_H(func)));
212 EMIT(PPC_RAW_ORI(_R0, _R0, IMM_L(func)));
213 EMIT(PPC_RAW_MTCTR(_R0));
214 EMIT(PPC_RAW_BCTRL());
215 }
216
217 return 0;
218 }
219
bpf_jit_emit_tail_call(u32 * image,struct codegen_context * ctx,u32 out)220 static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 out)
221 {
222 /*
223 * By now, the eBPF program has already setup parameters in r3-r6
224 * r3-r4/BPF_REG_1 - pointer to ctx -- passed as is to the next bpf program
225 * r5-r6/BPF_REG_2 - pointer to bpf_array
226 * r7-r8/BPF_REG_3 - index in bpf_array
227 */
228 int b2p_bpf_array = bpf_to_ppc(BPF_REG_2);
229 int b2p_index = bpf_to_ppc(BPF_REG_3);
230
231 /*
232 * if (index >= array->map.max_entries)
233 * goto out;
234 */
235 EMIT(PPC_RAW_LWZ(_R0, b2p_bpf_array, offsetof(struct bpf_array, map.max_entries)));
236 EMIT(PPC_RAW_CMPLW(b2p_index, _R0));
237 EMIT(PPC_RAW_LWZ(_R0, _R1, bpf_jit_stack_offsetof(ctx, BPF_PPC_TC)));
238 PPC_BCC_SHORT(COND_GE, out);
239
240 /*
241 * if (tail_call_cnt >= MAX_TAIL_CALL_CNT)
242 * goto out;
243 */
244 EMIT(PPC_RAW_CMPLWI(_R0, MAX_TAIL_CALL_CNT));
245 /* tail_call_cnt++; */
246 EMIT(PPC_RAW_ADDIC(_R0, _R0, 1));
247 PPC_BCC_SHORT(COND_GE, out);
248
249 /* prog = array->ptrs[index]; */
250 EMIT(PPC_RAW_RLWINM(_R3, b2p_index, 2, 0, 29));
251 EMIT(PPC_RAW_ADD(_R3, _R3, b2p_bpf_array));
252 EMIT(PPC_RAW_LWZ(_R3, _R3, offsetof(struct bpf_array, ptrs)));
253
254 /*
255 * if (prog == NULL)
256 * goto out;
257 */
258 EMIT(PPC_RAW_CMPLWI(_R3, 0));
259 PPC_BCC_SHORT(COND_EQ, out);
260
261 /* goto *(prog->bpf_func + prologue_size); */
262 EMIT(PPC_RAW_LWZ(_R3, _R3, offsetof(struct bpf_prog, bpf_func)));
263 EMIT(PPC_RAW_ADDIC(_R3, _R3, BPF_TAILCALL_PROLOGUE_SIZE));
264 EMIT(PPC_RAW_MTCTR(_R3));
265
266 EMIT(PPC_RAW_MR(_R3, bpf_to_ppc(BPF_REG_1)));
267
268 /* Put tail_call_cnt in r4 */
269 EMIT(PPC_RAW_MR(_R4, _R0));
270
271 /* tear restore NVRs, ... */
272 bpf_jit_emit_common_epilogue(image, ctx);
273
274 EMIT(PPC_RAW_BCTR());
275
276 /* out: */
277 return 0;
278 }
279
280 /* Assemble the body code between the prologue & epilogue */
bpf_jit_build_body(struct bpf_prog * fp,u32 * image,struct codegen_context * ctx,u32 * addrs,int pass,bool extra_pass)281 int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *ctx,
282 u32 *addrs, int pass, bool extra_pass)
283 {
284 const struct bpf_insn *insn = fp->insnsi;
285 int flen = fp->len;
286 int i, ret;
287
288 /* Start of epilogue code - will only be valid 2nd pass onwards */
289 u32 exit_addr = addrs[flen];
290
291 for (i = 0; i < flen; i++) {
292 u32 code = insn[i].code;
293 u32 prevcode = i ? insn[i - 1].code : 0;
294 u32 dst_reg = bpf_to_ppc(insn[i].dst_reg);
295 u32 dst_reg_h = dst_reg - 1;
296 u32 src_reg = bpf_to_ppc(insn[i].src_reg);
297 u32 src_reg_h = src_reg - 1;
298 u32 src2_reg = dst_reg;
299 u32 src2_reg_h = dst_reg_h;
300 u32 ax_reg = bpf_to_ppc(BPF_REG_AX);
301 u32 tmp_reg = bpf_to_ppc(TMP_REG);
302 u32 size = BPF_SIZE(code);
303 u32 save_reg, ret_reg;
304 s16 off = insn[i].off;
305 s32 imm = insn[i].imm;
306 bool func_addr_fixed;
307 u64 func_addr;
308 u32 true_cond;
309 u32 tmp_idx;
310 int j;
311
312 if (i && (BPF_CLASS(code) == BPF_ALU64 || BPF_CLASS(code) == BPF_ALU) &&
313 (BPF_CLASS(prevcode) == BPF_ALU64 || BPF_CLASS(prevcode) == BPF_ALU) &&
314 BPF_OP(prevcode) == BPF_MOV && BPF_SRC(prevcode) == BPF_X &&
315 insn[i - 1].dst_reg == insn[i].dst_reg && insn[i - 1].imm != 1) {
316 src2_reg = bpf_to_ppc(insn[i - 1].src_reg);
317 src2_reg_h = src2_reg - 1;
318 ctx->idx = addrs[i - 1] / 4;
319 }
320
321 /*
322 * addrs[] maps a BPF bytecode address into a real offset from
323 * the start of the body code.
324 */
325 addrs[i] = ctx->idx * 4;
326
327 /*
328 * As an optimization, we note down which registers
329 * are used so that we can only save/restore those in our
330 * prologue and epilogue. We do this here regardless of whether
331 * the actual BPF instruction uses src/dst registers or not
332 * (for instance, BPF_CALL does not use them). The expectation
333 * is that those instructions will have src_reg/dst_reg set to
334 * 0. Even otherwise, we just lose some prologue/epilogue
335 * optimization but everything else should work without
336 * any issues.
337 */
338 if (dst_reg >= 3 && dst_reg < 32) {
339 bpf_set_seen_register(ctx, dst_reg);
340 bpf_set_seen_register(ctx, dst_reg_h);
341 }
342
343 if (src_reg >= 3 && src_reg < 32) {
344 bpf_set_seen_register(ctx, src_reg);
345 bpf_set_seen_register(ctx, src_reg_h);
346 }
347
348 switch (code) {
349 /*
350 * Arithmetic operations: ADD/SUB/MUL/DIV/MOD/NEG
351 */
352 case BPF_ALU | BPF_ADD | BPF_X: /* (u32) dst += (u32) src */
353 EMIT(PPC_RAW_ADD(dst_reg, src2_reg, src_reg));
354 break;
355 case BPF_ALU64 | BPF_ADD | BPF_X: /* dst += src */
356 EMIT(PPC_RAW_ADDC(dst_reg, src2_reg, src_reg));
357 EMIT(PPC_RAW_ADDE(dst_reg_h, src2_reg_h, src_reg_h));
358 break;
359 case BPF_ALU | BPF_SUB | BPF_X: /* (u32) dst -= (u32) src */
360 EMIT(PPC_RAW_SUB(dst_reg, src2_reg, src_reg));
361 break;
362 case BPF_ALU64 | BPF_SUB | BPF_X: /* dst -= src */
363 EMIT(PPC_RAW_SUBFC(dst_reg, src_reg, src2_reg));
364 EMIT(PPC_RAW_SUBFE(dst_reg_h, src_reg_h, src2_reg_h));
365 break;
366 case BPF_ALU | BPF_SUB | BPF_K: /* (u32) dst -= (u32) imm */
367 imm = -imm;
368 fallthrough;
369 case BPF_ALU | BPF_ADD | BPF_K: /* (u32) dst += (u32) imm */
370 if (!imm) {
371 EMIT(PPC_RAW_MR(dst_reg, src2_reg));
372 } else if (IMM_HA(imm) & 0xffff) {
373 EMIT(PPC_RAW_ADDIS(dst_reg, src2_reg, IMM_HA(imm)));
374 src2_reg = dst_reg;
375 }
376 if (IMM_L(imm))
377 EMIT(PPC_RAW_ADDI(dst_reg, src2_reg, IMM_L(imm)));
378 break;
379 case BPF_ALU64 | BPF_SUB | BPF_K: /* dst -= imm */
380 imm = -imm;
381 fallthrough;
382 case BPF_ALU64 | BPF_ADD | BPF_K: /* dst += imm */
383 if (!imm) {
384 EMIT(PPC_RAW_MR(dst_reg, src2_reg));
385 EMIT(PPC_RAW_MR(dst_reg_h, src2_reg_h));
386 break;
387 }
388 if (imm >= -32768 && imm < 32768) {
389 EMIT(PPC_RAW_ADDIC(dst_reg, src2_reg, imm));
390 } else {
391 PPC_LI32(_R0, imm);
392 EMIT(PPC_RAW_ADDC(dst_reg, src2_reg, _R0));
393 }
394 if (imm >= 0 || (BPF_OP(code) == BPF_SUB && imm == 0x80000000))
395 EMIT(PPC_RAW_ADDZE(dst_reg_h, src2_reg_h));
396 else
397 EMIT(PPC_RAW_ADDME(dst_reg_h, src2_reg_h));
398 break;
399 case BPF_ALU64 | BPF_MUL | BPF_X: /* dst *= src */
400 bpf_set_seen_register(ctx, tmp_reg);
401 EMIT(PPC_RAW_MULW(_R0, src2_reg, src_reg_h));
402 EMIT(PPC_RAW_MULW(dst_reg_h, src2_reg_h, src_reg));
403 EMIT(PPC_RAW_MULHWU(tmp_reg, src2_reg, src_reg));
404 EMIT(PPC_RAW_MULW(dst_reg, src2_reg, src_reg));
405 EMIT(PPC_RAW_ADD(dst_reg_h, dst_reg_h, _R0));
406 EMIT(PPC_RAW_ADD(dst_reg_h, dst_reg_h, tmp_reg));
407 break;
408 case BPF_ALU | BPF_MUL | BPF_X: /* (u32) dst *= (u32) src */
409 EMIT(PPC_RAW_MULW(dst_reg, src2_reg, src_reg));
410 break;
411 case BPF_ALU | BPF_MUL | BPF_K: /* (u32) dst *= (u32) imm */
412 if (imm == 1) {
413 EMIT(PPC_RAW_MR(dst_reg, src2_reg));
414 } else if (imm == -1) {
415 EMIT(PPC_RAW_SUBFIC(dst_reg, src2_reg, 0));
416 } else if (is_power_of_2((u32)imm)) {
417 EMIT(PPC_RAW_SLWI(dst_reg, src2_reg, ilog2(imm)));
418 } else if (imm >= -32768 && imm < 32768) {
419 EMIT(PPC_RAW_MULI(dst_reg, src2_reg, imm));
420 } else {
421 PPC_LI32(_R0, imm);
422 EMIT(PPC_RAW_MULW(dst_reg, src2_reg, _R0));
423 }
424 break;
425 case BPF_ALU64 | BPF_MUL | BPF_K: /* dst *= imm */
426 if (!imm) {
427 PPC_LI32(dst_reg, 0);
428 PPC_LI32(dst_reg_h, 0);
429 } else if (imm == 1) {
430 EMIT(PPC_RAW_MR(dst_reg, src2_reg));
431 EMIT(PPC_RAW_MR(dst_reg_h, src2_reg_h));
432 } else if (imm == -1) {
433 EMIT(PPC_RAW_SUBFIC(dst_reg, src2_reg, 0));
434 EMIT(PPC_RAW_SUBFZE(dst_reg_h, src2_reg_h));
435 } else if (imm > 0 && is_power_of_2(imm)) {
436 imm = ilog2(imm);
437 EMIT(PPC_RAW_RLWINM(dst_reg_h, src2_reg_h, imm, 0, 31 - imm));
438 EMIT(PPC_RAW_RLWIMI(dst_reg_h, dst_reg, imm, 32 - imm, 31));
439 EMIT(PPC_RAW_SLWI(dst_reg, src2_reg, imm));
440 } else {
441 bpf_set_seen_register(ctx, tmp_reg);
442 PPC_LI32(tmp_reg, imm);
443 EMIT(PPC_RAW_MULW(dst_reg_h, src2_reg_h, tmp_reg));
444 if (imm < 0)
445 EMIT(PPC_RAW_SUB(dst_reg_h, dst_reg_h, src2_reg));
446 EMIT(PPC_RAW_MULHWU(_R0, src2_reg, tmp_reg));
447 EMIT(PPC_RAW_MULW(dst_reg, src2_reg, tmp_reg));
448 EMIT(PPC_RAW_ADD(dst_reg_h, dst_reg_h, _R0));
449 }
450 break;
451 case BPF_ALU | BPF_DIV | BPF_X: /* (u32) dst /= (u32) src */
452 EMIT(PPC_RAW_DIVWU(dst_reg, src2_reg, src_reg));
453 break;
454 case BPF_ALU | BPF_MOD | BPF_X: /* (u32) dst %= (u32) src */
455 EMIT(PPC_RAW_DIVWU(_R0, src2_reg, src_reg));
456 EMIT(PPC_RAW_MULW(_R0, src_reg, _R0));
457 EMIT(PPC_RAW_SUB(dst_reg, src2_reg, _R0));
458 break;
459 case BPF_ALU64 | BPF_DIV | BPF_X: /* dst /= src */
460 return -EOPNOTSUPP;
461 case BPF_ALU64 | BPF_MOD | BPF_X: /* dst %= src */
462 return -EOPNOTSUPP;
463 case BPF_ALU | BPF_DIV | BPF_K: /* (u32) dst /= (u32) imm */
464 if (!imm)
465 return -EINVAL;
466 if (imm == 1) {
467 EMIT(PPC_RAW_MR(dst_reg, src2_reg));
468 } else if (is_power_of_2((u32)imm)) {
469 EMIT(PPC_RAW_SRWI(dst_reg, src2_reg, ilog2(imm)));
470 } else {
471 PPC_LI32(_R0, imm);
472 EMIT(PPC_RAW_DIVWU(dst_reg, src2_reg, _R0));
473 }
474 break;
475 case BPF_ALU | BPF_MOD | BPF_K: /* (u32) dst %= (u32) imm */
476 if (!imm)
477 return -EINVAL;
478
479 if (!is_power_of_2((u32)imm)) {
480 bpf_set_seen_register(ctx, tmp_reg);
481 PPC_LI32(tmp_reg, imm);
482 EMIT(PPC_RAW_DIVWU(_R0, src2_reg, tmp_reg));
483 EMIT(PPC_RAW_MULW(_R0, tmp_reg, _R0));
484 EMIT(PPC_RAW_SUB(dst_reg, src2_reg, _R0));
485 } else if (imm == 1) {
486 EMIT(PPC_RAW_LI(dst_reg, 0));
487 } else {
488 imm = ilog2((u32)imm);
489 EMIT(PPC_RAW_RLWINM(dst_reg, src2_reg, 0, 32 - imm, 31));
490 }
491 break;
492 case BPF_ALU64 | BPF_MOD | BPF_K: /* dst %= imm */
493 if (!imm)
494 return -EINVAL;
495 if (imm < 0)
496 imm = -imm;
497 if (!is_power_of_2(imm))
498 return -EOPNOTSUPP;
499 if (imm == 1)
500 EMIT(PPC_RAW_LI(dst_reg, 0));
501 else
502 EMIT(PPC_RAW_RLWINM(dst_reg, src2_reg, 0, 32 - ilog2(imm), 31));
503 EMIT(PPC_RAW_LI(dst_reg_h, 0));
504 break;
505 case BPF_ALU64 | BPF_DIV | BPF_K: /* dst /= imm */
506 if (!imm)
507 return -EINVAL;
508 if (!is_power_of_2(abs(imm)))
509 return -EOPNOTSUPP;
510
511 if (imm < 0) {
512 EMIT(PPC_RAW_SUBFIC(dst_reg, src2_reg, 0));
513 EMIT(PPC_RAW_SUBFZE(dst_reg_h, src2_reg_h));
514 imm = -imm;
515 src2_reg = dst_reg;
516 }
517 if (imm == 1) {
518 EMIT(PPC_RAW_MR(dst_reg, src2_reg));
519 EMIT(PPC_RAW_MR(dst_reg_h, src2_reg_h));
520 } else {
521 imm = ilog2(imm);
522 EMIT(PPC_RAW_RLWINM(dst_reg, src2_reg, 32 - imm, imm, 31));
523 EMIT(PPC_RAW_RLWIMI(dst_reg, src2_reg_h, 32 - imm, 0, imm - 1));
524 EMIT(PPC_RAW_SRAWI(dst_reg_h, src2_reg_h, imm));
525 }
526 break;
527 case BPF_ALU | BPF_NEG: /* (u32) dst = -dst */
528 EMIT(PPC_RAW_NEG(dst_reg, src2_reg));
529 break;
530 case BPF_ALU64 | BPF_NEG: /* dst = -dst */
531 EMIT(PPC_RAW_SUBFIC(dst_reg, src2_reg, 0));
532 EMIT(PPC_RAW_SUBFZE(dst_reg_h, src2_reg_h));
533 break;
534
535 /*
536 * Logical operations: AND/OR/XOR/[A]LSH/[A]RSH
537 */
538 case BPF_ALU64 | BPF_AND | BPF_X: /* dst = dst & src */
539 EMIT(PPC_RAW_AND(dst_reg, src2_reg, src_reg));
540 EMIT(PPC_RAW_AND(dst_reg_h, src2_reg_h, src_reg_h));
541 break;
542 case BPF_ALU | BPF_AND | BPF_X: /* (u32) dst = dst & src */
543 EMIT(PPC_RAW_AND(dst_reg, src2_reg, src_reg));
544 break;
545 case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */
546 if (imm >= 0)
547 EMIT(PPC_RAW_LI(dst_reg_h, 0));
548 fallthrough;
549 case BPF_ALU | BPF_AND | BPF_K: /* (u32) dst = dst & imm */
550 if (!IMM_H(imm)) {
551 EMIT(PPC_RAW_ANDI(dst_reg, src2_reg, IMM_L(imm)));
552 } else if (!IMM_L(imm)) {
553 EMIT(PPC_RAW_ANDIS(dst_reg, src2_reg, IMM_H(imm)));
554 } else if (imm == (((1 << fls(imm)) - 1) ^ ((1 << (ffs(i) - 1)) - 1))) {
555 EMIT(PPC_RAW_RLWINM(dst_reg, src2_reg, 0,
556 32 - fls(imm), 32 - ffs(imm)));
557 } else {
558 PPC_LI32(_R0, imm);
559 EMIT(PPC_RAW_AND(dst_reg, src2_reg, _R0));
560 }
561 break;
562 case BPF_ALU64 | BPF_OR | BPF_X: /* dst = dst | src */
563 EMIT(PPC_RAW_OR(dst_reg, src2_reg, src_reg));
564 EMIT(PPC_RAW_OR(dst_reg_h, src2_reg_h, src_reg_h));
565 break;
566 case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */
567 EMIT(PPC_RAW_OR(dst_reg, src2_reg, src_reg));
568 break;
569 case BPF_ALU64 | BPF_OR | BPF_K:/* dst = dst | imm */
570 /* Sign-extended */
571 if (imm < 0)
572 EMIT(PPC_RAW_LI(dst_reg_h, -1));
573 fallthrough;
574 case BPF_ALU | BPF_OR | BPF_K:/* dst = (u32) dst | (u32) imm */
575 if (IMM_L(imm)) {
576 EMIT(PPC_RAW_ORI(dst_reg, src2_reg, IMM_L(imm)));
577 src2_reg = dst_reg;
578 }
579 if (IMM_H(imm))
580 EMIT(PPC_RAW_ORIS(dst_reg, src2_reg, IMM_H(imm)));
581 break;
582 case BPF_ALU64 | BPF_XOR | BPF_X: /* dst ^= src */
583 if (dst_reg == src_reg) {
584 EMIT(PPC_RAW_LI(dst_reg, 0));
585 EMIT(PPC_RAW_LI(dst_reg_h, 0));
586 } else {
587 EMIT(PPC_RAW_XOR(dst_reg, src2_reg, src_reg));
588 EMIT(PPC_RAW_XOR(dst_reg_h, src2_reg_h, src_reg_h));
589 }
590 break;
591 case BPF_ALU | BPF_XOR | BPF_X: /* (u32) dst ^= src */
592 if (dst_reg == src_reg)
593 EMIT(PPC_RAW_LI(dst_reg, 0));
594 else
595 EMIT(PPC_RAW_XOR(dst_reg, src2_reg, src_reg));
596 break;
597 case BPF_ALU64 | BPF_XOR | BPF_K: /* dst ^= imm */
598 if (imm < 0)
599 EMIT(PPC_RAW_NOR(dst_reg_h, src2_reg_h, src2_reg_h));
600 fallthrough;
601 case BPF_ALU | BPF_XOR | BPF_K: /* (u32) dst ^= (u32) imm */
602 if (IMM_L(imm)) {
603 EMIT(PPC_RAW_XORI(dst_reg, src2_reg, IMM_L(imm)));
604 src2_reg = dst_reg;
605 }
606 if (IMM_H(imm))
607 EMIT(PPC_RAW_XORIS(dst_reg, src2_reg, IMM_H(imm)));
608 break;
609 case BPF_ALU | BPF_LSH | BPF_X: /* (u32) dst <<= (u32) src */
610 EMIT(PPC_RAW_SLW(dst_reg, src2_reg, src_reg));
611 break;
612 case BPF_ALU64 | BPF_LSH | BPF_X: /* dst <<= src; */
613 bpf_set_seen_register(ctx, tmp_reg);
614 EMIT(PPC_RAW_SUBFIC(_R0, src_reg, 32));
615 EMIT(PPC_RAW_SLW(dst_reg_h, src2_reg_h, src_reg));
616 EMIT(PPC_RAW_ADDI(tmp_reg, src_reg, 32));
617 EMIT(PPC_RAW_SRW(_R0, src2_reg, _R0));
618 EMIT(PPC_RAW_SLW(tmp_reg, src2_reg, tmp_reg));
619 EMIT(PPC_RAW_OR(dst_reg_h, dst_reg_h, _R0));
620 EMIT(PPC_RAW_SLW(dst_reg, src2_reg, src_reg));
621 EMIT(PPC_RAW_OR(dst_reg_h, dst_reg_h, tmp_reg));
622 break;
623 case BPF_ALU | BPF_LSH | BPF_K: /* (u32) dst <<= (u32) imm */
624 if (imm)
625 EMIT(PPC_RAW_SLWI(dst_reg, src2_reg, imm));
626 else
627 EMIT(PPC_RAW_MR(dst_reg, src2_reg));
628 break;
629 case BPF_ALU64 | BPF_LSH | BPF_K: /* dst <<= imm */
630 if (imm < 0)
631 return -EINVAL;
632 if (!imm) {
633 EMIT(PPC_RAW_MR(dst_reg, src2_reg));
634 } else if (imm < 32) {
635 EMIT(PPC_RAW_RLWINM(dst_reg_h, src2_reg_h, imm, 0, 31 - imm));
636 EMIT(PPC_RAW_RLWIMI(dst_reg_h, src2_reg, imm, 32 - imm, 31));
637 EMIT(PPC_RAW_RLWINM(dst_reg, src2_reg, imm, 0, 31 - imm));
638 } else if (imm < 64) {
639 EMIT(PPC_RAW_RLWINM(dst_reg_h, src2_reg, imm, 0, 31 - imm));
640 EMIT(PPC_RAW_LI(dst_reg, 0));
641 } else {
642 EMIT(PPC_RAW_LI(dst_reg_h, 0));
643 EMIT(PPC_RAW_LI(dst_reg, 0));
644 }
645 break;
646 case BPF_ALU | BPF_RSH | BPF_X: /* (u32) dst >>= (u32) src */
647 EMIT(PPC_RAW_SRW(dst_reg, src2_reg, src_reg));
648 break;
649 case BPF_ALU64 | BPF_RSH | BPF_X: /* dst >>= src */
650 bpf_set_seen_register(ctx, tmp_reg);
651 EMIT(PPC_RAW_SUBFIC(_R0, src_reg, 32));
652 EMIT(PPC_RAW_SRW(dst_reg, src2_reg, src_reg));
653 EMIT(PPC_RAW_ADDI(tmp_reg, src_reg, 32));
654 EMIT(PPC_RAW_SLW(_R0, src2_reg_h, _R0));
655 EMIT(PPC_RAW_SRW(tmp_reg, dst_reg_h, tmp_reg));
656 EMIT(PPC_RAW_OR(dst_reg, dst_reg, _R0));
657 EMIT(PPC_RAW_SRW(dst_reg_h, src2_reg_h, src_reg));
658 EMIT(PPC_RAW_OR(dst_reg, dst_reg, tmp_reg));
659 break;
660 case BPF_ALU | BPF_RSH | BPF_K: /* (u32) dst >>= (u32) imm */
661 if (imm)
662 EMIT(PPC_RAW_SRWI(dst_reg, src2_reg, imm));
663 else
664 EMIT(PPC_RAW_MR(dst_reg, src2_reg));
665 break;
666 case BPF_ALU64 | BPF_RSH | BPF_K: /* dst >>= imm */
667 if (imm < 0)
668 return -EINVAL;
669 if (!imm) {
670 EMIT(PPC_RAW_MR(dst_reg, src2_reg));
671 EMIT(PPC_RAW_MR(dst_reg_h, src2_reg_h));
672 } else if (imm < 32) {
673 EMIT(PPC_RAW_RLWINM(dst_reg, src2_reg, 32 - imm, imm, 31));
674 EMIT(PPC_RAW_RLWIMI(dst_reg, src2_reg_h, 32 - imm, 0, imm - 1));
675 EMIT(PPC_RAW_RLWINM(dst_reg_h, src2_reg_h, 32 - imm, imm, 31));
676 } else if (imm < 64) {
677 EMIT(PPC_RAW_RLWINM(dst_reg, src2_reg_h, 64 - imm, imm - 32, 31));
678 EMIT(PPC_RAW_LI(dst_reg_h, 0));
679 } else {
680 EMIT(PPC_RAW_LI(dst_reg, 0));
681 EMIT(PPC_RAW_LI(dst_reg_h, 0));
682 }
683 break;
684 case BPF_ALU | BPF_ARSH | BPF_X: /* (s32) dst >>= src */
685 EMIT(PPC_RAW_SRAW(dst_reg, src2_reg, src_reg));
686 break;
687 case BPF_ALU64 | BPF_ARSH | BPF_X: /* (s64) dst >>= src */
688 bpf_set_seen_register(ctx, tmp_reg);
689 EMIT(PPC_RAW_SUBFIC(_R0, src_reg, 32));
690 EMIT(PPC_RAW_SRW(dst_reg, src2_reg, src_reg));
691 EMIT(PPC_RAW_SLW(_R0, src2_reg_h, _R0));
692 EMIT(PPC_RAW_ADDI(tmp_reg, src_reg, 32));
693 EMIT(PPC_RAW_OR(dst_reg, dst_reg, _R0));
694 EMIT(PPC_RAW_RLWINM(_R0, tmp_reg, 0, 26, 26));
695 EMIT(PPC_RAW_SRAW(tmp_reg, src2_reg_h, tmp_reg));
696 EMIT(PPC_RAW_SRAW(dst_reg_h, src2_reg_h, src_reg));
697 EMIT(PPC_RAW_SLW(tmp_reg, tmp_reg, _R0));
698 EMIT(PPC_RAW_OR(dst_reg, dst_reg, tmp_reg));
699 break;
700 case BPF_ALU | BPF_ARSH | BPF_K: /* (s32) dst >>= imm */
701 if (imm)
702 EMIT(PPC_RAW_SRAWI(dst_reg, src2_reg, imm));
703 else
704 EMIT(PPC_RAW_MR(dst_reg, src2_reg));
705 break;
706 case BPF_ALU64 | BPF_ARSH | BPF_K: /* (s64) dst >>= imm */
707 if (imm < 0)
708 return -EINVAL;
709 if (!imm) {
710 EMIT(PPC_RAW_MR(dst_reg, src2_reg));
711 EMIT(PPC_RAW_MR(dst_reg_h, src2_reg_h));
712 } else if (imm < 32) {
713 EMIT(PPC_RAW_RLWINM(dst_reg, src2_reg, 32 - imm, imm, 31));
714 EMIT(PPC_RAW_RLWIMI(dst_reg, src2_reg_h, 32 - imm, 0, imm - 1));
715 EMIT(PPC_RAW_SRAWI(dst_reg_h, src2_reg_h, imm));
716 } else if (imm < 64) {
717 EMIT(PPC_RAW_SRAWI(dst_reg, src2_reg_h, imm - 32));
718 EMIT(PPC_RAW_SRAWI(dst_reg_h, src2_reg_h, 31));
719 } else {
720 EMIT(PPC_RAW_SRAWI(dst_reg, src2_reg_h, 31));
721 EMIT(PPC_RAW_SRAWI(dst_reg_h, src2_reg_h, 31));
722 }
723 break;
724
725 /*
726 * MOV
727 */
728 case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */
729 if (dst_reg == src_reg)
730 break;
731 EMIT(PPC_RAW_MR(dst_reg, src_reg));
732 EMIT(PPC_RAW_MR(dst_reg_h, src_reg_h));
733 break;
734 case BPF_ALU | BPF_MOV | BPF_X: /* (u32) dst = src */
735 /* special mov32 for zext */
736 if (imm == 1)
737 EMIT(PPC_RAW_LI(dst_reg_h, 0));
738 else if (dst_reg != src_reg)
739 EMIT(PPC_RAW_MR(dst_reg, src_reg));
740 break;
741 case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = (s64) imm */
742 PPC_LI32(dst_reg, imm);
743 PPC_EX32(dst_reg_h, imm);
744 break;
745 case BPF_ALU | BPF_MOV | BPF_K: /* (u32) dst = imm */
746 PPC_LI32(dst_reg, imm);
747 break;
748
749 /*
750 * BPF_FROM_BE/LE
751 */
752 case BPF_ALU | BPF_END | BPF_FROM_LE:
753 switch (imm) {
754 case 16:
755 /* Copy 16 bits to upper part */
756 EMIT(PPC_RAW_RLWIMI(dst_reg, src2_reg, 16, 0, 15));
757 /* Rotate 8 bits right & mask */
758 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 24, 16, 31));
759 break;
760 case 32:
761 /*
762 * Rotate word left by 8 bits:
763 * 2 bytes are already in their final position
764 * -- byte 2 and 4 (of bytes 1, 2, 3 and 4)
765 */
766 EMIT(PPC_RAW_RLWINM(_R0, src2_reg, 8, 0, 31));
767 /* Rotate 24 bits and insert byte 1 */
768 EMIT(PPC_RAW_RLWIMI(_R0, src2_reg, 24, 0, 7));
769 /* Rotate 24 bits and insert byte 3 */
770 EMIT(PPC_RAW_RLWIMI(_R0, src2_reg, 24, 16, 23));
771 EMIT(PPC_RAW_MR(dst_reg, _R0));
772 break;
773 case 64:
774 bpf_set_seen_register(ctx, tmp_reg);
775 EMIT(PPC_RAW_RLWINM(tmp_reg, src2_reg, 8, 0, 31));
776 EMIT(PPC_RAW_RLWINM(_R0, src2_reg_h, 8, 0, 31));
777 /* Rotate 24 bits and insert byte 1 */
778 EMIT(PPC_RAW_RLWIMI(tmp_reg, src2_reg, 24, 0, 7));
779 EMIT(PPC_RAW_RLWIMI(_R0, src2_reg_h, 24, 0, 7));
780 /* Rotate 24 bits and insert byte 3 */
781 EMIT(PPC_RAW_RLWIMI(tmp_reg, src2_reg, 24, 16, 23));
782 EMIT(PPC_RAW_RLWIMI(_R0, src2_reg_h, 24, 16, 23));
783 EMIT(PPC_RAW_MR(dst_reg, _R0));
784 EMIT(PPC_RAW_MR(dst_reg_h, tmp_reg));
785 break;
786 }
787 break;
788 case BPF_ALU | BPF_END | BPF_FROM_BE:
789 switch (imm) {
790 case 16:
791 /* zero-extend 16 bits into 32 bits */
792 EMIT(PPC_RAW_RLWINM(dst_reg, src2_reg, 0, 16, 31));
793 break;
794 case 32:
795 case 64:
796 /* nop */
797 break;
798 }
799 break;
800
801 /*
802 * BPF_ST NOSPEC (speculation barrier)
803 */
804 case BPF_ST | BPF_NOSPEC:
805 break;
806
807 /*
808 * BPF_ST(X)
809 */
810 case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src */
811 EMIT(PPC_RAW_STB(src_reg, dst_reg, off));
812 break;
813 case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */
814 PPC_LI32(_R0, imm);
815 EMIT(PPC_RAW_STB(_R0, dst_reg, off));
816 break;
817 case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */
818 EMIT(PPC_RAW_STH(src_reg, dst_reg, off));
819 break;
820 case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */
821 PPC_LI32(_R0, imm);
822 EMIT(PPC_RAW_STH(_R0, dst_reg, off));
823 break;
824 case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */
825 EMIT(PPC_RAW_STW(src_reg, dst_reg, off));
826 break;
827 case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */
828 PPC_LI32(_R0, imm);
829 EMIT(PPC_RAW_STW(_R0, dst_reg, off));
830 break;
831 case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */
832 EMIT(PPC_RAW_STW(src_reg_h, dst_reg, off));
833 EMIT(PPC_RAW_STW(src_reg, dst_reg, off + 4));
834 break;
835 case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */
836 PPC_LI32(_R0, imm);
837 EMIT(PPC_RAW_STW(_R0, dst_reg, off + 4));
838 PPC_EX32(_R0, imm);
839 EMIT(PPC_RAW_STW(_R0, dst_reg, off));
840 break;
841
842 /*
843 * BPF_STX ATOMIC (atomic ops)
844 */
845 case BPF_STX | BPF_ATOMIC | BPF_W:
846 save_reg = _R0;
847 ret_reg = src_reg;
848
849 bpf_set_seen_register(ctx, tmp_reg);
850 bpf_set_seen_register(ctx, ax_reg);
851
852 /* Get offset into TMP_REG */
853 EMIT(PPC_RAW_LI(tmp_reg, off));
854 /*
855 * Enforce full ordering for operations with BPF_FETCH by emitting a 'sync'
856 * before and after the operation.
857 *
858 * This is a requirement in the Linux Kernel Memory Model.
859 * See __cmpxchg_u32() in asm/cmpxchg.h as an example.
860 */
861 if ((imm & BPF_FETCH) && IS_ENABLED(CONFIG_SMP))
862 EMIT(PPC_RAW_SYNC());
863 tmp_idx = ctx->idx * 4;
864 /* load value from memory into r0 */
865 EMIT(PPC_RAW_LWARX(_R0, tmp_reg, dst_reg, 0));
866
867 /* Save old value in BPF_REG_AX */
868 if (imm & BPF_FETCH)
869 EMIT(PPC_RAW_MR(ax_reg, _R0));
870
871 switch (imm) {
872 case BPF_ADD:
873 case BPF_ADD | BPF_FETCH:
874 EMIT(PPC_RAW_ADD(_R0, _R0, src_reg));
875 break;
876 case BPF_AND:
877 case BPF_AND | BPF_FETCH:
878 EMIT(PPC_RAW_AND(_R0, _R0, src_reg));
879 break;
880 case BPF_OR:
881 case BPF_OR | BPF_FETCH:
882 EMIT(PPC_RAW_OR(_R0, _R0, src_reg));
883 break;
884 case BPF_XOR:
885 case BPF_XOR | BPF_FETCH:
886 EMIT(PPC_RAW_XOR(_R0, _R0, src_reg));
887 break;
888 case BPF_CMPXCHG:
889 /*
890 * Return old value in BPF_REG_0 for BPF_CMPXCHG &
891 * in src_reg for other cases.
892 */
893 ret_reg = bpf_to_ppc(BPF_REG_0);
894
895 /* Compare with old value in BPF_REG_0 */
896 EMIT(PPC_RAW_CMPW(bpf_to_ppc(BPF_REG_0), _R0));
897 /* Don't set if different from old value */
898 PPC_BCC_SHORT(COND_NE, (ctx->idx + 3) * 4);
899 fallthrough;
900 case BPF_XCHG:
901 save_reg = src_reg;
902 break;
903 default:
904 pr_err_ratelimited("eBPF filter atomic op code %02x (@%d) unsupported\n",
905 code, i);
906 return -EOPNOTSUPP;
907 }
908
909 /* store new value */
910 EMIT(PPC_RAW_STWCX(save_reg, tmp_reg, dst_reg));
911 /* we're done if this succeeded */
912 PPC_BCC_SHORT(COND_NE, tmp_idx);
913
914 /* For the BPF_FETCH variant, get old data into src_reg */
915 if (imm & BPF_FETCH) {
916 /* Emit 'sync' to enforce full ordering */
917 if (IS_ENABLED(CONFIG_SMP))
918 EMIT(PPC_RAW_SYNC());
919 EMIT(PPC_RAW_MR(ret_reg, ax_reg));
920 if (!fp->aux->verifier_zext)
921 EMIT(PPC_RAW_LI(ret_reg - 1, 0)); /* higher 32-bit */
922 }
923 break;
924
925 case BPF_STX | BPF_ATOMIC | BPF_DW: /* *(u64 *)(dst + off) += src */
926 return -EOPNOTSUPP;
927
928 /*
929 * BPF_LDX
930 */
931 case BPF_LDX | BPF_MEM | BPF_B: /* dst = *(u8 *)(ul) (src + off) */
932 case BPF_LDX | BPF_PROBE_MEM | BPF_B:
933 case BPF_LDX | BPF_MEM | BPF_H: /* dst = *(u16 *)(ul) (src + off) */
934 case BPF_LDX | BPF_PROBE_MEM | BPF_H:
935 case BPF_LDX | BPF_MEM | BPF_W: /* dst = *(u32 *)(ul) (src + off) */
936 case BPF_LDX | BPF_PROBE_MEM | BPF_W:
937 case BPF_LDX | BPF_MEM | BPF_DW: /* dst = *(u64 *)(ul) (src + off) */
938 case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
939 /*
940 * As PTR_TO_BTF_ID that uses BPF_PROBE_MEM mode could either be a valid
941 * kernel pointer or NULL but not a userspace address, execute BPF_PROBE_MEM
942 * load only if addr is kernel address (see is_kernel_addr()), otherwise
943 * set dst_reg=0 and move on.
944 */
945 if (BPF_MODE(code) == BPF_PROBE_MEM) {
946 PPC_LI32(_R0, TASK_SIZE - off);
947 EMIT(PPC_RAW_CMPLW(src_reg, _R0));
948 PPC_BCC_SHORT(COND_GT, (ctx->idx + 4) * 4);
949 EMIT(PPC_RAW_LI(dst_reg, 0));
950 /*
951 * For BPF_DW case, "li reg_h,0" would be needed when
952 * !fp->aux->verifier_zext. Emit NOP otherwise.
953 *
954 * Note that "li reg_h,0" is emitted for BPF_B/H/W case,
955 * if necessary. So, jump there insted of emitting an
956 * additional "li reg_h,0" instruction.
957 */
958 if (size == BPF_DW && !fp->aux->verifier_zext)
959 EMIT(PPC_RAW_LI(dst_reg_h, 0));
960 else
961 EMIT(PPC_RAW_NOP());
962 /*
963 * Need to jump two instructions instead of one for BPF_DW case
964 * as there are two load instructions for dst_reg_h & dst_reg
965 * respectively.
966 */
967 if (size == BPF_DW)
968 PPC_JMP((ctx->idx + 3) * 4);
969 else
970 PPC_JMP((ctx->idx + 2) * 4);
971 }
972
973 switch (size) {
974 case BPF_B:
975 EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off));
976 break;
977 case BPF_H:
978 EMIT(PPC_RAW_LHZ(dst_reg, src_reg, off));
979 break;
980 case BPF_W:
981 EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off));
982 break;
983 case BPF_DW:
984 EMIT(PPC_RAW_LWZ(dst_reg_h, src_reg, off));
985 EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off + 4));
986 break;
987 }
988
989 if (size != BPF_DW && !fp->aux->verifier_zext)
990 EMIT(PPC_RAW_LI(dst_reg_h, 0));
991
992 if (BPF_MODE(code) == BPF_PROBE_MEM) {
993 int insn_idx = ctx->idx - 1;
994 int jmp_off = 4;
995
996 /*
997 * In case of BPF_DW, two lwz instructions are emitted, one
998 * for higher 32-bit and another for lower 32-bit. So, set
999 * ex->insn to the first of the two and jump over both
1000 * instructions in fixup.
1001 *
1002 * Similarly, with !verifier_zext, two instructions are
1003 * emitted for BPF_B/H/W case. So, set ex->insn to the
1004 * instruction that could fault and skip over both
1005 * instructions.
1006 */
1007 if (size == BPF_DW || !fp->aux->verifier_zext) {
1008 insn_idx -= 1;
1009 jmp_off += 4;
1010 }
1011
1012 ret = bpf_add_extable_entry(fp, image, pass, ctx, insn_idx,
1013 jmp_off, dst_reg);
1014 if (ret)
1015 return ret;
1016 }
1017 break;
1018
1019 /*
1020 * Doubleword load
1021 * 16 byte instruction that uses two 'struct bpf_insn'
1022 */
1023 case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */
1024 tmp_idx = ctx->idx;
1025 PPC_LI32(dst_reg_h, (u32)insn[i + 1].imm);
1026 PPC_LI32(dst_reg, (u32)insn[i].imm);
1027 /* padding to allow full 4 instructions for later patching */
1028 if (!image)
1029 for (j = ctx->idx - tmp_idx; j < 4; j++)
1030 EMIT(PPC_RAW_NOP());
1031 /* Adjust for two bpf instructions */
1032 addrs[++i] = ctx->idx * 4;
1033 break;
1034
1035 /*
1036 * Return/Exit
1037 */
1038 case BPF_JMP | BPF_EXIT:
1039 /*
1040 * If this isn't the very last instruction, branch to
1041 * the epilogue. If we _are_ the last instruction,
1042 * we'll just fall through to the epilogue.
1043 */
1044 if (i != flen - 1) {
1045 ret = bpf_jit_emit_exit_insn(image, ctx, _R0, exit_addr);
1046 if (ret)
1047 return ret;
1048 }
1049 /* else fall through to the epilogue */
1050 break;
1051
1052 /*
1053 * Call kernel helper or bpf function
1054 */
1055 case BPF_JMP | BPF_CALL:
1056 ctx->seen |= SEEN_FUNC;
1057
1058 ret = bpf_jit_get_func_addr(fp, &insn[i], extra_pass,
1059 &func_addr, &func_addr_fixed);
1060 if (ret < 0)
1061 return ret;
1062
1063 if (bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_5))) {
1064 EMIT(PPC_RAW_STW(bpf_to_ppc(BPF_REG_5) - 1, _R1, 8));
1065 EMIT(PPC_RAW_STW(bpf_to_ppc(BPF_REG_5), _R1, 12));
1066 }
1067
1068 ret = bpf_jit_emit_func_call_rel(image, ctx, func_addr);
1069 if (ret)
1070 return ret;
1071
1072 EMIT(PPC_RAW_MR(bpf_to_ppc(BPF_REG_0) - 1, _R3));
1073 EMIT(PPC_RAW_MR(bpf_to_ppc(BPF_REG_0), _R4));
1074 break;
1075
1076 /*
1077 * Jumps and branches
1078 */
1079 case BPF_JMP | BPF_JA:
1080 PPC_JMP(addrs[i + 1 + off]);
1081 break;
1082
1083 case BPF_JMP | BPF_JGT | BPF_K:
1084 case BPF_JMP | BPF_JGT | BPF_X:
1085 case BPF_JMP | BPF_JSGT | BPF_K:
1086 case BPF_JMP | BPF_JSGT | BPF_X:
1087 case BPF_JMP32 | BPF_JGT | BPF_K:
1088 case BPF_JMP32 | BPF_JGT | BPF_X:
1089 case BPF_JMP32 | BPF_JSGT | BPF_K:
1090 case BPF_JMP32 | BPF_JSGT | BPF_X:
1091 true_cond = COND_GT;
1092 goto cond_branch;
1093 case BPF_JMP | BPF_JLT | BPF_K:
1094 case BPF_JMP | BPF_JLT | BPF_X:
1095 case BPF_JMP | BPF_JSLT | BPF_K:
1096 case BPF_JMP | BPF_JSLT | BPF_X:
1097 case BPF_JMP32 | BPF_JLT | BPF_K:
1098 case BPF_JMP32 | BPF_JLT | BPF_X:
1099 case BPF_JMP32 | BPF_JSLT | BPF_K:
1100 case BPF_JMP32 | BPF_JSLT | BPF_X:
1101 true_cond = COND_LT;
1102 goto cond_branch;
1103 case BPF_JMP | BPF_JGE | BPF_K:
1104 case BPF_JMP | BPF_JGE | BPF_X:
1105 case BPF_JMP | BPF_JSGE | BPF_K:
1106 case BPF_JMP | BPF_JSGE | BPF_X:
1107 case BPF_JMP32 | BPF_JGE | BPF_K:
1108 case BPF_JMP32 | BPF_JGE | BPF_X:
1109 case BPF_JMP32 | BPF_JSGE | BPF_K:
1110 case BPF_JMP32 | BPF_JSGE | BPF_X:
1111 true_cond = COND_GE;
1112 goto cond_branch;
1113 case BPF_JMP | BPF_JLE | BPF_K:
1114 case BPF_JMP | BPF_JLE | BPF_X:
1115 case BPF_JMP | BPF_JSLE | BPF_K:
1116 case BPF_JMP | BPF_JSLE | BPF_X:
1117 case BPF_JMP32 | BPF_JLE | BPF_K:
1118 case BPF_JMP32 | BPF_JLE | BPF_X:
1119 case BPF_JMP32 | BPF_JSLE | BPF_K:
1120 case BPF_JMP32 | BPF_JSLE | BPF_X:
1121 true_cond = COND_LE;
1122 goto cond_branch;
1123 case BPF_JMP | BPF_JEQ | BPF_K:
1124 case BPF_JMP | BPF_JEQ | BPF_X:
1125 case BPF_JMP32 | BPF_JEQ | BPF_K:
1126 case BPF_JMP32 | BPF_JEQ | BPF_X:
1127 true_cond = COND_EQ;
1128 goto cond_branch;
1129 case BPF_JMP | BPF_JNE | BPF_K:
1130 case BPF_JMP | BPF_JNE | BPF_X:
1131 case BPF_JMP32 | BPF_JNE | BPF_K:
1132 case BPF_JMP32 | BPF_JNE | BPF_X:
1133 true_cond = COND_NE;
1134 goto cond_branch;
1135 case BPF_JMP | BPF_JSET | BPF_K:
1136 case BPF_JMP | BPF_JSET | BPF_X:
1137 case BPF_JMP32 | BPF_JSET | BPF_K:
1138 case BPF_JMP32 | BPF_JSET | BPF_X:
1139 true_cond = COND_NE;
1140 /* fallthrough; */
1141
1142 cond_branch:
1143 switch (code) {
1144 case BPF_JMP | BPF_JGT | BPF_X:
1145 case BPF_JMP | BPF_JLT | BPF_X:
1146 case BPF_JMP | BPF_JGE | BPF_X:
1147 case BPF_JMP | BPF_JLE | BPF_X:
1148 case BPF_JMP | BPF_JEQ | BPF_X:
1149 case BPF_JMP | BPF_JNE | BPF_X:
1150 /* unsigned comparison */
1151 EMIT(PPC_RAW_CMPLW(dst_reg_h, src_reg_h));
1152 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1153 EMIT(PPC_RAW_CMPLW(dst_reg, src_reg));
1154 break;
1155 case BPF_JMP32 | BPF_JGT | BPF_X:
1156 case BPF_JMP32 | BPF_JLT | BPF_X:
1157 case BPF_JMP32 | BPF_JGE | BPF_X:
1158 case BPF_JMP32 | BPF_JLE | BPF_X:
1159 case BPF_JMP32 | BPF_JEQ | BPF_X:
1160 case BPF_JMP32 | BPF_JNE | BPF_X:
1161 /* unsigned comparison */
1162 EMIT(PPC_RAW_CMPLW(dst_reg, src_reg));
1163 break;
1164 case BPF_JMP | BPF_JSGT | BPF_X:
1165 case BPF_JMP | BPF_JSLT | BPF_X:
1166 case BPF_JMP | BPF_JSGE | BPF_X:
1167 case BPF_JMP | BPF_JSLE | BPF_X:
1168 /* signed comparison */
1169 EMIT(PPC_RAW_CMPW(dst_reg_h, src_reg_h));
1170 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1171 EMIT(PPC_RAW_CMPLW(dst_reg, src_reg));
1172 break;
1173 case BPF_JMP32 | BPF_JSGT | BPF_X:
1174 case BPF_JMP32 | BPF_JSLT | BPF_X:
1175 case BPF_JMP32 | BPF_JSGE | BPF_X:
1176 case BPF_JMP32 | BPF_JSLE | BPF_X:
1177 /* signed comparison */
1178 EMIT(PPC_RAW_CMPW(dst_reg, src_reg));
1179 break;
1180 case BPF_JMP | BPF_JSET | BPF_X:
1181 EMIT(PPC_RAW_AND_DOT(_R0, dst_reg_h, src_reg_h));
1182 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1183 EMIT(PPC_RAW_AND_DOT(_R0, dst_reg, src_reg));
1184 break;
1185 case BPF_JMP32 | BPF_JSET | BPF_X: {
1186 EMIT(PPC_RAW_AND_DOT(_R0, dst_reg, src_reg));
1187 break;
1188 case BPF_JMP | BPF_JNE | BPF_K:
1189 case BPF_JMP | BPF_JEQ | BPF_K:
1190 case BPF_JMP | BPF_JGT | BPF_K:
1191 case BPF_JMP | BPF_JLT | BPF_K:
1192 case BPF_JMP | BPF_JGE | BPF_K:
1193 case BPF_JMP | BPF_JLE | BPF_K:
1194 /*
1195 * Need sign-extended load, so only positive
1196 * values can be used as imm in cmplwi
1197 */
1198 if (imm >= 0 && imm < 32768) {
1199 EMIT(PPC_RAW_CMPLWI(dst_reg_h, 0));
1200 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1201 EMIT(PPC_RAW_CMPLWI(dst_reg, imm));
1202 } else {
1203 /* sign-extending load ... but unsigned comparison */
1204 PPC_EX32(_R0, imm);
1205 EMIT(PPC_RAW_CMPLW(dst_reg_h, _R0));
1206 PPC_LI32(_R0, imm);
1207 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1208 EMIT(PPC_RAW_CMPLW(dst_reg, _R0));
1209 }
1210 break;
1211 case BPF_JMP32 | BPF_JNE | BPF_K:
1212 case BPF_JMP32 | BPF_JEQ | BPF_K:
1213 case BPF_JMP32 | BPF_JGT | BPF_K:
1214 case BPF_JMP32 | BPF_JLT | BPF_K:
1215 case BPF_JMP32 | BPF_JGE | BPF_K:
1216 case BPF_JMP32 | BPF_JLE | BPF_K:
1217 if (imm >= 0 && imm < 65536) {
1218 EMIT(PPC_RAW_CMPLWI(dst_reg, imm));
1219 } else {
1220 PPC_LI32(_R0, imm);
1221 EMIT(PPC_RAW_CMPLW(dst_reg, _R0));
1222 }
1223 break;
1224 }
1225 case BPF_JMP | BPF_JSGT | BPF_K:
1226 case BPF_JMP | BPF_JSLT | BPF_K:
1227 case BPF_JMP | BPF_JSGE | BPF_K:
1228 case BPF_JMP | BPF_JSLE | BPF_K:
1229 if (imm >= 0 && imm < 65536) {
1230 EMIT(PPC_RAW_CMPWI(dst_reg_h, imm < 0 ? -1 : 0));
1231 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1232 EMIT(PPC_RAW_CMPLWI(dst_reg, imm));
1233 } else {
1234 /* sign-extending load */
1235 EMIT(PPC_RAW_CMPWI(dst_reg_h, imm < 0 ? -1 : 0));
1236 PPC_LI32(_R0, imm);
1237 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1238 EMIT(PPC_RAW_CMPLW(dst_reg, _R0));
1239 }
1240 break;
1241 case BPF_JMP32 | BPF_JSGT | BPF_K:
1242 case BPF_JMP32 | BPF_JSLT | BPF_K:
1243 case BPF_JMP32 | BPF_JSGE | BPF_K:
1244 case BPF_JMP32 | BPF_JSLE | BPF_K:
1245 /*
1246 * signed comparison, so any 16-bit value
1247 * can be used in cmpwi
1248 */
1249 if (imm >= -32768 && imm < 32768) {
1250 EMIT(PPC_RAW_CMPWI(dst_reg, imm));
1251 } else {
1252 /* sign-extending load */
1253 PPC_LI32(_R0, imm);
1254 EMIT(PPC_RAW_CMPW(dst_reg, _R0));
1255 }
1256 break;
1257 case BPF_JMP | BPF_JSET | BPF_K:
1258 /* andi does not sign-extend the immediate */
1259 if (imm >= 0 && imm < 32768) {
1260 /* PPC_ANDI is _only/always_ dot-form */
1261 EMIT(PPC_RAW_ANDI(_R0, dst_reg, imm));
1262 } else {
1263 PPC_LI32(_R0, imm);
1264 if (imm < 0) {
1265 EMIT(PPC_RAW_CMPWI(dst_reg_h, 0));
1266 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4);
1267 }
1268 EMIT(PPC_RAW_AND_DOT(_R0, dst_reg, _R0));
1269 }
1270 break;
1271 case BPF_JMP32 | BPF_JSET | BPF_K:
1272 /* andi does not sign-extend the immediate */
1273 if (imm >= 0 && imm < 32768) {
1274 /* PPC_ANDI is _only/always_ dot-form */
1275 EMIT(PPC_RAW_ANDI(_R0, dst_reg, imm));
1276 } else {
1277 PPC_LI32(_R0, imm);
1278 EMIT(PPC_RAW_AND_DOT(_R0, dst_reg, _R0));
1279 }
1280 break;
1281 }
1282 PPC_BCC(true_cond, addrs[i + 1 + off]);
1283 break;
1284
1285 /*
1286 * Tail call
1287 */
1288 case BPF_JMP | BPF_TAIL_CALL:
1289 ctx->seen |= SEEN_TAILCALL;
1290 ret = bpf_jit_emit_tail_call(image, ctx, addrs[i + 1]);
1291 if (ret < 0)
1292 return ret;
1293 break;
1294
1295 default:
1296 /*
1297 * The filter contains something cruel & unusual.
1298 * We don't handle it, but also there shouldn't be
1299 * anything missing from our list.
1300 */
1301 pr_err_ratelimited("eBPF filter opcode %04x (@%d) unsupported\n", code, i);
1302 return -EOPNOTSUPP;
1303 }
1304 if (BPF_CLASS(code) == BPF_ALU && !fp->aux->verifier_zext &&
1305 !insn_is_zext(&insn[i + 1]) && !(BPF_OP(code) == BPF_END && imm == 64))
1306 EMIT(PPC_RAW_LI(dst_reg_h, 0));
1307 }
1308
1309 /* Set end-of-body-code address for exit. */
1310 addrs[i] = ctx->idx * 4;
1311
1312 return 0;
1313 }
1314