1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * eBPF JIT compiler for PPC32 4 * 5 * Copyright 2020 Christophe Leroy <christophe.leroy@csgroup.eu> 6 * CS GROUP France 7 * 8 * Based on PPC64 eBPF JIT compiler by Naveen N. Rao 9 */ 10 #include <linux/moduleloader.h> 11 #include <asm/cacheflush.h> 12 #include <asm/asm-compat.h> 13 #include <linux/netdevice.h> 14 #include <linux/filter.h> 15 #include <linux/if_vlan.h> 16 #include <asm/kprobes.h> 17 #include <linux/bpf.h> 18 19 #include "bpf_jit.h" 20 21 /* 22 * Stack layout: 23 * 24 * [ prev sp ] <------------- 25 * [ nv gpr save area ] 16 * 4 | 26 * fp (r31) --> [ ebpf stack space ] upto 512 | 27 * [ frame header ] 16 | 28 * sp (r1) ---> [ stack pointer ] -------------- 29 */ 30 31 /* for gpr non volatile registers r17 to r31 (14) + tail call */ 32 #define BPF_PPC_STACK_SAVE (15 * 4 + 4) 33 /* stack frame, ensure this is quadword aligned */ 34 #define BPF_PPC_STACKFRAME(ctx) (STACK_FRAME_MIN_SIZE + BPF_PPC_STACK_SAVE + (ctx)->stack_size) 35 36 #define PPC_EX32(r, i) EMIT(PPC_RAW_LI((r), (i) < 0 ? -1 : 0)) 37 38 /* PPC NVR range -- update this if we ever use NVRs below r17 */ 39 #define BPF_PPC_NVR_MIN _R17 40 #define BPF_PPC_TC _R16 41 42 /* BPF register usage */ 43 #define TMP_REG (MAX_BPF_JIT_REG + 0) 44 45 /* BPF to ppc register mappings */ 46 void bpf_jit_init_reg_mapping(struct codegen_context *ctx) 47 { 48 /* function return value */ 49 ctx->b2p[BPF_REG_0] = _R12; 50 /* function arguments */ 51 ctx->b2p[BPF_REG_1] = _R4; 52 ctx->b2p[BPF_REG_2] = _R6; 53 ctx->b2p[BPF_REG_3] = _R8; 54 ctx->b2p[BPF_REG_4] = _R10; 55 ctx->b2p[BPF_REG_5] = _R22; 56 /* non volatile registers */ 57 ctx->b2p[BPF_REG_6] = _R24; 58 ctx->b2p[BPF_REG_7] = _R26; 59 ctx->b2p[BPF_REG_8] = _R28; 60 ctx->b2p[BPF_REG_9] = _R30; 61 /* frame pointer aka BPF_REG_10 */ 62 ctx->b2p[BPF_REG_FP] = _R18; 63 /* eBPF jit internal registers */ 64 ctx->b2p[BPF_REG_AX] = _R20; 65 ctx->b2p[TMP_REG] = _R31; /* 32 bits */ 66 } 67 68 static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg) 69 { 70 if ((reg >= BPF_PPC_NVR_MIN && reg < 32) || reg == BPF_PPC_TC) 71 return BPF_PPC_STACKFRAME(ctx) - 4 * (32 - reg); 72 73 WARN(true, "BPF JIT is asking about unknown registers, will crash the stack"); 74 /* Use the hole we have left for alignment */ 75 return BPF_PPC_STACKFRAME(ctx) - 4; 76 } 77 78 #define SEEN_VREG_MASK 0x1ff80000 /* Volatile registers r3-r12 */ 79 #define SEEN_NVREG_FULL_MASK 0x0003ffff /* Non volatile registers r14-r31 */ 80 #define SEEN_NVREG_TEMP_MASK 0x00001e01 /* BPF_REG_5, BPF_REG_AX, TMP_REG */ 81 82 void bpf_jit_realloc_regs(struct codegen_context *ctx) 83 { 84 unsigned int nvreg_mask; 85 86 if (ctx->seen & SEEN_FUNC) 87 nvreg_mask = SEEN_NVREG_TEMP_MASK; 88 else 89 nvreg_mask = SEEN_NVREG_FULL_MASK; 90 91 while (ctx->seen & nvreg_mask && 92 (ctx->seen & SEEN_VREG_MASK) != SEEN_VREG_MASK) { 93 int old = 32 - fls(ctx->seen & (nvreg_mask & 0xaaaaaaab)); 94 int new = 32 - fls(~ctx->seen & (SEEN_VREG_MASK & 0xaaaaaaaa)); 95 int i; 96 97 for (i = BPF_REG_0; i <= TMP_REG; i++) { 98 if (ctx->b2p[i] != old) 99 continue; 100 ctx->b2p[i] = new; 101 bpf_set_seen_register(ctx, new); 102 bpf_clear_seen_register(ctx, old); 103 if (i != TMP_REG) { 104 bpf_set_seen_register(ctx, new - 1); 105 bpf_clear_seen_register(ctx, old - 1); 106 } 107 break; 108 } 109 } 110 } 111 112 void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx) 113 { 114 int i; 115 116 /* First arg comes in as a 32 bits pointer. */ 117 EMIT(PPC_RAW_MR(bpf_to_ppc(BPF_REG_1), _R3)); 118 EMIT(PPC_RAW_LI(bpf_to_ppc(BPF_REG_1) - 1, 0)); 119 EMIT(PPC_RAW_STWU(_R1, _R1, -BPF_PPC_STACKFRAME(ctx))); 120 121 /* 122 * Initialize tail_call_cnt in stack frame if we do tail calls. 123 * Otherwise, put in NOPs so that it can be skipped when we are 124 * invoked through a tail call. 125 */ 126 if (ctx->seen & SEEN_TAILCALL) 127 EMIT(PPC_RAW_STW(bpf_to_ppc(BPF_REG_1) - 1, _R1, 128 bpf_jit_stack_offsetof(ctx, BPF_PPC_TC))); 129 else 130 EMIT(PPC_RAW_NOP()); 131 132 #define BPF_TAILCALL_PROLOGUE_SIZE 16 133 134 /* 135 * We need a stack frame, but we don't necessarily need to 136 * save/restore LR unless we call other functions 137 */ 138 if (ctx->seen & SEEN_FUNC) 139 EMIT(PPC_RAW_MFLR(_R0)); 140 141 /* 142 * Back up non-volatile regs -- registers r18-r31 143 */ 144 for (i = BPF_PPC_NVR_MIN; i <= 31; i++) 145 if (bpf_is_seen_register(ctx, i)) 146 EMIT(PPC_RAW_STW(i, _R1, bpf_jit_stack_offsetof(ctx, i))); 147 148 /* If needed retrieve arguments 9 and 10, ie 5th 64 bits arg.*/ 149 if (bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_5))) { 150 EMIT(PPC_RAW_LWZ(bpf_to_ppc(BPF_REG_5) - 1, _R1, BPF_PPC_STACKFRAME(ctx)) + 8); 151 EMIT(PPC_RAW_LWZ(bpf_to_ppc(BPF_REG_5), _R1, BPF_PPC_STACKFRAME(ctx)) + 12); 152 } 153 154 /* Setup frame pointer to point to the bpf stack area */ 155 if (bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_FP))) { 156 EMIT(PPC_RAW_LI(bpf_to_ppc(BPF_REG_FP) - 1, 0)); 157 EMIT(PPC_RAW_ADDI(bpf_to_ppc(BPF_REG_FP), _R1, 158 STACK_FRAME_MIN_SIZE + ctx->stack_size)); 159 } 160 161 if (ctx->seen & SEEN_FUNC) 162 EMIT(PPC_RAW_STW(_R0, _R1, BPF_PPC_STACKFRAME(ctx) + PPC_LR_STKOFF)); 163 } 164 165 static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context *ctx) 166 { 167 int i; 168 169 /* Restore NVRs */ 170 for (i = BPF_PPC_NVR_MIN; i <= 31; i++) 171 if (bpf_is_seen_register(ctx, i)) 172 EMIT(PPC_RAW_LWZ(i, _R1, bpf_jit_stack_offsetof(ctx, i))); 173 } 174 175 void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx) 176 { 177 EMIT(PPC_RAW_MR(_R3, bpf_to_ppc(BPF_REG_0))); 178 179 bpf_jit_emit_common_epilogue(image, ctx); 180 181 /* Tear down our stack frame */ 182 183 if (ctx->seen & SEEN_FUNC) 184 EMIT(PPC_RAW_LWZ(_R0, _R1, BPF_PPC_STACKFRAME(ctx) + PPC_LR_STKOFF)); 185 186 EMIT(PPC_RAW_ADDI(_R1, _R1, BPF_PPC_STACKFRAME(ctx))); 187 188 if (ctx->seen & SEEN_FUNC) 189 EMIT(PPC_RAW_MTLR(_R0)); 190 191 EMIT(PPC_RAW_BLR()); 192 } 193 194 int bpf_jit_emit_func_call_rel(u32 *image, struct codegen_context *ctx, u64 func) 195 { 196 s32 rel = (s32)func - (s32)(image + ctx->idx); 197 198 if (image && rel < 0x2000000 && rel >= -0x2000000) { 199 PPC_BL(func); 200 EMIT(PPC_RAW_NOP()); 201 EMIT(PPC_RAW_NOP()); 202 EMIT(PPC_RAW_NOP()); 203 } else { 204 /* Load function address into r0 */ 205 EMIT(PPC_RAW_LIS(_R0, IMM_H(func))); 206 EMIT(PPC_RAW_ORI(_R0, _R0, IMM_L(func))); 207 EMIT(PPC_RAW_MTCTR(_R0)); 208 EMIT(PPC_RAW_BCTRL()); 209 } 210 211 return 0; 212 } 213 214 static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 out) 215 { 216 /* 217 * By now, the eBPF program has already setup parameters in r3-r6 218 * r3-r4/BPF_REG_1 - pointer to ctx -- passed as is to the next bpf program 219 * r5-r6/BPF_REG_2 - pointer to bpf_array 220 * r7-r8/BPF_REG_3 - index in bpf_array 221 */ 222 int b2p_bpf_array = bpf_to_ppc(BPF_REG_2); 223 int b2p_index = bpf_to_ppc(BPF_REG_3); 224 225 /* 226 * if (index >= array->map.max_entries) 227 * goto out; 228 */ 229 EMIT(PPC_RAW_LWZ(_R0, b2p_bpf_array, offsetof(struct bpf_array, map.max_entries))); 230 EMIT(PPC_RAW_CMPLW(b2p_index, _R0)); 231 EMIT(PPC_RAW_LWZ(_R0, _R1, bpf_jit_stack_offsetof(ctx, BPF_PPC_TC))); 232 PPC_BCC_SHORT(COND_GE, out); 233 234 /* 235 * if (tail_call_cnt >= MAX_TAIL_CALL_CNT) 236 * goto out; 237 */ 238 EMIT(PPC_RAW_CMPLWI(_R0, MAX_TAIL_CALL_CNT)); 239 /* tail_call_cnt++; */ 240 EMIT(PPC_RAW_ADDIC(_R0, _R0, 1)); 241 PPC_BCC_SHORT(COND_GE, out); 242 243 /* prog = array->ptrs[index]; */ 244 EMIT(PPC_RAW_RLWINM(_R3, b2p_index, 2, 0, 29)); 245 EMIT(PPC_RAW_ADD(_R3, _R3, b2p_bpf_array)); 246 EMIT(PPC_RAW_LWZ(_R3, _R3, offsetof(struct bpf_array, ptrs))); 247 EMIT(PPC_RAW_STW(_R0, _R1, bpf_jit_stack_offsetof(ctx, BPF_PPC_TC))); 248 249 /* 250 * if (prog == NULL) 251 * goto out; 252 */ 253 EMIT(PPC_RAW_CMPLWI(_R3, 0)); 254 PPC_BCC_SHORT(COND_EQ, out); 255 256 /* goto *(prog->bpf_func + prologue_size); */ 257 EMIT(PPC_RAW_LWZ(_R3, _R3, offsetof(struct bpf_prog, bpf_func))); 258 259 if (ctx->seen & SEEN_FUNC) 260 EMIT(PPC_RAW_LWZ(_R0, _R1, BPF_PPC_STACKFRAME(ctx) + PPC_LR_STKOFF)); 261 262 EMIT(PPC_RAW_ADDIC(_R3, _R3, BPF_TAILCALL_PROLOGUE_SIZE)); 263 264 if (ctx->seen & SEEN_FUNC) 265 EMIT(PPC_RAW_MTLR(_R0)); 266 267 EMIT(PPC_RAW_MTCTR(_R3)); 268 269 EMIT(PPC_RAW_MR(_R3, bpf_to_ppc(BPF_REG_1))); 270 271 /* tear restore NVRs, ... */ 272 bpf_jit_emit_common_epilogue(image, ctx); 273 274 EMIT(PPC_RAW_BCTR()); 275 276 /* out: */ 277 return 0; 278 } 279 280 /* Assemble the body code between the prologue & epilogue */ 281 int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *ctx, 282 u32 *addrs, int pass) 283 { 284 const struct bpf_insn *insn = fp->insnsi; 285 int flen = fp->len; 286 int i, ret; 287 288 /* Start of epilogue code - will only be valid 2nd pass onwards */ 289 u32 exit_addr = addrs[flen]; 290 291 for (i = 0; i < flen; i++) { 292 u32 code = insn[i].code; 293 u32 dst_reg = bpf_to_ppc(insn[i].dst_reg); 294 u32 dst_reg_h = dst_reg - 1; 295 u32 src_reg = bpf_to_ppc(insn[i].src_reg); 296 u32 src_reg_h = src_reg - 1; 297 u32 ax_reg = bpf_to_ppc(BPF_REG_AX); 298 u32 tmp_reg = bpf_to_ppc(TMP_REG); 299 u32 size = BPF_SIZE(code); 300 u32 save_reg, ret_reg; 301 s16 off = insn[i].off; 302 s32 imm = insn[i].imm; 303 bool func_addr_fixed; 304 u64 func_addr; 305 u32 true_cond; 306 u32 tmp_idx; 307 int j; 308 309 /* 310 * addrs[] maps a BPF bytecode address into a real offset from 311 * the start of the body code. 312 */ 313 addrs[i] = ctx->idx * 4; 314 315 /* 316 * As an optimization, we note down which registers 317 * are used so that we can only save/restore those in our 318 * prologue and epilogue. We do this here regardless of whether 319 * the actual BPF instruction uses src/dst registers or not 320 * (for instance, BPF_CALL does not use them). The expectation 321 * is that those instructions will have src_reg/dst_reg set to 322 * 0. Even otherwise, we just lose some prologue/epilogue 323 * optimization but everything else should work without 324 * any issues. 325 */ 326 if (dst_reg >= 3 && dst_reg < 32) { 327 bpf_set_seen_register(ctx, dst_reg); 328 bpf_set_seen_register(ctx, dst_reg_h); 329 } 330 331 if (src_reg >= 3 && src_reg < 32) { 332 bpf_set_seen_register(ctx, src_reg); 333 bpf_set_seen_register(ctx, src_reg_h); 334 } 335 336 switch (code) { 337 /* 338 * Arithmetic operations: ADD/SUB/MUL/DIV/MOD/NEG 339 */ 340 case BPF_ALU | BPF_ADD | BPF_X: /* (u32) dst += (u32) src */ 341 EMIT(PPC_RAW_ADD(dst_reg, dst_reg, src_reg)); 342 break; 343 case BPF_ALU64 | BPF_ADD | BPF_X: /* dst += src */ 344 EMIT(PPC_RAW_ADDC(dst_reg, dst_reg, src_reg)); 345 EMIT(PPC_RAW_ADDE(dst_reg_h, dst_reg_h, src_reg_h)); 346 break; 347 case BPF_ALU | BPF_SUB | BPF_X: /* (u32) dst -= (u32) src */ 348 EMIT(PPC_RAW_SUB(dst_reg, dst_reg, src_reg)); 349 break; 350 case BPF_ALU64 | BPF_SUB | BPF_X: /* dst -= src */ 351 EMIT(PPC_RAW_SUBFC(dst_reg, src_reg, dst_reg)); 352 EMIT(PPC_RAW_SUBFE(dst_reg_h, src_reg_h, dst_reg_h)); 353 break; 354 case BPF_ALU | BPF_SUB | BPF_K: /* (u32) dst -= (u32) imm */ 355 imm = -imm; 356 fallthrough; 357 case BPF_ALU | BPF_ADD | BPF_K: /* (u32) dst += (u32) imm */ 358 if (IMM_HA(imm) & 0xffff) 359 EMIT(PPC_RAW_ADDIS(dst_reg, dst_reg, IMM_HA(imm))); 360 if (IMM_L(imm)) 361 EMIT(PPC_RAW_ADDI(dst_reg, dst_reg, IMM_L(imm))); 362 break; 363 case BPF_ALU64 | BPF_SUB | BPF_K: /* dst -= imm */ 364 imm = -imm; 365 fallthrough; 366 case BPF_ALU64 | BPF_ADD | BPF_K: /* dst += imm */ 367 if (!imm) 368 break; 369 370 if (imm >= -32768 && imm < 32768) { 371 EMIT(PPC_RAW_ADDIC(dst_reg, dst_reg, imm)); 372 } else { 373 PPC_LI32(_R0, imm); 374 EMIT(PPC_RAW_ADDC(dst_reg, dst_reg, _R0)); 375 } 376 if (imm >= 0 || (BPF_OP(code) == BPF_SUB && imm == 0x80000000)) 377 EMIT(PPC_RAW_ADDZE(dst_reg_h, dst_reg_h)); 378 else 379 EMIT(PPC_RAW_ADDME(dst_reg_h, dst_reg_h)); 380 break; 381 case BPF_ALU64 | BPF_MUL | BPF_X: /* dst *= src */ 382 bpf_set_seen_register(ctx, tmp_reg); 383 EMIT(PPC_RAW_MULW(_R0, dst_reg, src_reg_h)); 384 EMIT(PPC_RAW_MULW(dst_reg_h, dst_reg_h, src_reg)); 385 EMIT(PPC_RAW_MULHWU(tmp_reg, dst_reg, src_reg)); 386 EMIT(PPC_RAW_MULW(dst_reg, dst_reg, src_reg)); 387 EMIT(PPC_RAW_ADD(dst_reg_h, dst_reg_h, _R0)); 388 EMIT(PPC_RAW_ADD(dst_reg_h, dst_reg_h, tmp_reg)); 389 break; 390 case BPF_ALU | BPF_MUL | BPF_X: /* (u32) dst *= (u32) src */ 391 EMIT(PPC_RAW_MULW(dst_reg, dst_reg, src_reg)); 392 break; 393 case BPF_ALU | BPF_MUL | BPF_K: /* (u32) dst *= (u32) imm */ 394 if (imm >= -32768 && imm < 32768) { 395 EMIT(PPC_RAW_MULI(dst_reg, dst_reg, imm)); 396 } else { 397 PPC_LI32(_R0, imm); 398 EMIT(PPC_RAW_MULW(dst_reg, dst_reg, _R0)); 399 } 400 break; 401 case BPF_ALU64 | BPF_MUL | BPF_K: /* dst *= imm */ 402 if (!imm) { 403 PPC_LI32(dst_reg, 0); 404 PPC_LI32(dst_reg_h, 0); 405 break; 406 } 407 if (imm == 1) 408 break; 409 if (imm == -1) { 410 EMIT(PPC_RAW_SUBFIC(dst_reg, dst_reg, 0)); 411 EMIT(PPC_RAW_SUBFZE(dst_reg_h, dst_reg_h)); 412 break; 413 } 414 bpf_set_seen_register(ctx, tmp_reg); 415 PPC_LI32(tmp_reg, imm); 416 EMIT(PPC_RAW_MULW(dst_reg_h, dst_reg_h, tmp_reg)); 417 if (imm < 0) 418 EMIT(PPC_RAW_SUB(dst_reg_h, dst_reg_h, dst_reg)); 419 EMIT(PPC_RAW_MULHWU(_R0, dst_reg, tmp_reg)); 420 EMIT(PPC_RAW_MULW(dst_reg, dst_reg, tmp_reg)); 421 EMIT(PPC_RAW_ADD(dst_reg_h, dst_reg_h, _R0)); 422 break; 423 case BPF_ALU | BPF_DIV | BPF_X: /* (u32) dst /= (u32) src */ 424 EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, src_reg)); 425 break; 426 case BPF_ALU | BPF_MOD | BPF_X: /* (u32) dst %= (u32) src */ 427 EMIT(PPC_RAW_DIVWU(_R0, dst_reg, src_reg)); 428 EMIT(PPC_RAW_MULW(_R0, src_reg, _R0)); 429 EMIT(PPC_RAW_SUB(dst_reg, dst_reg, _R0)); 430 break; 431 case BPF_ALU64 | BPF_DIV | BPF_X: /* dst /= src */ 432 return -EOPNOTSUPP; 433 case BPF_ALU64 | BPF_MOD | BPF_X: /* dst %= src */ 434 return -EOPNOTSUPP; 435 case BPF_ALU | BPF_DIV | BPF_K: /* (u32) dst /= (u32) imm */ 436 if (!imm) 437 return -EINVAL; 438 if (imm == 1) 439 break; 440 441 PPC_LI32(_R0, imm); 442 EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, _R0)); 443 break; 444 case BPF_ALU | BPF_MOD | BPF_K: /* (u32) dst %= (u32) imm */ 445 if (!imm) 446 return -EINVAL; 447 448 if (!is_power_of_2((u32)imm)) { 449 bpf_set_seen_register(ctx, tmp_reg); 450 PPC_LI32(tmp_reg, imm); 451 EMIT(PPC_RAW_DIVWU(_R0, dst_reg, tmp_reg)); 452 EMIT(PPC_RAW_MULW(_R0, tmp_reg, _R0)); 453 EMIT(PPC_RAW_SUB(dst_reg, dst_reg, _R0)); 454 break; 455 } 456 if (imm == 1) 457 EMIT(PPC_RAW_LI(dst_reg, 0)); 458 else 459 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 32 - ilog2((u32)imm), 31)); 460 461 break; 462 case BPF_ALU64 | BPF_MOD | BPF_K: /* dst %= imm */ 463 if (!imm) 464 return -EINVAL; 465 if (imm < 0) 466 imm = -imm; 467 if (!is_power_of_2(imm)) 468 return -EOPNOTSUPP; 469 if (imm == 1) 470 EMIT(PPC_RAW_LI(dst_reg, 0)); 471 else 472 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 32 - ilog2(imm), 31)); 473 EMIT(PPC_RAW_LI(dst_reg_h, 0)); 474 break; 475 case BPF_ALU64 | BPF_DIV | BPF_K: /* dst /= imm */ 476 if (!imm) 477 return -EINVAL; 478 if (!is_power_of_2(abs(imm))) 479 return -EOPNOTSUPP; 480 481 if (imm < 0) { 482 EMIT(PPC_RAW_SUBFIC(dst_reg, dst_reg, 0)); 483 EMIT(PPC_RAW_SUBFZE(dst_reg_h, dst_reg_h)); 484 imm = -imm; 485 } 486 if (imm == 1) 487 break; 488 imm = ilog2(imm); 489 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 32 - imm, imm, 31)); 490 EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg_h, 32 - imm, 0, imm - 1)); 491 EMIT(PPC_RAW_SRAWI(dst_reg_h, dst_reg_h, imm)); 492 break; 493 case BPF_ALU | BPF_NEG: /* (u32) dst = -dst */ 494 EMIT(PPC_RAW_NEG(dst_reg, dst_reg)); 495 break; 496 case BPF_ALU64 | BPF_NEG: /* dst = -dst */ 497 EMIT(PPC_RAW_SUBFIC(dst_reg, dst_reg, 0)); 498 EMIT(PPC_RAW_SUBFZE(dst_reg_h, dst_reg_h)); 499 break; 500 501 /* 502 * Logical operations: AND/OR/XOR/[A]LSH/[A]RSH 503 */ 504 case BPF_ALU64 | BPF_AND | BPF_X: /* dst = dst & src */ 505 EMIT(PPC_RAW_AND(dst_reg, dst_reg, src_reg)); 506 EMIT(PPC_RAW_AND(dst_reg_h, dst_reg_h, src_reg_h)); 507 break; 508 case BPF_ALU | BPF_AND | BPF_X: /* (u32) dst = dst & src */ 509 EMIT(PPC_RAW_AND(dst_reg, dst_reg, src_reg)); 510 break; 511 case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */ 512 if (imm >= 0) 513 EMIT(PPC_RAW_LI(dst_reg_h, 0)); 514 fallthrough; 515 case BPF_ALU | BPF_AND | BPF_K: /* (u32) dst = dst & imm */ 516 if (!IMM_H(imm)) { 517 EMIT(PPC_RAW_ANDI(dst_reg, dst_reg, IMM_L(imm))); 518 } else if (!IMM_L(imm)) { 519 EMIT(PPC_RAW_ANDIS(dst_reg, dst_reg, IMM_H(imm))); 520 } else if (imm == (((1 << fls(imm)) - 1) ^ ((1 << (ffs(i) - 1)) - 1))) { 521 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 522 32 - fls(imm), 32 - ffs(imm))); 523 } else { 524 PPC_LI32(_R0, imm); 525 EMIT(PPC_RAW_AND(dst_reg, dst_reg, _R0)); 526 } 527 break; 528 case BPF_ALU64 | BPF_OR | BPF_X: /* dst = dst | src */ 529 EMIT(PPC_RAW_OR(dst_reg, dst_reg, src_reg)); 530 EMIT(PPC_RAW_OR(dst_reg_h, dst_reg_h, src_reg_h)); 531 break; 532 case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */ 533 EMIT(PPC_RAW_OR(dst_reg, dst_reg, src_reg)); 534 break; 535 case BPF_ALU64 | BPF_OR | BPF_K:/* dst = dst | imm */ 536 /* Sign-extended */ 537 if (imm < 0) 538 EMIT(PPC_RAW_LI(dst_reg_h, -1)); 539 fallthrough; 540 case BPF_ALU | BPF_OR | BPF_K:/* dst = (u32) dst | (u32) imm */ 541 if (IMM_L(imm)) 542 EMIT(PPC_RAW_ORI(dst_reg, dst_reg, IMM_L(imm))); 543 if (IMM_H(imm)) 544 EMIT(PPC_RAW_ORIS(dst_reg, dst_reg, IMM_H(imm))); 545 break; 546 case BPF_ALU64 | BPF_XOR | BPF_X: /* dst ^= src */ 547 if (dst_reg == src_reg) { 548 EMIT(PPC_RAW_LI(dst_reg, 0)); 549 EMIT(PPC_RAW_LI(dst_reg_h, 0)); 550 } else { 551 EMIT(PPC_RAW_XOR(dst_reg, dst_reg, src_reg)); 552 EMIT(PPC_RAW_XOR(dst_reg_h, dst_reg_h, src_reg_h)); 553 } 554 break; 555 case BPF_ALU | BPF_XOR | BPF_X: /* (u32) dst ^= src */ 556 if (dst_reg == src_reg) 557 EMIT(PPC_RAW_LI(dst_reg, 0)); 558 else 559 EMIT(PPC_RAW_XOR(dst_reg, dst_reg, src_reg)); 560 break; 561 case BPF_ALU64 | BPF_XOR | BPF_K: /* dst ^= imm */ 562 if (imm < 0) 563 EMIT(PPC_RAW_NOR(dst_reg_h, dst_reg_h, dst_reg_h)); 564 fallthrough; 565 case BPF_ALU | BPF_XOR | BPF_K: /* (u32) dst ^= (u32) imm */ 566 if (IMM_L(imm)) 567 EMIT(PPC_RAW_XORI(dst_reg, dst_reg, IMM_L(imm))); 568 if (IMM_H(imm)) 569 EMIT(PPC_RAW_XORIS(dst_reg, dst_reg, IMM_H(imm))); 570 break; 571 case BPF_ALU | BPF_LSH | BPF_X: /* (u32) dst <<= (u32) src */ 572 EMIT(PPC_RAW_SLW(dst_reg, dst_reg, src_reg)); 573 break; 574 case BPF_ALU64 | BPF_LSH | BPF_X: /* dst <<= src; */ 575 bpf_set_seen_register(ctx, tmp_reg); 576 EMIT(PPC_RAW_SUBFIC(_R0, src_reg, 32)); 577 EMIT(PPC_RAW_SLW(dst_reg_h, dst_reg_h, src_reg)); 578 EMIT(PPC_RAW_ADDI(tmp_reg, src_reg, 32)); 579 EMIT(PPC_RAW_SRW(_R0, dst_reg, _R0)); 580 EMIT(PPC_RAW_SLW(tmp_reg, dst_reg, tmp_reg)); 581 EMIT(PPC_RAW_OR(dst_reg_h, dst_reg_h, _R0)); 582 EMIT(PPC_RAW_SLW(dst_reg, dst_reg, src_reg)); 583 EMIT(PPC_RAW_OR(dst_reg_h, dst_reg_h, tmp_reg)); 584 break; 585 case BPF_ALU | BPF_LSH | BPF_K: /* (u32) dst <<= (u32) imm */ 586 if (!imm) 587 break; 588 EMIT(PPC_RAW_SLWI(dst_reg, dst_reg, imm)); 589 break; 590 case BPF_ALU64 | BPF_LSH | BPF_K: /* dst <<= imm */ 591 if (imm < 0) 592 return -EINVAL; 593 if (!imm) 594 break; 595 if (imm < 32) { 596 EMIT(PPC_RAW_RLWINM(dst_reg_h, dst_reg_h, imm, 0, 31 - imm)); 597 EMIT(PPC_RAW_RLWIMI(dst_reg_h, dst_reg, imm, 32 - imm, 31)); 598 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, imm, 0, 31 - imm)); 599 break; 600 } 601 if (imm < 64) 602 EMIT(PPC_RAW_RLWINM(dst_reg_h, dst_reg, imm, 0, 31 - imm)); 603 else 604 EMIT(PPC_RAW_LI(dst_reg_h, 0)); 605 EMIT(PPC_RAW_LI(dst_reg, 0)); 606 break; 607 case BPF_ALU | BPF_RSH | BPF_X: /* (u32) dst >>= (u32) src */ 608 EMIT(PPC_RAW_SRW(dst_reg, dst_reg, src_reg)); 609 break; 610 case BPF_ALU64 | BPF_RSH | BPF_X: /* dst >>= src */ 611 bpf_set_seen_register(ctx, tmp_reg); 612 EMIT(PPC_RAW_SUBFIC(_R0, src_reg, 32)); 613 EMIT(PPC_RAW_SRW(dst_reg, dst_reg, src_reg)); 614 EMIT(PPC_RAW_ADDI(tmp_reg, src_reg, 32)); 615 EMIT(PPC_RAW_SLW(_R0, dst_reg_h, _R0)); 616 EMIT(PPC_RAW_SRW(tmp_reg, dst_reg_h, tmp_reg)); 617 EMIT(PPC_RAW_OR(dst_reg, dst_reg, _R0)); 618 EMIT(PPC_RAW_SRW(dst_reg_h, dst_reg_h, src_reg)); 619 EMIT(PPC_RAW_OR(dst_reg, dst_reg, tmp_reg)); 620 break; 621 case BPF_ALU | BPF_RSH | BPF_K: /* (u32) dst >>= (u32) imm */ 622 if (!imm) 623 break; 624 EMIT(PPC_RAW_SRWI(dst_reg, dst_reg, imm)); 625 break; 626 case BPF_ALU64 | BPF_RSH | BPF_K: /* dst >>= imm */ 627 if (imm < 0) 628 return -EINVAL; 629 if (!imm) 630 break; 631 if (imm < 32) { 632 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 32 - imm, imm, 31)); 633 EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg_h, 32 - imm, 0, imm - 1)); 634 EMIT(PPC_RAW_RLWINM(dst_reg_h, dst_reg_h, 32 - imm, imm, 31)); 635 break; 636 } 637 if (imm < 64) 638 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg_h, 64 - imm, imm - 32, 31)); 639 else 640 EMIT(PPC_RAW_LI(dst_reg, 0)); 641 EMIT(PPC_RAW_LI(dst_reg_h, 0)); 642 break; 643 case BPF_ALU | BPF_ARSH | BPF_X: /* (s32) dst >>= src */ 644 EMIT(PPC_RAW_SRAW(dst_reg, dst_reg, src_reg)); 645 break; 646 case BPF_ALU64 | BPF_ARSH | BPF_X: /* (s64) dst >>= src */ 647 bpf_set_seen_register(ctx, tmp_reg); 648 EMIT(PPC_RAW_SUBFIC(_R0, src_reg, 32)); 649 EMIT(PPC_RAW_SRW(dst_reg, dst_reg, src_reg)); 650 EMIT(PPC_RAW_SLW(_R0, dst_reg_h, _R0)); 651 EMIT(PPC_RAW_ADDI(tmp_reg, src_reg, 32)); 652 EMIT(PPC_RAW_OR(dst_reg, dst_reg, _R0)); 653 EMIT(PPC_RAW_RLWINM(_R0, tmp_reg, 0, 26, 26)); 654 EMIT(PPC_RAW_SRAW(tmp_reg, dst_reg_h, tmp_reg)); 655 EMIT(PPC_RAW_SRAW(dst_reg_h, dst_reg_h, src_reg)); 656 EMIT(PPC_RAW_SLW(tmp_reg, tmp_reg, _R0)); 657 EMIT(PPC_RAW_OR(dst_reg, dst_reg, tmp_reg)); 658 break; 659 case BPF_ALU | BPF_ARSH | BPF_K: /* (s32) dst >>= imm */ 660 if (!imm) 661 break; 662 EMIT(PPC_RAW_SRAWI(dst_reg, dst_reg, imm)); 663 break; 664 case BPF_ALU64 | BPF_ARSH | BPF_K: /* (s64) dst >>= imm */ 665 if (imm < 0) 666 return -EINVAL; 667 if (!imm) 668 break; 669 if (imm < 32) { 670 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 32 - imm, imm, 31)); 671 EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg_h, 32 - imm, 0, imm - 1)); 672 EMIT(PPC_RAW_SRAWI(dst_reg_h, dst_reg_h, imm)); 673 break; 674 } 675 if (imm < 64) 676 EMIT(PPC_RAW_SRAWI(dst_reg, dst_reg_h, imm - 32)); 677 else 678 EMIT(PPC_RAW_SRAWI(dst_reg, dst_reg_h, 31)); 679 EMIT(PPC_RAW_SRAWI(dst_reg_h, dst_reg_h, 31)); 680 break; 681 682 /* 683 * MOV 684 */ 685 case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */ 686 if (dst_reg == src_reg) 687 break; 688 EMIT(PPC_RAW_MR(dst_reg, src_reg)); 689 EMIT(PPC_RAW_MR(dst_reg_h, src_reg_h)); 690 break; 691 case BPF_ALU | BPF_MOV | BPF_X: /* (u32) dst = src */ 692 /* special mov32 for zext */ 693 if (imm == 1) 694 EMIT(PPC_RAW_LI(dst_reg_h, 0)); 695 else if (dst_reg != src_reg) 696 EMIT(PPC_RAW_MR(dst_reg, src_reg)); 697 break; 698 case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = (s64) imm */ 699 PPC_LI32(dst_reg, imm); 700 PPC_EX32(dst_reg_h, imm); 701 break; 702 case BPF_ALU | BPF_MOV | BPF_K: /* (u32) dst = imm */ 703 PPC_LI32(dst_reg, imm); 704 break; 705 706 /* 707 * BPF_FROM_BE/LE 708 */ 709 case BPF_ALU | BPF_END | BPF_FROM_LE: 710 switch (imm) { 711 case 16: 712 /* Copy 16 bits to upper part */ 713 EMIT(PPC_RAW_RLWIMI(dst_reg, dst_reg, 16, 0, 15)); 714 /* Rotate 8 bits right & mask */ 715 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 24, 16, 31)); 716 break; 717 case 32: 718 /* 719 * Rotate word left by 8 bits: 720 * 2 bytes are already in their final position 721 * -- byte 2 and 4 (of bytes 1, 2, 3 and 4) 722 */ 723 EMIT(PPC_RAW_RLWINM(_R0, dst_reg, 8, 0, 31)); 724 /* Rotate 24 bits and insert byte 1 */ 725 EMIT(PPC_RAW_RLWIMI(_R0, dst_reg, 24, 0, 7)); 726 /* Rotate 24 bits and insert byte 3 */ 727 EMIT(PPC_RAW_RLWIMI(_R0, dst_reg, 24, 16, 23)); 728 EMIT(PPC_RAW_MR(dst_reg, _R0)); 729 break; 730 case 64: 731 bpf_set_seen_register(ctx, tmp_reg); 732 EMIT(PPC_RAW_RLWINM(tmp_reg, dst_reg, 8, 0, 31)); 733 EMIT(PPC_RAW_RLWINM(_R0, dst_reg_h, 8, 0, 31)); 734 /* Rotate 24 bits and insert byte 1 */ 735 EMIT(PPC_RAW_RLWIMI(tmp_reg, dst_reg, 24, 0, 7)); 736 EMIT(PPC_RAW_RLWIMI(_R0, dst_reg_h, 24, 0, 7)); 737 /* Rotate 24 bits and insert byte 3 */ 738 EMIT(PPC_RAW_RLWIMI(tmp_reg, dst_reg, 24, 16, 23)); 739 EMIT(PPC_RAW_RLWIMI(_R0, dst_reg_h, 24, 16, 23)); 740 EMIT(PPC_RAW_MR(dst_reg, _R0)); 741 EMIT(PPC_RAW_MR(dst_reg_h, tmp_reg)); 742 break; 743 } 744 break; 745 case BPF_ALU | BPF_END | BPF_FROM_BE: 746 switch (imm) { 747 case 16: 748 /* zero-extend 16 bits into 32 bits */ 749 EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 16, 31)); 750 break; 751 case 32: 752 case 64: 753 /* nop */ 754 break; 755 } 756 break; 757 758 /* 759 * BPF_ST NOSPEC (speculation barrier) 760 */ 761 case BPF_ST | BPF_NOSPEC: 762 break; 763 764 /* 765 * BPF_ST(X) 766 */ 767 case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src */ 768 EMIT(PPC_RAW_STB(src_reg, dst_reg, off)); 769 break; 770 case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */ 771 PPC_LI32(_R0, imm); 772 EMIT(PPC_RAW_STB(_R0, dst_reg, off)); 773 break; 774 case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */ 775 EMIT(PPC_RAW_STH(src_reg, dst_reg, off)); 776 break; 777 case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */ 778 PPC_LI32(_R0, imm); 779 EMIT(PPC_RAW_STH(_R0, dst_reg, off)); 780 break; 781 case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */ 782 EMIT(PPC_RAW_STW(src_reg, dst_reg, off)); 783 break; 784 case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */ 785 PPC_LI32(_R0, imm); 786 EMIT(PPC_RAW_STW(_R0, dst_reg, off)); 787 break; 788 case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */ 789 EMIT(PPC_RAW_STW(src_reg_h, dst_reg, off)); 790 EMIT(PPC_RAW_STW(src_reg, dst_reg, off + 4)); 791 break; 792 case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */ 793 PPC_LI32(_R0, imm); 794 EMIT(PPC_RAW_STW(_R0, dst_reg, off + 4)); 795 PPC_EX32(_R0, imm); 796 EMIT(PPC_RAW_STW(_R0, dst_reg, off)); 797 break; 798 799 /* 800 * BPF_STX ATOMIC (atomic ops) 801 */ 802 case BPF_STX | BPF_ATOMIC | BPF_W: 803 save_reg = _R0; 804 ret_reg = src_reg; 805 806 bpf_set_seen_register(ctx, tmp_reg); 807 bpf_set_seen_register(ctx, ax_reg); 808 809 /* Get offset into TMP_REG */ 810 EMIT(PPC_RAW_LI(tmp_reg, off)); 811 tmp_idx = ctx->idx * 4; 812 /* load value from memory into r0 */ 813 EMIT(PPC_RAW_LWARX(_R0, tmp_reg, dst_reg, 0)); 814 815 /* Save old value in BPF_REG_AX */ 816 if (imm & BPF_FETCH) 817 EMIT(PPC_RAW_MR(ax_reg, _R0)); 818 819 switch (imm) { 820 case BPF_ADD: 821 case BPF_ADD | BPF_FETCH: 822 EMIT(PPC_RAW_ADD(_R0, _R0, src_reg)); 823 break; 824 case BPF_AND: 825 case BPF_AND | BPF_FETCH: 826 EMIT(PPC_RAW_AND(_R0, _R0, src_reg)); 827 break; 828 case BPF_OR: 829 case BPF_OR | BPF_FETCH: 830 EMIT(PPC_RAW_OR(_R0, _R0, src_reg)); 831 break; 832 case BPF_XOR: 833 case BPF_XOR | BPF_FETCH: 834 EMIT(PPC_RAW_XOR(_R0, _R0, src_reg)); 835 break; 836 case BPF_CMPXCHG: 837 /* 838 * Return old value in BPF_REG_0 for BPF_CMPXCHG & 839 * in src_reg for other cases. 840 */ 841 ret_reg = bpf_to_ppc(BPF_REG_0); 842 843 /* Compare with old value in BPF_REG_0 */ 844 EMIT(PPC_RAW_CMPW(bpf_to_ppc(BPF_REG_0), _R0)); 845 /* Don't set if different from old value */ 846 PPC_BCC_SHORT(COND_NE, (ctx->idx + 3) * 4); 847 fallthrough; 848 case BPF_XCHG: 849 save_reg = src_reg; 850 break; 851 default: 852 pr_err_ratelimited("eBPF filter atomic op code %02x (@%d) unsupported\n", 853 code, i); 854 return -EOPNOTSUPP; 855 } 856 857 /* store new value */ 858 EMIT(PPC_RAW_STWCX(save_reg, tmp_reg, dst_reg)); 859 /* we're done if this succeeded */ 860 PPC_BCC_SHORT(COND_NE, tmp_idx); 861 862 /* For the BPF_FETCH variant, get old data into src_reg */ 863 if (imm & BPF_FETCH) { 864 EMIT(PPC_RAW_MR(ret_reg, ax_reg)); 865 if (!fp->aux->verifier_zext) 866 EMIT(PPC_RAW_LI(ret_reg - 1, 0)); /* higher 32-bit */ 867 } 868 break; 869 870 case BPF_STX | BPF_ATOMIC | BPF_DW: /* *(u64 *)(dst + off) += src */ 871 return -EOPNOTSUPP; 872 873 /* 874 * BPF_LDX 875 */ 876 case BPF_LDX | BPF_MEM | BPF_B: /* dst = *(u8 *)(ul) (src + off) */ 877 case BPF_LDX | BPF_PROBE_MEM | BPF_B: 878 case BPF_LDX | BPF_MEM | BPF_H: /* dst = *(u16 *)(ul) (src + off) */ 879 case BPF_LDX | BPF_PROBE_MEM | BPF_H: 880 case BPF_LDX | BPF_MEM | BPF_W: /* dst = *(u32 *)(ul) (src + off) */ 881 case BPF_LDX | BPF_PROBE_MEM | BPF_W: 882 case BPF_LDX | BPF_MEM | BPF_DW: /* dst = *(u64 *)(ul) (src + off) */ 883 case BPF_LDX | BPF_PROBE_MEM | BPF_DW: 884 /* 885 * As PTR_TO_BTF_ID that uses BPF_PROBE_MEM mode could either be a valid 886 * kernel pointer or NULL but not a userspace address, execute BPF_PROBE_MEM 887 * load only if addr is kernel address (see is_kernel_addr()), otherwise 888 * set dst_reg=0 and move on. 889 */ 890 if (BPF_MODE(code) == BPF_PROBE_MEM) { 891 PPC_LI32(_R0, TASK_SIZE - off); 892 EMIT(PPC_RAW_CMPLW(src_reg, _R0)); 893 PPC_BCC_SHORT(COND_GT, (ctx->idx + 4) * 4); 894 EMIT(PPC_RAW_LI(dst_reg, 0)); 895 /* 896 * For BPF_DW case, "li reg_h,0" would be needed when 897 * !fp->aux->verifier_zext. Emit NOP otherwise. 898 * 899 * Note that "li reg_h,0" is emitted for BPF_B/H/W case, 900 * if necessary. So, jump there insted of emitting an 901 * additional "li reg_h,0" instruction. 902 */ 903 if (size == BPF_DW && !fp->aux->verifier_zext) 904 EMIT(PPC_RAW_LI(dst_reg_h, 0)); 905 else 906 EMIT(PPC_RAW_NOP()); 907 /* 908 * Need to jump two instructions instead of one for BPF_DW case 909 * as there are two load instructions for dst_reg_h & dst_reg 910 * respectively. 911 */ 912 if (size == BPF_DW) 913 PPC_JMP((ctx->idx + 3) * 4); 914 else 915 PPC_JMP((ctx->idx + 2) * 4); 916 } 917 918 switch (size) { 919 case BPF_B: 920 EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off)); 921 break; 922 case BPF_H: 923 EMIT(PPC_RAW_LHZ(dst_reg, src_reg, off)); 924 break; 925 case BPF_W: 926 EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off)); 927 break; 928 case BPF_DW: 929 EMIT(PPC_RAW_LWZ(dst_reg_h, src_reg, off)); 930 EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off + 4)); 931 break; 932 } 933 934 if (size != BPF_DW && !fp->aux->verifier_zext) 935 EMIT(PPC_RAW_LI(dst_reg_h, 0)); 936 937 if (BPF_MODE(code) == BPF_PROBE_MEM) { 938 int insn_idx = ctx->idx - 1; 939 int jmp_off = 4; 940 941 /* 942 * In case of BPF_DW, two lwz instructions are emitted, one 943 * for higher 32-bit and another for lower 32-bit. So, set 944 * ex->insn to the first of the two and jump over both 945 * instructions in fixup. 946 * 947 * Similarly, with !verifier_zext, two instructions are 948 * emitted for BPF_B/H/W case. So, set ex->insn to the 949 * instruction that could fault and skip over both 950 * instructions. 951 */ 952 if (size == BPF_DW || !fp->aux->verifier_zext) { 953 insn_idx -= 1; 954 jmp_off += 4; 955 } 956 957 ret = bpf_add_extable_entry(fp, image, pass, ctx, insn_idx, 958 jmp_off, dst_reg); 959 if (ret) 960 return ret; 961 } 962 break; 963 964 /* 965 * Doubleword load 966 * 16 byte instruction that uses two 'struct bpf_insn' 967 */ 968 case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */ 969 tmp_idx = ctx->idx; 970 PPC_LI32(dst_reg_h, (u32)insn[i + 1].imm); 971 PPC_LI32(dst_reg, (u32)insn[i].imm); 972 /* padding to allow full 4 instructions for later patching */ 973 for (j = ctx->idx - tmp_idx; j < 4; j++) 974 EMIT(PPC_RAW_NOP()); 975 /* Adjust for two bpf instructions */ 976 addrs[++i] = ctx->idx * 4; 977 break; 978 979 /* 980 * Return/Exit 981 */ 982 case BPF_JMP | BPF_EXIT: 983 /* 984 * If this isn't the very last instruction, branch to 985 * the epilogue. If we _are_ the last instruction, 986 * we'll just fall through to the epilogue. 987 */ 988 if (i != flen - 1) { 989 ret = bpf_jit_emit_exit_insn(image, ctx, _R0, exit_addr); 990 if (ret) 991 return ret; 992 } 993 /* else fall through to the epilogue */ 994 break; 995 996 /* 997 * Call kernel helper or bpf function 998 */ 999 case BPF_JMP | BPF_CALL: 1000 ctx->seen |= SEEN_FUNC; 1001 1002 ret = bpf_jit_get_func_addr(fp, &insn[i], false, 1003 &func_addr, &func_addr_fixed); 1004 if (ret < 0) 1005 return ret; 1006 1007 if (bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_5))) { 1008 EMIT(PPC_RAW_STW(bpf_to_ppc(BPF_REG_5) - 1, _R1, 8)); 1009 EMIT(PPC_RAW_STW(bpf_to_ppc(BPF_REG_5), _R1, 12)); 1010 } 1011 1012 ret = bpf_jit_emit_func_call_rel(image, ctx, func_addr); 1013 if (ret) 1014 return ret; 1015 1016 EMIT(PPC_RAW_MR(bpf_to_ppc(BPF_REG_0) - 1, _R3)); 1017 EMIT(PPC_RAW_MR(bpf_to_ppc(BPF_REG_0), _R4)); 1018 break; 1019 1020 /* 1021 * Jumps and branches 1022 */ 1023 case BPF_JMP | BPF_JA: 1024 PPC_JMP(addrs[i + 1 + off]); 1025 break; 1026 1027 case BPF_JMP | BPF_JGT | BPF_K: 1028 case BPF_JMP | BPF_JGT | BPF_X: 1029 case BPF_JMP | BPF_JSGT | BPF_K: 1030 case BPF_JMP | BPF_JSGT | BPF_X: 1031 case BPF_JMP32 | BPF_JGT | BPF_K: 1032 case BPF_JMP32 | BPF_JGT | BPF_X: 1033 case BPF_JMP32 | BPF_JSGT | BPF_K: 1034 case BPF_JMP32 | BPF_JSGT | BPF_X: 1035 true_cond = COND_GT; 1036 goto cond_branch; 1037 case BPF_JMP | BPF_JLT | BPF_K: 1038 case BPF_JMP | BPF_JLT | BPF_X: 1039 case BPF_JMP | BPF_JSLT | BPF_K: 1040 case BPF_JMP | BPF_JSLT | BPF_X: 1041 case BPF_JMP32 | BPF_JLT | BPF_K: 1042 case BPF_JMP32 | BPF_JLT | BPF_X: 1043 case BPF_JMP32 | BPF_JSLT | BPF_K: 1044 case BPF_JMP32 | BPF_JSLT | BPF_X: 1045 true_cond = COND_LT; 1046 goto cond_branch; 1047 case BPF_JMP | BPF_JGE | BPF_K: 1048 case BPF_JMP | BPF_JGE | BPF_X: 1049 case BPF_JMP | BPF_JSGE | BPF_K: 1050 case BPF_JMP | BPF_JSGE | BPF_X: 1051 case BPF_JMP32 | BPF_JGE | BPF_K: 1052 case BPF_JMP32 | BPF_JGE | BPF_X: 1053 case BPF_JMP32 | BPF_JSGE | BPF_K: 1054 case BPF_JMP32 | BPF_JSGE | BPF_X: 1055 true_cond = COND_GE; 1056 goto cond_branch; 1057 case BPF_JMP | BPF_JLE | BPF_K: 1058 case BPF_JMP | BPF_JLE | BPF_X: 1059 case BPF_JMP | BPF_JSLE | BPF_K: 1060 case BPF_JMP | BPF_JSLE | BPF_X: 1061 case BPF_JMP32 | BPF_JLE | BPF_K: 1062 case BPF_JMP32 | BPF_JLE | BPF_X: 1063 case BPF_JMP32 | BPF_JSLE | BPF_K: 1064 case BPF_JMP32 | BPF_JSLE | BPF_X: 1065 true_cond = COND_LE; 1066 goto cond_branch; 1067 case BPF_JMP | BPF_JEQ | BPF_K: 1068 case BPF_JMP | BPF_JEQ | BPF_X: 1069 case BPF_JMP32 | BPF_JEQ | BPF_K: 1070 case BPF_JMP32 | BPF_JEQ | BPF_X: 1071 true_cond = COND_EQ; 1072 goto cond_branch; 1073 case BPF_JMP | BPF_JNE | BPF_K: 1074 case BPF_JMP | BPF_JNE | BPF_X: 1075 case BPF_JMP32 | BPF_JNE | BPF_K: 1076 case BPF_JMP32 | BPF_JNE | BPF_X: 1077 true_cond = COND_NE; 1078 goto cond_branch; 1079 case BPF_JMP | BPF_JSET | BPF_K: 1080 case BPF_JMP | BPF_JSET | BPF_X: 1081 case BPF_JMP32 | BPF_JSET | BPF_K: 1082 case BPF_JMP32 | BPF_JSET | BPF_X: 1083 true_cond = COND_NE; 1084 /* fallthrough; */ 1085 1086 cond_branch: 1087 switch (code) { 1088 case BPF_JMP | BPF_JGT | BPF_X: 1089 case BPF_JMP | BPF_JLT | BPF_X: 1090 case BPF_JMP | BPF_JGE | BPF_X: 1091 case BPF_JMP | BPF_JLE | BPF_X: 1092 case BPF_JMP | BPF_JEQ | BPF_X: 1093 case BPF_JMP | BPF_JNE | BPF_X: 1094 /* unsigned comparison */ 1095 EMIT(PPC_RAW_CMPLW(dst_reg_h, src_reg_h)); 1096 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4); 1097 EMIT(PPC_RAW_CMPLW(dst_reg, src_reg)); 1098 break; 1099 case BPF_JMP32 | BPF_JGT | BPF_X: 1100 case BPF_JMP32 | BPF_JLT | BPF_X: 1101 case BPF_JMP32 | BPF_JGE | BPF_X: 1102 case BPF_JMP32 | BPF_JLE | BPF_X: 1103 case BPF_JMP32 | BPF_JEQ | BPF_X: 1104 case BPF_JMP32 | BPF_JNE | BPF_X: 1105 /* unsigned comparison */ 1106 EMIT(PPC_RAW_CMPLW(dst_reg, src_reg)); 1107 break; 1108 case BPF_JMP | BPF_JSGT | BPF_X: 1109 case BPF_JMP | BPF_JSLT | BPF_X: 1110 case BPF_JMP | BPF_JSGE | BPF_X: 1111 case BPF_JMP | BPF_JSLE | BPF_X: 1112 /* signed comparison */ 1113 EMIT(PPC_RAW_CMPW(dst_reg_h, src_reg_h)); 1114 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4); 1115 EMIT(PPC_RAW_CMPLW(dst_reg, src_reg)); 1116 break; 1117 case BPF_JMP32 | BPF_JSGT | BPF_X: 1118 case BPF_JMP32 | BPF_JSLT | BPF_X: 1119 case BPF_JMP32 | BPF_JSGE | BPF_X: 1120 case BPF_JMP32 | BPF_JSLE | BPF_X: 1121 /* signed comparison */ 1122 EMIT(PPC_RAW_CMPW(dst_reg, src_reg)); 1123 break; 1124 case BPF_JMP | BPF_JSET | BPF_X: 1125 EMIT(PPC_RAW_AND_DOT(_R0, dst_reg_h, src_reg_h)); 1126 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4); 1127 EMIT(PPC_RAW_AND_DOT(_R0, dst_reg, src_reg)); 1128 break; 1129 case BPF_JMP32 | BPF_JSET | BPF_X: { 1130 EMIT(PPC_RAW_AND_DOT(_R0, dst_reg, src_reg)); 1131 break; 1132 case BPF_JMP | BPF_JNE | BPF_K: 1133 case BPF_JMP | BPF_JEQ | BPF_K: 1134 case BPF_JMP | BPF_JGT | BPF_K: 1135 case BPF_JMP | BPF_JLT | BPF_K: 1136 case BPF_JMP | BPF_JGE | BPF_K: 1137 case BPF_JMP | BPF_JLE | BPF_K: 1138 /* 1139 * Need sign-extended load, so only positive 1140 * values can be used as imm in cmplwi 1141 */ 1142 if (imm >= 0 && imm < 32768) { 1143 EMIT(PPC_RAW_CMPLWI(dst_reg_h, 0)); 1144 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4); 1145 EMIT(PPC_RAW_CMPLWI(dst_reg, imm)); 1146 } else { 1147 /* sign-extending load ... but unsigned comparison */ 1148 PPC_EX32(_R0, imm); 1149 EMIT(PPC_RAW_CMPLW(dst_reg_h, _R0)); 1150 PPC_LI32(_R0, imm); 1151 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4); 1152 EMIT(PPC_RAW_CMPLW(dst_reg, _R0)); 1153 } 1154 break; 1155 case BPF_JMP32 | BPF_JNE | BPF_K: 1156 case BPF_JMP32 | BPF_JEQ | BPF_K: 1157 case BPF_JMP32 | BPF_JGT | BPF_K: 1158 case BPF_JMP32 | BPF_JLT | BPF_K: 1159 case BPF_JMP32 | BPF_JGE | BPF_K: 1160 case BPF_JMP32 | BPF_JLE | BPF_K: 1161 if (imm >= 0 && imm < 65536) { 1162 EMIT(PPC_RAW_CMPLWI(dst_reg, imm)); 1163 } else { 1164 PPC_LI32(_R0, imm); 1165 EMIT(PPC_RAW_CMPLW(dst_reg, _R0)); 1166 } 1167 break; 1168 } 1169 case BPF_JMP | BPF_JSGT | BPF_K: 1170 case BPF_JMP | BPF_JSLT | BPF_K: 1171 case BPF_JMP | BPF_JSGE | BPF_K: 1172 case BPF_JMP | BPF_JSLE | BPF_K: 1173 if (imm >= 0 && imm < 65536) { 1174 EMIT(PPC_RAW_CMPWI(dst_reg_h, imm < 0 ? -1 : 0)); 1175 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4); 1176 EMIT(PPC_RAW_CMPLWI(dst_reg, imm)); 1177 } else { 1178 /* sign-extending load */ 1179 EMIT(PPC_RAW_CMPWI(dst_reg_h, imm < 0 ? -1 : 0)); 1180 PPC_LI32(_R0, imm); 1181 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4); 1182 EMIT(PPC_RAW_CMPLW(dst_reg, _R0)); 1183 } 1184 break; 1185 case BPF_JMP32 | BPF_JSGT | BPF_K: 1186 case BPF_JMP32 | BPF_JSLT | BPF_K: 1187 case BPF_JMP32 | BPF_JSGE | BPF_K: 1188 case BPF_JMP32 | BPF_JSLE | BPF_K: 1189 /* 1190 * signed comparison, so any 16-bit value 1191 * can be used in cmpwi 1192 */ 1193 if (imm >= -32768 && imm < 32768) { 1194 EMIT(PPC_RAW_CMPWI(dst_reg, imm)); 1195 } else { 1196 /* sign-extending load */ 1197 PPC_LI32(_R0, imm); 1198 EMIT(PPC_RAW_CMPW(dst_reg, _R0)); 1199 } 1200 break; 1201 case BPF_JMP | BPF_JSET | BPF_K: 1202 /* andi does not sign-extend the immediate */ 1203 if (imm >= 0 && imm < 32768) { 1204 /* PPC_ANDI is _only/always_ dot-form */ 1205 EMIT(PPC_RAW_ANDI(_R0, dst_reg, imm)); 1206 } else { 1207 PPC_LI32(_R0, imm); 1208 if (imm < 0) { 1209 EMIT(PPC_RAW_CMPWI(dst_reg_h, 0)); 1210 PPC_BCC_SHORT(COND_NE, (ctx->idx + 2) * 4); 1211 } 1212 EMIT(PPC_RAW_AND_DOT(_R0, dst_reg, _R0)); 1213 } 1214 break; 1215 case BPF_JMP32 | BPF_JSET | BPF_K: 1216 /* andi does not sign-extend the immediate */ 1217 if (imm >= 0 && imm < 32768) { 1218 /* PPC_ANDI is _only/always_ dot-form */ 1219 EMIT(PPC_RAW_ANDI(_R0, dst_reg, imm)); 1220 } else { 1221 PPC_LI32(_R0, imm); 1222 EMIT(PPC_RAW_AND_DOT(_R0, dst_reg, _R0)); 1223 } 1224 break; 1225 } 1226 PPC_BCC(true_cond, addrs[i + 1 + off]); 1227 break; 1228 1229 /* 1230 * Tail call 1231 */ 1232 case BPF_JMP | BPF_TAIL_CALL: 1233 ctx->seen |= SEEN_TAILCALL; 1234 ret = bpf_jit_emit_tail_call(image, ctx, addrs[i + 1]); 1235 if (ret < 0) 1236 return ret; 1237 break; 1238 1239 default: 1240 /* 1241 * The filter contains something cruel & unusual. 1242 * We don't handle it, but also there shouldn't be 1243 * anything missing from our list. 1244 */ 1245 pr_err_ratelimited("eBPF filter opcode %04x (@%d) unsupported\n", code, i); 1246 return -EOPNOTSUPP; 1247 } 1248 if (BPF_CLASS(code) == BPF_ALU && !fp->aux->verifier_zext && 1249 !insn_is_zext(&insn[i + 1]) && !(BPF_OP(code) == BPF_END && imm == 64)) 1250 EMIT(PPC_RAW_LI(dst_reg_h, 0)); 1251 } 1252 1253 /* Set end-of-body-code address for exit. */ 1254 addrs[i] = ctx->idx * 4; 1255 1256 return 0; 1257 } 1258