1 /* 2 * bpf_jit_comp64.c: eBPF JIT compiler 3 * 4 * Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com> 5 * IBM Corporation 6 * 7 * Based on the powerpc classic BPF JIT compiler by Matt Evans 8 * 9 * This program is free software; you can redistribute it and/or 10 * modify it under the terms of the GNU General Public License 11 * as published by the Free Software Foundation; version 2 12 * of the License. 13 */ 14 #include <linux/moduleloader.h> 15 #include <asm/cacheflush.h> 16 #include <asm/asm-compat.h> 17 #include <linux/netdevice.h> 18 #include <linux/filter.h> 19 #include <linux/if_vlan.h> 20 #include <asm/kprobes.h> 21 #include <linux/bpf.h> 22 23 #include "bpf_jit64.h" 24 25 static void bpf_jit_fill_ill_insns(void *area, unsigned int size) 26 { 27 memset32(area, BREAKPOINT_INSTRUCTION, size/4); 28 } 29 30 static inline void bpf_flush_icache(void *start, void *end) 31 { 32 smp_wmb(); 33 flush_icache_range((unsigned long)start, (unsigned long)end); 34 } 35 36 static inline bool bpf_is_seen_register(struct codegen_context *ctx, int i) 37 { 38 return (ctx->seen & (1 << (31 - b2p[i]))); 39 } 40 41 static inline void bpf_set_seen_register(struct codegen_context *ctx, int i) 42 { 43 ctx->seen |= (1 << (31 - b2p[i])); 44 } 45 46 static inline bool bpf_has_stack_frame(struct codegen_context *ctx) 47 { 48 /* 49 * We only need a stack frame if: 50 * - we call other functions (kernel helpers), or 51 * - the bpf program uses its stack area 52 * The latter condition is deduced from the usage of BPF_REG_FP 53 */ 54 return ctx->seen & SEEN_FUNC || bpf_is_seen_register(ctx, BPF_REG_FP); 55 } 56 57 /* 58 * When not setting up our own stackframe, the redzone usage is: 59 * 60 * [ prev sp ] <------------- 61 * [ ... ] | 62 * sp (r1) ---> [ stack pointer ] -------------- 63 * [ nv gpr save area ] 6*8 64 * [ tail_call_cnt ] 8 65 * [ local_tmp_var ] 8 66 * [ unused red zone ] 208 bytes protected 67 */ 68 static int bpf_jit_stack_local(struct codegen_context *ctx) 69 { 70 if (bpf_has_stack_frame(ctx)) 71 return STACK_FRAME_MIN_SIZE + ctx->stack_size; 72 else 73 return -(BPF_PPC_STACK_SAVE + 16); 74 } 75 76 static int bpf_jit_stack_tailcallcnt(struct codegen_context *ctx) 77 { 78 return bpf_jit_stack_local(ctx) + 8; 79 } 80 81 static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg) 82 { 83 if (reg >= BPF_PPC_NVR_MIN && reg < 32) 84 return (bpf_has_stack_frame(ctx) ? 85 (BPF_PPC_STACKFRAME + ctx->stack_size) : 0) 86 - (8 * (32 - reg)); 87 88 pr_err("BPF JIT is asking about unknown registers"); 89 BUG(); 90 } 91 92 static void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx) 93 { 94 int i; 95 96 /* 97 * Initialize tail_call_cnt if we do tail calls. 98 * Otherwise, put in NOPs so that it can be skipped when we are 99 * invoked through a tail call. 100 */ 101 if (ctx->seen & SEEN_TAILCALL) { 102 PPC_LI(b2p[TMP_REG_1], 0); 103 /* this goes in the redzone */ 104 PPC_BPF_STL(b2p[TMP_REG_1], 1, -(BPF_PPC_STACK_SAVE + 8)); 105 } else { 106 PPC_NOP(); 107 PPC_NOP(); 108 } 109 110 #define BPF_TAILCALL_PROLOGUE_SIZE 8 111 112 if (bpf_has_stack_frame(ctx)) { 113 /* 114 * We need a stack frame, but we don't necessarily need to 115 * save/restore LR unless we call other functions 116 */ 117 if (ctx->seen & SEEN_FUNC) { 118 EMIT(PPC_INST_MFLR | __PPC_RT(R0)); 119 PPC_BPF_STL(0, 1, PPC_LR_STKOFF); 120 } 121 122 PPC_BPF_STLU(1, 1, -(BPF_PPC_STACKFRAME + ctx->stack_size)); 123 } 124 125 /* 126 * Back up non-volatile regs -- BPF registers 6-10 127 * If we haven't created our own stack frame, we save these 128 * in the protected zone below the previous stack frame 129 */ 130 for (i = BPF_REG_6; i <= BPF_REG_10; i++) 131 if (bpf_is_seen_register(ctx, i)) 132 PPC_BPF_STL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i])); 133 134 /* Setup frame pointer to point to the bpf stack area */ 135 if (bpf_is_seen_register(ctx, BPF_REG_FP)) 136 PPC_ADDI(b2p[BPF_REG_FP], 1, 137 STACK_FRAME_MIN_SIZE + ctx->stack_size); 138 } 139 140 static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context *ctx) 141 { 142 int i; 143 144 /* Restore NVRs */ 145 for (i = BPF_REG_6; i <= BPF_REG_10; i++) 146 if (bpf_is_seen_register(ctx, i)) 147 PPC_BPF_LL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i])); 148 149 /* Tear down our stack frame */ 150 if (bpf_has_stack_frame(ctx)) { 151 PPC_ADDI(1, 1, BPF_PPC_STACKFRAME + ctx->stack_size); 152 if (ctx->seen & SEEN_FUNC) { 153 PPC_BPF_LL(0, 1, PPC_LR_STKOFF); 154 PPC_MTLR(0); 155 } 156 } 157 } 158 159 static void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx) 160 { 161 bpf_jit_emit_common_epilogue(image, ctx); 162 163 /* Move result to r3 */ 164 PPC_MR(3, b2p[BPF_REG_0]); 165 166 PPC_BLR(); 167 } 168 169 static void bpf_jit_emit_func_call(u32 *image, struct codegen_context *ctx, u64 func) 170 { 171 unsigned int i, ctx_idx = ctx->idx; 172 173 /* Load function address into r12 */ 174 PPC_LI64(12, func); 175 176 /* For bpf-to-bpf function calls, the callee's address is unknown 177 * until the last extra pass. As seen above, we use PPC_LI64() to 178 * load the callee's address, but this may optimize the number of 179 * instructions required based on the nature of the address. 180 * 181 * Since we don't want the number of instructions emitted to change, 182 * we pad the optimized PPC_LI64() call with NOPs to guarantee that 183 * we always have a five-instruction sequence, which is the maximum 184 * that PPC_LI64() can emit. 185 */ 186 for (i = ctx->idx - ctx_idx; i < 5; i++) 187 PPC_NOP(); 188 189 #ifdef PPC64_ELF_ABI_v1 190 /* 191 * Load TOC from function descriptor at offset 8. 192 * We can clobber r2 since we get called through a 193 * function pointer (so caller will save/restore r2) 194 * and since we don't use a TOC ourself. 195 */ 196 PPC_BPF_LL(2, 12, 8); 197 /* Load actual entry point from function descriptor */ 198 PPC_BPF_LL(12, 12, 0); 199 #endif 200 201 PPC_MTLR(12); 202 PPC_BLRL(); 203 } 204 205 static void bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 out) 206 { 207 /* 208 * By now, the eBPF program has already setup parameters in r3, r4 and r5 209 * r3/BPF_REG_1 - pointer to ctx -- passed as is to the next bpf program 210 * r4/BPF_REG_2 - pointer to bpf_array 211 * r5/BPF_REG_3 - index in bpf_array 212 */ 213 int b2p_bpf_array = b2p[BPF_REG_2]; 214 int b2p_index = b2p[BPF_REG_3]; 215 216 /* 217 * if (index >= array->map.max_entries) 218 * goto out; 219 */ 220 PPC_LWZ(b2p[TMP_REG_1], b2p_bpf_array, offsetof(struct bpf_array, map.max_entries)); 221 PPC_RLWINM(b2p_index, b2p_index, 0, 0, 31); 222 PPC_CMPLW(b2p_index, b2p[TMP_REG_1]); 223 PPC_BCC(COND_GE, out); 224 225 /* 226 * if (tail_call_cnt > MAX_TAIL_CALL_CNT) 227 * goto out; 228 */ 229 PPC_LD(b2p[TMP_REG_1], 1, bpf_jit_stack_tailcallcnt(ctx)); 230 PPC_CMPLWI(b2p[TMP_REG_1], MAX_TAIL_CALL_CNT); 231 PPC_BCC(COND_GT, out); 232 233 /* 234 * tail_call_cnt++; 235 */ 236 PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], 1); 237 PPC_BPF_STL(b2p[TMP_REG_1], 1, bpf_jit_stack_tailcallcnt(ctx)); 238 239 /* prog = array->ptrs[index]; */ 240 PPC_MULI(b2p[TMP_REG_1], b2p_index, 8); 241 PPC_ADD(b2p[TMP_REG_1], b2p[TMP_REG_1], b2p_bpf_array); 242 PPC_LD(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_array, ptrs)); 243 244 /* 245 * if (prog == NULL) 246 * goto out; 247 */ 248 PPC_CMPLDI(b2p[TMP_REG_1], 0); 249 PPC_BCC(COND_EQ, out); 250 251 /* goto *(prog->bpf_func + prologue_size); */ 252 PPC_LD(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_prog, bpf_func)); 253 #ifdef PPC64_ELF_ABI_v1 254 /* skip past the function descriptor */ 255 PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], 256 FUNCTION_DESCR_SIZE + BPF_TAILCALL_PROLOGUE_SIZE); 257 #else 258 PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], BPF_TAILCALL_PROLOGUE_SIZE); 259 #endif 260 PPC_MTCTR(b2p[TMP_REG_1]); 261 262 /* tear down stack, restore NVRs, ... */ 263 bpf_jit_emit_common_epilogue(image, ctx); 264 265 PPC_BCTR(); 266 /* out: */ 267 } 268 269 /* Assemble the body code between the prologue & epilogue */ 270 static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, 271 struct codegen_context *ctx, 272 u32 *addrs, bool extra_pass) 273 { 274 const struct bpf_insn *insn = fp->insnsi; 275 int flen = fp->len; 276 int i; 277 278 /* Start of epilogue code - will only be valid 2nd pass onwards */ 279 u32 exit_addr = addrs[flen]; 280 281 for (i = 0; i < flen; i++) { 282 u32 code = insn[i].code; 283 u32 dst_reg = b2p[insn[i].dst_reg]; 284 u32 src_reg = b2p[insn[i].src_reg]; 285 s16 off = insn[i].off; 286 s32 imm = insn[i].imm; 287 u64 imm64; 288 u8 *func; 289 u32 true_cond; 290 u32 tmp_idx; 291 292 /* 293 * addrs[] maps a BPF bytecode address into a real offset from 294 * the start of the body code. 295 */ 296 addrs[i] = ctx->idx * 4; 297 298 /* 299 * As an optimization, we note down which non-volatile registers 300 * are used so that we can only save/restore those in our 301 * prologue and epilogue. We do this here regardless of whether 302 * the actual BPF instruction uses src/dst registers or not 303 * (for instance, BPF_CALL does not use them). The expectation 304 * is that those instructions will have src_reg/dst_reg set to 305 * 0. Even otherwise, we just lose some prologue/epilogue 306 * optimization but everything else should work without 307 * any issues. 308 */ 309 if (dst_reg >= BPF_PPC_NVR_MIN && dst_reg < 32) 310 bpf_set_seen_register(ctx, insn[i].dst_reg); 311 if (src_reg >= BPF_PPC_NVR_MIN && src_reg < 32) 312 bpf_set_seen_register(ctx, insn[i].src_reg); 313 314 switch (code) { 315 /* 316 * Arithmetic operations: ADD/SUB/MUL/DIV/MOD/NEG 317 */ 318 case BPF_ALU | BPF_ADD | BPF_X: /* (u32) dst += (u32) src */ 319 case BPF_ALU64 | BPF_ADD | BPF_X: /* dst += src */ 320 PPC_ADD(dst_reg, dst_reg, src_reg); 321 goto bpf_alu32_trunc; 322 case BPF_ALU | BPF_SUB | BPF_X: /* (u32) dst -= (u32) src */ 323 case BPF_ALU64 | BPF_SUB | BPF_X: /* dst -= src */ 324 PPC_SUB(dst_reg, dst_reg, src_reg); 325 goto bpf_alu32_trunc; 326 case BPF_ALU | BPF_ADD | BPF_K: /* (u32) dst += (u32) imm */ 327 case BPF_ALU | BPF_SUB | BPF_K: /* (u32) dst -= (u32) imm */ 328 case BPF_ALU64 | BPF_ADD | BPF_K: /* dst += imm */ 329 case BPF_ALU64 | BPF_SUB | BPF_K: /* dst -= imm */ 330 if (BPF_OP(code) == BPF_SUB) 331 imm = -imm; 332 if (imm) { 333 if (imm >= -32768 && imm < 32768) 334 PPC_ADDI(dst_reg, dst_reg, IMM_L(imm)); 335 else { 336 PPC_LI32(b2p[TMP_REG_1], imm); 337 PPC_ADD(dst_reg, dst_reg, b2p[TMP_REG_1]); 338 } 339 } 340 goto bpf_alu32_trunc; 341 case BPF_ALU | BPF_MUL | BPF_X: /* (u32) dst *= (u32) src */ 342 case BPF_ALU64 | BPF_MUL | BPF_X: /* dst *= src */ 343 if (BPF_CLASS(code) == BPF_ALU) 344 PPC_MULW(dst_reg, dst_reg, src_reg); 345 else 346 PPC_MULD(dst_reg, dst_reg, src_reg); 347 goto bpf_alu32_trunc; 348 case BPF_ALU | BPF_MUL | BPF_K: /* (u32) dst *= (u32) imm */ 349 case BPF_ALU64 | BPF_MUL | BPF_K: /* dst *= imm */ 350 if (imm >= -32768 && imm < 32768) 351 PPC_MULI(dst_reg, dst_reg, IMM_L(imm)); 352 else { 353 PPC_LI32(b2p[TMP_REG_1], imm); 354 if (BPF_CLASS(code) == BPF_ALU) 355 PPC_MULW(dst_reg, dst_reg, 356 b2p[TMP_REG_1]); 357 else 358 PPC_MULD(dst_reg, dst_reg, 359 b2p[TMP_REG_1]); 360 } 361 goto bpf_alu32_trunc; 362 case BPF_ALU | BPF_DIV | BPF_X: /* (u32) dst /= (u32) src */ 363 case BPF_ALU | BPF_MOD | BPF_X: /* (u32) dst %= (u32) src */ 364 if (BPF_OP(code) == BPF_MOD) { 365 PPC_DIVWU(b2p[TMP_REG_1], dst_reg, src_reg); 366 PPC_MULW(b2p[TMP_REG_1], src_reg, 367 b2p[TMP_REG_1]); 368 PPC_SUB(dst_reg, dst_reg, b2p[TMP_REG_1]); 369 } else 370 PPC_DIVWU(dst_reg, dst_reg, src_reg); 371 goto bpf_alu32_trunc; 372 case BPF_ALU64 | BPF_DIV | BPF_X: /* dst /= src */ 373 case BPF_ALU64 | BPF_MOD | BPF_X: /* dst %= src */ 374 if (BPF_OP(code) == BPF_MOD) { 375 PPC_DIVD(b2p[TMP_REG_1], dst_reg, src_reg); 376 PPC_MULD(b2p[TMP_REG_1], src_reg, 377 b2p[TMP_REG_1]); 378 PPC_SUB(dst_reg, dst_reg, b2p[TMP_REG_1]); 379 } else 380 PPC_DIVD(dst_reg, dst_reg, src_reg); 381 break; 382 case BPF_ALU | BPF_MOD | BPF_K: /* (u32) dst %= (u32) imm */ 383 case BPF_ALU | BPF_DIV | BPF_K: /* (u32) dst /= (u32) imm */ 384 case BPF_ALU64 | BPF_MOD | BPF_K: /* dst %= imm */ 385 case BPF_ALU64 | BPF_DIV | BPF_K: /* dst /= imm */ 386 if (imm == 0) 387 return -EINVAL; 388 else if (imm == 1) 389 goto bpf_alu32_trunc; 390 391 PPC_LI32(b2p[TMP_REG_1], imm); 392 switch (BPF_CLASS(code)) { 393 case BPF_ALU: 394 if (BPF_OP(code) == BPF_MOD) { 395 PPC_DIVWU(b2p[TMP_REG_2], dst_reg, 396 b2p[TMP_REG_1]); 397 PPC_MULW(b2p[TMP_REG_1], 398 b2p[TMP_REG_1], 399 b2p[TMP_REG_2]); 400 PPC_SUB(dst_reg, dst_reg, 401 b2p[TMP_REG_1]); 402 } else 403 PPC_DIVWU(dst_reg, dst_reg, 404 b2p[TMP_REG_1]); 405 break; 406 case BPF_ALU64: 407 if (BPF_OP(code) == BPF_MOD) { 408 PPC_DIVD(b2p[TMP_REG_2], dst_reg, 409 b2p[TMP_REG_1]); 410 PPC_MULD(b2p[TMP_REG_1], 411 b2p[TMP_REG_1], 412 b2p[TMP_REG_2]); 413 PPC_SUB(dst_reg, dst_reg, 414 b2p[TMP_REG_1]); 415 } else 416 PPC_DIVD(dst_reg, dst_reg, 417 b2p[TMP_REG_1]); 418 break; 419 } 420 goto bpf_alu32_trunc; 421 case BPF_ALU | BPF_NEG: /* (u32) dst = -dst */ 422 case BPF_ALU64 | BPF_NEG: /* dst = -dst */ 423 PPC_NEG(dst_reg, dst_reg); 424 goto bpf_alu32_trunc; 425 426 /* 427 * Logical operations: AND/OR/XOR/[A]LSH/[A]RSH 428 */ 429 case BPF_ALU | BPF_AND | BPF_X: /* (u32) dst = dst & src */ 430 case BPF_ALU64 | BPF_AND | BPF_X: /* dst = dst & src */ 431 PPC_AND(dst_reg, dst_reg, src_reg); 432 goto bpf_alu32_trunc; 433 case BPF_ALU | BPF_AND | BPF_K: /* (u32) dst = dst & imm */ 434 case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */ 435 if (!IMM_H(imm)) 436 PPC_ANDI(dst_reg, dst_reg, IMM_L(imm)); 437 else { 438 /* Sign-extended */ 439 PPC_LI32(b2p[TMP_REG_1], imm); 440 PPC_AND(dst_reg, dst_reg, b2p[TMP_REG_1]); 441 } 442 goto bpf_alu32_trunc; 443 case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */ 444 case BPF_ALU64 | BPF_OR | BPF_X: /* dst = dst | src */ 445 PPC_OR(dst_reg, dst_reg, src_reg); 446 goto bpf_alu32_trunc; 447 case BPF_ALU | BPF_OR | BPF_K:/* dst = (u32) dst | (u32) imm */ 448 case BPF_ALU64 | BPF_OR | BPF_K:/* dst = dst | imm */ 449 if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) { 450 /* Sign-extended */ 451 PPC_LI32(b2p[TMP_REG_1], imm); 452 PPC_OR(dst_reg, dst_reg, b2p[TMP_REG_1]); 453 } else { 454 if (IMM_L(imm)) 455 PPC_ORI(dst_reg, dst_reg, IMM_L(imm)); 456 if (IMM_H(imm)) 457 PPC_ORIS(dst_reg, dst_reg, IMM_H(imm)); 458 } 459 goto bpf_alu32_trunc; 460 case BPF_ALU | BPF_XOR | BPF_X: /* (u32) dst ^= src */ 461 case BPF_ALU64 | BPF_XOR | BPF_X: /* dst ^= src */ 462 PPC_XOR(dst_reg, dst_reg, src_reg); 463 goto bpf_alu32_trunc; 464 case BPF_ALU | BPF_XOR | BPF_K: /* (u32) dst ^= (u32) imm */ 465 case BPF_ALU64 | BPF_XOR | BPF_K: /* dst ^= imm */ 466 if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) { 467 /* Sign-extended */ 468 PPC_LI32(b2p[TMP_REG_1], imm); 469 PPC_XOR(dst_reg, dst_reg, b2p[TMP_REG_1]); 470 } else { 471 if (IMM_L(imm)) 472 PPC_XORI(dst_reg, dst_reg, IMM_L(imm)); 473 if (IMM_H(imm)) 474 PPC_XORIS(dst_reg, dst_reg, IMM_H(imm)); 475 } 476 goto bpf_alu32_trunc; 477 case BPF_ALU | BPF_LSH | BPF_X: /* (u32) dst <<= (u32) src */ 478 /* slw clears top 32 bits */ 479 PPC_SLW(dst_reg, dst_reg, src_reg); 480 break; 481 case BPF_ALU64 | BPF_LSH | BPF_X: /* dst <<= src; */ 482 PPC_SLD(dst_reg, dst_reg, src_reg); 483 break; 484 case BPF_ALU | BPF_LSH | BPF_K: /* (u32) dst <<== (u32) imm */ 485 /* with imm 0, we still need to clear top 32 bits */ 486 PPC_SLWI(dst_reg, dst_reg, imm); 487 break; 488 case BPF_ALU64 | BPF_LSH | BPF_K: /* dst <<== imm */ 489 if (imm != 0) 490 PPC_SLDI(dst_reg, dst_reg, imm); 491 break; 492 case BPF_ALU | BPF_RSH | BPF_X: /* (u32) dst >>= (u32) src */ 493 PPC_SRW(dst_reg, dst_reg, src_reg); 494 break; 495 case BPF_ALU64 | BPF_RSH | BPF_X: /* dst >>= src */ 496 PPC_SRD(dst_reg, dst_reg, src_reg); 497 break; 498 case BPF_ALU | BPF_RSH | BPF_K: /* (u32) dst >>= (u32) imm */ 499 PPC_SRWI(dst_reg, dst_reg, imm); 500 break; 501 case BPF_ALU64 | BPF_RSH | BPF_K: /* dst >>= imm */ 502 if (imm != 0) 503 PPC_SRDI(dst_reg, dst_reg, imm); 504 break; 505 case BPF_ALU64 | BPF_ARSH | BPF_X: /* (s64) dst >>= src */ 506 PPC_SRAD(dst_reg, dst_reg, src_reg); 507 break; 508 case BPF_ALU64 | BPF_ARSH | BPF_K: /* (s64) dst >>= imm */ 509 if (imm != 0) 510 PPC_SRADI(dst_reg, dst_reg, imm); 511 break; 512 513 /* 514 * MOV 515 */ 516 case BPF_ALU | BPF_MOV | BPF_X: /* (u32) dst = src */ 517 case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */ 518 PPC_MR(dst_reg, src_reg); 519 goto bpf_alu32_trunc; 520 case BPF_ALU | BPF_MOV | BPF_K: /* (u32) dst = imm */ 521 case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = (s64) imm */ 522 PPC_LI32(dst_reg, imm); 523 if (imm < 0) 524 goto bpf_alu32_trunc; 525 break; 526 527 bpf_alu32_trunc: 528 /* Truncate to 32-bits */ 529 if (BPF_CLASS(code) == BPF_ALU) 530 PPC_RLWINM(dst_reg, dst_reg, 0, 0, 31); 531 break; 532 533 /* 534 * BPF_FROM_BE/LE 535 */ 536 case BPF_ALU | BPF_END | BPF_FROM_LE: 537 case BPF_ALU | BPF_END | BPF_FROM_BE: 538 #ifdef __BIG_ENDIAN__ 539 if (BPF_SRC(code) == BPF_FROM_BE) 540 goto emit_clear; 541 #else /* !__BIG_ENDIAN__ */ 542 if (BPF_SRC(code) == BPF_FROM_LE) 543 goto emit_clear; 544 #endif 545 switch (imm) { 546 case 16: 547 /* Rotate 8 bits left & mask with 0x0000ff00 */ 548 PPC_RLWINM(b2p[TMP_REG_1], dst_reg, 8, 16, 23); 549 /* Rotate 8 bits right & insert LSB to reg */ 550 PPC_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 24, 31); 551 /* Move result back to dst_reg */ 552 PPC_MR(dst_reg, b2p[TMP_REG_1]); 553 break; 554 case 32: 555 /* 556 * Rotate word left by 8 bits: 557 * 2 bytes are already in their final position 558 * -- byte 2 and 4 (of bytes 1, 2, 3 and 4) 559 */ 560 PPC_RLWINM(b2p[TMP_REG_1], dst_reg, 8, 0, 31); 561 /* Rotate 24 bits and insert byte 1 */ 562 PPC_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 0, 7); 563 /* Rotate 24 bits and insert byte 3 */ 564 PPC_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 16, 23); 565 PPC_MR(dst_reg, b2p[TMP_REG_1]); 566 break; 567 case 64: 568 /* 569 * Way easier and faster(?) to store the value 570 * into stack and then use ldbrx 571 * 572 * ctx->seen will be reliable in pass2, but 573 * the instructions generated will remain the 574 * same across all passes 575 */ 576 PPC_STD(dst_reg, 1, bpf_jit_stack_local(ctx)); 577 PPC_ADDI(b2p[TMP_REG_1], 1, bpf_jit_stack_local(ctx)); 578 PPC_LDBRX(dst_reg, 0, b2p[TMP_REG_1]); 579 break; 580 } 581 break; 582 583 emit_clear: 584 switch (imm) { 585 case 16: 586 /* zero-extend 16 bits into 64 bits */ 587 PPC_RLDICL(dst_reg, dst_reg, 0, 48); 588 break; 589 case 32: 590 /* zero-extend 32 bits into 64 bits */ 591 PPC_RLDICL(dst_reg, dst_reg, 0, 32); 592 break; 593 case 64: 594 /* nop */ 595 break; 596 } 597 break; 598 599 /* 600 * BPF_ST(X) 601 */ 602 case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src */ 603 case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */ 604 if (BPF_CLASS(code) == BPF_ST) { 605 PPC_LI(b2p[TMP_REG_1], imm); 606 src_reg = b2p[TMP_REG_1]; 607 } 608 PPC_STB(src_reg, dst_reg, off); 609 break; 610 case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */ 611 case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */ 612 if (BPF_CLASS(code) == BPF_ST) { 613 PPC_LI(b2p[TMP_REG_1], imm); 614 src_reg = b2p[TMP_REG_1]; 615 } 616 PPC_STH(src_reg, dst_reg, off); 617 break; 618 case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */ 619 case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */ 620 if (BPF_CLASS(code) == BPF_ST) { 621 PPC_LI32(b2p[TMP_REG_1], imm); 622 src_reg = b2p[TMP_REG_1]; 623 } 624 PPC_STW(src_reg, dst_reg, off); 625 break; 626 case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */ 627 case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */ 628 if (BPF_CLASS(code) == BPF_ST) { 629 PPC_LI32(b2p[TMP_REG_1], imm); 630 src_reg = b2p[TMP_REG_1]; 631 } 632 PPC_STD(src_reg, dst_reg, off); 633 break; 634 635 /* 636 * BPF_STX XADD (atomic_add) 637 */ 638 /* *(u32 *)(dst + off) += src */ 639 case BPF_STX | BPF_XADD | BPF_W: 640 /* Get EA into TMP_REG_1 */ 641 PPC_ADDI(b2p[TMP_REG_1], dst_reg, off); 642 tmp_idx = ctx->idx * 4; 643 /* load value from memory into TMP_REG_2 */ 644 PPC_BPF_LWARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0); 645 /* add value from src_reg into this */ 646 PPC_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg); 647 /* store result back */ 648 PPC_BPF_STWCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]); 649 /* we're done if this succeeded */ 650 PPC_BCC_SHORT(COND_NE, tmp_idx); 651 break; 652 /* *(u64 *)(dst + off) += src */ 653 case BPF_STX | BPF_XADD | BPF_DW: 654 PPC_ADDI(b2p[TMP_REG_1], dst_reg, off); 655 tmp_idx = ctx->idx * 4; 656 PPC_BPF_LDARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0); 657 PPC_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg); 658 PPC_BPF_STDCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]); 659 PPC_BCC_SHORT(COND_NE, tmp_idx); 660 break; 661 662 /* 663 * BPF_LDX 664 */ 665 /* dst = *(u8 *)(ul) (src + off) */ 666 case BPF_LDX | BPF_MEM | BPF_B: 667 PPC_LBZ(dst_reg, src_reg, off); 668 break; 669 /* dst = *(u16 *)(ul) (src + off) */ 670 case BPF_LDX | BPF_MEM | BPF_H: 671 PPC_LHZ(dst_reg, src_reg, off); 672 break; 673 /* dst = *(u32 *)(ul) (src + off) */ 674 case BPF_LDX | BPF_MEM | BPF_W: 675 PPC_LWZ(dst_reg, src_reg, off); 676 break; 677 /* dst = *(u64 *)(ul) (src + off) */ 678 case BPF_LDX | BPF_MEM | BPF_DW: 679 PPC_LD(dst_reg, src_reg, off); 680 break; 681 682 /* 683 * Doubleword load 684 * 16 byte instruction that uses two 'struct bpf_insn' 685 */ 686 case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */ 687 imm64 = ((u64)(u32) insn[i].imm) | 688 (((u64)(u32) insn[i+1].imm) << 32); 689 /* Adjust for two bpf instructions */ 690 addrs[++i] = ctx->idx * 4; 691 PPC_LI64(dst_reg, imm64); 692 break; 693 694 /* 695 * Return/Exit 696 */ 697 case BPF_JMP | BPF_EXIT: 698 /* 699 * If this isn't the very last instruction, branch to 700 * the epilogue. If we _are_ the last instruction, 701 * we'll just fall through to the epilogue. 702 */ 703 if (i != flen - 1) 704 PPC_JMP(exit_addr); 705 /* else fall through to the epilogue */ 706 break; 707 708 /* 709 * Call kernel helper or bpf function 710 */ 711 case BPF_JMP | BPF_CALL: 712 ctx->seen |= SEEN_FUNC; 713 714 /* bpf function call */ 715 if (insn[i].src_reg == BPF_PSEUDO_CALL) 716 if (!extra_pass) 717 func = NULL; 718 else if (fp->aux->func && off < fp->aux->func_cnt) 719 /* use the subprog id from the off 720 * field to lookup the callee address 721 */ 722 func = (u8 *) fp->aux->func[off]->bpf_func; 723 else 724 return -EINVAL; 725 /* kernel helper call */ 726 else 727 func = (u8 *) __bpf_call_base + imm; 728 729 bpf_jit_emit_func_call(image, ctx, (u64)func); 730 731 /* move return value from r3 to BPF_REG_0 */ 732 PPC_MR(b2p[BPF_REG_0], 3); 733 break; 734 735 /* 736 * Jumps and branches 737 */ 738 case BPF_JMP | BPF_JA: 739 PPC_JMP(addrs[i + 1 + off]); 740 break; 741 742 case BPF_JMP | BPF_JGT | BPF_K: 743 case BPF_JMP | BPF_JGT | BPF_X: 744 case BPF_JMP | BPF_JSGT | BPF_K: 745 case BPF_JMP | BPF_JSGT | BPF_X: 746 true_cond = COND_GT; 747 goto cond_branch; 748 case BPF_JMP | BPF_JLT | BPF_K: 749 case BPF_JMP | BPF_JLT | BPF_X: 750 case BPF_JMP | BPF_JSLT | BPF_K: 751 case BPF_JMP | BPF_JSLT | BPF_X: 752 true_cond = COND_LT; 753 goto cond_branch; 754 case BPF_JMP | BPF_JGE | BPF_K: 755 case BPF_JMP | BPF_JGE | BPF_X: 756 case BPF_JMP | BPF_JSGE | BPF_K: 757 case BPF_JMP | BPF_JSGE | BPF_X: 758 true_cond = COND_GE; 759 goto cond_branch; 760 case BPF_JMP | BPF_JLE | BPF_K: 761 case BPF_JMP | BPF_JLE | BPF_X: 762 case BPF_JMP | BPF_JSLE | BPF_K: 763 case BPF_JMP | BPF_JSLE | BPF_X: 764 true_cond = COND_LE; 765 goto cond_branch; 766 case BPF_JMP | BPF_JEQ | BPF_K: 767 case BPF_JMP | BPF_JEQ | BPF_X: 768 true_cond = COND_EQ; 769 goto cond_branch; 770 case BPF_JMP | BPF_JNE | BPF_K: 771 case BPF_JMP | BPF_JNE | BPF_X: 772 true_cond = COND_NE; 773 goto cond_branch; 774 case BPF_JMP | BPF_JSET | BPF_K: 775 case BPF_JMP | BPF_JSET | BPF_X: 776 true_cond = COND_NE; 777 /* Fall through */ 778 779 cond_branch: 780 switch (code) { 781 case BPF_JMP | BPF_JGT | BPF_X: 782 case BPF_JMP | BPF_JLT | BPF_X: 783 case BPF_JMP | BPF_JGE | BPF_X: 784 case BPF_JMP | BPF_JLE | BPF_X: 785 case BPF_JMP | BPF_JEQ | BPF_X: 786 case BPF_JMP | BPF_JNE | BPF_X: 787 /* unsigned comparison */ 788 PPC_CMPLD(dst_reg, src_reg); 789 break; 790 case BPF_JMP | BPF_JSGT | BPF_X: 791 case BPF_JMP | BPF_JSLT | BPF_X: 792 case BPF_JMP | BPF_JSGE | BPF_X: 793 case BPF_JMP | BPF_JSLE | BPF_X: 794 /* signed comparison */ 795 PPC_CMPD(dst_reg, src_reg); 796 break; 797 case BPF_JMP | BPF_JSET | BPF_X: 798 PPC_AND_DOT(b2p[TMP_REG_1], dst_reg, src_reg); 799 break; 800 case BPF_JMP | BPF_JNE | BPF_K: 801 case BPF_JMP | BPF_JEQ | BPF_K: 802 case BPF_JMP | BPF_JGT | BPF_K: 803 case BPF_JMP | BPF_JLT | BPF_K: 804 case BPF_JMP | BPF_JGE | BPF_K: 805 case BPF_JMP | BPF_JLE | BPF_K: 806 /* 807 * Need sign-extended load, so only positive 808 * values can be used as imm in cmpldi 809 */ 810 if (imm >= 0 && imm < 32768) 811 PPC_CMPLDI(dst_reg, imm); 812 else { 813 /* sign-extending load */ 814 PPC_LI32(b2p[TMP_REG_1], imm); 815 /* ... but unsigned comparison */ 816 PPC_CMPLD(dst_reg, b2p[TMP_REG_1]); 817 } 818 break; 819 case BPF_JMP | BPF_JSGT | BPF_K: 820 case BPF_JMP | BPF_JSLT | BPF_K: 821 case BPF_JMP | BPF_JSGE | BPF_K: 822 case BPF_JMP | BPF_JSLE | BPF_K: 823 /* 824 * signed comparison, so any 16-bit value 825 * can be used in cmpdi 826 */ 827 if (imm >= -32768 && imm < 32768) 828 PPC_CMPDI(dst_reg, imm); 829 else { 830 PPC_LI32(b2p[TMP_REG_1], imm); 831 PPC_CMPD(dst_reg, b2p[TMP_REG_1]); 832 } 833 break; 834 case BPF_JMP | BPF_JSET | BPF_K: 835 /* andi does not sign-extend the immediate */ 836 if (imm >= 0 && imm < 32768) 837 /* PPC_ANDI is _only/always_ dot-form */ 838 PPC_ANDI(b2p[TMP_REG_1], dst_reg, imm); 839 else { 840 PPC_LI32(b2p[TMP_REG_1], imm); 841 PPC_AND_DOT(b2p[TMP_REG_1], dst_reg, 842 b2p[TMP_REG_1]); 843 } 844 break; 845 } 846 PPC_BCC(true_cond, addrs[i + 1 + off]); 847 break; 848 849 /* 850 * Tail call 851 */ 852 case BPF_JMP | BPF_TAIL_CALL: 853 ctx->seen |= SEEN_TAILCALL; 854 bpf_jit_emit_tail_call(image, ctx, addrs[i + 1]); 855 break; 856 857 default: 858 /* 859 * The filter contains something cruel & unusual. 860 * We don't handle it, but also there shouldn't be 861 * anything missing from our list. 862 */ 863 pr_err_ratelimited("eBPF filter opcode %04x (@%d) unsupported\n", 864 code, i); 865 return -ENOTSUPP; 866 } 867 } 868 869 /* Set end-of-body-code address for exit. */ 870 addrs[i] = ctx->idx * 4; 871 872 return 0; 873 } 874 875 struct powerpc64_jit_data { 876 struct bpf_binary_header *header; 877 u32 *addrs; 878 u8 *image; 879 u32 proglen; 880 struct codegen_context ctx; 881 }; 882 883 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) 884 { 885 u32 proglen; 886 u32 alloclen; 887 u8 *image = NULL; 888 u32 *code_base; 889 u32 *addrs; 890 struct powerpc64_jit_data *jit_data; 891 struct codegen_context cgctx; 892 int pass; 893 int flen; 894 struct bpf_binary_header *bpf_hdr; 895 struct bpf_prog *org_fp = fp; 896 struct bpf_prog *tmp_fp; 897 bool bpf_blinded = false; 898 bool extra_pass = false; 899 900 if (!fp->jit_requested) 901 return org_fp; 902 903 tmp_fp = bpf_jit_blind_constants(org_fp); 904 if (IS_ERR(tmp_fp)) 905 return org_fp; 906 907 if (tmp_fp != org_fp) { 908 bpf_blinded = true; 909 fp = tmp_fp; 910 } 911 912 jit_data = fp->aux->jit_data; 913 if (!jit_data) { 914 jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL); 915 if (!jit_data) { 916 fp = org_fp; 917 goto out; 918 } 919 fp->aux->jit_data = jit_data; 920 } 921 922 flen = fp->len; 923 addrs = jit_data->addrs; 924 if (addrs) { 925 cgctx = jit_data->ctx; 926 image = jit_data->image; 927 bpf_hdr = jit_data->header; 928 proglen = jit_data->proglen; 929 alloclen = proglen + FUNCTION_DESCR_SIZE; 930 extra_pass = true; 931 goto skip_init_ctx; 932 } 933 934 addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL); 935 if (addrs == NULL) { 936 fp = org_fp; 937 goto out_addrs; 938 } 939 940 memset(&cgctx, 0, sizeof(struct codegen_context)); 941 942 /* Make sure that the stack is quadword aligned. */ 943 cgctx.stack_size = round_up(fp->aux->stack_depth, 16); 944 945 /* Scouting faux-generate pass 0 */ 946 if (bpf_jit_build_body(fp, 0, &cgctx, addrs, false)) { 947 /* We hit something illegal or unsupported. */ 948 fp = org_fp; 949 goto out_addrs; 950 } 951 952 /* 953 * Pretend to build prologue, given the features we've seen. This will 954 * update ctgtx.idx as it pretends to output instructions, then we can 955 * calculate total size from idx. 956 */ 957 bpf_jit_build_prologue(0, &cgctx); 958 bpf_jit_build_epilogue(0, &cgctx); 959 960 proglen = cgctx.idx * 4; 961 alloclen = proglen + FUNCTION_DESCR_SIZE; 962 963 bpf_hdr = bpf_jit_binary_alloc(alloclen, &image, 4, 964 bpf_jit_fill_ill_insns); 965 if (!bpf_hdr) { 966 fp = org_fp; 967 goto out_addrs; 968 } 969 970 skip_init_ctx: 971 code_base = (u32 *)(image + FUNCTION_DESCR_SIZE); 972 973 /* Code generation passes 1-2 */ 974 for (pass = 1; pass < 3; pass++) { 975 /* Now build the prologue, body code & epilogue for real. */ 976 cgctx.idx = 0; 977 bpf_jit_build_prologue(code_base, &cgctx); 978 bpf_jit_build_body(fp, code_base, &cgctx, addrs, extra_pass); 979 bpf_jit_build_epilogue(code_base, &cgctx); 980 981 if (bpf_jit_enable > 1) 982 pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass, 983 proglen - (cgctx.idx * 4), cgctx.seen); 984 } 985 986 if (bpf_jit_enable > 1) 987 /* 988 * Note that we output the base address of the code_base 989 * rather than image, since opcodes are in code_base. 990 */ 991 bpf_jit_dump(flen, proglen, pass, code_base); 992 993 #ifdef PPC64_ELF_ABI_v1 994 /* Function descriptor nastiness: Address + TOC */ 995 ((u64 *)image)[0] = (u64)code_base; 996 ((u64 *)image)[1] = local_paca->kernel_toc; 997 #endif 998 999 fp->bpf_func = (void *)image; 1000 fp->jited = 1; 1001 fp->jited_len = alloclen; 1002 1003 bpf_flush_icache(bpf_hdr, (u8 *)bpf_hdr + (bpf_hdr->pages * PAGE_SIZE)); 1004 if (!fp->is_func || extra_pass) { 1005 out_addrs: 1006 kfree(addrs); 1007 kfree(jit_data); 1008 fp->aux->jit_data = NULL; 1009 } else { 1010 jit_data->addrs = addrs; 1011 jit_data->ctx = cgctx; 1012 jit_data->proglen = proglen; 1013 jit_data->image = image; 1014 jit_data->header = bpf_hdr; 1015 } 1016 1017 out: 1018 if (bpf_blinded) 1019 bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp); 1020 1021 return fp; 1022 } 1023 1024 /* Overriding bpf_jit_free() as we don't set images read-only. */ 1025 void bpf_jit_free(struct bpf_prog *fp) 1026 { 1027 unsigned long addr = (unsigned long)fp->bpf_func & PAGE_MASK; 1028 struct bpf_binary_header *bpf_hdr = (void *)addr; 1029 1030 if (fp->jited) 1031 bpf_jit_binary_free(bpf_hdr); 1032 1033 bpf_prog_unlock_free(fp); 1034 } 1035