1 /* 2 * bpf_jit_comp64.c: eBPF JIT compiler 3 * 4 * Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com> 5 * IBM Corporation 6 * 7 * Based on the powerpc classic BPF JIT compiler by Matt Evans 8 * 9 * This program is free software; you can redistribute it and/or 10 * modify it under the terms of the GNU General Public License 11 * as published by the Free Software Foundation; version 2 12 * of the License. 13 */ 14 #include <linux/moduleloader.h> 15 #include <asm/cacheflush.h> 16 #include <linux/netdevice.h> 17 #include <linux/filter.h> 18 #include <linux/if_vlan.h> 19 #include <asm/kprobes.h> 20 #include <linux/bpf.h> 21 22 #include "bpf_jit64.h" 23 24 static void bpf_jit_fill_ill_insns(void *area, unsigned int size) 25 { 26 memset32(area, BREAKPOINT_INSTRUCTION, size/4); 27 } 28 29 static inline void bpf_flush_icache(void *start, void *end) 30 { 31 smp_wmb(); 32 flush_icache_range((unsigned long)start, (unsigned long)end); 33 } 34 35 static inline bool bpf_is_seen_register(struct codegen_context *ctx, int i) 36 { 37 return (ctx->seen & (1 << (31 - b2p[i]))); 38 } 39 40 static inline void bpf_set_seen_register(struct codegen_context *ctx, int i) 41 { 42 ctx->seen |= (1 << (31 - b2p[i])); 43 } 44 45 static inline bool bpf_has_stack_frame(struct codegen_context *ctx) 46 { 47 /* 48 * We only need a stack frame if: 49 * - we call other functions (kernel helpers), or 50 * - the bpf program uses its stack area 51 * The latter condition is deduced from the usage of BPF_REG_FP 52 */ 53 return ctx->seen & SEEN_FUNC || bpf_is_seen_register(ctx, BPF_REG_FP); 54 } 55 56 /* 57 * When not setting up our own stackframe, the redzone usage is: 58 * 59 * [ prev sp ] <------------- 60 * [ ... ] | 61 * sp (r1) ---> [ stack pointer ] -------------- 62 * [ nv gpr save area ] 6*8 63 * [ tail_call_cnt ] 8 64 * [ local_tmp_var ] 8 65 * [ unused red zone ] 208 bytes protected 66 */ 67 static int bpf_jit_stack_local(struct codegen_context *ctx) 68 { 69 if (bpf_has_stack_frame(ctx)) 70 return STACK_FRAME_MIN_SIZE + ctx->stack_size; 71 else 72 return -(BPF_PPC_STACK_SAVE + 16); 73 } 74 75 static int bpf_jit_stack_tailcallcnt(struct codegen_context *ctx) 76 { 77 return bpf_jit_stack_local(ctx) + 8; 78 } 79 80 static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg) 81 { 82 if (reg >= BPF_PPC_NVR_MIN && reg < 32) 83 return (bpf_has_stack_frame(ctx) ? 84 (BPF_PPC_STACKFRAME + ctx->stack_size) : 0) 85 - (8 * (32 - reg)); 86 87 pr_err("BPF JIT is asking about unknown registers"); 88 BUG(); 89 } 90 91 static void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx) 92 { 93 int i; 94 95 /* 96 * Initialize tail_call_cnt if we do tail calls. 97 * Otherwise, put in NOPs so that it can be skipped when we are 98 * invoked through a tail call. 99 */ 100 if (ctx->seen & SEEN_TAILCALL) { 101 PPC_LI(b2p[TMP_REG_1], 0); 102 /* this goes in the redzone */ 103 PPC_BPF_STL(b2p[TMP_REG_1], 1, -(BPF_PPC_STACK_SAVE + 8)); 104 } else { 105 PPC_NOP(); 106 PPC_NOP(); 107 } 108 109 #define BPF_TAILCALL_PROLOGUE_SIZE 8 110 111 if (bpf_has_stack_frame(ctx)) { 112 /* 113 * We need a stack frame, but we don't necessarily need to 114 * save/restore LR unless we call other functions 115 */ 116 if (ctx->seen & SEEN_FUNC) { 117 EMIT(PPC_INST_MFLR | __PPC_RT(R0)); 118 PPC_BPF_STL(0, 1, PPC_LR_STKOFF); 119 } 120 121 PPC_BPF_STLU(1, 1, -(BPF_PPC_STACKFRAME + ctx->stack_size)); 122 } 123 124 /* 125 * Back up non-volatile regs -- BPF registers 6-10 126 * If we haven't created our own stack frame, we save these 127 * in the protected zone below the previous stack frame 128 */ 129 for (i = BPF_REG_6; i <= BPF_REG_10; i++) 130 if (bpf_is_seen_register(ctx, i)) 131 PPC_BPF_STL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i])); 132 133 /* Setup frame pointer to point to the bpf stack area */ 134 if (bpf_is_seen_register(ctx, BPF_REG_FP)) 135 PPC_ADDI(b2p[BPF_REG_FP], 1, 136 STACK_FRAME_MIN_SIZE + ctx->stack_size); 137 } 138 139 static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context *ctx) 140 { 141 int i; 142 143 /* Restore NVRs */ 144 for (i = BPF_REG_6; i <= BPF_REG_10; i++) 145 if (bpf_is_seen_register(ctx, i)) 146 PPC_BPF_LL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i])); 147 148 /* Tear down our stack frame */ 149 if (bpf_has_stack_frame(ctx)) { 150 PPC_ADDI(1, 1, BPF_PPC_STACKFRAME + ctx->stack_size); 151 if (ctx->seen & SEEN_FUNC) { 152 PPC_BPF_LL(0, 1, PPC_LR_STKOFF); 153 PPC_MTLR(0); 154 } 155 } 156 } 157 158 static void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx) 159 { 160 bpf_jit_emit_common_epilogue(image, ctx); 161 162 /* Move result to r3 */ 163 PPC_MR(3, b2p[BPF_REG_0]); 164 165 PPC_BLR(); 166 } 167 168 static void bpf_jit_emit_func_call(u32 *image, struct codegen_context *ctx, u64 func) 169 { 170 #ifdef PPC64_ELF_ABI_v1 171 /* func points to the function descriptor */ 172 PPC_LI64(b2p[TMP_REG_2], func); 173 /* Load actual entry point from function descriptor */ 174 PPC_BPF_LL(b2p[TMP_REG_1], b2p[TMP_REG_2], 0); 175 /* ... and move it to LR */ 176 PPC_MTLR(b2p[TMP_REG_1]); 177 /* 178 * Load TOC from function descriptor at offset 8. 179 * We can clobber r2 since we get called through a 180 * function pointer (so caller will save/restore r2) 181 * and since we don't use a TOC ourself. 182 */ 183 PPC_BPF_LL(2, b2p[TMP_REG_2], 8); 184 #else 185 /* We can clobber r12 */ 186 PPC_FUNC_ADDR(12, func); 187 PPC_MTLR(12); 188 #endif 189 PPC_BLRL(); 190 } 191 192 static void bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 out) 193 { 194 /* 195 * By now, the eBPF program has already setup parameters in r3, r4 and r5 196 * r3/BPF_REG_1 - pointer to ctx -- passed as is to the next bpf program 197 * r4/BPF_REG_2 - pointer to bpf_array 198 * r5/BPF_REG_3 - index in bpf_array 199 */ 200 int b2p_bpf_array = b2p[BPF_REG_2]; 201 int b2p_index = b2p[BPF_REG_3]; 202 203 /* 204 * if (index >= array->map.max_entries) 205 * goto out; 206 */ 207 PPC_LWZ(b2p[TMP_REG_1], b2p_bpf_array, offsetof(struct bpf_array, map.max_entries)); 208 PPC_RLWINM(b2p_index, b2p_index, 0, 0, 31); 209 PPC_CMPLW(b2p_index, b2p[TMP_REG_1]); 210 PPC_BCC(COND_GE, out); 211 212 /* 213 * if (tail_call_cnt > MAX_TAIL_CALL_CNT) 214 * goto out; 215 */ 216 PPC_LD(b2p[TMP_REG_1], 1, bpf_jit_stack_tailcallcnt(ctx)); 217 PPC_CMPLWI(b2p[TMP_REG_1], MAX_TAIL_CALL_CNT); 218 PPC_BCC(COND_GT, out); 219 220 /* 221 * tail_call_cnt++; 222 */ 223 PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], 1); 224 PPC_BPF_STL(b2p[TMP_REG_1], 1, bpf_jit_stack_tailcallcnt(ctx)); 225 226 /* prog = array->ptrs[index]; */ 227 PPC_MULI(b2p[TMP_REG_1], b2p_index, 8); 228 PPC_ADD(b2p[TMP_REG_1], b2p[TMP_REG_1], b2p_bpf_array); 229 PPC_LD(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_array, ptrs)); 230 231 /* 232 * if (prog == NULL) 233 * goto out; 234 */ 235 PPC_CMPLDI(b2p[TMP_REG_1], 0); 236 PPC_BCC(COND_EQ, out); 237 238 /* goto *(prog->bpf_func + prologue_size); */ 239 PPC_LD(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_prog, bpf_func)); 240 #ifdef PPC64_ELF_ABI_v1 241 /* skip past the function descriptor */ 242 PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], 243 FUNCTION_DESCR_SIZE + BPF_TAILCALL_PROLOGUE_SIZE); 244 #else 245 PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], BPF_TAILCALL_PROLOGUE_SIZE); 246 #endif 247 PPC_MTCTR(b2p[TMP_REG_1]); 248 249 /* tear down stack, restore NVRs, ... */ 250 bpf_jit_emit_common_epilogue(image, ctx); 251 252 PPC_BCTR(); 253 /* out: */ 254 } 255 256 /* Assemble the body code between the prologue & epilogue */ 257 static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, 258 struct codegen_context *ctx, 259 u32 *addrs) 260 { 261 const struct bpf_insn *insn = fp->insnsi; 262 int flen = fp->len; 263 int i; 264 265 /* Start of epilogue code - will only be valid 2nd pass onwards */ 266 u32 exit_addr = addrs[flen]; 267 268 for (i = 0; i < flen; i++) { 269 u32 code = insn[i].code; 270 u32 dst_reg = b2p[insn[i].dst_reg]; 271 u32 src_reg = b2p[insn[i].src_reg]; 272 s16 off = insn[i].off; 273 s32 imm = insn[i].imm; 274 u64 imm64; 275 u8 *func; 276 u32 true_cond; 277 278 /* 279 * addrs[] maps a BPF bytecode address into a real offset from 280 * the start of the body code. 281 */ 282 addrs[i] = ctx->idx * 4; 283 284 /* 285 * As an optimization, we note down which non-volatile registers 286 * are used so that we can only save/restore those in our 287 * prologue and epilogue. We do this here regardless of whether 288 * the actual BPF instruction uses src/dst registers or not 289 * (for instance, BPF_CALL does not use them). The expectation 290 * is that those instructions will have src_reg/dst_reg set to 291 * 0. Even otherwise, we just lose some prologue/epilogue 292 * optimization but everything else should work without 293 * any issues. 294 */ 295 if (dst_reg >= BPF_PPC_NVR_MIN && dst_reg < 32) 296 bpf_set_seen_register(ctx, insn[i].dst_reg); 297 if (src_reg >= BPF_PPC_NVR_MIN && src_reg < 32) 298 bpf_set_seen_register(ctx, insn[i].src_reg); 299 300 switch (code) { 301 /* 302 * Arithmetic operations: ADD/SUB/MUL/DIV/MOD/NEG 303 */ 304 case BPF_ALU | BPF_ADD | BPF_X: /* (u32) dst += (u32) src */ 305 case BPF_ALU64 | BPF_ADD | BPF_X: /* dst += src */ 306 PPC_ADD(dst_reg, dst_reg, src_reg); 307 goto bpf_alu32_trunc; 308 case BPF_ALU | BPF_SUB | BPF_X: /* (u32) dst -= (u32) src */ 309 case BPF_ALU64 | BPF_SUB | BPF_X: /* dst -= src */ 310 PPC_SUB(dst_reg, dst_reg, src_reg); 311 goto bpf_alu32_trunc; 312 case BPF_ALU | BPF_ADD | BPF_K: /* (u32) dst += (u32) imm */ 313 case BPF_ALU | BPF_SUB | BPF_K: /* (u32) dst -= (u32) imm */ 314 case BPF_ALU64 | BPF_ADD | BPF_K: /* dst += imm */ 315 case BPF_ALU64 | BPF_SUB | BPF_K: /* dst -= imm */ 316 if (BPF_OP(code) == BPF_SUB) 317 imm = -imm; 318 if (imm) { 319 if (imm >= -32768 && imm < 32768) 320 PPC_ADDI(dst_reg, dst_reg, IMM_L(imm)); 321 else { 322 PPC_LI32(b2p[TMP_REG_1], imm); 323 PPC_ADD(dst_reg, dst_reg, b2p[TMP_REG_1]); 324 } 325 } 326 goto bpf_alu32_trunc; 327 case BPF_ALU | BPF_MUL | BPF_X: /* (u32) dst *= (u32) src */ 328 case BPF_ALU64 | BPF_MUL | BPF_X: /* dst *= src */ 329 if (BPF_CLASS(code) == BPF_ALU) 330 PPC_MULW(dst_reg, dst_reg, src_reg); 331 else 332 PPC_MULD(dst_reg, dst_reg, src_reg); 333 goto bpf_alu32_trunc; 334 case BPF_ALU | BPF_MUL | BPF_K: /* (u32) dst *= (u32) imm */ 335 case BPF_ALU64 | BPF_MUL | BPF_K: /* dst *= imm */ 336 if (imm >= -32768 && imm < 32768) 337 PPC_MULI(dst_reg, dst_reg, IMM_L(imm)); 338 else { 339 PPC_LI32(b2p[TMP_REG_1], imm); 340 if (BPF_CLASS(code) == BPF_ALU) 341 PPC_MULW(dst_reg, dst_reg, 342 b2p[TMP_REG_1]); 343 else 344 PPC_MULD(dst_reg, dst_reg, 345 b2p[TMP_REG_1]); 346 } 347 goto bpf_alu32_trunc; 348 case BPF_ALU | BPF_DIV | BPF_X: /* (u32) dst /= (u32) src */ 349 case BPF_ALU | BPF_MOD | BPF_X: /* (u32) dst %= (u32) src */ 350 if (BPF_OP(code) == BPF_MOD) { 351 PPC_DIVWU(b2p[TMP_REG_1], dst_reg, src_reg); 352 PPC_MULW(b2p[TMP_REG_1], src_reg, 353 b2p[TMP_REG_1]); 354 PPC_SUB(dst_reg, dst_reg, b2p[TMP_REG_1]); 355 } else 356 PPC_DIVWU(dst_reg, dst_reg, src_reg); 357 goto bpf_alu32_trunc; 358 case BPF_ALU64 | BPF_DIV | BPF_X: /* dst /= src */ 359 case BPF_ALU64 | BPF_MOD | BPF_X: /* dst %= src */ 360 if (BPF_OP(code) == BPF_MOD) { 361 PPC_DIVD(b2p[TMP_REG_1], dst_reg, src_reg); 362 PPC_MULD(b2p[TMP_REG_1], src_reg, 363 b2p[TMP_REG_1]); 364 PPC_SUB(dst_reg, dst_reg, b2p[TMP_REG_1]); 365 } else 366 PPC_DIVD(dst_reg, dst_reg, src_reg); 367 break; 368 case BPF_ALU | BPF_MOD | BPF_K: /* (u32) dst %= (u32) imm */ 369 case BPF_ALU | BPF_DIV | BPF_K: /* (u32) dst /= (u32) imm */ 370 case BPF_ALU64 | BPF_MOD | BPF_K: /* dst %= imm */ 371 case BPF_ALU64 | BPF_DIV | BPF_K: /* dst /= imm */ 372 if (imm == 0) 373 return -EINVAL; 374 else if (imm == 1) 375 goto bpf_alu32_trunc; 376 377 PPC_LI32(b2p[TMP_REG_1], imm); 378 switch (BPF_CLASS(code)) { 379 case BPF_ALU: 380 if (BPF_OP(code) == BPF_MOD) { 381 PPC_DIVWU(b2p[TMP_REG_2], dst_reg, 382 b2p[TMP_REG_1]); 383 PPC_MULW(b2p[TMP_REG_1], 384 b2p[TMP_REG_1], 385 b2p[TMP_REG_2]); 386 PPC_SUB(dst_reg, dst_reg, 387 b2p[TMP_REG_1]); 388 } else 389 PPC_DIVWU(dst_reg, dst_reg, 390 b2p[TMP_REG_1]); 391 break; 392 case BPF_ALU64: 393 if (BPF_OP(code) == BPF_MOD) { 394 PPC_DIVD(b2p[TMP_REG_2], dst_reg, 395 b2p[TMP_REG_1]); 396 PPC_MULD(b2p[TMP_REG_1], 397 b2p[TMP_REG_1], 398 b2p[TMP_REG_2]); 399 PPC_SUB(dst_reg, dst_reg, 400 b2p[TMP_REG_1]); 401 } else 402 PPC_DIVD(dst_reg, dst_reg, 403 b2p[TMP_REG_1]); 404 break; 405 } 406 goto bpf_alu32_trunc; 407 case BPF_ALU | BPF_NEG: /* (u32) dst = -dst */ 408 case BPF_ALU64 | BPF_NEG: /* dst = -dst */ 409 PPC_NEG(dst_reg, dst_reg); 410 goto bpf_alu32_trunc; 411 412 /* 413 * Logical operations: AND/OR/XOR/[A]LSH/[A]RSH 414 */ 415 case BPF_ALU | BPF_AND | BPF_X: /* (u32) dst = dst & src */ 416 case BPF_ALU64 | BPF_AND | BPF_X: /* dst = dst & src */ 417 PPC_AND(dst_reg, dst_reg, src_reg); 418 goto bpf_alu32_trunc; 419 case BPF_ALU | BPF_AND | BPF_K: /* (u32) dst = dst & imm */ 420 case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */ 421 if (!IMM_H(imm)) 422 PPC_ANDI(dst_reg, dst_reg, IMM_L(imm)); 423 else { 424 /* Sign-extended */ 425 PPC_LI32(b2p[TMP_REG_1], imm); 426 PPC_AND(dst_reg, dst_reg, b2p[TMP_REG_1]); 427 } 428 goto bpf_alu32_trunc; 429 case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */ 430 case BPF_ALU64 | BPF_OR | BPF_X: /* dst = dst | src */ 431 PPC_OR(dst_reg, dst_reg, src_reg); 432 goto bpf_alu32_trunc; 433 case BPF_ALU | BPF_OR | BPF_K:/* dst = (u32) dst | (u32) imm */ 434 case BPF_ALU64 | BPF_OR | BPF_K:/* dst = dst | imm */ 435 if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) { 436 /* Sign-extended */ 437 PPC_LI32(b2p[TMP_REG_1], imm); 438 PPC_OR(dst_reg, dst_reg, b2p[TMP_REG_1]); 439 } else { 440 if (IMM_L(imm)) 441 PPC_ORI(dst_reg, dst_reg, IMM_L(imm)); 442 if (IMM_H(imm)) 443 PPC_ORIS(dst_reg, dst_reg, IMM_H(imm)); 444 } 445 goto bpf_alu32_trunc; 446 case BPF_ALU | BPF_XOR | BPF_X: /* (u32) dst ^= src */ 447 case BPF_ALU64 | BPF_XOR | BPF_X: /* dst ^= src */ 448 PPC_XOR(dst_reg, dst_reg, src_reg); 449 goto bpf_alu32_trunc; 450 case BPF_ALU | BPF_XOR | BPF_K: /* (u32) dst ^= (u32) imm */ 451 case BPF_ALU64 | BPF_XOR | BPF_K: /* dst ^= imm */ 452 if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) { 453 /* Sign-extended */ 454 PPC_LI32(b2p[TMP_REG_1], imm); 455 PPC_XOR(dst_reg, dst_reg, b2p[TMP_REG_1]); 456 } else { 457 if (IMM_L(imm)) 458 PPC_XORI(dst_reg, dst_reg, IMM_L(imm)); 459 if (IMM_H(imm)) 460 PPC_XORIS(dst_reg, dst_reg, IMM_H(imm)); 461 } 462 goto bpf_alu32_trunc; 463 case BPF_ALU | BPF_LSH | BPF_X: /* (u32) dst <<= (u32) src */ 464 /* slw clears top 32 bits */ 465 PPC_SLW(dst_reg, dst_reg, src_reg); 466 break; 467 case BPF_ALU64 | BPF_LSH | BPF_X: /* dst <<= src; */ 468 PPC_SLD(dst_reg, dst_reg, src_reg); 469 break; 470 case BPF_ALU | BPF_LSH | BPF_K: /* (u32) dst <<== (u32) imm */ 471 /* with imm 0, we still need to clear top 32 bits */ 472 PPC_SLWI(dst_reg, dst_reg, imm); 473 break; 474 case BPF_ALU64 | BPF_LSH | BPF_K: /* dst <<== imm */ 475 if (imm != 0) 476 PPC_SLDI(dst_reg, dst_reg, imm); 477 break; 478 case BPF_ALU | BPF_RSH | BPF_X: /* (u32) dst >>= (u32) src */ 479 PPC_SRW(dst_reg, dst_reg, src_reg); 480 break; 481 case BPF_ALU64 | BPF_RSH | BPF_X: /* dst >>= src */ 482 PPC_SRD(dst_reg, dst_reg, src_reg); 483 break; 484 case BPF_ALU | BPF_RSH | BPF_K: /* (u32) dst >>= (u32) imm */ 485 PPC_SRWI(dst_reg, dst_reg, imm); 486 break; 487 case BPF_ALU64 | BPF_RSH | BPF_K: /* dst >>= imm */ 488 if (imm != 0) 489 PPC_SRDI(dst_reg, dst_reg, imm); 490 break; 491 case BPF_ALU64 | BPF_ARSH | BPF_X: /* (s64) dst >>= src */ 492 PPC_SRAD(dst_reg, dst_reg, src_reg); 493 break; 494 case BPF_ALU64 | BPF_ARSH | BPF_K: /* (s64) dst >>= imm */ 495 if (imm != 0) 496 PPC_SRADI(dst_reg, dst_reg, imm); 497 break; 498 499 /* 500 * MOV 501 */ 502 case BPF_ALU | BPF_MOV | BPF_X: /* (u32) dst = src */ 503 case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */ 504 PPC_MR(dst_reg, src_reg); 505 goto bpf_alu32_trunc; 506 case BPF_ALU | BPF_MOV | BPF_K: /* (u32) dst = imm */ 507 case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = (s64) imm */ 508 PPC_LI32(dst_reg, imm); 509 if (imm < 0) 510 goto bpf_alu32_trunc; 511 break; 512 513 bpf_alu32_trunc: 514 /* Truncate to 32-bits */ 515 if (BPF_CLASS(code) == BPF_ALU) 516 PPC_RLWINM(dst_reg, dst_reg, 0, 0, 31); 517 break; 518 519 /* 520 * BPF_FROM_BE/LE 521 */ 522 case BPF_ALU | BPF_END | BPF_FROM_LE: 523 case BPF_ALU | BPF_END | BPF_FROM_BE: 524 #ifdef __BIG_ENDIAN__ 525 if (BPF_SRC(code) == BPF_FROM_BE) 526 goto emit_clear; 527 #else /* !__BIG_ENDIAN__ */ 528 if (BPF_SRC(code) == BPF_FROM_LE) 529 goto emit_clear; 530 #endif 531 switch (imm) { 532 case 16: 533 /* Rotate 8 bits left & mask with 0x0000ff00 */ 534 PPC_RLWINM(b2p[TMP_REG_1], dst_reg, 8, 16, 23); 535 /* Rotate 8 bits right & insert LSB to reg */ 536 PPC_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 24, 31); 537 /* Move result back to dst_reg */ 538 PPC_MR(dst_reg, b2p[TMP_REG_1]); 539 break; 540 case 32: 541 /* 542 * Rotate word left by 8 bits: 543 * 2 bytes are already in their final position 544 * -- byte 2 and 4 (of bytes 1, 2, 3 and 4) 545 */ 546 PPC_RLWINM(b2p[TMP_REG_1], dst_reg, 8, 0, 31); 547 /* Rotate 24 bits and insert byte 1 */ 548 PPC_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 0, 7); 549 /* Rotate 24 bits and insert byte 3 */ 550 PPC_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 16, 23); 551 PPC_MR(dst_reg, b2p[TMP_REG_1]); 552 break; 553 case 64: 554 /* 555 * Way easier and faster(?) to store the value 556 * into stack and then use ldbrx 557 * 558 * ctx->seen will be reliable in pass2, but 559 * the instructions generated will remain the 560 * same across all passes 561 */ 562 PPC_STD(dst_reg, 1, bpf_jit_stack_local(ctx)); 563 PPC_ADDI(b2p[TMP_REG_1], 1, bpf_jit_stack_local(ctx)); 564 PPC_LDBRX(dst_reg, 0, b2p[TMP_REG_1]); 565 break; 566 } 567 break; 568 569 emit_clear: 570 switch (imm) { 571 case 16: 572 /* zero-extend 16 bits into 64 bits */ 573 PPC_RLDICL(dst_reg, dst_reg, 0, 48); 574 break; 575 case 32: 576 /* zero-extend 32 bits into 64 bits */ 577 PPC_RLDICL(dst_reg, dst_reg, 0, 32); 578 break; 579 case 64: 580 /* nop */ 581 break; 582 } 583 break; 584 585 /* 586 * BPF_ST(X) 587 */ 588 case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src */ 589 case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */ 590 if (BPF_CLASS(code) == BPF_ST) { 591 PPC_LI(b2p[TMP_REG_1], imm); 592 src_reg = b2p[TMP_REG_1]; 593 } 594 PPC_STB(src_reg, dst_reg, off); 595 break; 596 case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */ 597 case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */ 598 if (BPF_CLASS(code) == BPF_ST) { 599 PPC_LI(b2p[TMP_REG_1], imm); 600 src_reg = b2p[TMP_REG_1]; 601 } 602 PPC_STH(src_reg, dst_reg, off); 603 break; 604 case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */ 605 case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */ 606 if (BPF_CLASS(code) == BPF_ST) { 607 PPC_LI32(b2p[TMP_REG_1], imm); 608 src_reg = b2p[TMP_REG_1]; 609 } 610 PPC_STW(src_reg, dst_reg, off); 611 break; 612 case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */ 613 case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */ 614 if (BPF_CLASS(code) == BPF_ST) { 615 PPC_LI32(b2p[TMP_REG_1], imm); 616 src_reg = b2p[TMP_REG_1]; 617 } 618 PPC_STD(src_reg, dst_reg, off); 619 break; 620 621 /* 622 * BPF_STX XADD (atomic_add) 623 */ 624 /* *(u32 *)(dst + off) += src */ 625 case BPF_STX | BPF_XADD | BPF_W: 626 /* Get EA into TMP_REG_1 */ 627 PPC_ADDI(b2p[TMP_REG_1], dst_reg, off); 628 /* error if EA is not word-aligned */ 629 PPC_ANDI(b2p[TMP_REG_2], b2p[TMP_REG_1], 0x03); 630 PPC_BCC_SHORT(COND_EQ, (ctx->idx * 4) + 12); 631 PPC_LI(b2p[BPF_REG_0], 0); 632 PPC_JMP(exit_addr); 633 /* load value from memory into TMP_REG_2 */ 634 PPC_BPF_LWARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0); 635 /* add value from src_reg into this */ 636 PPC_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg); 637 /* store result back */ 638 PPC_BPF_STWCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]); 639 /* we're done if this succeeded */ 640 PPC_BCC_SHORT(COND_EQ, (ctx->idx * 4) + (7*4)); 641 /* otherwise, let's try once more */ 642 PPC_BPF_LWARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0); 643 PPC_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg); 644 PPC_BPF_STWCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]); 645 /* exit if the store was not successful */ 646 PPC_LI(b2p[BPF_REG_0], 0); 647 PPC_BCC(COND_NE, exit_addr); 648 break; 649 /* *(u64 *)(dst + off) += src */ 650 case BPF_STX | BPF_XADD | BPF_DW: 651 PPC_ADDI(b2p[TMP_REG_1], dst_reg, off); 652 /* error if EA is not doubleword-aligned */ 653 PPC_ANDI(b2p[TMP_REG_2], b2p[TMP_REG_1], 0x07); 654 PPC_BCC_SHORT(COND_EQ, (ctx->idx * 4) + (3*4)); 655 PPC_LI(b2p[BPF_REG_0], 0); 656 PPC_JMP(exit_addr); 657 PPC_BPF_LDARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0); 658 PPC_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg); 659 PPC_BPF_STDCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]); 660 PPC_BCC_SHORT(COND_EQ, (ctx->idx * 4) + (7*4)); 661 PPC_BPF_LDARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0); 662 PPC_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg); 663 PPC_BPF_STDCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]); 664 PPC_LI(b2p[BPF_REG_0], 0); 665 PPC_BCC(COND_NE, exit_addr); 666 break; 667 668 /* 669 * BPF_LDX 670 */ 671 /* dst = *(u8 *)(ul) (src + off) */ 672 case BPF_LDX | BPF_MEM | BPF_B: 673 PPC_LBZ(dst_reg, src_reg, off); 674 break; 675 /* dst = *(u16 *)(ul) (src + off) */ 676 case BPF_LDX | BPF_MEM | BPF_H: 677 PPC_LHZ(dst_reg, src_reg, off); 678 break; 679 /* dst = *(u32 *)(ul) (src + off) */ 680 case BPF_LDX | BPF_MEM | BPF_W: 681 PPC_LWZ(dst_reg, src_reg, off); 682 break; 683 /* dst = *(u64 *)(ul) (src + off) */ 684 case BPF_LDX | BPF_MEM | BPF_DW: 685 PPC_LD(dst_reg, src_reg, off); 686 break; 687 688 /* 689 * Doubleword load 690 * 16 byte instruction that uses two 'struct bpf_insn' 691 */ 692 case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */ 693 imm64 = ((u64)(u32) insn[i].imm) | 694 (((u64)(u32) insn[i+1].imm) << 32); 695 /* Adjust for two bpf instructions */ 696 addrs[++i] = ctx->idx * 4; 697 PPC_LI64(dst_reg, imm64); 698 break; 699 700 /* 701 * Return/Exit 702 */ 703 case BPF_JMP | BPF_EXIT: 704 /* 705 * If this isn't the very last instruction, branch to 706 * the epilogue. If we _are_ the last instruction, 707 * we'll just fall through to the epilogue. 708 */ 709 if (i != flen - 1) 710 PPC_JMP(exit_addr); 711 /* else fall through to the epilogue */ 712 break; 713 714 /* 715 * Call kernel helper 716 */ 717 case BPF_JMP | BPF_CALL: 718 ctx->seen |= SEEN_FUNC; 719 func = (u8 *) __bpf_call_base + imm; 720 721 bpf_jit_emit_func_call(image, ctx, (u64)func); 722 723 /* move return value from r3 to BPF_REG_0 */ 724 PPC_MR(b2p[BPF_REG_0], 3); 725 break; 726 727 /* 728 * Jumps and branches 729 */ 730 case BPF_JMP | BPF_JA: 731 PPC_JMP(addrs[i + 1 + off]); 732 break; 733 734 case BPF_JMP | BPF_JGT | BPF_K: 735 case BPF_JMP | BPF_JGT | BPF_X: 736 case BPF_JMP | BPF_JSGT | BPF_K: 737 case BPF_JMP | BPF_JSGT | BPF_X: 738 true_cond = COND_GT; 739 goto cond_branch; 740 case BPF_JMP | BPF_JLT | BPF_K: 741 case BPF_JMP | BPF_JLT | BPF_X: 742 case BPF_JMP | BPF_JSLT | BPF_K: 743 case BPF_JMP | BPF_JSLT | BPF_X: 744 true_cond = COND_LT; 745 goto cond_branch; 746 case BPF_JMP | BPF_JGE | BPF_K: 747 case BPF_JMP | BPF_JGE | BPF_X: 748 case BPF_JMP | BPF_JSGE | BPF_K: 749 case BPF_JMP | BPF_JSGE | BPF_X: 750 true_cond = COND_GE; 751 goto cond_branch; 752 case BPF_JMP | BPF_JLE | BPF_K: 753 case BPF_JMP | BPF_JLE | BPF_X: 754 case BPF_JMP | BPF_JSLE | BPF_K: 755 case BPF_JMP | BPF_JSLE | BPF_X: 756 true_cond = COND_LE; 757 goto cond_branch; 758 case BPF_JMP | BPF_JEQ | BPF_K: 759 case BPF_JMP | BPF_JEQ | BPF_X: 760 true_cond = COND_EQ; 761 goto cond_branch; 762 case BPF_JMP | BPF_JNE | BPF_K: 763 case BPF_JMP | BPF_JNE | BPF_X: 764 true_cond = COND_NE; 765 goto cond_branch; 766 case BPF_JMP | BPF_JSET | BPF_K: 767 case BPF_JMP | BPF_JSET | BPF_X: 768 true_cond = COND_NE; 769 /* Fall through */ 770 771 cond_branch: 772 switch (code) { 773 case BPF_JMP | BPF_JGT | BPF_X: 774 case BPF_JMP | BPF_JLT | BPF_X: 775 case BPF_JMP | BPF_JGE | BPF_X: 776 case BPF_JMP | BPF_JLE | BPF_X: 777 case BPF_JMP | BPF_JEQ | BPF_X: 778 case BPF_JMP | BPF_JNE | BPF_X: 779 /* unsigned comparison */ 780 PPC_CMPLD(dst_reg, src_reg); 781 break; 782 case BPF_JMP | BPF_JSGT | BPF_X: 783 case BPF_JMP | BPF_JSLT | BPF_X: 784 case BPF_JMP | BPF_JSGE | BPF_X: 785 case BPF_JMP | BPF_JSLE | BPF_X: 786 /* signed comparison */ 787 PPC_CMPD(dst_reg, src_reg); 788 break; 789 case BPF_JMP | BPF_JSET | BPF_X: 790 PPC_AND_DOT(b2p[TMP_REG_1], dst_reg, src_reg); 791 break; 792 case BPF_JMP | BPF_JNE | BPF_K: 793 case BPF_JMP | BPF_JEQ | BPF_K: 794 case BPF_JMP | BPF_JGT | BPF_K: 795 case BPF_JMP | BPF_JLT | BPF_K: 796 case BPF_JMP | BPF_JGE | BPF_K: 797 case BPF_JMP | BPF_JLE | BPF_K: 798 /* 799 * Need sign-extended load, so only positive 800 * values can be used as imm in cmpldi 801 */ 802 if (imm >= 0 && imm < 32768) 803 PPC_CMPLDI(dst_reg, imm); 804 else { 805 /* sign-extending load */ 806 PPC_LI32(b2p[TMP_REG_1], imm); 807 /* ... but unsigned comparison */ 808 PPC_CMPLD(dst_reg, b2p[TMP_REG_1]); 809 } 810 break; 811 case BPF_JMP | BPF_JSGT | BPF_K: 812 case BPF_JMP | BPF_JSLT | BPF_K: 813 case BPF_JMP | BPF_JSGE | BPF_K: 814 case BPF_JMP | BPF_JSLE | BPF_K: 815 /* 816 * signed comparison, so any 16-bit value 817 * can be used in cmpdi 818 */ 819 if (imm >= -32768 && imm < 32768) 820 PPC_CMPDI(dst_reg, imm); 821 else { 822 PPC_LI32(b2p[TMP_REG_1], imm); 823 PPC_CMPD(dst_reg, b2p[TMP_REG_1]); 824 } 825 break; 826 case BPF_JMP | BPF_JSET | BPF_K: 827 /* andi does not sign-extend the immediate */ 828 if (imm >= 0 && imm < 32768) 829 /* PPC_ANDI is _only/always_ dot-form */ 830 PPC_ANDI(b2p[TMP_REG_1], dst_reg, imm); 831 else { 832 PPC_LI32(b2p[TMP_REG_1], imm); 833 PPC_AND_DOT(b2p[TMP_REG_1], dst_reg, 834 b2p[TMP_REG_1]); 835 } 836 break; 837 } 838 PPC_BCC(true_cond, addrs[i + 1 + off]); 839 break; 840 841 /* 842 * Tail call 843 */ 844 case BPF_JMP | BPF_TAIL_CALL: 845 ctx->seen |= SEEN_TAILCALL; 846 bpf_jit_emit_tail_call(image, ctx, addrs[i + 1]); 847 break; 848 849 default: 850 /* 851 * The filter contains something cruel & unusual. 852 * We don't handle it, but also there shouldn't be 853 * anything missing from our list. 854 */ 855 pr_err_ratelimited("eBPF filter opcode %04x (@%d) unsupported\n", 856 code, i); 857 return -ENOTSUPP; 858 } 859 } 860 861 /* Set end-of-body-code address for exit. */ 862 addrs[i] = ctx->idx * 4; 863 864 return 0; 865 } 866 867 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) 868 { 869 u32 proglen; 870 u32 alloclen; 871 u8 *image = NULL; 872 u32 *code_base; 873 u32 *addrs; 874 struct codegen_context cgctx; 875 int pass; 876 int flen; 877 struct bpf_binary_header *bpf_hdr; 878 struct bpf_prog *org_fp = fp; 879 struct bpf_prog *tmp_fp; 880 bool bpf_blinded = false; 881 882 if (!fp->jit_requested) 883 return org_fp; 884 885 tmp_fp = bpf_jit_blind_constants(org_fp); 886 if (IS_ERR(tmp_fp)) 887 return org_fp; 888 889 if (tmp_fp != org_fp) { 890 bpf_blinded = true; 891 fp = tmp_fp; 892 } 893 894 flen = fp->len; 895 addrs = kzalloc((flen+1) * sizeof(*addrs), GFP_KERNEL); 896 if (addrs == NULL) { 897 fp = org_fp; 898 goto out; 899 } 900 901 memset(&cgctx, 0, sizeof(struct codegen_context)); 902 903 /* Make sure that the stack is quadword aligned. */ 904 cgctx.stack_size = round_up(fp->aux->stack_depth, 16); 905 906 /* Scouting faux-generate pass 0 */ 907 if (bpf_jit_build_body(fp, 0, &cgctx, addrs)) { 908 /* We hit something illegal or unsupported. */ 909 fp = org_fp; 910 goto out; 911 } 912 913 /* 914 * Pretend to build prologue, given the features we've seen. This will 915 * update ctgtx.idx as it pretends to output instructions, then we can 916 * calculate total size from idx. 917 */ 918 bpf_jit_build_prologue(0, &cgctx); 919 bpf_jit_build_epilogue(0, &cgctx); 920 921 proglen = cgctx.idx * 4; 922 alloclen = proglen + FUNCTION_DESCR_SIZE; 923 924 bpf_hdr = bpf_jit_binary_alloc(alloclen, &image, 4, 925 bpf_jit_fill_ill_insns); 926 if (!bpf_hdr) { 927 fp = org_fp; 928 goto out; 929 } 930 931 code_base = (u32 *)(image + FUNCTION_DESCR_SIZE); 932 933 /* Code generation passes 1-2 */ 934 for (pass = 1; pass < 3; pass++) { 935 /* Now build the prologue, body code & epilogue for real. */ 936 cgctx.idx = 0; 937 bpf_jit_build_prologue(code_base, &cgctx); 938 bpf_jit_build_body(fp, code_base, &cgctx, addrs); 939 bpf_jit_build_epilogue(code_base, &cgctx); 940 941 if (bpf_jit_enable > 1) 942 pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass, 943 proglen - (cgctx.idx * 4), cgctx.seen); 944 } 945 946 if (bpf_jit_enable > 1) 947 /* 948 * Note that we output the base address of the code_base 949 * rather than image, since opcodes are in code_base. 950 */ 951 bpf_jit_dump(flen, proglen, pass, code_base); 952 953 #ifdef PPC64_ELF_ABI_v1 954 /* Function descriptor nastiness: Address + TOC */ 955 ((u64 *)image)[0] = (u64)code_base; 956 ((u64 *)image)[1] = local_paca->kernel_toc; 957 #endif 958 959 fp->bpf_func = (void *)image; 960 fp->jited = 1; 961 fp->jited_len = alloclen; 962 963 bpf_flush_icache(bpf_hdr, (u8 *)bpf_hdr + (bpf_hdr->pages * PAGE_SIZE)); 964 965 out: 966 kfree(addrs); 967 968 if (bpf_blinded) 969 bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp); 970 971 return fp; 972 } 973 974 /* Overriding bpf_jit_free() as we don't set images read-only. */ 975 void bpf_jit_free(struct bpf_prog *fp) 976 { 977 unsigned long addr = (unsigned long)fp->bpf_func & PAGE_MASK; 978 struct bpf_binary_header *bpf_hdr = (void *)addr; 979 980 if (fp->jited) 981 bpf_jit_binary_free(bpf_hdr); 982 983 bpf_prog_unlock_free(fp); 984 } 985