1 /* 2 * bpf_jit_comp64.c: eBPF JIT compiler 3 * 4 * Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com> 5 * IBM Corporation 6 * 7 * Based on the powerpc classic BPF JIT compiler by Matt Evans 8 * 9 * This program is free software; you can redistribute it and/or 10 * modify it under the terms of the GNU General Public License 11 * as published by the Free Software Foundation; version 2 12 * of the License. 13 */ 14 #include <linux/moduleloader.h> 15 #include <asm/cacheflush.h> 16 #include <linux/netdevice.h> 17 #include <linux/filter.h> 18 #include <linux/if_vlan.h> 19 #include <asm/kprobes.h> 20 #include <linux/bpf.h> 21 22 #include "bpf_jit64.h" 23 24 int bpf_jit_enable __read_mostly; 25 26 static void bpf_jit_fill_ill_insns(void *area, unsigned int size) 27 { 28 int *p = area; 29 30 /* Fill whole space with trap instructions */ 31 while (p < (int *)((char *)area + size)) 32 *p++ = BREAKPOINT_INSTRUCTION; 33 } 34 35 static inline void bpf_flush_icache(void *start, void *end) 36 { 37 smp_wmb(); 38 flush_icache_range((unsigned long)start, (unsigned long)end); 39 } 40 41 static inline bool bpf_is_seen_register(struct codegen_context *ctx, int i) 42 { 43 return (ctx->seen & (1 << (31 - b2p[i]))); 44 } 45 46 static inline void bpf_set_seen_register(struct codegen_context *ctx, int i) 47 { 48 ctx->seen |= (1 << (31 - b2p[i])); 49 } 50 51 static inline bool bpf_has_stack_frame(struct codegen_context *ctx) 52 { 53 /* 54 * We only need a stack frame if: 55 * - we call other functions (kernel helpers), or 56 * - the bpf program uses its stack area 57 * The latter condition is deduced from the usage of BPF_REG_FP 58 */ 59 return ctx->seen & SEEN_FUNC || bpf_is_seen_register(ctx, BPF_REG_FP); 60 } 61 62 /* 63 * When not setting up our own stackframe, the redzone usage is: 64 * 65 * [ prev sp ] <------------- 66 * [ ... ] | 67 * sp (r1) ---> [ stack pointer ] -------------- 68 * [ nv gpr save area ] 8*8 69 * [ tail_call_cnt ] 8 70 * [ local_tmp_var ] 8 71 * [ unused red zone ] 208 bytes protected 72 */ 73 static int bpf_jit_stack_local(struct codegen_context *ctx) 74 { 75 if (bpf_has_stack_frame(ctx)) 76 return STACK_FRAME_MIN_SIZE + MAX_BPF_STACK; 77 else 78 return -(BPF_PPC_STACK_SAVE + 16); 79 } 80 81 static int bpf_jit_stack_tailcallcnt(struct codegen_context *ctx) 82 { 83 return bpf_jit_stack_local(ctx) + 8; 84 } 85 86 static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg) 87 { 88 if (reg >= BPF_PPC_NVR_MIN && reg < 32) 89 return (bpf_has_stack_frame(ctx) ? BPF_PPC_STACKFRAME : 0) 90 - (8 * (32 - reg)); 91 92 pr_err("BPF JIT is asking about unknown registers"); 93 BUG(); 94 } 95 96 static void bpf_jit_emit_skb_loads(u32 *image, struct codegen_context *ctx) 97 { 98 /* 99 * Load skb->len and skb->data_len 100 * r3 points to skb 101 */ 102 PPC_LWZ(b2p[SKB_HLEN_REG], 3, offsetof(struct sk_buff, len)); 103 PPC_LWZ(b2p[TMP_REG_1], 3, offsetof(struct sk_buff, data_len)); 104 /* header_len = len - data_len */ 105 PPC_SUB(b2p[SKB_HLEN_REG], b2p[SKB_HLEN_REG], b2p[TMP_REG_1]); 106 107 /* skb->data pointer */ 108 PPC_BPF_LL(b2p[SKB_DATA_REG], 3, offsetof(struct sk_buff, data)); 109 } 110 111 static void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx) 112 { 113 int i; 114 115 /* 116 * Initialize tail_call_cnt if we do tail calls. 117 * Otherwise, put in NOPs so that it can be skipped when we are 118 * invoked through a tail call. 119 */ 120 if (ctx->seen & SEEN_TAILCALL) { 121 PPC_LI(b2p[TMP_REG_1], 0); 122 /* this goes in the redzone */ 123 PPC_BPF_STL(b2p[TMP_REG_1], 1, -(BPF_PPC_STACK_SAVE + 8)); 124 } else { 125 PPC_NOP(); 126 PPC_NOP(); 127 } 128 129 #define BPF_TAILCALL_PROLOGUE_SIZE 8 130 131 if (bpf_has_stack_frame(ctx)) { 132 /* 133 * We need a stack frame, but we don't necessarily need to 134 * save/restore LR unless we call other functions 135 */ 136 if (ctx->seen & SEEN_FUNC) { 137 EMIT(PPC_INST_MFLR | __PPC_RT(R0)); 138 PPC_BPF_STL(0, 1, PPC_LR_STKOFF); 139 } 140 141 PPC_BPF_STLU(1, 1, -BPF_PPC_STACKFRAME); 142 } 143 144 /* 145 * Back up non-volatile regs -- BPF registers 6-10 146 * If we haven't created our own stack frame, we save these 147 * in the protected zone below the previous stack frame 148 */ 149 for (i = BPF_REG_6; i <= BPF_REG_10; i++) 150 if (bpf_is_seen_register(ctx, i)) 151 PPC_BPF_STL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i])); 152 153 /* 154 * Save additional non-volatile regs if we cache skb 155 * Also, setup skb data 156 */ 157 if (ctx->seen & SEEN_SKB) { 158 PPC_BPF_STL(b2p[SKB_HLEN_REG], 1, 159 bpf_jit_stack_offsetof(ctx, b2p[SKB_HLEN_REG])); 160 PPC_BPF_STL(b2p[SKB_DATA_REG], 1, 161 bpf_jit_stack_offsetof(ctx, b2p[SKB_DATA_REG])); 162 bpf_jit_emit_skb_loads(image, ctx); 163 } 164 165 /* Setup frame pointer to point to the bpf stack area */ 166 if (bpf_is_seen_register(ctx, BPF_REG_FP)) 167 PPC_ADDI(b2p[BPF_REG_FP], 1, 168 STACK_FRAME_MIN_SIZE + MAX_BPF_STACK); 169 } 170 171 static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context *ctx) 172 { 173 int i; 174 175 /* Restore NVRs */ 176 for (i = BPF_REG_6; i <= BPF_REG_10; i++) 177 if (bpf_is_seen_register(ctx, i)) 178 PPC_BPF_LL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i])); 179 180 /* Restore non-volatile registers used for skb cache */ 181 if (ctx->seen & SEEN_SKB) { 182 PPC_BPF_LL(b2p[SKB_HLEN_REG], 1, 183 bpf_jit_stack_offsetof(ctx, b2p[SKB_HLEN_REG])); 184 PPC_BPF_LL(b2p[SKB_DATA_REG], 1, 185 bpf_jit_stack_offsetof(ctx, b2p[SKB_DATA_REG])); 186 } 187 188 /* Tear down our stack frame */ 189 if (bpf_has_stack_frame(ctx)) { 190 PPC_ADDI(1, 1, BPF_PPC_STACKFRAME); 191 if (ctx->seen & SEEN_FUNC) { 192 PPC_BPF_LL(0, 1, PPC_LR_STKOFF); 193 PPC_MTLR(0); 194 } 195 } 196 } 197 198 static void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx) 199 { 200 bpf_jit_emit_common_epilogue(image, ctx); 201 202 /* Move result to r3 */ 203 PPC_MR(3, b2p[BPF_REG_0]); 204 205 PPC_BLR(); 206 } 207 208 static void bpf_jit_emit_func_call(u32 *image, struct codegen_context *ctx, u64 func) 209 { 210 #ifdef PPC64_ELF_ABI_v1 211 /* func points to the function descriptor */ 212 PPC_LI64(b2p[TMP_REG_2], func); 213 /* Load actual entry point from function descriptor */ 214 PPC_BPF_LL(b2p[TMP_REG_1], b2p[TMP_REG_2], 0); 215 /* ... and move it to LR */ 216 PPC_MTLR(b2p[TMP_REG_1]); 217 /* 218 * Load TOC from function descriptor at offset 8. 219 * We can clobber r2 since we get called through a 220 * function pointer (so caller will save/restore r2) 221 * and since we don't use a TOC ourself. 222 */ 223 PPC_BPF_LL(2, b2p[TMP_REG_2], 8); 224 #else 225 /* We can clobber r12 */ 226 PPC_FUNC_ADDR(12, func); 227 PPC_MTLR(12); 228 #endif 229 PPC_BLRL(); 230 } 231 232 static void bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 out) 233 { 234 /* 235 * By now, the eBPF program has already setup parameters in r3, r4 and r5 236 * r3/BPF_REG_1 - pointer to ctx -- passed as is to the next bpf program 237 * r4/BPF_REG_2 - pointer to bpf_array 238 * r5/BPF_REG_3 - index in bpf_array 239 */ 240 int b2p_bpf_array = b2p[BPF_REG_2]; 241 int b2p_index = b2p[BPF_REG_3]; 242 243 /* 244 * if (index >= array->map.max_entries) 245 * goto out; 246 */ 247 PPC_LWZ(b2p[TMP_REG_1], b2p_bpf_array, offsetof(struct bpf_array, map.max_entries)); 248 PPC_CMPLW(b2p_index, b2p[TMP_REG_1]); 249 PPC_BCC(COND_GE, out); 250 251 /* 252 * if (tail_call_cnt > MAX_TAIL_CALL_CNT) 253 * goto out; 254 */ 255 PPC_LD(b2p[TMP_REG_1], 1, bpf_jit_stack_tailcallcnt(ctx)); 256 PPC_CMPLWI(b2p[TMP_REG_1], MAX_TAIL_CALL_CNT); 257 PPC_BCC(COND_GT, out); 258 259 /* 260 * tail_call_cnt++; 261 */ 262 PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], 1); 263 PPC_BPF_STL(b2p[TMP_REG_1], 1, bpf_jit_stack_tailcallcnt(ctx)); 264 265 /* prog = array->ptrs[index]; */ 266 PPC_MULI(b2p[TMP_REG_1], b2p_index, 8); 267 PPC_ADD(b2p[TMP_REG_1], b2p[TMP_REG_1], b2p_bpf_array); 268 PPC_LD(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_array, ptrs)); 269 270 /* 271 * if (prog == NULL) 272 * goto out; 273 */ 274 PPC_CMPLDI(b2p[TMP_REG_1], 0); 275 PPC_BCC(COND_EQ, out); 276 277 /* goto *(prog->bpf_func + prologue_size); */ 278 PPC_LD(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_prog, bpf_func)); 279 #ifdef PPC64_ELF_ABI_v1 280 /* skip past the function descriptor */ 281 PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], 282 FUNCTION_DESCR_SIZE + BPF_TAILCALL_PROLOGUE_SIZE); 283 #else 284 PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], BPF_TAILCALL_PROLOGUE_SIZE); 285 #endif 286 PPC_MTCTR(b2p[TMP_REG_1]); 287 288 /* tear down stack, restore NVRs, ... */ 289 bpf_jit_emit_common_epilogue(image, ctx); 290 291 PPC_BCTR(); 292 /* out: */ 293 } 294 295 /* Assemble the body code between the prologue & epilogue */ 296 static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, 297 struct codegen_context *ctx, 298 u32 *addrs) 299 { 300 const struct bpf_insn *insn = fp->insnsi; 301 int flen = fp->len; 302 int i; 303 304 /* Start of epilogue code - will only be valid 2nd pass onwards */ 305 u32 exit_addr = addrs[flen]; 306 307 for (i = 0; i < flen; i++) { 308 u32 code = insn[i].code; 309 u32 dst_reg = b2p[insn[i].dst_reg]; 310 u32 src_reg = b2p[insn[i].src_reg]; 311 s16 off = insn[i].off; 312 s32 imm = insn[i].imm; 313 u64 imm64; 314 u8 *func; 315 u32 true_cond; 316 317 /* 318 * addrs[] maps a BPF bytecode address into a real offset from 319 * the start of the body code. 320 */ 321 addrs[i] = ctx->idx * 4; 322 323 /* 324 * As an optimization, we note down which non-volatile registers 325 * are used so that we can only save/restore those in our 326 * prologue and epilogue. We do this here regardless of whether 327 * the actual BPF instruction uses src/dst registers or not 328 * (for instance, BPF_CALL does not use them). The expectation 329 * is that those instructions will have src_reg/dst_reg set to 330 * 0. Even otherwise, we just lose some prologue/epilogue 331 * optimization but everything else should work without 332 * any issues. 333 */ 334 if (dst_reg >= BPF_PPC_NVR_MIN && dst_reg < 32) 335 bpf_set_seen_register(ctx, insn[i].dst_reg); 336 if (src_reg >= BPF_PPC_NVR_MIN && src_reg < 32) 337 bpf_set_seen_register(ctx, insn[i].src_reg); 338 339 switch (code) { 340 /* 341 * Arithmetic operations: ADD/SUB/MUL/DIV/MOD/NEG 342 */ 343 case BPF_ALU | BPF_ADD | BPF_X: /* (u32) dst += (u32) src */ 344 case BPF_ALU64 | BPF_ADD | BPF_X: /* dst += src */ 345 PPC_ADD(dst_reg, dst_reg, src_reg); 346 goto bpf_alu32_trunc; 347 case BPF_ALU | BPF_SUB | BPF_X: /* (u32) dst -= (u32) src */ 348 case BPF_ALU64 | BPF_SUB | BPF_X: /* dst -= src */ 349 PPC_SUB(dst_reg, dst_reg, src_reg); 350 goto bpf_alu32_trunc; 351 case BPF_ALU | BPF_ADD | BPF_K: /* (u32) dst += (u32) imm */ 352 case BPF_ALU | BPF_SUB | BPF_K: /* (u32) dst -= (u32) imm */ 353 case BPF_ALU64 | BPF_ADD | BPF_K: /* dst += imm */ 354 case BPF_ALU64 | BPF_SUB | BPF_K: /* dst -= imm */ 355 if (BPF_OP(code) == BPF_SUB) 356 imm = -imm; 357 if (imm) { 358 if (imm >= -32768 && imm < 32768) 359 PPC_ADDI(dst_reg, dst_reg, IMM_L(imm)); 360 else { 361 PPC_LI32(b2p[TMP_REG_1], imm); 362 PPC_ADD(dst_reg, dst_reg, b2p[TMP_REG_1]); 363 } 364 } 365 goto bpf_alu32_trunc; 366 case BPF_ALU | BPF_MUL | BPF_X: /* (u32) dst *= (u32) src */ 367 case BPF_ALU64 | BPF_MUL | BPF_X: /* dst *= src */ 368 if (BPF_CLASS(code) == BPF_ALU) 369 PPC_MULW(dst_reg, dst_reg, src_reg); 370 else 371 PPC_MULD(dst_reg, dst_reg, src_reg); 372 goto bpf_alu32_trunc; 373 case BPF_ALU | BPF_MUL | BPF_K: /* (u32) dst *= (u32) imm */ 374 case BPF_ALU64 | BPF_MUL | BPF_K: /* dst *= imm */ 375 if (imm >= -32768 && imm < 32768) 376 PPC_MULI(dst_reg, dst_reg, IMM_L(imm)); 377 else { 378 PPC_LI32(b2p[TMP_REG_1], imm); 379 if (BPF_CLASS(code) == BPF_ALU) 380 PPC_MULW(dst_reg, dst_reg, 381 b2p[TMP_REG_1]); 382 else 383 PPC_MULD(dst_reg, dst_reg, 384 b2p[TMP_REG_1]); 385 } 386 goto bpf_alu32_trunc; 387 case BPF_ALU | BPF_DIV | BPF_X: /* (u32) dst /= (u32) src */ 388 case BPF_ALU | BPF_MOD | BPF_X: /* (u32) dst %= (u32) src */ 389 PPC_CMPWI(src_reg, 0); 390 PPC_BCC_SHORT(COND_NE, (ctx->idx * 4) + 12); 391 PPC_LI(b2p[BPF_REG_0], 0); 392 PPC_JMP(exit_addr); 393 if (BPF_OP(code) == BPF_MOD) { 394 PPC_DIVWU(b2p[TMP_REG_1], dst_reg, src_reg); 395 PPC_MULW(b2p[TMP_REG_1], src_reg, 396 b2p[TMP_REG_1]); 397 PPC_SUB(dst_reg, dst_reg, b2p[TMP_REG_1]); 398 } else 399 PPC_DIVWU(dst_reg, dst_reg, src_reg); 400 goto bpf_alu32_trunc; 401 case BPF_ALU64 | BPF_DIV | BPF_X: /* dst /= src */ 402 case BPF_ALU64 | BPF_MOD | BPF_X: /* dst %= src */ 403 PPC_CMPDI(src_reg, 0); 404 PPC_BCC_SHORT(COND_NE, (ctx->idx * 4) + 12); 405 PPC_LI(b2p[BPF_REG_0], 0); 406 PPC_JMP(exit_addr); 407 if (BPF_OP(code) == BPF_MOD) { 408 PPC_DIVD(b2p[TMP_REG_1], dst_reg, src_reg); 409 PPC_MULD(b2p[TMP_REG_1], src_reg, 410 b2p[TMP_REG_1]); 411 PPC_SUB(dst_reg, dst_reg, b2p[TMP_REG_1]); 412 } else 413 PPC_DIVD(dst_reg, dst_reg, src_reg); 414 break; 415 case BPF_ALU | BPF_MOD | BPF_K: /* (u32) dst %= (u32) imm */ 416 case BPF_ALU | BPF_DIV | BPF_K: /* (u32) dst /= (u32) imm */ 417 case BPF_ALU64 | BPF_MOD | BPF_K: /* dst %= imm */ 418 case BPF_ALU64 | BPF_DIV | BPF_K: /* dst /= imm */ 419 if (imm == 0) 420 return -EINVAL; 421 else if (imm == 1) 422 goto bpf_alu32_trunc; 423 424 PPC_LI32(b2p[TMP_REG_1], imm); 425 switch (BPF_CLASS(code)) { 426 case BPF_ALU: 427 if (BPF_OP(code) == BPF_MOD) { 428 PPC_DIVWU(b2p[TMP_REG_2], dst_reg, 429 b2p[TMP_REG_1]); 430 PPC_MULW(b2p[TMP_REG_1], 431 b2p[TMP_REG_1], 432 b2p[TMP_REG_2]); 433 PPC_SUB(dst_reg, dst_reg, 434 b2p[TMP_REG_1]); 435 } else 436 PPC_DIVWU(dst_reg, dst_reg, 437 b2p[TMP_REG_1]); 438 break; 439 case BPF_ALU64: 440 if (BPF_OP(code) == BPF_MOD) { 441 PPC_DIVD(b2p[TMP_REG_2], dst_reg, 442 b2p[TMP_REG_1]); 443 PPC_MULD(b2p[TMP_REG_1], 444 b2p[TMP_REG_1], 445 b2p[TMP_REG_2]); 446 PPC_SUB(dst_reg, dst_reg, 447 b2p[TMP_REG_1]); 448 } else 449 PPC_DIVD(dst_reg, dst_reg, 450 b2p[TMP_REG_1]); 451 break; 452 } 453 goto bpf_alu32_trunc; 454 case BPF_ALU | BPF_NEG: /* (u32) dst = -dst */ 455 case BPF_ALU64 | BPF_NEG: /* dst = -dst */ 456 PPC_NEG(dst_reg, dst_reg); 457 goto bpf_alu32_trunc; 458 459 /* 460 * Logical operations: AND/OR/XOR/[A]LSH/[A]RSH 461 */ 462 case BPF_ALU | BPF_AND | BPF_X: /* (u32) dst = dst & src */ 463 case BPF_ALU64 | BPF_AND | BPF_X: /* dst = dst & src */ 464 PPC_AND(dst_reg, dst_reg, src_reg); 465 goto bpf_alu32_trunc; 466 case BPF_ALU | BPF_AND | BPF_K: /* (u32) dst = dst & imm */ 467 case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */ 468 if (!IMM_H(imm)) 469 PPC_ANDI(dst_reg, dst_reg, IMM_L(imm)); 470 else { 471 /* Sign-extended */ 472 PPC_LI32(b2p[TMP_REG_1], imm); 473 PPC_AND(dst_reg, dst_reg, b2p[TMP_REG_1]); 474 } 475 goto bpf_alu32_trunc; 476 case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */ 477 case BPF_ALU64 | BPF_OR | BPF_X: /* dst = dst | src */ 478 PPC_OR(dst_reg, dst_reg, src_reg); 479 goto bpf_alu32_trunc; 480 case BPF_ALU | BPF_OR | BPF_K:/* dst = (u32) dst | (u32) imm */ 481 case BPF_ALU64 | BPF_OR | BPF_K:/* dst = dst | imm */ 482 if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) { 483 /* Sign-extended */ 484 PPC_LI32(b2p[TMP_REG_1], imm); 485 PPC_OR(dst_reg, dst_reg, b2p[TMP_REG_1]); 486 } else { 487 if (IMM_L(imm)) 488 PPC_ORI(dst_reg, dst_reg, IMM_L(imm)); 489 if (IMM_H(imm)) 490 PPC_ORIS(dst_reg, dst_reg, IMM_H(imm)); 491 } 492 goto bpf_alu32_trunc; 493 case BPF_ALU | BPF_XOR | BPF_X: /* (u32) dst ^= src */ 494 case BPF_ALU64 | BPF_XOR | BPF_X: /* dst ^= src */ 495 PPC_XOR(dst_reg, dst_reg, src_reg); 496 goto bpf_alu32_trunc; 497 case BPF_ALU | BPF_XOR | BPF_K: /* (u32) dst ^= (u32) imm */ 498 case BPF_ALU64 | BPF_XOR | BPF_K: /* dst ^= imm */ 499 if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) { 500 /* Sign-extended */ 501 PPC_LI32(b2p[TMP_REG_1], imm); 502 PPC_XOR(dst_reg, dst_reg, b2p[TMP_REG_1]); 503 } else { 504 if (IMM_L(imm)) 505 PPC_XORI(dst_reg, dst_reg, IMM_L(imm)); 506 if (IMM_H(imm)) 507 PPC_XORIS(dst_reg, dst_reg, IMM_H(imm)); 508 } 509 goto bpf_alu32_trunc; 510 case BPF_ALU | BPF_LSH | BPF_X: /* (u32) dst <<= (u32) src */ 511 /* slw clears top 32 bits */ 512 PPC_SLW(dst_reg, dst_reg, src_reg); 513 break; 514 case BPF_ALU64 | BPF_LSH | BPF_X: /* dst <<= src; */ 515 PPC_SLD(dst_reg, dst_reg, src_reg); 516 break; 517 case BPF_ALU | BPF_LSH | BPF_K: /* (u32) dst <<== (u32) imm */ 518 /* with imm 0, we still need to clear top 32 bits */ 519 PPC_SLWI(dst_reg, dst_reg, imm); 520 break; 521 case BPF_ALU64 | BPF_LSH | BPF_K: /* dst <<== imm */ 522 if (imm != 0) 523 PPC_SLDI(dst_reg, dst_reg, imm); 524 break; 525 case BPF_ALU | BPF_RSH | BPF_X: /* (u32) dst >>= (u32) src */ 526 PPC_SRW(dst_reg, dst_reg, src_reg); 527 break; 528 case BPF_ALU64 | BPF_RSH | BPF_X: /* dst >>= src */ 529 PPC_SRD(dst_reg, dst_reg, src_reg); 530 break; 531 case BPF_ALU | BPF_RSH | BPF_K: /* (u32) dst >>= (u32) imm */ 532 PPC_SRWI(dst_reg, dst_reg, imm); 533 break; 534 case BPF_ALU64 | BPF_RSH | BPF_K: /* dst >>= imm */ 535 if (imm != 0) 536 PPC_SRDI(dst_reg, dst_reg, imm); 537 break; 538 case BPF_ALU64 | BPF_ARSH | BPF_X: /* (s64) dst >>= src */ 539 PPC_SRAD(dst_reg, dst_reg, src_reg); 540 break; 541 case BPF_ALU64 | BPF_ARSH | BPF_K: /* (s64) dst >>= imm */ 542 if (imm != 0) 543 PPC_SRADI(dst_reg, dst_reg, imm); 544 break; 545 546 /* 547 * MOV 548 */ 549 case BPF_ALU | BPF_MOV | BPF_X: /* (u32) dst = src */ 550 case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */ 551 PPC_MR(dst_reg, src_reg); 552 goto bpf_alu32_trunc; 553 case BPF_ALU | BPF_MOV | BPF_K: /* (u32) dst = imm */ 554 case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = (s64) imm */ 555 PPC_LI32(dst_reg, imm); 556 if (imm < 0) 557 goto bpf_alu32_trunc; 558 break; 559 560 bpf_alu32_trunc: 561 /* Truncate to 32-bits */ 562 if (BPF_CLASS(code) == BPF_ALU) 563 PPC_RLWINM(dst_reg, dst_reg, 0, 0, 31); 564 break; 565 566 /* 567 * BPF_FROM_BE/LE 568 */ 569 case BPF_ALU | BPF_END | BPF_FROM_LE: 570 case BPF_ALU | BPF_END | BPF_FROM_BE: 571 #ifdef __BIG_ENDIAN__ 572 if (BPF_SRC(code) == BPF_FROM_BE) 573 goto emit_clear; 574 #else /* !__BIG_ENDIAN__ */ 575 if (BPF_SRC(code) == BPF_FROM_LE) 576 goto emit_clear; 577 #endif 578 switch (imm) { 579 case 16: 580 /* Rotate 8 bits left & mask with 0x0000ff00 */ 581 PPC_RLWINM(b2p[TMP_REG_1], dst_reg, 8, 16, 23); 582 /* Rotate 8 bits right & insert LSB to reg */ 583 PPC_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 24, 31); 584 /* Move result back to dst_reg */ 585 PPC_MR(dst_reg, b2p[TMP_REG_1]); 586 break; 587 case 32: 588 /* 589 * Rotate word left by 8 bits: 590 * 2 bytes are already in their final position 591 * -- byte 2 and 4 (of bytes 1, 2, 3 and 4) 592 */ 593 PPC_RLWINM(b2p[TMP_REG_1], dst_reg, 8, 0, 31); 594 /* Rotate 24 bits and insert byte 1 */ 595 PPC_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 0, 7); 596 /* Rotate 24 bits and insert byte 3 */ 597 PPC_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 16, 23); 598 PPC_MR(dst_reg, b2p[TMP_REG_1]); 599 break; 600 case 64: 601 /* 602 * Way easier and faster(?) to store the value 603 * into stack and then use ldbrx 604 * 605 * ctx->seen will be reliable in pass2, but 606 * the instructions generated will remain the 607 * same across all passes 608 */ 609 PPC_STD(dst_reg, 1, bpf_jit_stack_local(ctx)); 610 PPC_ADDI(b2p[TMP_REG_1], 1, bpf_jit_stack_local(ctx)); 611 PPC_LDBRX(dst_reg, 0, b2p[TMP_REG_1]); 612 break; 613 } 614 break; 615 616 emit_clear: 617 switch (imm) { 618 case 16: 619 /* zero-extend 16 bits into 64 bits */ 620 PPC_RLDICL(dst_reg, dst_reg, 0, 48); 621 break; 622 case 32: 623 /* zero-extend 32 bits into 64 bits */ 624 PPC_RLDICL(dst_reg, dst_reg, 0, 32); 625 break; 626 case 64: 627 /* nop */ 628 break; 629 } 630 break; 631 632 /* 633 * BPF_ST(X) 634 */ 635 case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src */ 636 case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */ 637 if (BPF_CLASS(code) == BPF_ST) { 638 PPC_LI(b2p[TMP_REG_1], imm); 639 src_reg = b2p[TMP_REG_1]; 640 } 641 PPC_STB(src_reg, dst_reg, off); 642 break; 643 case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */ 644 case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */ 645 if (BPF_CLASS(code) == BPF_ST) { 646 PPC_LI(b2p[TMP_REG_1], imm); 647 src_reg = b2p[TMP_REG_1]; 648 } 649 PPC_STH(src_reg, dst_reg, off); 650 break; 651 case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */ 652 case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */ 653 if (BPF_CLASS(code) == BPF_ST) { 654 PPC_LI32(b2p[TMP_REG_1], imm); 655 src_reg = b2p[TMP_REG_1]; 656 } 657 PPC_STW(src_reg, dst_reg, off); 658 break; 659 case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */ 660 case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */ 661 if (BPF_CLASS(code) == BPF_ST) { 662 PPC_LI32(b2p[TMP_REG_1], imm); 663 src_reg = b2p[TMP_REG_1]; 664 } 665 PPC_STD(src_reg, dst_reg, off); 666 break; 667 668 /* 669 * BPF_STX XADD (atomic_add) 670 */ 671 /* *(u32 *)(dst + off) += src */ 672 case BPF_STX | BPF_XADD | BPF_W: 673 /* Get EA into TMP_REG_1 */ 674 PPC_ADDI(b2p[TMP_REG_1], dst_reg, off); 675 /* error if EA is not word-aligned */ 676 PPC_ANDI(b2p[TMP_REG_2], b2p[TMP_REG_1], 0x03); 677 PPC_BCC_SHORT(COND_EQ, (ctx->idx * 4) + 12); 678 PPC_LI(b2p[BPF_REG_0], 0); 679 PPC_JMP(exit_addr); 680 /* load value from memory into TMP_REG_2 */ 681 PPC_BPF_LWARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0); 682 /* add value from src_reg into this */ 683 PPC_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg); 684 /* store result back */ 685 PPC_BPF_STWCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]); 686 /* we're done if this succeeded */ 687 PPC_BCC_SHORT(COND_EQ, (ctx->idx * 4) + (7*4)); 688 /* otherwise, let's try once more */ 689 PPC_BPF_LWARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0); 690 PPC_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg); 691 PPC_BPF_STWCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]); 692 /* exit if the store was not successful */ 693 PPC_LI(b2p[BPF_REG_0], 0); 694 PPC_BCC(COND_NE, exit_addr); 695 break; 696 /* *(u64 *)(dst + off) += src */ 697 case BPF_STX | BPF_XADD | BPF_DW: 698 PPC_ADDI(b2p[TMP_REG_1], dst_reg, off); 699 /* error if EA is not doubleword-aligned */ 700 PPC_ANDI(b2p[TMP_REG_2], b2p[TMP_REG_1], 0x07); 701 PPC_BCC_SHORT(COND_EQ, (ctx->idx * 4) + (3*4)); 702 PPC_LI(b2p[BPF_REG_0], 0); 703 PPC_JMP(exit_addr); 704 PPC_BPF_LDARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0); 705 PPC_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg); 706 PPC_BPF_STDCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]); 707 PPC_BCC_SHORT(COND_EQ, (ctx->idx * 4) + (7*4)); 708 PPC_BPF_LDARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0); 709 PPC_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg); 710 PPC_BPF_STDCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]); 711 PPC_LI(b2p[BPF_REG_0], 0); 712 PPC_BCC(COND_NE, exit_addr); 713 break; 714 715 /* 716 * BPF_LDX 717 */ 718 /* dst = *(u8 *)(ul) (src + off) */ 719 case BPF_LDX | BPF_MEM | BPF_B: 720 PPC_LBZ(dst_reg, src_reg, off); 721 break; 722 /* dst = *(u16 *)(ul) (src + off) */ 723 case BPF_LDX | BPF_MEM | BPF_H: 724 PPC_LHZ(dst_reg, src_reg, off); 725 break; 726 /* dst = *(u32 *)(ul) (src + off) */ 727 case BPF_LDX | BPF_MEM | BPF_W: 728 PPC_LWZ(dst_reg, src_reg, off); 729 break; 730 /* dst = *(u64 *)(ul) (src + off) */ 731 case BPF_LDX | BPF_MEM | BPF_DW: 732 PPC_LD(dst_reg, src_reg, off); 733 break; 734 735 /* 736 * Doubleword load 737 * 16 byte instruction that uses two 'struct bpf_insn' 738 */ 739 case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */ 740 imm64 = ((u64)(u32) insn[i].imm) | 741 (((u64)(u32) insn[i+1].imm) << 32); 742 /* Adjust for two bpf instructions */ 743 addrs[++i] = ctx->idx * 4; 744 PPC_LI64(dst_reg, imm64); 745 break; 746 747 /* 748 * Return/Exit 749 */ 750 case BPF_JMP | BPF_EXIT: 751 /* 752 * If this isn't the very last instruction, branch to 753 * the epilogue. If we _are_ the last instruction, 754 * we'll just fall through to the epilogue. 755 */ 756 if (i != flen - 1) 757 PPC_JMP(exit_addr); 758 /* else fall through to the epilogue */ 759 break; 760 761 /* 762 * Call kernel helper 763 */ 764 case BPF_JMP | BPF_CALL: 765 ctx->seen |= SEEN_FUNC; 766 func = (u8 *) __bpf_call_base + imm; 767 768 /* Save skb pointer if we need to re-cache skb data */ 769 if (bpf_helper_changes_skb_data(func)) 770 PPC_BPF_STL(3, 1, bpf_jit_stack_local(ctx)); 771 772 bpf_jit_emit_func_call(image, ctx, (u64)func); 773 774 /* move return value from r3 to BPF_REG_0 */ 775 PPC_MR(b2p[BPF_REG_0], 3); 776 777 /* refresh skb cache */ 778 if (bpf_helper_changes_skb_data(func)) { 779 /* reload skb pointer to r3 */ 780 PPC_BPF_LL(3, 1, bpf_jit_stack_local(ctx)); 781 bpf_jit_emit_skb_loads(image, ctx); 782 } 783 break; 784 785 /* 786 * Jumps and branches 787 */ 788 case BPF_JMP | BPF_JA: 789 PPC_JMP(addrs[i + 1 + off]); 790 break; 791 792 case BPF_JMP | BPF_JGT | BPF_K: 793 case BPF_JMP | BPF_JGT | BPF_X: 794 case BPF_JMP | BPF_JSGT | BPF_K: 795 case BPF_JMP | BPF_JSGT | BPF_X: 796 true_cond = COND_GT; 797 goto cond_branch; 798 case BPF_JMP | BPF_JGE | BPF_K: 799 case BPF_JMP | BPF_JGE | BPF_X: 800 case BPF_JMP | BPF_JSGE | BPF_K: 801 case BPF_JMP | BPF_JSGE | BPF_X: 802 true_cond = COND_GE; 803 goto cond_branch; 804 case BPF_JMP | BPF_JEQ | BPF_K: 805 case BPF_JMP | BPF_JEQ | BPF_X: 806 true_cond = COND_EQ; 807 goto cond_branch; 808 case BPF_JMP | BPF_JNE | BPF_K: 809 case BPF_JMP | BPF_JNE | BPF_X: 810 true_cond = COND_NE; 811 goto cond_branch; 812 case BPF_JMP | BPF_JSET | BPF_K: 813 case BPF_JMP | BPF_JSET | BPF_X: 814 true_cond = COND_NE; 815 /* Fall through */ 816 817 cond_branch: 818 switch (code) { 819 case BPF_JMP | BPF_JGT | BPF_X: 820 case BPF_JMP | BPF_JGE | BPF_X: 821 case BPF_JMP | BPF_JEQ | BPF_X: 822 case BPF_JMP | BPF_JNE | BPF_X: 823 /* unsigned comparison */ 824 PPC_CMPLD(dst_reg, src_reg); 825 break; 826 case BPF_JMP | BPF_JSGT | BPF_X: 827 case BPF_JMP | BPF_JSGE | BPF_X: 828 /* signed comparison */ 829 PPC_CMPD(dst_reg, src_reg); 830 break; 831 case BPF_JMP | BPF_JSET | BPF_X: 832 PPC_AND_DOT(b2p[TMP_REG_1], dst_reg, src_reg); 833 break; 834 case BPF_JMP | BPF_JNE | BPF_K: 835 case BPF_JMP | BPF_JEQ | BPF_K: 836 case BPF_JMP | BPF_JGT | BPF_K: 837 case BPF_JMP | BPF_JGE | BPF_K: 838 /* 839 * Need sign-extended load, so only positive 840 * values can be used as imm in cmpldi 841 */ 842 if (imm >= 0 && imm < 32768) 843 PPC_CMPLDI(dst_reg, imm); 844 else { 845 /* sign-extending load */ 846 PPC_LI32(b2p[TMP_REG_1], imm); 847 /* ... but unsigned comparison */ 848 PPC_CMPLD(dst_reg, b2p[TMP_REG_1]); 849 } 850 break; 851 case BPF_JMP | BPF_JSGT | BPF_K: 852 case BPF_JMP | BPF_JSGE | BPF_K: 853 /* 854 * signed comparison, so any 16-bit value 855 * can be used in cmpdi 856 */ 857 if (imm >= -32768 && imm < 32768) 858 PPC_CMPDI(dst_reg, imm); 859 else { 860 PPC_LI32(b2p[TMP_REG_1], imm); 861 PPC_CMPD(dst_reg, b2p[TMP_REG_1]); 862 } 863 break; 864 case BPF_JMP | BPF_JSET | BPF_K: 865 /* andi does not sign-extend the immediate */ 866 if (imm >= 0 && imm < 32768) 867 /* PPC_ANDI is _only/always_ dot-form */ 868 PPC_ANDI(b2p[TMP_REG_1], dst_reg, imm); 869 else { 870 PPC_LI32(b2p[TMP_REG_1], imm); 871 PPC_AND_DOT(b2p[TMP_REG_1], dst_reg, 872 b2p[TMP_REG_1]); 873 } 874 break; 875 } 876 PPC_BCC(true_cond, addrs[i + 1 + off]); 877 break; 878 879 /* 880 * Loads from packet header/data 881 * Assume 32-bit input value in imm and X (src_reg) 882 */ 883 884 /* Absolute loads */ 885 case BPF_LD | BPF_W | BPF_ABS: 886 func = (u8 *)CHOOSE_LOAD_FUNC(imm, sk_load_word); 887 goto common_load_abs; 888 case BPF_LD | BPF_H | BPF_ABS: 889 func = (u8 *)CHOOSE_LOAD_FUNC(imm, sk_load_half); 890 goto common_load_abs; 891 case BPF_LD | BPF_B | BPF_ABS: 892 func = (u8 *)CHOOSE_LOAD_FUNC(imm, sk_load_byte); 893 common_load_abs: 894 /* 895 * Load from [imm] 896 * Load into r4, which can just be passed onto 897 * skb load helpers as the second parameter 898 */ 899 PPC_LI32(4, imm); 900 goto common_load; 901 902 /* Indirect loads */ 903 case BPF_LD | BPF_W | BPF_IND: 904 func = (u8 *)sk_load_word; 905 goto common_load_ind; 906 case BPF_LD | BPF_H | BPF_IND: 907 func = (u8 *)sk_load_half; 908 goto common_load_ind; 909 case BPF_LD | BPF_B | BPF_IND: 910 func = (u8 *)sk_load_byte; 911 common_load_ind: 912 /* 913 * Load from [src_reg + imm] 914 * Treat src_reg as a 32-bit value 915 */ 916 PPC_EXTSW(4, src_reg); 917 if (imm) { 918 if (imm >= -32768 && imm < 32768) 919 PPC_ADDI(4, 4, IMM_L(imm)); 920 else { 921 PPC_LI32(b2p[TMP_REG_1], imm); 922 PPC_ADD(4, 4, b2p[TMP_REG_1]); 923 } 924 } 925 926 common_load: 927 ctx->seen |= SEEN_SKB; 928 ctx->seen |= SEEN_FUNC; 929 bpf_jit_emit_func_call(image, ctx, (u64)func); 930 931 /* 932 * Helper returns 'lt' condition on error, and an 933 * appropriate return value in BPF_REG_0 934 */ 935 PPC_BCC(COND_LT, exit_addr); 936 break; 937 938 /* 939 * Tail call 940 */ 941 case BPF_JMP | BPF_CALL | BPF_X: 942 ctx->seen |= SEEN_TAILCALL; 943 bpf_jit_emit_tail_call(image, ctx, addrs[i + 1]); 944 break; 945 946 default: 947 /* 948 * The filter contains something cruel & unusual. 949 * We don't handle it, but also there shouldn't be 950 * anything missing from our list. 951 */ 952 pr_err_ratelimited("eBPF filter opcode %04x (@%d) unsupported\n", 953 code, i); 954 return -ENOTSUPP; 955 } 956 } 957 958 /* Set end-of-body-code address for exit. */ 959 addrs[i] = ctx->idx * 4; 960 961 return 0; 962 } 963 964 void bpf_jit_compile(struct bpf_prog *fp) { } 965 966 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) 967 { 968 u32 proglen; 969 u32 alloclen; 970 u8 *image = NULL; 971 u32 *code_base; 972 u32 *addrs; 973 struct codegen_context cgctx; 974 int pass; 975 int flen; 976 struct bpf_binary_header *bpf_hdr; 977 struct bpf_prog *org_fp = fp; 978 struct bpf_prog *tmp_fp; 979 bool bpf_blinded = false; 980 981 if (!bpf_jit_enable) 982 return org_fp; 983 984 tmp_fp = bpf_jit_blind_constants(org_fp); 985 if (IS_ERR(tmp_fp)) 986 return org_fp; 987 988 if (tmp_fp != org_fp) { 989 bpf_blinded = true; 990 fp = tmp_fp; 991 } 992 993 flen = fp->len; 994 addrs = kzalloc((flen+1) * sizeof(*addrs), GFP_KERNEL); 995 if (addrs == NULL) { 996 fp = org_fp; 997 goto out; 998 } 999 1000 memset(&cgctx, 0, sizeof(struct codegen_context)); 1001 1002 /* Scouting faux-generate pass 0 */ 1003 if (bpf_jit_build_body(fp, 0, &cgctx, addrs)) { 1004 /* We hit something illegal or unsupported. */ 1005 fp = org_fp; 1006 goto out; 1007 } 1008 1009 /* 1010 * Pretend to build prologue, given the features we've seen. This will 1011 * update ctgtx.idx as it pretends to output instructions, then we can 1012 * calculate total size from idx. 1013 */ 1014 bpf_jit_build_prologue(0, &cgctx); 1015 bpf_jit_build_epilogue(0, &cgctx); 1016 1017 proglen = cgctx.idx * 4; 1018 alloclen = proglen + FUNCTION_DESCR_SIZE; 1019 1020 bpf_hdr = bpf_jit_binary_alloc(alloclen, &image, 4, 1021 bpf_jit_fill_ill_insns); 1022 if (!bpf_hdr) { 1023 fp = org_fp; 1024 goto out; 1025 } 1026 1027 code_base = (u32 *)(image + FUNCTION_DESCR_SIZE); 1028 1029 /* Code generation passes 1-2 */ 1030 for (pass = 1; pass < 3; pass++) { 1031 /* Now build the prologue, body code & epilogue for real. */ 1032 cgctx.idx = 0; 1033 bpf_jit_build_prologue(code_base, &cgctx); 1034 bpf_jit_build_body(fp, code_base, &cgctx, addrs); 1035 bpf_jit_build_epilogue(code_base, &cgctx); 1036 1037 if (bpf_jit_enable > 1) 1038 pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass, 1039 proglen - (cgctx.idx * 4), cgctx.seen); 1040 } 1041 1042 if (bpf_jit_enable > 1) 1043 /* 1044 * Note that we output the base address of the code_base 1045 * rather than image, since opcodes are in code_base. 1046 */ 1047 bpf_jit_dump(flen, proglen, pass, code_base); 1048 1049 if (image) { 1050 bpf_flush_icache(bpf_hdr, image + alloclen); 1051 #ifdef PPC64_ELF_ABI_v1 1052 /* Function descriptor nastiness: Address + TOC */ 1053 ((u64 *)image)[0] = (u64)code_base; 1054 ((u64 *)image)[1] = local_paca->kernel_toc; 1055 #endif 1056 fp->bpf_func = (void *)image; 1057 fp->jited = 1; 1058 } 1059 1060 out: 1061 kfree(addrs); 1062 1063 if (bpf_blinded) 1064 bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp); 1065 1066 return fp; 1067 } 1068 1069 void bpf_jit_free(struct bpf_prog *fp) 1070 { 1071 unsigned long addr = (unsigned long)fp->bpf_func & PAGE_MASK; 1072 struct bpf_binary_header *bpf_hdr = (void *)addr; 1073 1074 if (fp->jited) 1075 bpf_jit_binary_free(bpf_hdr); 1076 1077 bpf_prog_unlock_free(fp); 1078 } 1079