1 #include <linux/moduleloader.h> 2 #include <linux/workqueue.h> 3 #include <linux/netdevice.h> 4 #include <linux/filter.h> 5 #include <linux/bpf.h> 6 #include <linux/cache.h> 7 #include <linux/if_vlan.h> 8 9 #include <asm/cacheflush.h> 10 #include <asm/ptrace.h> 11 12 #include "bpf_jit_64.h" 13 14 int bpf_jit_enable __read_mostly; 15 16 static inline bool is_simm13(unsigned int value) 17 { 18 return value + 0x1000 < 0x2000; 19 } 20 21 static inline bool is_simm10(unsigned int value) 22 { 23 return value + 0x200 < 0x400; 24 } 25 26 static inline bool is_simm5(unsigned int value) 27 { 28 return value + 0x10 < 0x20; 29 } 30 31 static inline bool is_sethi(unsigned int value) 32 { 33 return (value & ~0x3fffff) == 0; 34 } 35 36 static void bpf_flush_icache(void *start_, void *end_) 37 { 38 /* Cheetah's I-cache is fully coherent. */ 39 if (tlb_type == spitfire) { 40 unsigned long start = (unsigned long) start_; 41 unsigned long end = (unsigned long) end_; 42 43 start &= ~7UL; 44 end = (end + 7UL) & ~7UL; 45 while (start < end) { 46 flushi(start); 47 start += 32; 48 } 49 } 50 } 51 52 #define SEEN_DATAREF 1 /* might call external helpers */ 53 #define SEEN_XREG 2 /* ebx is used */ 54 #define SEEN_MEM 4 /* use mem[] for temporary storage */ 55 56 #define S13(X) ((X) & 0x1fff) 57 #define S5(X) ((X) & 0x1f) 58 #define IMMED 0x00002000 59 #define RD(X) ((X) << 25) 60 #define RS1(X) ((X) << 14) 61 #define RS2(X) ((X)) 62 #define OP(X) ((X) << 30) 63 #define OP2(X) ((X) << 22) 64 #define OP3(X) ((X) << 19) 65 #define COND(X) (((X) & 0xf) << 25) 66 #define CBCOND(X) (((X) & 0x1f) << 25) 67 #define F1(X) OP(X) 68 #define F2(X, Y) (OP(X) | OP2(Y)) 69 #define F3(X, Y) (OP(X) | OP3(Y)) 70 #define ASI(X) (((X) & 0xff) << 5) 71 72 #define CONDN COND(0x0) 73 #define CONDE COND(0x1) 74 #define CONDLE COND(0x2) 75 #define CONDL COND(0x3) 76 #define CONDLEU COND(0x4) 77 #define CONDCS COND(0x5) 78 #define CONDNEG COND(0x6) 79 #define CONDVC COND(0x7) 80 #define CONDA COND(0x8) 81 #define CONDNE COND(0x9) 82 #define CONDG COND(0xa) 83 #define CONDGE COND(0xb) 84 #define CONDGU COND(0xc) 85 #define CONDCC COND(0xd) 86 #define CONDPOS COND(0xe) 87 #define CONDVS COND(0xf) 88 89 #define CONDGEU CONDCC 90 #define CONDLU CONDCS 91 92 #define WDISP22(X) (((X) >> 2) & 0x3fffff) 93 #define WDISP19(X) (((X) >> 2) & 0x7ffff) 94 95 /* The 10-bit branch displacement for CBCOND is split into two fields */ 96 static u32 WDISP10(u32 off) 97 { 98 u32 ret = ((off >> 2) & 0xff) << 5; 99 100 ret |= ((off >> (2 + 8)) & 0x03) << 19; 101 102 return ret; 103 } 104 105 #define CBCONDE CBCOND(0x09) 106 #define CBCONDLE CBCOND(0x0a) 107 #define CBCONDL CBCOND(0x0b) 108 #define CBCONDLEU CBCOND(0x0c) 109 #define CBCONDCS CBCOND(0x0d) 110 #define CBCONDN CBCOND(0x0e) 111 #define CBCONDVS CBCOND(0x0f) 112 #define CBCONDNE CBCOND(0x19) 113 #define CBCONDG CBCOND(0x1a) 114 #define CBCONDGE CBCOND(0x1b) 115 #define CBCONDGU CBCOND(0x1c) 116 #define CBCONDCC CBCOND(0x1d) 117 #define CBCONDPOS CBCOND(0x1e) 118 #define CBCONDVC CBCOND(0x1f) 119 120 #define CBCONDGEU CBCONDCC 121 #define CBCONDLU CBCONDCS 122 123 #define ANNUL (1 << 29) 124 #define XCC (1 << 21) 125 126 #define BRANCH (F2(0, 1) | XCC) 127 #define CBCOND_OP (F2(0, 3) | XCC) 128 129 #define BA (BRANCH | CONDA) 130 #define BG (BRANCH | CONDG) 131 #define BGU (BRANCH | CONDGU) 132 #define BLEU (BRANCH | CONDLEU) 133 #define BGE (BRANCH | CONDGE) 134 #define BGEU (BRANCH | CONDGEU) 135 #define BLU (BRANCH | CONDLU) 136 #define BE (BRANCH | CONDE) 137 #define BNE (BRANCH | CONDNE) 138 139 #define SETHI(K, REG) \ 140 (F2(0, 0x4) | RD(REG) | (((K) >> 10) & 0x3fffff)) 141 #define OR_LO(K, REG) \ 142 (F3(2, 0x02) | IMMED | RS1(REG) | ((K) & 0x3ff) | RD(REG)) 143 144 #define ADD F3(2, 0x00) 145 #define AND F3(2, 0x01) 146 #define ANDCC F3(2, 0x11) 147 #define OR F3(2, 0x02) 148 #define XOR F3(2, 0x03) 149 #define SUB F3(2, 0x04) 150 #define SUBCC F3(2, 0x14) 151 #define MUL F3(2, 0x0a) 152 #define MULX F3(2, 0x09) 153 #define UDIVX F3(2, 0x0d) 154 #define DIV F3(2, 0x0e) 155 #define SLL F3(2, 0x25) 156 #define SLLX (F3(2, 0x25)|(1<<12)) 157 #define SRA F3(2, 0x27) 158 #define SRAX (F3(2, 0x27)|(1<<12)) 159 #define SRL F3(2, 0x26) 160 #define SRLX (F3(2, 0x26)|(1<<12)) 161 #define JMPL F3(2, 0x38) 162 #define SAVE F3(2, 0x3c) 163 #define RESTORE F3(2, 0x3d) 164 #define CALL F1(1) 165 #define BR F2(0, 0x01) 166 #define RD_Y F3(2, 0x28) 167 #define WR_Y F3(2, 0x30) 168 169 #define LD32 F3(3, 0x00) 170 #define LD8 F3(3, 0x01) 171 #define LD16 F3(3, 0x02) 172 #define LD64 F3(3, 0x0b) 173 #define LD64A F3(3, 0x1b) 174 #define ST8 F3(3, 0x05) 175 #define ST16 F3(3, 0x06) 176 #define ST32 F3(3, 0x04) 177 #define ST64 F3(3, 0x0e) 178 179 #define CAS F3(3, 0x3c) 180 #define CASX F3(3, 0x3e) 181 182 #define LDPTR LD64 183 #define BASE_STACKFRAME 176 184 185 #define LD32I (LD32 | IMMED) 186 #define LD8I (LD8 | IMMED) 187 #define LD16I (LD16 | IMMED) 188 #define LD64I (LD64 | IMMED) 189 #define LDPTRI (LDPTR | IMMED) 190 #define ST32I (ST32 | IMMED) 191 192 struct jit_ctx { 193 struct bpf_prog *prog; 194 unsigned int *offset; 195 int idx; 196 int epilogue_offset; 197 bool tmp_1_used; 198 bool tmp_2_used; 199 bool tmp_3_used; 200 bool saw_ld_abs_ind; 201 bool saw_frame_pointer; 202 bool saw_call; 203 bool saw_tail_call; 204 u32 *image; 205 }; 206 207 #define TMP_REG_1 (MAX_BPF_JIT_REG + 0) 208 #define TMP_REG_2 (MAX_BPF_JIT_REG + 1) 209 #define SKB_HLEN_REG (MAX_BPF_JIT_REG + 2) 210 #define SKB_DATA_REG (MAX_BPF_JIT_REG + 3) 211 #define TMP_REG_3 (MAX_BPF_JIT_REG + 4) 212 213 /* Map BPF registers to SPARC registers */ 214 static const int bpf2sparc[] = { 215 /* return value from in-kernel function, and exit value from eBPF */ 216 [BPF_REG_0] = O5, 217 218 /* arguments from eBPF program to in-kernel function */ 219 [BPF_REG_1] = O0, 220 [BPF_REG_2] = O1, 221 [BPF_REG_3] = O2, 222 [BPF_REG_4] = O3, 223 [BPF_REG_5] = O4, 224 225 /* callee saved registers that in-kernel function will preserve */ 226 [BPF_REG_6] = L0, 227 [BPF_REG_7] = L1, 228 [BPF_REG_8] = L2, 229 [BPF_REG_9] = L3, 230 231 /* read-only frame pointer to access stack */ 232 [BPF_REG_FP] = L6, 233 234 [BPF_REG_AX] = G7, 235 236 /* temporary register for internal BPF JIT */ 237 [TMP_REG_1] = G1, 238 [TMP_REG_2] = G2, 239 [TMP_REG_3] = G3, 240 241 [SKB_HLEN_REG] = L4, 242 [SKB_DATA_REG] = L5, 243 }; 244 245 static void emit(const u32 insn, struct jit_ctx *ctx) 246 { 247 if (ctx->image != NULL) 248 ctx->image[ctx->idx] = insn; 249 250 ctx->idx++; 251 } 252 253 static void emit_call(u32 *func, struct jit_ctx *ctx) 254 { 255 if (ctx->image != NULL) { 256 void *here = &ctx->image[ctx->idx]; 257 unsigned int off; 258 259 off = (void *)func - here; 260 ctx->image[ctx->idx] = CALL | ((off >> 2) & 0x3fffffff); 261 } 262 ctx->idx++; 263 } 264 265 static void emit_nop(struct jit_ctx *ctx) 266 { 267 emit(SETHI(0, G0), ctx); 268 } 269 270 static void emit_reg_move(u32 from, u32 to, struct jit_ctx *ctx) 271 { 272 emit(OR | RS1(G0) | RS2(from) | RD(to), ctx); 273 } 274 275 /* Emit 32-bit constant, zero extended. */ 276 static void emit_set_const(s32 K, u32 reg, struct jit_ctx *ctx) 277 { 278 emit(SETHI(K, reg), ctx); 279 emit(OR_LO(K, reg), ctx); 280 } 281 282 /* Emit 32-bit constant, sign extended. */ 283 static void emit_set_const_sext(s32 K, u32 reg, struct jit_ctx *ctx) 284 { 285 if (K >= 0) { 286 emit(SETHI(K, reg), ctx); 287 emit(OR_LO(K, reg), ctx); 288 } else { 289 u32 hbits = ~(u32) K; 290 u32 lbits = -0x400 | (u32) K; 291 292 emit(SETHI(hbits, reg), ctx); 293 emit(XOR | IMMED | RS1(reg) | S13(lbits) | RD(reg), ctx); 294 } 295 } 296 297 static void emit_alu(u32 opcode, u32 src, u32 dst, struct jit_ctx *ctx) 298 { 299 emit(opcode | RS1(dst) | RS2(src) | RD(dst), ctx); 300 } 301 302 static void emit_alu3(u32 opcode, u32 a, u32 b, u32 c, struct jit_ctx *ctx) 303 { 304 emit(opcode | RS1(a) | RS2(b) | RD(c), ctx); 305 } 306 307 static void emit_alu_K(unsigned int opcode, unsigned int dst, unsigned int imm, 308 struct jit_ctx *ctx) 309 { 310 bool small_immed = is_simm13(imm); 311 unsigned int insn = opcode; 312 313 insn |= RS1(dst) | RD(dst); 314 if (small_immed) { 315 emit(insn | IMMED | S13(imm), ctx); 316 } else { 317 unsigned int tmp = bpf2sparc[TMP_REG_1]; 318 319 ctx->tmp_1_used = true; 320 321 emit_set_const_sext(imm, tmp, ctx); 322 emit(insn | RS2(tmp), ctx); 323 } 324 } 325 326 static void emit_alu3_K(unsigned int opcode, unsigned int src, unsigned int imm, 327 unsigned int dst, struct jit_ctx *ctx) 328 { 329 bool small_immed = is_simm13(imm); 330 unsigned int insn = opcode; 331 332 insn |= RS1(src) | RD(dst); 333 if (small_immed) { 334 emit(insn | IMMED | S13(imm), ctx); 335 } else { 336 unsigned int tmp = bpf2sparc[TMP_REG_1]; 337 338 ctx->tmp_1_used = true; 339 340 emit_set_const_sext(imm, tmp, ctx); 341 emit(insn | RS2(tmp), ctx); 342 } 343 } 344 345 static void emit_loadimm32(s32 K, unsigned int dest, struct jit_ctx *ctx) 346 { 347 if (K >= 0 && is_simm13(K)) { 348 /* or %g0, K, DEST */ 349 emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx); 350 } else { 351 emit_set_const(K, dest, ctx); 352 } 353 } 354 355 static void emit_loadimm(s32 K, unsigned int dest, struct jit_ctx *ctx) 356 { 357 if (is_simm13(K)) { 358 /* or %g0, K, DEST */ 359 emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx); 360 } else { 361 emit_set_const(K, dest, ctx); 362 } 363 } 364 365 static void emit_loadimm_sext(s32 K, unsigned int dest, struct jit_ctx *ctx) 366 { 367 if (is_simm13(K)) { 368 /* or %g0, K, DEST */ 369 emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx); 370 } else { 371 emit_set_const_sext(K, dest, ctx); 372 } 373 } 374 375 static void analyze_64bit_constant(u32 high_bits, u32 low_bits, 376 int *hbsp, int *lbsp, int *abbasp) 377 { 378 int lowest_bit_set, highest_bit_set, all_bits_between_are_set; 379 int i; 380 381 lowest_bit_set = highest_bit_set = -1; 382 i = 0; 383 do { 384 if ((lowest_bit_set == -1) && ((low_bits >> i) & 1)) 385 lowest_bit_set = i; 386 if ((highest_bit_set == -1) && ((high_bits >> (32 - i - 1)) & 1)) 387 highest_bit_set = (64 - i - 1); 388 } while (++i < 32 && (highest_bit_set == -1 || 389 lowest_bit_set == -1)); 390 if (i == 32) { 391 i = 0; 392 do { 393 if (lowest_bit_set == -1 && ((high_bits >> i) & 1)) 394 lowest_bit_set = i + 32; 395 if (highest_bit_set == -1 && 396 ((low_bits >> (32 - i - 1)) & 1)) 397 highest_bit_set = 32 - i - 1; 398 } while (++i < 32 && (highest_bit_set == -1 || 399 lowest_bit_set == -1)); 400 } 401 402 all_bits_between_are_set = 1; 403 for (i = lowest_bit_set; i <= highest_bit_set; i++) { 404 if (i < 32) { 405 if ((low_bits & (1 << i)) != 0) 406 continue; 407 } else { 408 if ((high_bits & (1 << (i - 32))) != 0) 409 continue; 410 } 411 all_bits_between_are_set = 0; 412 break; 413 } 414 *hbsp = highest_bit_set; 415 *lbsp = lowest_bit_set; 416 *abbasp = all_bits_between_are_set; 417 } 418 419 static unsigned long create_simple_focus_bits(unsigned long high_bits, 420 unsigned long low_bits, 421 int lowest_bit_set, int shift) 422 { 423 long hi, lo; 424 425 if (lowest_bit_set < 32) { 426 lo = (low_bits >> lowest_bit_set) << shift; 427 hi = ((high_bits << (32 - lowest_bit_set)) << shift); 428 } else { 429 lo = 0; 430 hi = ((high_bits >> (lowest_bit_set - 32)) << shift); 431 } 432 return hi | lo; 433 } 434 435 static bool const64_is_2insns(unsigned long high_bits, 436 unsigned long low_bits) 437 { 438 int highest_bit_set, lowest_bit_set, all_bits_between_are_set; 439 440 if (high_bits == 0 || high_bits == 0xffffffff) 441 return true; 442 443 analyze_64bit_constant(high_bits, low_bits, 444 &highest_bit_set, &lowest_bit_set, 445 &all_bits_between_are_set); 446 447 if ((highest_bit_set == 63 || lowest_bit_set == 0) && 448 all_bits_between_are_set != 0) 449 return true; 450 451 if (highest_bit_set - lowest_bit_set < 21) 452 return true; 453 454 return false; 455 } 456 457 static void sparc_emit_set_const64_quick2(unsigned long high_bits, 458 unsigned long low_imm, 459 unsigned int dest, 460 int shift_count, struct jit_ctx *ctx) 461 { 462 emit_loadimm32(high_bits, dest, ctx); 463 464 /* Now shift it up into place. */ 465 emit_alu_K(SLLX, dest, shift_count, ctx); 466 467 /* If there is a low immediate part piece, finish up by 468 * putting that in as well. 469 */ 470 if (low_imm != 0) 471 emit(OR | IMMED | RS1(dest) | S13(low_imm) | RD(dest), ctx); 472 } 473 474 static void emit_loadimm64(u64 K, unsigned int dest, struct jit_ctx *ctx) 475 { 476 int all_bits_between_are_set, lowest_bit_set, highest_bit_set; 477 unsigned int tmp = bpf2sparc[TMP_REG_1]; 478 u32 low_bits = (K & 0xffffffff); 479 u32 high_bits = (K >> 32); 480 481 /* These two tests also take care of all of the one 482 * instruction cases. 483 */ 484 if (high_bits == 0xffffffff && (low_bits & 0x80000000)) 485 return emit_loadimm_sext(K, dest, ctx); 486 if (high_bits == 0x00000000) 487 return emit_loadimm32(K, dest, ctx); 488 489 analyze_64bit_constant(high_bits, low_bits, &highest_bit_set, 490 &lowest_bit_set, &all_bits_between_are_set); 491 492 /* 1) mov -1, %reg 493 * sllx %reg, shift, %reg 494 * 2) mov -1, %reg 495 * srlx %reg, shift, %reg 496 * 3) mov some_small_const, %reg 497 * sllx %reg, shift, %reg 498 */ 499 if (((highest_bit_set == 63 || lowest_bit_set == 0) && 500 all_bits_between_are_set != 0) || 501 ((highest_bit_set - lowest_bit_set) < 12)) { 502 int shift = lowest_bit_set; 503 long the_const = -1; 504 505 if ((highest_bit_set != 63 && lowest_bit_set != 0) || 506 all_bits_between_are_set == 0) { 507 the_const = 508 create_simple_focus_bits(high_bits, low_bits, 509 lowest_bit_set, 0); 510 } else if (lowest_bit_set == 0) 511 shift = -(63 - highest_bit_set); 512 513 emit(OR | IMMED | RS1(G0) | S13(the_const) | RD(dest), ctx); 514 if (shift > 0) 515 emit_alu_K(SLLX, dest, shift, ctx); 516 else if (shift < 0) 517 emit_alu_K(SRLX, dest, -shift, ctx); 518 519 return; 520 } 521 522 /* Now a range of 22 or less bits set somewhere. 523 * 1) sethi %hi(focus_bits), %reg 524 * sllx %reg, shift, %reg 525 * 2) sethi %hi(focus_bits), %reg 526 * srlx %reg, shift, %reg 527 */ 528 if ((highest_bit_set - lowest_bit_set) < 21) { 529 unsigned long focus_bits = 530 create_simple_focus_bits(high_bits, low_bits, 531 lowest_bit_set, 10); 532 533 emit(SETHI(focus_bits, dest), ctx); 534 535 /* If lowest_bit_set == 10 then a sethi alone could 536 * have done it. 537 */ 538 if (lowest_bit_set < 10) 539 emit_alu_K(SRLX, dest, 10 - lowest_bit_set, ctx); 540 else if (lowest_bit_set > 10) 541 emit_alu_K(SLLX, dest, lowest_bit_set - 10, ctx); 542 return; 543 } 544 545 /* Ok, now 3 instruction sequences. */ 546 if (low_bits == 0) { 547 emit_loadimm32(high_bits, dest, ctx); 548 emit_alu_K(SLLX, dest, 32, ctx); 549 return; 550 } 551 552 /* We may be able to do something quick 553 * when the constant is negated, so try that. 554 */ 555 if (const64_is_2insns((~high_bits) & 0xffffffff, 556 (~low_bits) & 0xfffffc00)) { 557 /* NOTE: The trailing bits get XOR'd so we need the 558 * non-negated bits, not the negated ones. 559 */ 560 unsigned long trailing_bits = low_bits & 0x3ff; 561 562 if ((((~high_bits) & 0xffffffff) == 0 && 563 ((~low_bits) & 0x80000000) == 0) || 564 (((~high_bits) & 0xffffffff) == 0xffffffff && 565 ((~low_bits) & 0x80000000) != 0)) { 566 unsigned long fast_int = (~low_bits & 0xffffffff); 567 568 if ((is_sethi(fast_int) && 569 (~high_bits & 0xffffffff) == 0)) { 570 emit(SETHI(fast_int, dest), ctx); 571 } else if (is_simm13(fast_int)) { 572 emit(OR | IMMED | RS1(G0) | S13(fast_int) | RD(dest), ctx); 573 } else { 574 emit_loadimm64(fast_int, dest, ctx); 575 } 576 } else { 577 u64 n = ((~low_bits) & 0xfffffc00) | 578 (((unsigned long)((~high_bits) & 0xffffffff))<<32); 579 emit_loadimm64(n, dest, ctx); 580 } 581 582 low_bits = -0x400 | trailing_bits; 583 584 emit(XOR | IMMED | RS1(dest) | S13(low_bits) | RD(dest), ctx); 585 return; 586 } 587 588 /* 1) sethi %hi(xxx), %reg 589 * or %reg, %lo(xxx), %reg 590 * sllx %reg, yyy, %reg 591 */ 592 if ((highest_bit_set - lowest_bit_set) < 32) { 593 unsigned long focus_bits = 594 create_simple_focus_bits(high_bits, low_bits, 595 lowest_bit_set, 0); 596 597 /* So what we know is that the set bits straddle the 598 * middle of the 64-bit word. 599 */ 600 sparc_emit_set_const64_quick2(focus_bits, 0, dest, 601 lowest_bit_set, ctx); 602 return; 603 } 604 605 /* 1) sethi %hi(high_bits), %reg 606 * or %reg, %lo(high_bits), %reg 607 * sllx %reg, 32, %reg 608 * or %reg, low_bits, %reg 609 */ 610 if (is_simm13(low_bits) && ((int)low_bits > 0)) { 611 sparc_emit_set_const64_quick2(high_bits, low_bits, 612 dest, 32, ctx); 613 return; 614 } 615 616 /* Oh well, we tried... Do a full 64-bit decomposition. */ 617 ctx->tmp_1_used = true; 618 619 emit_loadimm32(high_bits, tmp, ctx); 620 emit_loadimm32(low_bits, dest, ctx); 621 emit_alu_K(SLLX, tmp, 32, ctx); 622 emit(OR | RS1(dest) | RS2(tmp) | RD(dest), ctx); 623 } 624 625 static void emit_branch(unsigned int br_opc, unsigned int from_idx, unsigned int to_idx, 626 struct jit_ctx *ctx) 627 { 628 unsigned int off = to_idx - from_idx; 629 630 if (br_opc & XCC) 631 emit(br_opc | WDISP19(off << 2), ctx); 632 else 633 emit(br_opc | WDISP22(off << 2), ctx); 634 } 635 636 static void emit_cbcond(unsigned int cb_opc, unsigned int from_idx, unsigned int to_idx, 637 const u8 dst, const u8 src, struct jit_ctx *ctx) 638 { 639 unsigned int off = to_idx - from_idx; 640 641 emit(cb_opc | WDISP10(off << 2) | RS1(dst) | RS2(src), ctx); 642 } 643 644 static void emit_cbcondi(unsigned int cb_opc, unsigned int from_idx, unsigned int to_idx, 645 const u8 dst, s32 imm, struct jit_ctx *ctx) 646 { 647 unsigned int off = to_idx - from_idx; 648 649 emit(cb_opc | IMMED | WDISP10(off << 2) | RS1(dst) | S5(imm), ctx); 650 } 651 652 #define emit_read_y(REG, CTX) emit(RD_Y | RD(REG), CTX) 653 #define emit_write_y(REG, CTX) emit(WR_Y | IMMED | RS1(REG) | S13(0), CTX) 654 655 #define emit_cmp(R1, R2, CTX) \ 656 emit(SUBCC | RS1(R1) | RS2(R2) | RD(G0), CTX) 657 658 #define emit_cmpi(R1, IMM, CTX) \ 659 emit(SUBCC | IMMED | RS1(R1) | S13(IMM) | RD(G0), CTX) 660 661 #define emit_btst(R1, R2, CTX) \ 662 emit(ANDCC | RS1(R1) | RS2(R2) | RD(G0), CTX) 663 664 #define emit_btsti(R1, IMM, CTX) \ 665 emit(ANDCC | IMMED | RS1(R1) | S13(IMM) | RD(G0), CTX) 666 667 static int emit_compare_and_branch(const u8 code, const u8 dst, u8 src, 668 const s32 imm, bool is_imm, int branch_dst, 669 struct jit_ctx *ctx) 670 { 671 bool use_cbcond = (sparc64_elf_hwcap & AV_SPARC_CBCOND) != 0; 672 const u8 tmp = bpf2sparc[TMP_REG_1]; 673 674 branch_dst = ctx->offset[branch_dst]; 675 676 if (!is_simm10(branch_dst - ctx->idx) || 677 BPF_OP(code) == BPF_JSET) 678 use_cbcond = false; 679 680 if (is_imm) { 681 bool fits = true; 682 683 if (use_cbcond) { 684 if (!is_simm5(imm)) 685 fits = false; 686 } else if (!is_simm13(imm)) { 687 fits = false; 688 } 689 if (!fits) { 690 ctx->tmp_1_used = true; 691 emit_loadimm_sext(imm, tmp, ctx); 692 src = tmp; 693 is_imm = false; 694 } 695 } 696 697 if (!use_cbcond) { 698 u32 br_opcode; 699 700 if (BPF_OP(code) == BPF_JSET) { 701 if (is_imm) 702 emit_btsti(dst, imm, ctx); 703 else 704 emit_btst(dst, src, ctx); 705 } else { 706 if (is_imm) 707 emit_cmpi(dst, imm, ctx); 708 else 709 emit_cmp(dst, src, ctx); 710 } 711 switch (BPF_OP(code)) { 712 case BPF_JEQ: 713 br_opcode = BE; 714 break; 715 case BPF_JGT: 716 br_opcode = BGU; 717 break; 718 case BPF_JGE: 719 br_opcode = BGEU; 720 break; 721 case BPF_JSET: 722 case BPF_JNE: 723 br_opcode = BNE; 724 break; 725 case BPF_JSGT: 726 br_opcode = BG; 727 break; 728 case BPF_JSGE: 729 br_opcode = BGE; 730 break; 731 default: 732 /* Make sure we dont leak kernel information to the 733 * user. 734 */ 735 return -EFAULT; 736 } 737 emit_branch(br_opcode, ctx->idx, branch_dst, ctx); 738 emit_nop(ctx); 739 } else { 740 u32 cbcond_opcode; 741 742 switch (BPF_OP(code)) { 743 case BPF_JEQ: 744 cbcond_opcode = CBCONDE; 745 break; 746 case BPF_JGT: 747 cbcond_opcode = CBCONDGU; 748 break; 749 case BPF_JGE: 750 cbcond_opcode = CBCONDGEU; 751 break; 752 case BPF_JNE: 753 cbcond_opcode = CBCONDNE; 754 break; 755 case BPF_JSGT: 756 cbcond_opcode = CBCONDG; 757 break; 758 case BPF_JSGE: 759 cbcond_opcode = CBCONDGE; 760 break; 761 default: 762 /* Make sure we dont leak kernel information to the 763 * user. 764 */ 765 return -EFAULT; 766 } 767 cbcond_opcode |= CBCOND_OP; 768 if (is_imm) 769 emit_cbcondi(cbcond_opcode, ctx->idx, branch_dst, 770 dst, imm, ctx); 771 else 772 emit_cbcond(cbcond_opcode, ctx->idx, branch_dst, 773 dst, src, ctx); 774 } 775 return 0; 776 } 777 778 static void load_skb_regs(struct jit_ctx *ctx, u8 r_skb) 779 { 780 const u8 r_headlen = bpf2sparc[SKB_HLEN_REG]; 781 const u8 r_data = bpf2sparc[SKB_DATA_REG]; 782 const u8 r_tmp = bpf2sparc[TMP_REG_1]; 783 unsigned int off; 784 785 off = offsetof(struct sk_buff, len); 786 emit(LD32I | RS1(r_skb) | S13(off) | RD(r_headlen), ctx); 787 788 off = offsetof(struct sk_buff, data_len); 789 emit(LD32I | RS1(r_skb) | S13(off) | RD(r_tmp), ctx); 790 791 emit(SUB | RS1(r_headlen) | RS2(r_tmp) | RD(r_headlen), ctx); 792 793 off = offsetof(struct sk_buff, data); 794 emit(LDPTRI | RS1(r_skb) | S13(off) | RD(r_data), ctx); 795 } 796 797 /* Just skip the save instruction and the ctx register move. */ 798 #define BPF_TAILCALL_PROLOGUE_SKIP 16 799 #define BPF_TAILCALL_CNT_SP_OFF (STACK_BIAS + 128) 800 801 static void build_prologue(struct jit_ctx *ctx) 802 { 803 s32 stack_needed = BASE_STACKFRAME; 804 805 if (ctx->saw_frame_pointer || ctx->saw_tail_call) { 806 struct bpf_prog *prog = ctx->prog; 807 u32 stack_depth; 808 809 stack_depth = prog->aux->stack_depth; 810 stack_needed += round_up(stack_depth, 16); 811 } 812 813 if (ctx->saw_tail_call) 814 stack_needed += 8; 815 816 /* save %sp, -176, %sp */ 817 emit(SAVE | IMMED | RS1(SP) | S13(-stack_needed) | RD(SP), ctx); 818 819 /* tail_call_cnt = 0 */ 820 if (ctx->saw_tail_call) { 821 u32 off = BPF_TAILCALL_CNT_SP_OFF; 822 823 emit(ST32 | IMMED | RS1(SP) | S13(off) | RD(G0), ctx); 824 } else { 825 emit_nop(ctx); 826 } 827 if (ctx->saw_frame_pointer) { 828 const u8 vfp = bpf2sparc[BPF_REG_FP]; 829 830 emit(ADD | IMMED | RS1(FP) | S13(STACK_BIAS) | RD(vfp), ctx); 831 } 832 833 emit_reg_move(I0, O0, ctx); 834 /* If you add anything here, adjust BPF_TAILCALL_PROLOGUE_SKIP above. */ 835 836 if (ctx->saw_ld_abs_ind) 837 load_skb_regs(ctx, bpf2sparc[BPF_REG_1]); 838 } 839 840 static void build_epilogue(struct jit_ctx *ctx) 841 { 842 ctx->epilogue_offset = ctx->idx; 843 844 /* ret (jmpl %i7 + 8, %g0) */ 845 emit(JMPL | IMMED | RS1(I7) | S13(8) | RD(G0), ctx); 846 847 /* restore %i5, %g0, %o0 */ 848 emit(RESTORE | RS1(bpf2sparc[BPF_REG_0]) | RS2(G0) | RD(O0), ctx); 849 } 850 851 static void emit_tail_call(struct jit_ctx *ctx) 852 { 853 const u8 bpf_array = bpf2sparc[BPF_REG_2]; 854 const u8 bpf_index = bpf2sparc[BPF_REG_3]; 855 const u8 tmp = bpf2sparc[TMP_REG_1]; 856 u32 off; 857 858 ctx->saw_tail_call = true; 859 860 off = offsetof(struct bpf_array, map.max_entries); 861 emit(LD32 | IMMED | RS1(bpf_array) | S13(off) | RD(tmp), ctx); 862 emit_cmp(bpf_index, tmp, ctx); 863 #define OFFSET1 17 864 emit_branch(BGEU, ctx->idx, ctx->idx + OFFSET1, ctx); 865 emit_nop(ctx); 866 867 off = BPF_TAILCALL_CNT_SP_OFF; 868 emit(LD32 | IMMED | RS1(SP) | S13(off) | RD(tmp), ctx); 869 emit_cmpi(tmp, MAX_TAIL_CALL_CNT, ctx); 870 #define OFFSET2 13 871 emit_branch(BGU, ctx->idx, ctx->idx + OFFSET2, ctx); 872 emit_nop(ctx); 873 874 emit_alu_K(ADD, tmp, 1, ctx); 875 off = BPF_TAILCALL_CNT_SP_OFF; 876 emit(ST32 | IMMED | RS1(SP) | S13(off) | RD(tmp), ctx); 877 878 emit_alu3_K(SLL, bpf_index, 3, tmp, ctx); 879 emit_alu(ADD, bpf_array, tmp, ctx); 880 off = offsetof(struct bpf_array, ptrs); 881 emit(LD64 | IMMED | RS1(tmp) | S13(off) | RD(tmp), ctx); 882 883 emit_cmpi(tmp, 0, ctx); 884 #define OFFSET3 5 885 emit_branch(BE, ctx->idx, ctx->idx + OFFSET3, ctx); 886 emit_nop(ctx); 887 888 off = offsetof(struct bpf_prog, bpf_func); 889 emit(LD64 | IMMED | RS1(tmp) | S13(off) | RD(tmp), ctx); 890 891 off = BPF_TAILCALL_PROLOGUE_SKIP; 892 emit(JMPL | IMMED | RS1(tmp) | S13(off) | RD(G0), ctx); 893 emit_nop(ctx); 894 } 895 896 static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) 897 { 898 const u8 code = insn->code; 899 const u8 dst = bpf2sparc[insn->dst_reg]; 900 const u8 src = bpf2sparc[insn->src_reg]; 901 const int i = insn - ctx->prog->insnsi; 902 const s16 off = insn->off; 903 const s32 imm = insn->imm; 904 u32 *func; 905 906 if (insn->src_reg == BPF_REG_FP) 907 ctx->saw_frame_pointer = true; 908 909 switch (code) { 910 /* dst = src */ 911 case BPF_ALU | BPF_MOV | BPF_X: 912 emit_alu3_K(SRL, src, 0, dst, ctx); 913 break; 914 case BPF_ALU64 | BPF_MOV | BPF_X: 915 emit_reg_move(src, dst, ctx); 916 break; 917 /* dst = dst OP src */ 918 case BPF_ALU | BPF_ADD | BPF_X: 919 case BPF_ALU64 | BPF_ADD | BPF_X: 920 emit_alu(ADD, src, dst, ctx); 921 goto do_alu32_trunc; 922 case BPF_ALU | BPF_SUB | BPF_X: 923 case BPF_ALU64 | BPF_SUB | BPF_X: 924 emit_alu(SUB, src, dst, ctx); 925 goto do_alu32_trunc; 926 case BPF_ALU | BPF_AND | BPF_X: 927 case BPF_ALU64 | BPF_AND | BPF_X: 928 emit_alu(AND, src, dst, ctx); 929 goto do_alu32_trunc; 930 case BPF_ALU | BPF_OR | BPF_X: 931 case BPF_ALU64 | BPF_OR | BPF_X: 932 emit_alu(OR, src, dst, ctx); 933 goto do_alu32_trunc; 934 case BPF_ALU | BPF_XOR | BPF_X: 935 case BPF_ALU64 | BPF_XOR | BPF_X: 936 emit_alu(XOR, src, dst, ctx); 937 goto do_alu32_trunc; 938 case BPF_ALU | BPF_MUL | BPF_X: 939 emit_alu(MUL, src, dst, ctx); 940 goto do_alu32_trunc; 941 case BPF_ALU64 | BPF_MUL | BPF_X: 942 emit_alu(MULX, src, dst, ctx); 943 break; 944 case BPF_ALU | BPF_DIV | BPF_X: 945 emit_cmp(src, G0, ctx); 946 emit_branch(BE|ANNUL, ctx->idx, ctx->epilogue_offset, ctx); 947 emit_loadimm(0, bpf2sparc[BPF_REG_0], ctx); 948 949 emit_write_y(G0, ctx); 950 emit_alu(DIV, src, dst, ctx); 951 break; 952 953 case BPF_ALU64 | BPF_DIV | BPF_X: 954 emit_cmp(src, G0, ctx); 955 emit_branch(BE|ANNUL, ctx->idx, ctx->epilogue_offset, ctx); 956 emit_loadimm(0, bpf2sparc[BPF_REG_0], ctx); 957 958 emit_alu(UDIVX, src, dst, ctx); 959 break; 960 961 case BPF_ALU | BPF_MOD | BPF_X: { 962 const u8 tmp = bpf2sparc[TMP_REG_1]; 963 964 ctx->tmp_1_used = true; 965 966 emit_cmp(src, G0, ctx); 967 emit_branch(BE|ANNUL, ctx->idx, ctx->epilogue_offset, ctx); 968 emit_loadimm(0, bpf2sparc[BPF_REG_0], ctx); 969 970 emit_write_y(G0, ctx); 971 emit_alu3(DIV, dst, src, tmp, ctx); 972 emit_alu3(MULX, tmp, src, tmp, ctx); 973 emit_alu3(SUB, dst, tmp, dst, ctx); 974 goto do_alu32_trunc; 975 } 976 case BPF_ALU64 | BPF_MOD | BPF_X: { 977 const u8 tmp = bpf2sparc[TMP_REG_1]; 978 979 ctx->tmp_1_used = true; 980 981 emit_cmp(src, G0, ctx); 982 emit_branch(BE|ANNUL, ctx->idx, ctx->epilogue_offset, ctx); 983 emit_loadimm(0, bpf2sparc[BPF_REG_0], ctx); 984 985 emit_alu3(UDIVX, dst, src, tmp, ctx); 986 emit_alu3(MULX, tmp, src, tmp, ctx); 987 emit_alu3(SUB, dst, tmp, dst, ctx); 988 break; 989 } 990 case BPF_ALU | BPF_LSH | BPF_X: 991 emit_alu(SLL, src, dst, ctx); 992 goto do_alu32_trunc; 993 case BPF_ALU64 | BPF_LSH | BPF_X: 994 emit_alu(SLLX, src, dst, ctx); 995 break; 996 case BPF_ALU | BPF_RSH | BPF_X: 997 emit_alu(SRL, src, dst, ctx); 998 break; 999 case BPF_ALU64 | BPF_RSH | BPF_X: 1000 emit_alu(SRLX, src, dst, ctx); 1001 break; 1002 case BPF_ALU | BPF_ARSH | BPF_X: 1003 emit_alu(SRA, src, dst, ctx); 1004 goto do_alu32_trunc; 1005 case BPF_ALU64 | BPF_ARSH | BPF_X: 1006 emit_alu(SRAX, src, dst, ctx); 1007 break; 1008 1009 /* dst = -dst */ 1010 case BPF_ALU | BPF_NEG: 1011 case BPF_ALU64 | BPF_NEG: 1012 emit(SUB | RS1(0) | RS2(dst) | RD(dst), ctx); 1013 goto do_alu32_trunc; 1014 1015 case BPF_ALU | BPF_END | BPF_FROM_BE: 1016 switch (imm) { 1017 case 16: 1018 emit_alu_K(SLL, dst, 16, ctx); 1019 emit_alu_K(SRL, dst, 16, ctx); 1020 break; 1021 case 32: 1022 emit_alu_K(SRL, dst, 0, ctx); 1023 break; 1024 case 64: 1025 /* nop */ 1026 break; 1027 1028 } 1029 break; 1030 1031 /* dst = BSWAP##imm(dst) */ 1032 case BPF_ALU | BPF_END | BPF_FROM_LE: { 1033 const u8 tmp = bpf2sparc[TMP_REG_1]; 1034 const u8 tmp2 = bpf2sparc[TMP_REG_2]; 1035 1036 ctx->tmp_1_used = true; 1037 switch (imm) { 1038 case 16: 1039 emit_alu3_K(AND, dst, 0xff, tmp, ctx); 1040 emit_alu3_K(SRL, dst, 8, dst, ctx); 1041 emit_alu3_K(AND, dst, 0xff, dst, ctx); 1042 emit_alu3_K(SLL, tmp, 8, tmp, ctx); 1043 emit_alu(OR, tmp, dst, ctx); 1044 break; 1045 1046 case 32: 1047 ctx->tmp_2_used = true; 1048 emit_alu3_K(SRL, dst, 24, tmp, ctx); /* tmp = dst >> 24 */ 1049 emit_alu3_K(SRL, dst, 16, tmp2, ctx); /* tmp2 = dst >> 16 */ 1050 emit_alu3_K(AND, tmp2, 0xff, tmp2, ctx);/* tmp2 = tmp2 & 0xff */ 1051 emit_alu3_K(SLL, tmp2, 8, tmp2, ctx); /* tmp2 = tmp2 << 8 */ 1052 emit_alu(OR, tmp2, tmp, ctx); /* tmp = tmp | tmp2 */ 1053 emit_alu3_K(SRL, dst, 8, tmp2, ctx); /* tmp2 = dst >> 8 */ 1054 emit_alu3_K(AND, tmp2, 0xff, tmp2, ctx);/* tmp2 = tmp2 & 0xff */ 1055 emit_alu3_K(SLL, tmp2, 16, tmp2, ctx); /* tmp2 = tmp2 << 16 */ 1056 emit_alu(OR, tmp2, tmp, ctx); /* tmp = tmp | tmp2 */ 1057 emit_alu3_K(AND, dst, 0xff, dst, ctx); /* dst = dst & 0xff */ 1058 emit_alu3_K(SLL, dst, 24, dst, ctx); /* dst = dst << 24 */ 1059 emit_alu(OR, tmp, dst, ctx); /* dst = dst | tmp */ 1060 break; 1061 1062 case 64: 1063 emit_alu3_K(ADD, SP, STACK_BIAS + 128, tmp, ctx); 1064 emit(ST64 | RS1(tmp) | RS2(G0) | RD(dst), ctx); 1065 emit(LD64A | ASI(ASI_PL) | RS1(tmp) | RS2(G0) | RD(dst), ctx); 1066 break; 1067 } 1068 break; 1069 } 1070 /* dst = imm */ 1071 case BPF_ALU | BPF_MOV | BPF_K: 1072 emit_loadimm32(imm, dst, ctx); 1073 break; 1074 case BPF_ALU64 | BPF_MOV | BPF_K: 1075 emit_loadimm_sext(imm, dst, ctx); 1076 break; 1077 /* dst = dst OP imm */ 1078 case BPF_ALU | BPF_ADD | BPF_K: 1079 case BPF_ALU64 | BPF_ADD | BPF_K: 1080 emit_alu_K(ADD, dst, imm, ctx); 1081 goto do_alu32_trunc; 1082 case BPF_ALU | BPF_SUB | BPF_K: 1083 case BPF_ALU64 | BPF_SUB | BPF_K: 1084 emit_alu_K(SUB, dst, imm, ctx); 1085 goto do_alu32_trunc; 1086 case BPF_ALU | BPF_AND | BPF_K: 1087 case BPF_ALU64 | BPF_AND | BPF_K: 1088 emit_alu_K(AND, dst, imm, ctx); 1089 goto do_alu32_trunc; 1090 case BPF_ALU | BPF_OR | BPF_K: 1091 case BPF_ALU64 | BPF_OR | BPF_K: 1092 emit_alu_K(OR, dst, imm, ctx); 1093 goto do_alu32_trunc; 1094 case BPF_ALU | BPF_XOR | BPF_K: 1095 case BPF_ALU64 | BPF_XOR | BPF_K: 1096 emit_alu_K(XOR, dst, imm, ctx); 1097 goto do_alu32_trunc; 1098 case BPF_ALU | BPF_MUL | BPF_K: 1099 emit_alu_K(MUL, dst, imm, ctx); 1100 goto do_alu32_trunc; 1101 case BPF_ALU64 | BPF_MUL | BPF_K: 1102 emit_alu_K(MULX, dst, imm, ctx); 1103 break; 1104 case BPF_ALU | BPF_DIV | BPF_K: 1105 if (imm == 0) 1106 return -EINVAL; 1107 1108 emit_write_y(G0, ctx); 1109 emit_alu_K(DIV, dst, imm, ctx); 1110 goto do_alu32_trunc; 1111 case BPF_ALU64 | BPF_DIV | BPF_K: 1112 if (imm == 0) 1113 return -EINVAL; 1114 1115 emit_alu_K(UDIVX, dst, imm, ctx); 1116 break; 1117 case BPF_ALU64 | BPF_MOD | BPF_K: 1118 case BPF_ALU | BPF_MOD | BPF_K: { 1119 const u8 tmp = bpf2sparc[TMP_REG_2]; 1120 unsigned int div; 1121 1122 if (imm == 0) 1123 return -EINVAL; 1124 1125 div = (BPF_CLASS(code) == BPF_ALU64) ? UDIVX : DIV; 1126 1127 ctx->tmp_2_used = true; 1128 1129 if (BPF_CLASS(code) != BPF_ALU64) 1130 emit_write_y(G0, ctx); 1131 if (is_simm13(imm)) { 1132 emit(div | IMMED | RS1(dst) | S13(imm) | RD(tmp), ctx); 1133 emit(MULX | IMMED | RS1(tmp) | S13(imm) | RD(tmp), ctx); 1134 emit(SUB | RS1(dst) | RS2(tmp) | RD(dst), ctx); 1135 } else { 1136 const u8 tmp1 = bpf2sparc[TMP_REG_1]; 1137 1138 ctx->tmp_1_used = true; 1139 1140 emit_set_const_sext(imm, tmp1, ctx); 1141 emit(div | RS1(dst) | RS2(tmp1) | RD(tmp), ctx); 1142 emit(MULX | RS1(tmp) | RS2(tmp1) | RD(tmp), ctx); 1143 emit(SUB | RS1(dst) | RS2(tmp) | RD(dst), ctx); 1144 } 1145 goto do_alu32_trunc; 1146 } 1147 case BPF_ALU | BPF_LSH | BPF_K: 1148 emit_alu_K(SLL, dst, imm, ctx); 1149 goto do_alu32_trunc; 1150 case BPF_ALU64 | BPF_LSH | BPF_K: 1151 emit_alu_K(SLLX, dst, imm, ctx); 1152 break; 1153 case BPF_ALU | BPF_RSH | BPF_K: 1154 emit_alu_K(SRL, dst, imm, ctx); 1155 break; 1156 case BPF_ALU64 | BPF_RSH | BPF_K: 1157 emit_alu_K(SRLX, dst, imm, ctx); 1158 break; 1159 case BPF_ALU | BPF_ARSH | BPF_K: 1160 emit_alu_K(SRA, dst, imm, ctx); 1161 goto do_alu32_trunc; 1162 case BPF_ALU64 | BPF_ARSH | BPF_K: 1163 emit_alu_K(SRAX, dst, imm, ctx); 1164 break; 1165 1166 do_alu32_trunc: 1167 if (BPF_CLASS(code) == BPF_ALU) 1168 emit_alu_K(SRL, dst, 0, ctx); 1169 break; 1170 1171 /* JUMP off */ 1172 case BPF_JMP | BPF_JA: 1173 emit_branch(BA, ctx->idx, ctx->offset[i + off], ctx); 1174 emit_nop(ctx); 1175 break; 1176 /* IF (dst COND src) JUMP off */ 1177 case BPF_JMP | BPF_JEQ | BPF_X: 1178 case BPF_JMP | BPF_JGT | BPF_X: 1179 case BPF_JMP | BPF_JGE | BPF_X: 1180 case BPF_JMP | BPF_JNE | BPF_X: 1181 case BPF_JMP | BPF_JSGT | BPF_X: 1182 case BPF_JMP | BPF_JSGE | BPF_X: 1183 case BPF_JMP | BPF_JSET | BPF_X: { 1184 int err; 1185 1186 err = emit_compare_and_branch(code, dst, src, 0, false, i + off, ctx); 1187 if (err) 1188 return err; 1189 break; 1190 } 1191 /* IF (dst COND imm) JUMP off */ 1192 case BPF_JMP | BPF_JEQ | BPF_K: 1193 case BPF_JMP | BPF_JGT | BPF_K: 1194 case BPF_JMP | BPF_JGE | BPF_K: 1195 case BPF_JMP | BPF_JNE | BPF_K: 1196 case BPF_JMP | BPF_JSGT | BPF_K: 1197 case BPF_JMP | BPF_JSGE | BPF_K: 1198 case BPF_JMP | BPF_JSET | BPF_K: { 1199 int err; 1200 1201 err = emit_compare_and_branch(code, dst, 0, imm, true, i + off, ctx); 1202 if (err) 1203 return err; 1204 break; 1205 } 1206 1207 /* function call */ 1208 case BPF_JMP | BPF_CALL: 1209 { 1210 u8 *func = ((u8 *)__bpf_call_base) + imm; 1211 1212 ctx->saw_call = true; 1213 1214 emit_call((u32 *)func, ctx); 1215 emit_nop(ctx); 1216 1217 emit_reg_move(O0, bpf2sparc[BPF_REG_0], ctx); 1218 1219 if (bpf_helper_changes_pkt_data(func) && ctx->saw_ld_abs_ind) 1220 load_skb_regs(ctx, bpf2sparc[BPF_REG_6]); 1221 break; 1222 } 1223 1224 /* tail call */ 1225 case BPF_JMP | BPF_TAIL_CALL: 1226 emit_tail_call(ctx); 1227 break; 1228 1229 /* function return */ 1230 case BPF_JMP | BPF_EXIT: 1231 /* Optimization: when last instruction is EXIT, 1232 simply fallthrough to epilogue. */ 1233 if (i == ctx->prog->len - 1) 1234 break; 1235 emit_branch(BA, ctx->idx, ctx->epilogue_offset, ctx); 1236 emit_nop(ctx); 1237 break; 1238 1239 /* dst = imm64 */ 1240 case BPF_LD | BPF_IMM | BPF_DW: 1241 { 1242 const struct bpf_insn insn1 = insn[1]; 1243 u64 imm64; 1244 1245 imm64 = (u64)insn1.imm << 32 | (u32)imm; 1246 emit_loadimm64(imm64, dst, ctx); 1247 1248 return 1; 1249 } 1250 1251 /* LDX: dst = *(size *)(src + off) */ 1252 case BPF_LDX | BPF_MEM | BPF_W: 1253 case BPF_LDX | BPF_MEM | BPF_H: 1254 case BPF_LDX | BPF_MEM | BPF_B: 1255 case BPF_LDX | BPF_MEM | BPF_DW: { 1256 const u8 tmp = bpf2sparc[TMP_REG_1]; 1257 u32 opcode = 0, rs2; 1258 1259 ctx->tmp_1_used = true; 1260 switch (BPF_SIZE(code)) { 1261 case BPF_W: 1262 opcode = LD32; 1263 break; 1264 case BPF_H: 1265 opcode = LD16; 1266 break; 1267 case BPF_B: 1268 opcode = LD8; 1269 break; 1270 case BPF_DW: 1271 opcode = LD64; 1272 break; 1273 } 1274 1275 if (is_simm13(off)) { 1276 opcode |= IMMED; 1277 rs2 = S13(off); 1278 } else { 1279 emit_loadimm(off, tmp, ctx); 1280 rs2 = RS2(tmp); 1281 } 1282 emit(opcode | RS1(src) | rs2 | RD(dst), ctx); 1283 break; 1284 } 1285 /* ST: *(size *)(dst + off) = imm */ 1286 case BPF_ST | BPF_MEM | BPF_W: 1287 case BPF_ST | BPF_MEM | BPF_H: 1288 case BPF_ST | BPF_MEM | BPF_B: 1289 case BPF_ST | BPF_MEM | BPF_DW: { 1290 const u8 tmp = bpf2sparc[TMP_REG_1]; 1291 const u8 tmp2 = bpf2sparc[TMP_REG_2]; 1292 u32 opcode = 0, rs2; 1293 1294 ctx->tmp_2_used = true; 1295 emit_loadimm(imm, tmp2, ctx); 1296 1297 switch (BPF_SIZE(code)) { 1298 case BPF_W: 1299 opcode = ST32; 1300 break; 1301 case BPF_H: 1302 opcode = ST16; 1303 break; 1304 case BPF_B: 1305 opcode = ST8; 1306 break; 1307 case BPF_DW: 1308 opcode = ST64; 1309 break; 1310 } 1311 1312 if (is_simm13(off)) { 1313 opcode |= IMMED; 1314 rs2 = S13(off); 1315 } else { 1316 ctx->tmp_1_used = true; 1317 emit_loadimm(off, tmp, ctx); 1318 rs2 = RS2(tmp); 1319 } 1320 emit(opcode | RS1(dst) | rs2 | RD(tmp2), ctx); 1321 break; 1322 } 1323 1324 /* STX: *(size *)(dst + off) = src */ 1325 case BPF_STX | BPF_MEM | BPF_W: 1326 case BPF_STX | BPF_MEM | BPF_H: 1327 case BPF_STX | BPF_MEM | BPF_B: 1328 case BPF_STX | BPF_MEM | BPF_DW: { 1329 const u8 tmp = bpf2sparc[TMP_REG_1]; 1330 u32 opcode = 0, rs2; 1331 1332 switch (BPF_SIZE(code)) { 1333 case BPF_W: 1334 opcode = ST32; 1335 break; 1336 case BPF_H: 1337 opcode = ST16; 1338 break; 1339 case BPF_B: 1340 opcode = ST8; 1341 break; 1342 case BPF_DW: 1343 opcode = ST64; 1344 break; 1345 } 1346 if (is_simm13(off)) { 1347 opcode |= IMMED; 1348 rs2 = S13(off); 1349 } else { 1350 ctx->tmp_1_used = true; 1351 emit_loadimm(off, tmp, ctx); 1352 rs2 = RS2(tmp); 1353 } 1354 emit(opcode | RS1(dst) | rs2 | RD(src), ctx); 1355 break; 1356 } 1357 1358 /* STX XADD: lock *(u32 *)(dst + off) += src */ 1359 case BPF_STX | BPF_XADD | BPF_W: { 1360 const u8 tmp = bpf2sparc[TMP_REG_1]; 1361 const u8 tmp2 = bpf2sparc[TMP_REG_2]; 1362 const u8 tmp3 = bpf2sparc[TMP_REG_3]; 1363 1364 ctx->tmp_1_used = true; 1365 ctx->tmp_2_used = true; 1366 ctx->tmp_3_used = true; 1367 emit_loadimm(off, tmp, ctx); 1368 emit_alu3(ADD, dst, tmp, tmp, ctx); 1369 1370 emit(LD32 | RS1(tmp) | RS2(G0) | RD(tmp2), ctx); 1371 emit_alu3(ADD, tmp2, src, tmp3, ctx); 1372 emit(CAS | ASI(ASI_P) | RS1(tmp) | RS2(tmp2) | RD(tmp3), ctx); 1373 emit_cmp(tmp2, tmp3, ctx); 1374 emit_branch(BNE, 4, 0, ctx); 1375 emit_nop(ctx); 1376 break; 1377 } 1378 /* STX XADD: lock *(u64 *)(dst + off) += src */ 1379 case BPF_STX | BPF_XADD | BPF_DW: { 1380 const u8 tmp = bpf2sparc[TMP_REG_1]; 1381 const u8 tmp2 = bpf2sparc[TMP_REG_2]; 1382 const u8 tmp3 = bpf2sparc[TMP_REG_3]; 1383 1384 ctx->tmp_1_used = true; 1385 ctx->tmp_2_used = true; 1386 ctx->tmp_3_used = true; 1387 emit_loadimm(off, tmp, ctx); 1388 emit_alu3(ADD, dst, tmp, tmp, ctx); 1389 1390 emit(LD64 | RS1(tmp) | RS2(G0) | RD(tmp2), ctx); 1391 emit_alu3(ADD, tmp2, src, tmp3, ctx); 1392 emit(CASX | ASI(ASI_P) | RS1(tmp) | RS2(tmp2) | RD(tmp3), ctx); 1393 emit_cmp(tmp2, tmp3, ctx); 1394 emit_branch(BNE, 4, 0, ctx); 1395 emit_nop(ctx); 1396 break; 1397 } 1398 #define CHOOSE_LOAD_FUNC(K, func) \ 1399 ((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative_offset : func) : func##_positive_offset) 1400 1401 /* R0 = ntohx(*(size *)(((struct sk_buff *)R6)->data + imm)) */ 1402 case BPF_LD | BPF_ABS | BPF_W: 1403 func = CHOOSE_LOAD_FUNC(imm, bpf_jit_load_word); 1404 goto common_load; 1405 case BPF_LD | BPF_ABS | BPF_H: 1406 func = CHOOSE_LOAD_FUNC(imm, bpf_jit_load_half); 1407 goto common_load; 1408 case BPF_LD | BPF_ABS | BPF_B: 1409 func = CHOOSE_LOAD_FUNC(imm, bpf_jit_load_byte); 1410 goto common_load; 1411 /* R0 = ntohx(*(size *)(((struct sk_buff *)R6)->data + src + imm)) */ 1412 case BPF_LD | BPF_IND | BPF_W: 1413 func = bpf_jit_load_word; 1414 goto common_load; 1415 case BPF_LD | BPF_IND | BPF_H: 1416 func = bpf_jit_load_half; 1417 goto common_load; 1418 1419 case BPF_LD | BPF_IND | BPF_B: 1420 func = bpf_jit_load_byte; 1421 common_load: 1422 ctx->saw_ld_abs_ind = true; 1423 1424 emit_reg_move(bpf2sparc[BPF_REG_6], O0, ctx); 1425 emit_loadimm(imm, O1, ctx); 1426 1427 if (BPF_MODE(code) == BPF_IND) 1428 emit_alu(ADD, src, O1, ctx); 1429 1430 emit_call(func, ctx); 1431 emit_alu_K(SRA, O1, 0, ctx); 1432 1433 emit_reg_move(O0, bpf2sparc[BPF_REG_0], ctx); 1434 break; 1435 1436 default: 1437 pr_err_once("unknown opcode %02x\n", code); 1438 return -EINVAL; 1439 } 1440 1441 return 0; 1442 } 1443 1444 static int build_body(struct jit_ctx *ctx) 1445 { 1446 const struct bpf_prog *prog = ctx->prog; 1447 int i; 1448 1449 for (i = 0; i < prog->len; i++) { 1450 const struct bpf_insn *insn = &prog->insnsi[i]; 1451 int ret; 1452 1453 ret = build_insn(insn, ctx); 1454 1455 if (ret > 0) { 1456 i++; 1457 ctx->offset[i] = ctx->idx; 1458 continue; 1459 } 1460 ctx->offset[i] = ctx->idx; 1461 if (ret) 1462 return ret; 1463 } 1464 return 0; 1465 } 1466 1467 static void jit_fill_hole(void *area, unsigned int size) 1468 { 1469 u32 *ptr; 1470 /* We are guaranteed to have aligned memory. */ 1471 for (ptr = area; size >= sizeof(u32); size -= sizeof(u32)) 1472 *ptr++ = 0x91d02005; /* ta 5 */ 1473 } 1474 1475 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 1476 { 1477 struct bpf_prog *tmp, *orig_prog = prog; 1478 struct bpf_binary_header *header; 1479 bool tmp_blinded = false; 1480 struct jit_ctx ctx; 1481 u32 image_size; 1482 u8 *image_ptr; 1483 int pass; 1484 1485 if (!bpf_jit_enable) 1486 return orig_prog; 1487 1488 tmp = bpf_jit_blind_constants(prog); 1489 /* If blinding was requested and we failed during blinding, 1490 * we must fall back to the interpreter. 1491 */ 1492 if (IS_ERR(tmp)) 1493 return orig_prog; 1494 if (tmp != prog) { 1495 tmp_blinded = true; 1496 prog = tmp; 1497 } 1498 1499 memset(&ctx, 0, sizeof(ctx)); 1500 ctx.prog = prog; 1501 1502 ctx.offset = kcalloc(prog->len, sizeof(unsigned int), GFP_KERNEL); 1503 if (ctx.offset == NULL) { 1504 prog = orig_prog; 1505 goto out; 1506 } 1507 1508 /* Fake pass to detect features used, and get an accurate assessment 1509 * of what the final image size will be. 1510 */ 1511 if (build_body(&ctx)) { 1512 prog = orig_prog; 1513 goto out_off; 1514 } 1515 build_prologue(&ctx); 1516 build_epilogue(&ctx); 1517 1518 /* Now we know the actual image size. */ 1519 image_size = sizeof(u32) * ctx.idx; 1520 header = bpf_jit_binary_alloc(image_size, &image_ptr, 1521 sizeof(u32), jit_fill_hole); 1522 if (header == NULL) { 1523 prog = orig_prog; 1524 goto out_off; 1525 } 1526 1527 ctx.image = (u32 *)image_ptr; 1528 1529 for (pass = 1; pass < 3; pass++) { 1530 ctx.idx = 0; 1531 1532 build_prologue(&ctx); 1533 1534 if (build_body(&ctx)) { 1535 bpf_jit_binary_free(header); 1536 prog = orig_prog; 1537 goto out_off; 1538 } 1539 1540 build_epilogue(&ctx); 1541 1542 if (bpf_jit_enable > 1) 1543 pr_info("Pass %d: shrink = %d, seen = [%c%c%c%c%c%c%c]\n", pass, 1544 image_size - (ctx.idx * 4), 1545 ctx.tmp_1_used ? '1' : ' ', 1546 ctx.tmp_2_used ? '2' : ' ', 1547 ctx.tmp_3_used ? '3' : ' ', 1548 ctx.saw_ld_abs_ind ? 'L' : ' ', 1549 ctx.saw_frame_pointer ? 'F' : ' ', 1550 ctx.saw_call ? 'C' : ' ', 1551 ctx.saw_tail_call ? 'T' : ' '); 1552 } 1553 1554 if (bpf_jit_enable > 1) 1555 bpf_jit_dump(prog->len, image_size, pass, ctx.image); 1556 1557 bpf_flush_icache(header, (u8 *)header + (header->pages * PAGE_SIZE)); 1558 1559 bpf_jit_binary_lock_ro(header); 1560 1561 prog->bpf_func = (void *)ctx.image; 1562 prog->jited = 1; 1563 prog->jited_len = image_size; 1564 1565 out_off: 1566 kfree(ctx.offset); 1567 out: 1568 if (tmp_blinded) 1569 bpf_jit_prog_release_other(prog, prog == orig_prog ? 1570 tmp : orig_prog); 1571 return prog; 1572 } 1573