1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/moduleloader.h> 3 #include <linux/workqueue.h> 4 #include <linux/netdevice.h> 5 #include <linux/filter.h> 6 #include <linux/cache.h> 7 #include <linux/if_vlan.h> 8 9 #include <asm/cacheflush.h> 10 #include <asm/ptrace.h> 11 12 #include "bpf_jit_32.h" 13 14 int bpf_jit_enable __read_mostly; 15 16 static inline bool is_simm13(unsigned int value) 17 { 18 return value + 0x1000 < 0x2000; 19 } 20 21 #define SEEN_DATAREF 1 /* might call external helpers */ 22 #define SEEN_XREG 2 /* ebx is used */ 23 #define SEEN_MEM 4 /* use mem[] for temporary storage */ 24 25 #define S13(X) ((X) & 0x1fff) 26 #define IMMED 0x00002000 27 #define RD(X) ((X) << 25) 28 #define RS1(X) ((X) << 14) 29 #define RS2(X) ((X)) 30 #define OP(X) ((X) << 30) 31 #define OP2(X) ((X) << 22) 32 #define OP3(X) ((X) << 19) 33 #define COND(X) ((X) << 25) 34 #define F1(X) OP(X) 35 #define F2(X, Y) (OP(X) | OP2(Y)) 36 #define F3(X, Y) (OP(X) | OP3(Y)) 37 38 #define CONDN COND(0x0) 39 #define CONDE COND(0x1) 40 #define CONDLE COND(0x2) 41 #define CONDL COND(0x3) 42 #define CONDLEU COND(0x4) 43 #define CONDCS COND(0x5) 44 #define CONDNEG COND(0x6) 45 #define CONDVC COND(0x7) 46 #define CONDA COND(0x8) 47 #define CONDNE COND(0x9) 48 #define CONDG COND(0xa) 49 #define CONDGE COND(0xb) 50 #define CONDGU COND(0xc) 51 #define CONDCC COND(0xd) 52 #define CONDPOS COND(0xe) 53 #define CONDVS COND(0xf) 54 55 #define CONDGEU CONDCC 56 #define CONDLU CONDCS 57 58 #define WDISP22(X) (((X) >> 2) & 0x3fffff) 59 60 #define BA (F2(0, 2) | CONDA) 61 #define BGU (F2(0, 2) | CONDGU) 62 #define BLEU (F2(0, 2) | CONDLEU) 63 #define BGEU (F2(0, 2) | CONDGEU) 64 #define BLU (F2(0, 2) | CONDLU) 65 #define BE (F2(0, 2) | CONDE) 66 #define BNE (F2(0, 2) | CONDNE) 67 68 #define BE_PTR BE 69 70 #define SETHI(K, REG) \ 71 (F2(0, 0x4) | RD(REG) | (((K) >> 10) & 0x3fffff)) 72 #define OR_LO(K, REG) \ 73 (F3(2, 0x02) | IMMED | RS1(REG) | ((K) & 0x3ff) | RD(REG)) 74 75 #define ADD F3(2, 0x00) 76 #define AND F3(2, 0x01) 77 #define ANDCC F3(2, 0x11) 78 #define OR F3(2, 0x02) 79 #define XOR F3(2, 0x03) 80 #define SUB F3(2, 0x04) 81 #define SUBCC F3(2, 0x14) 82 #define MUL F3(2, 0x0a) /* umul */ 83 #define DIV F3(2, 0x0e) /* udiv */ 84 #define SLL F3(2, 0x25) 85 #define SRL F3(2, 0x26) 86 #define JMPL F3(2, 0x38) 87 #define CALL F1(1) 88 #define BR F2(0, 0x01) 89 #define RD_Y F3(2, 0x28) 90 #define WR_Y F3(2, 0x30) 91 92 #define LD32 F3(3, 0x00) 93 #define LD8 F3(3, 0x01) 94 #define LD16 F3(3, 0x02) 95 #define LD64 F3(3, 0x0b) 96 #define ST32 F3(3, 0x04) 97 98 #define LDPTR LD32 99 #define BASE_STACKFRAME 96 100 101 #define LD32I (LD32 | IMMED) 102 #define LD8I (LD8 | IMMED) 103 #define LD16I (LD16 | IMMED) 104 #define LD64I (LD64 | IMMED) 105 #define LDPTRI (LDPTR | IMMED) 106 #define ST32I (ST32 | IMMED) 107 108 #define emit_nop() \ 109 do { \ 110 *prog++ = SETHI(0, G0); \ 111 } while (0) 112 113 #define emit_neg() \ 114 do { /* sub %g0, r_A, r_A */ \ 115 *prog++ = SUB | RS1(G0) | RS2(r_A) | RD(r_A); \ 116 } while (0) 117 118 #define emit_reg_move(FROM, TO) \ 119 do { /* or %g0, FROM, TO */ \ 120 *prog++ = OR | RS1(G0) | RS2(FROM) | RD(TO); \ 121 } while (0) 122 123 #define emit_clear(REG) \ 124 do { /* or %g0, %g0, REG */ \ 125 *prog++ = OR | RS1(G0) | RS2(G0) | RD(REG); \ 126 } while (0) 127 128 #define emit_set_const(K, REG) \ 129 do { /* sethi %hi(K), REG */ \ 130 *prog++ = SETHI(K, REG); \ 131 /* or REG, %lo(K), REG */ \ 132 *prog++ = OR_LO(K, REG); \ 133 } while (0) 134 135 /* Emit 136 * 137 * OP r_A, r_X, r_A 138 */ 139 #define emit_alu_X(OPCODE) \ 140 do { \ 141 seen |= SEEN_XREG; \ 142 *prog++ = OPCODE | RS1(r_A) | RS2(r_X) | RD(r_A); \ 143 } while (0) 144 145 /* Emit either: 146 * 147 * OP r_A, K, r_A 148 * 149 * or 150 * 151 * sethi %hi(K), r_TMP 152 * or r_TMP, %lo(K), r_TMP 153 * OP r_A, r_TMP, r_A 154 * 155 * depending upon whether K fits in a signed 13-bit 156 * immediate instruction field. Emit nothing if K 157 * is zero. 158 */ 159 #define emit_alu_K(OPCODE, K) \ 160 do { \ 161 if (K || OPCODE == AND || OPCODE == MUL) { \ 162 unsigned int _insn = OPCODE; \ 163 _insn |= RS1(r_A) | RD(r_A); \ 164 if (is_simm13(K)) { \ 165 *prog++ = _insn | IMMED | S13(K); \ 166 } else { \ 167 emit_set_const(K, r_TMP); \ 168 *prog++ = _insn | RS2(r_TMP); \ 169 } \ 170 } \ 171 } while (0) 172 173 #define emit_loadimm(K, DEST) \ 174 do { \ 175 if (is_simm13(K)) { \ 176 /* or %g0, K, DEST */ \ 177 *prog++ = OR | IMMED | RS1(G0) | S13(K) | RD(DEST); \ 178 } else { \ 179 emit_set_const(K, DEST); \ 180 } \ 181 } while (0) 182 183 #define emit_loadptr(BASE, STRUCT, FIELD, DEST) \ 184 do { unsigned int _off = offsetof(STRUCT, FIELD); \ 185 BUILD_BUG_ON(FIELD_SIZEOF(STRUCT, FIELD) != sizeof(void *)); \ 186 *prog++ = LDPTRI | RS1(BASE) | S13(_off) | RD(DEST); \ 187 } while (0) 188 189 #define emit_load32(BASE, STRUCT, FIELD, DEST) \ 190 do { unsigned int _off = offsetof(STRUCT, FIELD); \ 191 BUILD_BUG_ON(FIELD_SIZEOF(STRUCT, FIELD) != sizeof(u32)); \ 192 *prog++ = LD32I | RS1(BASE) | S13(_off) | RD(DEST); \ 193 } while (0) 194 195 #define emit_load16(BASE, STRUCT, FIELD, DEST) \ 196 do { unsigned int _off = offsetof(STRUCT, FIELD); \ 197 BUILD_BUG_ON(FIELD_SIZEOF(STRUCT, FIELD) != sizeof(u16)); \ 198 *prog++ = LD16I | RS1(BASE) | S13(_off) | RD(DEST); \ 199 } while (0) 200 201 #define __emit_load8(BASE, STRUCT, FIELD, DEST) \ 202 do { unsigned int _off = offsetof(STRUCT, FIELD); \ 203 *prog++ = LD8I | RS1(BASE) | S13(_off) | RD(DEST); \ 204 } while (0) 205 206 #define emit_load8(BASE, STRUCT, FIELD, DEST) \ 207 do { BUILD_BUG_ON(FIELD_SIZEOF(STRUCT, FIELD) != sizeof(u8)); \ 208 __emit_load8(BASE, STRUCT, FIELD, DEST); \ 209 } while (0) 210 211 #define BIAS (-4) 212 213 #define emit_ldmem(OFF, DEST) \ 214 do { *prog++ = LD32I | RS1(SP) | S13(BIAS - (OFF)) | RD(DEST); \ 215 } while (0) 216 217 #define emit_stmem(OFF, SRC) \ 218 do { *prog++ = ST32I | RS1(SP) | S13(BIAS - (OFF)) | RD(SRC); \ 219 } while (0) 220 221 #ifdef CONFIG_SMP 222 #define emit_load_cpu(REG) \ 223 emit_load32(G6, struct thread_info, cpu, REG) 224 #else 225 #define emit_load_cpu(REG) emit_clear(REG) 226 #endif 227 228 #define emit_skb_loadptr(FIELD, DEST) \ 229 emit_loadptr(r_SKB, struct sk_buff, FIELD, DEST) 230 #define emit_skb_load32(FIELD, DEST) \ 231 emit_load32(r_SKB, struct sk_buff, FIELD, DEST) 232 #define emit_skb_load16(FIELD, DEST) \ 233 emit_load16(r_SKB, struct sk_buff, FIELD, DEST) 234 #define __emit_skb_load8(FIELD, DEST) \ 235 __emit_load8(r_SKB, struct sk_buff, FIELD, DEST) 236 #define emit_skb_load8(FIELD, DEST) \ 237 emit_load8(r_SKB, struct sk_buff, FIELD, DEST) 238 239 #define emit_jmpl(BASE, IMM_OFF, LREG) \ 240 *prog++ = (JMPL | IMMED | RS1(BASE) | S13(IMM_OFF) | RD(LREG)) 241 242 #define emit_call(FUNC) \ 243 do { void *_here = image + addrs[i] - 8; \ 244 unsigned int _off = (void *)(FUNC) - _here; \ 245 *prog++ = CALL | (((_off) >> 2) & 0x3fffffff); \ 246 emit_nop(); \ 247 } while (0) 248 249 #define emit_branch(BR_OPC, DEST) \ 250 do { unsigned int _here = addrs[i] - 8; \ 251 *prog++ = BR_OPC | WDISP22((DEST) - _here); \ 252 } while (0) 253 254 #define emit_branch_off(BR_OPC, OFF) \ 255 do { *prog++ = BR_OPC | WDISP22(OFF); \ 256 } while (0) 257 258 #define emit_jump(DEST) emit_branch(BA, DEST) 259 260 #define emit_read_y(REG) *prog++ = RD_Y | RD(REG) 261 #define emit_write_y(REG) *prog++ = WR_Y | IMMED | RS1(REG) | S13(0) 262 263 #define emit_cmp(R1, R2) \ 264 *prog++ = (SUBCC | RS1(R1) | RS2(R2) | RD(G0)) 265 266 #define emit_cmpi(R1, IMM) \ 267 *prog++ = (SUBCC | IMMED | RS1(R1) | S13(IMM) | RD(G0)); 268 269 #define emit_btst(R1, R2) \ 270 *prog++ = (ANDCC | RS1(R1) | RS2(R2) | RD(G0)) 271 272 #define emit_btsti(R1, IMM) \ 273 *prog++ = (ANDCC | IMMED | RS1(R1) | S13(IMM) | RD(G0)); 274 275 #define emit_sub(R1, R2, R3) \ 276 *prog++ = (SUB | RS1(R1) | RS2(R2) | RD(R3)) 277 278 #define emit_subi(R1, IMM, R3) \ 279 *prog++ = (SUB | IMMED | RS1(R1) | S13(IMM) | RD(R3)) 280 281 #define emit_add(R1, R2, R3) \ 282 *prog++ = (ADD | RS1(R1) | RS2(R2) | RD(R3)) 283 284 #define emit_addi(R1, IMM, R3) \ 285 *prog++ = (ADD | IMMED | RS1(R1) | S13(IMM) | RD(R3)) 286 287 #define emit_and(R1, R2, R3) \ 288 *prog++ = (AND | RS1(R1) | RS2(R2) | RD(R3)) 289 290 #define emit_andi(R1, IMM, R3) \ 291 *prog++ = (AND | IMMED | RS1(R1) | S13(IMM) | RD(R3)) 292 293 #define emit_alloc_stack(SZ) \ 294 *prog++ = (SUB | IMMED | RS1(SP) | S13(SZ) | RD(SP)) 295 296 #define emit_release_stack(SZ) \ 297 *prog++ = (ADD | IMMED | RS1(SP) | S13(SZ) | RD(SP)) 298 299 /* A note about branch offset calculations. The addrs[] array, 300 * indexed by BPF instruction, records the address after all the 301 * sparc instructions emitted for that BPF instruction. 302 * 303 * The most common case is to emit a branch at the end of such 304 * a code sequence. So this would be two instructions, the 305 * branch and it's delay slot. 306 * 307 * Therefore by default the branch emitters calculate the branch 308 * offset field as: 309 * 310 * destination - (addrs[i] - 8) 311 * 312 * This "addrs[i] - 8" is the address of the branch itself or 313 * what "." would be in assembler notation. The "8" part is 314 * how we take into consideration the branch and it's delay 315 * slot mentioned above. 316 * 317 * Sometimes we need to emit a branch earlier in the code 318 * sequence. And in these situations we adjust "destination" 319 * to accommodate this difference. For example, if we needed 320 * to emit a branch (and it's delay slot) right before the 321 * final instruction emitted for a BPF opcode, we'd use 322 * "destination + 4" instead of just plain "destination" above. 323 * 324 * This is why you see all of these funny emit_branch() and 325 * emit_jump() calls with adjusted offsets. 326 */ 327 328 void bpf_jit_compile(struct bpf_prog *fp) 329 { 330 unsigned int cleanup_addr, proglen, oldproglen = 0; 331 u32 temp[8], *prog, *func, seen = 0, pass; 332 const struct sock_filter *filter = fp->insns; 333 int i, flen = fp->len, pc_ret0 = -1; 334 unsigned int *addrs; 335 void *image; 336 337 if (!bpf_jit_enable) 338 return; 339 340 addrs = kmalloc(flen * sizeof(*addrs), GFP_KERNEL); 341 if (addrs == NULL) 342 return; 343 344 /* Before first pass, make a rough estimation of addrs[] 345 * each bpf instruction is translated to less than 64 bytes 346 */ 347 for (proglen = 0, i = 0; i < flen; i++) { 348 proglen += 64; 349 addrs[i] = proglen; 350 } 351 cleanup_addr = proglen; /* epilogue address */ 352 image = NULL; 353 for (pass = 0; pass < 10; pass++) { 354 u8 seen_or_pass0 = (pass == 0) ? (SEEN_XREG | SEEN_DATAREF | SEEN_MEM) : seen; 355 356 /* no prologue/epilogue for trivial filters (RET something) */ 357 proglen = 0; 358 prog = temp; 359 360 /* Prologue */ 361 if (seen_or_pass0) { 362 if (seen_or_pass0 & SEEN_MEM) { 363 unsigned int sz = BASE_STACKFRAME; 364 sz += BPF_MEMWORDS * sizeof(u32); 365 emit_alloc_stack(sz); 366 } 367 368 /* Make sure we dont leek kernel memory. */ 369 if (seen_or_pass0 & SEEN_XREG) 370 emit_clear(r_X); 371 372 /* If this filter needs to access skb data, 373 * load %o4 and %o5 with: 374 * %o4 = skb->len - skb->data_len 375 * %o5 = skb->data 376 * And also back up %o7 into r_saved_O7 so we can 377 * invoke the stubs using 'call'. 378 */ 379 if (seen_or_pass0 & SEEN_DATAREF) { 380 emit_load32(r_SKB, struct sk_buff, len, r_HEADLEN); 381 emit_load32(r_SKB, struct sk_buff, data_len, r_TMP); 382 emit_sub(r_HEADLEN, r_TMP, r_HEADLEN); 383 emit_loadptr(r_SKB, struct sk_buff, data, r_SKB_DATA); 384 } 385 } 386 emit_reg_move(O7, r_saved_O7); 387 388 /* Make sure we dont leak kernel information to the user. */ 389 if (bpf_needs_clear_a(&filter[0])) 390 emit_clear(r_A); /* A = 0 */ 391 392 for (i = 0; i < flen; i++) { 393 unsigned int K = filter[i].k; 394 unsigned int t_offset; 395 unsigned int f_offset; 396 u32 t_op, f_op; 397 u16 code = bpf_anc_helper(&filter[i]); 398 int ilen; 399 400 switch (code) { 401 case BPF_ALU | BPF_ADD | BPF_X: /* A += X; */ 402 emit_alu_X(ADD); 403 break; 404 case BPF_ALU | BPF_ADD | BPF_K: /* A += K; */ 405 emit_alu_K(ADD, K); 406 break; 407 case BPF_ALU | BPF_SUB | BPF_X: /* A -= X; */ 408 emit_alu_X(SUB); 409 break; 410 case BPF_ALU | BPF_SUB | BPF_K: /* A -= K */ 411 emit_alu_K(SUB, K); 412 break; 413 case BPF_ALU | BPF_AND | BPF_X: /* A &= X */ 414 emit_alu_X(AND); 415 break; 416 case BPF_ALU | BPF_AND | BPF_K: /* A &= K */ 417 emit_alu_K(AND, K); 418 break; 419 case BPF_ALU | BPF_OR | BPF_X: /* A |= X */ 420 emit_alu_X(OR); 421 break; 422 case BPF_ALU | BPF_OR | BPF_K: /* A |= K */ 423 emit_alu_K(OR, K); 424 break; 425 case BPF_ANC | SKF_AD_ALU_XOR_X: /* A ^= X; */ 426 case BPF_ALU | BPF_XOR | BPF_X: 427 emit_alu_X(XOR); 428 break; 429 case BPF_ALU | BPF_XOR | BPF_K: /* A ^= K */ 430 emit_alu_K(XOR, K); 431 break; 432 case BPF_ALU | BPF_LSH | BPF_X: /* A <<= X */ 433 emit_alu_X(SLL); 434 break; 435 case BPF_ALU | BPF_LSH | BPF_K: /* A <<= K */ 436 emit_alu_K(SLL, K); 437 break; 438 case BPF_ALU | BPF_RSH | BPF_X: /* A >>= X */ 439 emit_alu_X(SRL); 440 break; 441 case BPF_ALU | BPF_RSH | BPF_K: /* A >>= K */ 442 emit_alu_K(SRL, K); 443 break; 444 case BPF_ALU | BPF_MUL | BPF_X: /* A *= X; */ 445 emit_alu_X(MUL); 446 break; 447 case BPF_ALU | BPF_MUL | BPF_K: /* A *= K */ 448 emit_alu_K(MUL, K); 449 break; 450 case BPF_ALU | BPF_DIV | BPF_K: /* A /= K with K != 0*/ 451 if (K == 1) 452 break; 453 emit_write_y(G0); 454 /* The Sparc v8 architecture requires 455 * three instructions between a %y 456 * register write and the first use. 457 */ 458 emit_nop(); 459 emit_nop(); 460 emit_nop(); 461 emit_alu_K(DIV, K); 462 break; 463 case BPF_ALU | BPF_DIV | BPF_X: /* A /= X; */ 464 emit_cmpi(r_X, 0); 465 if (pc_ret0 > 0) { 466 t_offset = addrs[pc_ret0 - 1]; 467 emit_branch(BE, t_offset + 20); 468 emit_nop(); /* delay slot */ 469 } else { 470 emit_branch_off(BNE, 16); 471 emit_nop(); 472 emit_jump(cleanup_addr + 20); 473 emit_clear(r_A); 474 } 475 emit_write_y(G0); 476 /* The Sparc v8 architecture requires 477 * three instructions between a %y 478 * register write and the first use. 479 */ 480 emit_nop(); 481 emit_nop(); 482 emit_nop(); 483 emit_alu_X(DIV); 484 break; 485 case BPF_ALU | BPF_NEG: 486 emit_neg(); 487 break; 488 case BPF_RET | BPF_K: 489 if (!K) { 490 if (pc_ret0 == -1) 491 pc_ret0 = i; 492 emit_clear(r_A); 493 } else { 494 emit_loadimm(K, r_A); 495 } 496 /* Fallthrough */ 497 case BPF_RET | BPF_A: 498 if (seen_or_pass0) { 499 if (i != flen - 1) { 500 emit_jump(cleanup_addr); 501 emit_nop(); 502 break; 503 } 504 if (seen_or_pass0 & SEEN_MEM) { 505 unsigned int sz = BASE_STACKFRAME; 506 sz += BPF_MEMWORDS * sizeof(u32); 507 emit_release_stack(sz); 508 } 509 } 510 /* jmpl %r_saved_O7 + 8, %g0 */ 511 emit_jmpl(r_saved_O7, 8, G0); 512 emit_reg_move(r_A, O0); /* delay slot */ 513 break; 514 case BPF_MISC | BPF_TAX: 515 seen |= SEEN_XREG; 516 emit_reg_move(r_A, r_X); 517 break; 518 case BPF_MISC | BPF_TXA: 519 seen |= SEEN_XREG; 520 emit_reg_move(r_X, r_A); 521 break; 522 case BPF_ANC | SKF_AD_CPU: 523 emit_load_cpu(r_A); 524 break; 525 case BPF_ANC | SKF_AD_PROTOCOL: 526 emit_skb_load16(protocol, r_A); 527 break; 528 case BPF_ANC | SKF_AD_PKTTYPE: 529 __emit_skb_load8(__pkt_type_offset, r_A); 530 emit_andi(r_A, PKT_TYPE_MAX, r_A); 531 emit_alu_K(SRL, 5); 532 break; 533 case BPF_ANC | SKF_AD_IFINDEX: 534 emit_skb_loadptr(dev, r_A); 535 emit_cmpi(r_A, 0); 536 emit_branch(BE_PTR, cleanup_addr + 4); 537 emit_nop(); 538 emit_load32(r_A, struct net_device, ifindex, r_A); 539 break; 540 case BPF_ANC | SKF_AD_MARK: 541 emit_skb_load32(mark, r_A); 542 break; 543 case BPF_ANC | SKF_AD_QUEUE: 544 emit_skb_load16(queue_mapping, r_A); 545 break; 546 case BPF_ANC | SKF_AD_HATYPE: 547 emit_skb_loadptr(dev, r_A); 548 emit_cmpi(r_A, 0); 549 emit_branch(BE_PTR, cleanup_addr + 4); 550 emit_nop(); 551 emit_load16(r_A, struct net_device, type, r_A); 552 break; 553 case BPF_ANC | SKF_AD_RXHASH: 554 emit_skb_load32(hash, r_A); 555 break; 556 case BPF_ANC | SKF_AD_VLAN_TAG: 557 case BPF_ANC | SKF_AD_VLAN_TAG_PRESENT: 558 emit_skb_load16(vlan_tci, r_A); 559 if (code != (BPF_ANC | SKF_AD_VLAN_TAG)) { 560 emit_alu_K(SRL, 12); 561 emit_andi(r_A, 1, r_A); 562 } else { 563 emit_loadimm(~VLAN_TAG_PRESENT, r_TMP); 564 emit_and(r_A, r_TMP, r_A); 565 } 566 break; 567 case BPF_LD | BPF_W | BPF_LEN: 568 emit_skb_load32(len, r_A); 569 break; 570 case BPF_LDX | BPF_W | BPF_LEN: 571 emit_skb_load32(len, r_X); 572 break; 573 case BPF_LD | BPF_IMM: 574 emit_loadimm(K, r_A); 575 break; 576 case BPF_LDX | BPF_IMM: 577 emit_loadimm(K, r_X); 578 break; 579 case BPF_LD | BPF_MEM: 580 seen |= SEEN_MEM; 581 emit_ldmem(K * 4, r_A); 582 break; 583 case BPF_LDX | BPF_MEM: 584 seen |= SEEN_MEM | SEEN_XREG; 585 emit_ldmem(K * 4, r_X); 586 break; 587 case BPF_ST: 588 seen |= SEEN_MEM; 589 emit_stmem(K * 4, r_A); 590 break; 591 case BPF_STX: 592 seen |= SEEN_MEM | SEEN_XREG; 593 emit_stmem(K * 4, r_X); 594 break; 595 596 #define CHOOSE_LOAD_FUNC(K, func) \ 597 ((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative_offset : func) : func##_positive_offset) 598 599 case BPF_LD | BPF_W | BPF_ABS: 600 func = CHOOSE_LOAD_FUNC(K, bpf_jit_load_word); 601 common_load: seen |= SEEN_DATAREF; 602 emit_loadimm(K, r_OFF); 603 emit_call(func); 604 break; 605 case BPF_LD | BPF_H | BPF_ABS: 606 func = CHOOSE_LOAD_FUNC(K, bpf_jit_load_half); 607 goto common_load; 608 case BPF_LD | BPF_B | BPF_ABS: 609 func = CHOOSE_LOAD_FUNC(K, bpf_jit_load_byte); 610 goto common_load; 611 case BPF_LDX | BPF_B | BPF_MSH: 612 func = CHOOSE_LOAD_FUNC(K, bpf_jit_load_byte_msh); 613 goto common_load; 614 case BPF_LD | BPF_W | BPF_IND: 615 func = bpf_jit_load_word; 616 common_load_ind: seen |= SEEN_DATAREF | SEEN_XREG; 617 if (K) { 618 if (is_simm13(K)) { 619 emit_addi(r_X, K, r_OFF); 620 } else { 621 emit_loadimm(K, r_TMP); 622 emit_add(r_X, r_TMP, r_OFF); 623 } 624 } else { 625 emit_reg_move(r_X, r_OFF); 626 } 627 emit_call(func); 628 break; 629 case BPF_LD | BPF_H | BPF_IND: 630 func = bpf_jit_load_half; 631 goto common_load_ind; 632 case BPF_LD | BPF_B | BPF_IND: 633 func = bpf_jit_load_byte; 634 goto common_load_ind; 635 case BPF_JMP | BPF_JA: 636 emit_jump(addrs[i + K]); 637 emit_nop(); 638 break; 639 640 #define COND_SEL(CODE, TOP, FOP) \ 641 case CODE: \ 642 t_op = TOP; \ 643 f_op = FOP; \ 644 goto cond_branch 645 646 COND_SEL(BPF_JMP | BPF_JGT | BPF_K, BGU, BLEU); 647 COND_SEL(BPF_JMP | BPF_JGE | BPF_K, BGEU, BLU); 648 COND_SEL(BPF_JMP | BPF_JEQ | BPF_K, BE, BNE); 649 COND_SEL(BPF_JMP | BPF_JSET | BPF_K, BNE, BE); 650 COND_SEL(BPF_JMP | BPF_JGT | BPF_X, BGU, BLEU); 651 COND_SEL(BPF_JMP | BPF_JGE | BPF_X, BGEU, BLU); 652 COND_SEL(BPF_JMP | BPF_JEQ | BPF_X, BE, BNE); 653 COND_SEL(BPF_JMP | BPF_JSET | BPF_X, BNE, BE); 654 655 cond_branch: f_offset = addrs[i + filter[i].jf]; 656 t_offset = addrs[i + filter[i].jt]; 657 658 /* same targets, can avoid doing the test :) */ 659 if (filter[i].jt == filter[i].jf) { 660 emit_jump(t_offset); 661 emit_nop(); 662 break; 663 } 664 665 switch (code) { 666 case BPF_JMP | BPF_JGT | BPF_X: 667 case BPF_JMP | BPF_JGE | BPF_X: 668 case BPF_JMP | BPF_JEQ | BPF_X: 669 seen |= SEEN_XREG; 670 emit_cmp(r_A, r_X); 671 break; 672 case BPF_JMP | BPF_JSET | BPF_X: 673 seen |= SEEN_XREG; 674 emit_btst(r_A, r_X); 675 break; 676 case BPF_JMP | BPF_JEQ | BPF_K: 677 case BPF_JMP | BPF_JGT | BPF_K: 678 case BPF_JMP | BPF_JGE | BPF_K: 679 if (is_simm13(K)) { 680 emit_cmpi(r_A, K); 681 } else { 682 emit_loadimm(K, r_TMP); 683 emit_cmp(r_A, r_TMP); 684 } 685 break; 686 case BPF_JMP | BPF_JSET | BPF_K: 687 if (is_simm13(K)) { 688 emit_btsti(r_A, K); 689 } else { 690 emit_loadimm(K, r_TMP); 691 emit_btst(r_A, r_TMP); 692 } 693 break; 694 } 695 if (filter[i].jt != 0) { 696 if (filter[i].jf) 697 t_offset += 8; 698 emit_branch(t_op, t_offset); 699 emit_nop(); /* delay slot */ 700 if (filter[i].jf) { 701 emit_jump(f_offset); 702 emit_nop(); 703 } 704 break; 705 } 706 emit_branch(f_op, f_offset); 707 emit_nop(); /* delay slot */ 708 break; 709 710 default: 711 /* hmm, too complex filter, give up with jit compiler */ 712 goto out; 713 } 714 ilen = (void *) prog - (void *) temp; 715 if (image) { 716 if (unlikely(proglen + ilen > oldproglen)) { 717 pr_err("bpb_jit_compile fatal error\n"); 718 kfree(addrs); 719 module_memfree(image); 720 return; 721 } 722 memcpy(image + proglen, temp, ilen); 723 } 724 proglen += ilen; 725 addrs[i] = proglen; 726 prog = temp; 727 } 728 /* last bpf instruction is always a RET : 729 * use it to give the cleanup instruction(s) addr 730 */ 731 cleanup_addr = proglen - 8; /* jmpl; mov r_A,%o0; */ 732 if (seen_or_pass0 & SEEN_MEM) 733 cleanup_addr -= 4; /* add %sp, X, %sp; */ 734 735 if (image) { 736 if (proglen != oldproglen) 737 pr_err("bpb_jit_compile proglen=%u != oldproglen=%u\n", 738 proglen, oldproglen); 739 break; 740 } 741 if (proglen == oldproglen) { 742 image = module_alloc(proglen); 743 if (!image) 744 goto out; 745 } 746 oldproglen = proglen; 747 } 748 749 if (bpf_jit_enable > 1) 750 bpf_jit_dump(flen, proglen, pass + 1, image); 751 752 if (image) { 753 fp->bpf_func = (void *)image; 754 fp->jited = 1; 755 } 756 out: 757 kfree(addrs); 758 return; 759 } 760 761 void bpf_jit_free(struct bpf_prog *fp) 762 { 763 if (fp->jited) 764 module_memfree(fp->bpf_func); 765 766 bpf_prog_unlock_free(fp); 767 } 768