1 /* 2 * BPF Jit compiler for s390. 3 * 4 * Copyright IBM Corp. 2012 5 * 6 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> 7 */ 8 #include <linux/netdevice.h> 9 #include <linux/if_vlan.h> 10 #include <linux/filter.h> 11 #include <linux/init.h> 12 #include <asm/cacheflush.h> 13 #include <asm/facility.h> 14 #include <asm/dis.h> 15 16 /* 17 * Conventions: 18 * %r2 = skb pointer 19 * %r3 = offset parameter 20 * %r4 = scratch register / length parameter 21 * %r5 = BPF A accumulator 22 * %r8 = return address 23 * %r9 = save register for skb pointer 24 * %r10 = skb->data 25 * %r11 = skb->len - skb->data_len (headlen) 26 * %r12 = BPF X accumulator 27 * %r13 = literal pool pointer 28 * 0(%r15) - 63(%r15) scratch memory array with BPF_MEMWORDS 29 */ 30 int bpf_jit_enable __read_mostly; 31 32 /* 33 * assembly code in arch/x86/net/bpf_jit.S 34 */ 35 extern u8 sk_load_word[], sk_load_half[], sk_load_byte[], sk_load_byte_msh[]; 36 extern u8 sk_load_word_ind[], sk_load_half_ind[], sk_load_byte_ind[]; 37 38 struct bpf_jit { 39 unsigned int seen; 40 u8 *start; 41 u8 *prg; 42 u8 *mid; 43 u8 *lit; 44 u8 *end; 45 u8 *base_ip; 46 u8 *ret0_ip; 47 u8 *exit_ip; 48 unsigned int off_load_word; 49 unsigned int off_load_half; 50 unsigned int off_load_byte; 51 unsigned int off_load_bmsh; 52 unsigned int off_load_iword; 53 unsigned int off_load_ihalf; 54 unsigned int off_load_ibyte; 55 }; 56 57 #define BPF_SIZE_MAX 4096 /* Max size for program */ 58 59 #define SEEN_DATAREF 1 /* might call external helpers */ 60 #define SEEN_XREG 2 /* ebx is used */ 61 #define SEEN_MEM 4 /* use mem[] for temporary storage */ 62 #define SEEN_RET0 8 /* pc_ret0 points to a valid return 0 */ 63 #define SEEN_LITERAL 16 /* code uses literals */ 64 #define SEEN_LOAD_WORD 32 /* code uses sk_load_word */ 65 #define SEEN_LOAD_HALF 64 /* code uses sk_load_half */ 66 #define SEEN_LOAD_BYTE 128 /* code uses sk_load_byte */ 67 #define SEEN_LOAD_BMSH 256 /* code uses sk_load_byte_msh */ 68 #define SEEN_LOAD_IWORD 512 /* code uses sk_load_word_ind */ 69 #define SEEN_LOAD_IHALF 1024 /* code uses sk_load_half_ind */ 70 #define SEEN_LOAD_IBYTE 2048 /* code uses sk_load_byte_ind */ 71 72 #define EMIT2(op) \ 73 ({ \ 74 if (jit->prg + 2 <= jit->mid) \ 75 *(u16 *) jit->prg = op; \ 76 jit->prg += 2; \ 77 }) 78 79 #define EMIT4(op) \ 80 ({ \ 81 if (jit->prg + 4 <= jit->mid) \ 82 *(u32 *) jit->prg = op; \ 83 jit->prg += 4; \ 84 }) 85 86 #define EMIT4_DISP(op, disp) \ 87 ({ \ 88 unsigned int __disp = (disp) & 0xfff; \ 89 EMIT4(op | __disp); \ 90 }) 91 92 #define EMIT4_IMM(op, imm) \ 93 ({ \ 94 unsigned int __imm = (imm) & 0xffff; \ 95 EMIT4(op | __imm); \ 96 }) 97 98 #define EMIT4_PCREL(op, pcrel) \ 99 ({ \ 100 long __pcrel = ((pcrel) >> 1) & 0xffff; \ 101 EMIT4(op | __pcrel); \ 102 }) 103 104 #define EMIT6(op1, op2) \ 105 ({ \ 106 if (jit->prg + 6 <= jit->mid) { \ 107 *(u32 *) jit->prg = op1; \ 108 *(u16 *) (jit->prg + 4) = op2; \ 109 } \ 110 jit->prg += 6; \ 111 }) 112 113 #define EMIT6_DISP(op1, op2, disp) \ 114 ({ \ 115 unsigned int __disp = (disp) & 0xfff; \ 116 EMIT6(op1 | __disp, op2); \ 117 }) 118 119 #define EMIT6_IMM(op, imm) \ 120 ({ \ 121 unsigned int __imm = (imm); \ 122 EMIT6(op | (__imm >> 16), __imm & 0xffff); \ 123 }) 124 125 #define EMIT_CONST(val) \ 126 ({ \ 127 unsigned int ret; \ 128 ret = (unsigned int) (jit->lit - jit->base_ip); \ 129 jit->seen |= SEEN_LITERAL; \ 130 if (jit->lit + 4 <= jit->end) \ 131 *(u32 *) jit->lit = val; \ 132 jit->lit += 4; \ 133 ret; \ 134 }) 135 136 #define EMIT_FN_CONST(bit, fn) \ 137 ({ \ 138 unsigned int ret; \ 139 ret = (unsigned int) (jit->lit - jit->base_ip); \ 140 if (jit->seen & bit) { \ 141 jit->seen |= SEEN_LITERAL; \ 142 if (jit->lit + 8 <= jit->end) \ 143 *(void **) jit->lit = fn; \ 144 jit->lit += 8; \ 145 } \ 146 ret; \ 147 }) 148 149 static void bpf_jit_fill_hole(void *area, unsigned int size) 150 { 151 /* Fill whole space with illegal instructions */ 152 memset(area, 0, size); 153 } 154 155 static void bpf_jit_prologue(struct bpf_jit *jit) 156 { 157 /* Save registers and create stack frame if necessary */ 158 if (jit->seen & SEEN_DATAREF) { 159 /* stmg %r8,%r15,88(%r15) */ 160 EMIT6(0xeb8ff058, 0x0024); 161 /* lgr %r14,%r15 */ 162 EMIT4(0xb90400ef); 163 /* aghi %r15,<offset> */ 164 EMIT4_IMM(0xa7fb0000, (jit->seen & SEEN_MEM) ? -112 : -80); 165 /* stg %r14,152(%r15) */ 166 EMIT6(0xe3e0f098, 0x0024); 167 } else if ((jit->seen & SEEN_XREG) && (jit->seen & SEEN_LITERAL)) 168 /* stmg %r12,%r13,120(%r15) */ 169 EMIT6(0xebcdf078, 0x0024); 170 else if (jit->seen & SEEN_XREG) 171 /* stg %r12,120(%r15) */ 172 EMIT6(0xe3c0f078, 0x0024); 173 else if (jit->seen & SEEN_LITERAL) 174 /* stg %r13,128(%r15) */ 175 EMIT6(0xe3d0f080, 0x0024); 176 177 /* Setup literal pool */ 178 if (jit->seen & SEEN_LITERAL) { 179 /* basr %r13,0 */ 180 EMIT2(0x0dd0); 181 jit->base_ip = jit->prg; 182 } 183 jit->off_load_word = EMIT_FN_CONST(SEEN_LOAD_WORD, sk_load_word); 184 jit->off_load_half = EMIT_FN_CONST(SEEN_LOAD_HALF, sk_load_half); 185 jit->off_load_byte = EMIT_FN_CONST(SEEN_LOAD_BYTE, sk_load_byte); 186 jit->off_load_bmsh = EMIT_FN_CONST(SEEN_LOAD_BMSH, sk_load_byte_msh); 187 jit->off_load_iword = EMIT_FN_CONST(SEEN_LOAD_IWORD, sk_load_word_ind); 188 jit->off_load_ihalf = EMIT_FN_CONST(SEEN_LOAD_IHALF, sk_load_half_ind); 189 jit->off_load_ibyte = EMIT_FN_CONST(SEEN_LOAD_IBYTE, sk_load_byte_ind); 190 191 /* Filter needs to access skb data */ 192 if (jit->seen & SEEN_DATAREF) { 193 /* l %r11,<len>(%r2) */ 194 EMIT4_DISP(0x58b02000, offsetof(struct sk_buff, len)); 195 /* s %r11,<data_len>(%r2) */ 196 EMIT4_DISP(0x5bb02000, offsetof(struct sk_buff, data_len)); 197 /* lg %r10,<data>(%r2) */ 198 EMIT6_DISP(0xe3a02000, 0x0004, 199 offsetof(struct sk_buff, data)); 200 } 201 } 202 203 static void bpf_jit_epilogue(struct bpf_jit *jit) 204 { 205 /* Return 0 */ 206 if (jit->seen & SEEN_RET0) { 207 jit->ret0_ip = jit->prg; 208 /* lghi %r2,0 */ 209 EMIT4(0xa7290000); 210 } 211 jit->exit_ip = jit->prg; 212 /* Restore registers */ 213 if (jit->seen & SEEN_DATAREF) 214 /* lmg %r8,%r15,<offset>(%r15) */ 215 EMIT6_DISP(0xeb8ff000, 0x0004, 216 (jit->seen & SEEN_MEM) ? 200 : 168); 217 else if ((jit->seen & SEEN_XREG) && (jit->seen & SEEN_LITERAL)) 218 /* lmg %r12,%r13,120(%r15) */ 219 EMIT6(0xebcdf078, 0x0004); 220 else if (jit->seen & SEEN_XREG) 221 /* lg %r12,120(%r15) */ 222 EMIT6(0xe3c0f078, 0x0004); 223 else if (jit->seen & SEEN_LITERAL) 224 /* lg %r13,128(%r15) */ 225 EMIT6(0xe3d0f080, 0x0004); 226 /* br %r14 */ 227 EMIT2(0x07fe); 228 } 229 230 /* 231 * make sure we dont leak kernel information to user 232 */ 233 static void bpf_jit_noleaks(struct bpf_jit *jit, struct sock_filter *filter) 234 { 235 /* Clear temporary memory if (seen & SEEN_MEM) */ 236 if (jit->seen & SEEN_MEM) 237 /* xc 0(64,%r15),0(%r15) */ 238 EMIT6(0xd73ff000, 0xf000); 239 /* Clear X if (seen & SEEN_XREG) */ 240 if (jit->seen & SEEN_XREG) 241 /* lhi %r12,0 */ 242 EMIT4(0xa7c80000); 243 /* Clear A if the first register does not set it. */ 244 switch (filter[0].code) { 245 case BPF_LD | BPF_W | BPF_ABS: 246 case BPF_LD | BPF_H | BPF_ABS: 247 case BPF_LD | BPF_B | BPF_ABS: 248 case BPF_LD | BPF_W | BPF_LEN: 249 case BPF_LD | BPF_W | BPF_IND: 250 case BPF_LD | BPF_H | BPF_IND: 251 case BPF_LD | BPF_B | BPF_IND: 252 case BPF_LD | BPF_IMM: 253 case BPF_LD | BPF_MEM: 254 case BPF_MISC | BPF_TXA: 255 case BPF_RET | BPF_K: 256 /* first instruction sets A register */ 257 break; 258 default: /* A = 0 */ 259 /* lhi %r5,0 */ 260 EMIT4(0xa7580000); 261 } 262 } 263 264 static int bpf_jit_insn(struct bpf_jit *jit, struct sock_filter *filter, 265 unsigned int *addrs, int i, int last) 266 { 267 unsigned int K; 268 int offset; 269 unsigned int mask; 270 u16 code; 271 272 K = filter->k; 273 code = bpf_anc_helper(filter); 274 275 switch (code) { 276 case BPF_ALU | BPF_ADD | BPF_X: /* A += X */ 277 jit->seen |= SEEN_XREG; 278 /* ar %r5,%r12 */ 279 EMIT2(0x1a5c); 280 break; 281 case BPF_ALU | BPF_ADD | BPF_K: /* A += K */ 282 if (!K) 283 break; 284 if (K <= 16383) 285 /* ahi %r5,<K> */ 286 EMIT4_IMM(0xa75a0000, K); 287 else if (test_facility(21)) 288 /* alfi %r5,<K> */ 289 EMIT6_IMM(0xc25b0000, K); 290 else 291 /* a %r5,<d(K)>(%r13) */ 292 EMIT4_DISP(0x5a50d000, EMIT_CONST(K)); 293 break; 294 case BPF_ALU | BPF_SUB | BPF_X: /* A -= X */ 295 jit->seen |= SEEN_XREG; 296 /* sr %r5,%r12 */ 297 EMIT2(0x1b5c); 298 break; 299 case BPF_ALU | BPF_SUB | BPF_K: /* A -= K */ 300 if (!K) 301 break; 302 if (K <= 16384) 303 /* ahi %r5,-K */ 304 EMIT4_IMM(0xa75a0000, -K); 305 else if (test_facility(21)) 306 /* alfi %r5,-K */ 307 EMIT6_IMM(0xc25b0000, -K); 308 else 309 /* s %r5,<d(K)>(%r13) */ 310 EMIT4_DISP(0x5b50d000, EMIT_CONST(K)); 311 break; 312 case BPF_ALU | BPF_MUL | BPF_X: /* A *= X */ 313 jit->seen |= SEEN_XREG; 314 /* msr %r5,%r12 */ 315 EMIT4(0xb252005c); 316 break; 317 case BPF_ALU | BPF_MUL | BPF_K: /* A *= K */ 318 if (K <= 16383) 319 /* mhi %r5,K */ 320 EMIT4_IMM(0xa75c0000, K); 321 else if (test_facility(34)) 322 /* msfi %r5,<K> */ 323 EMIT6_IMM(0xc2510000, K); 324 else 325 /* ms %r5,<d(K)>(%r13) */ 326 EMIT4_DISP(0x7150d000, EMIT_CONST(K)); 327 break; 328 case BPF_ALU | BPF_DIV | BPF_X: /* A /= X */ 329 jit->seen |= SEEN_XREG | SEEN_RET0; 330 /* ltr %r12,%r12 */ 331 EMIT2(0x12cc); 332 /* jz <ret0> */ 333 EMIT4_PCREL(0xa7840000, (jit->ret0_ip - jit->prg)); 334 /* lhi %r4,0 */ 335 EMIT4(0xa7480000); 336 /* dlr %r4,%r12 */ 337 EMIT4(0xb997004c); 338 break; 339 case BPF_ALU | BPF_DIV | BPF_K: /* A /= K */ 340 if (K == 1) 341 break; 342 /* lhi %r4,0 */ 343 EMIT4(0xa7480000); 344 /* dl %r4,<d(K)>(%r13) */ 345 EMIT6_DISP(0xe340d000, 0x0097, EMIT_CONST(K)); 346 break; 347 case BPF_ALU | BPF_MOD | BPF_X: /* A %= X */ 348 jit->seen |= SEEN_XREG | SEEN_RET0; 349 /* ltr %r12,%r12 */ 350 EMIT2(0x12cc); 351 /* jz <ret0> */ 352 EMIT4_PCREL(0xa7840000, (jit->ret0_ip - jit->prg)); 353 /* lhi %r4,0 */ 354 EMIT4(0xa7480000); 355 /* dlr %r4,%r12 */ 356 EMIT4(0xb997004c); 357 /* lr %r5,%r4 */ 358 EMIT2(0x1854); 359 break; 360 case BPF_ALU | BPF_MOD | BPF_K: /* A %= K */ 361 if (K == 1) { 362 /* lhi %r5,0 */ 363 EMIT4(0xa7580000); 364 break; 365 } 366 /* lhi %r4,0 */ 367 EMIT4(0xa7480000); 368 /* dl %r4,<d(K)>(%r13) */ 369 EMIT6_DISP(0xe340d000, 0x0097, EMIT_CONST(K)); 370 /* lr %r5,%r4 */ 371 EMIT2(0x1854); 372 break; 373 case BPF_ALU | BPF_AND | BPF_X: /* A &= X */ 374 jit->seen |= SEEN_XREG; 375 /* nr %r5,%r12 */ 376 EMIT2(0x145c); 377 break; 378 case BPF_ALU | BPF_AND | BPF_K: /* A &= K */ 379 if (test_facility(21)) 380 /* nilf %r5,<K> */ 381 EMIT6_IMM(0xc05b0000, K); 382 else 383 /* n %r5,<d(K)>(%r13) */ 384 EMIT4_DISP(0x5450d000, EMIT_CONST(K)); 385 break; 386 case BPF_ALU | BPF_OR | BPF_X: /* A |= X */ 387 jit->seen |= SEEN_XREG; 388 /* or %r5,%r12 */ 389 EMIT2(0x165c); 390 break; 391 case BPF_ALU | BPF_OR | BPF_K: /* A |= K */ 392 if (test_facility(21)) 393 /* oilf %r5,<K> */ 394 EMIT6_IMM(0xc05d0000, K); 395 else 396 /* o %r5,<d(K)>(%r13) */ 397 EMIT4_DISP(0x5650d000, EMIT_CONST(K)); 398 break; 399 case BPF_ANC | SKF_AD_ALU_XOR_X: /* A ^= X; */ 400 case BPF_ALU | BPF_XOR | BPF_X: 401 jit->seen |= SEEN_XREG; 402 /* xr %r5,%r12 */ 403 EMIT2(0x175c); 404 break; 405 case BPF_ALU | BPF_XOR | BPF_K: /* A ^= K */ 406 if (!K) 407 break; 408 /* x %r5,<d(K)>(%r13) */ 409 EMIT4_DISP(0x5750d000, EMIT_CONST(K)); 410 break; 411 case BPF_ALU | BPF_LSH | BPF_X: /* A <<= X; */ 412 jit->seen |= SEEN_XREG; 413 /* sll %r5,0(%r12) */ 414 EMIT4(0x8950c000); 415 break; 416 case BPF_ALU | BPF_LSH | BPF_K: /* A <<= K */ 417 if (K == 0) 418 break; 419 /* sll %r5,K */ 420 EMIT4_DISP(0x89500000, K); 421 break; 422 case BPF_ALU | BPF_RSH | BPF_X: /* A >>= X; */ 423 jit->seen |= SEEN_XREG; 424 /* srl %r5,0(%r12) */ 425 EMIT4(0x8850c000); 426 break; 427 case BPF_ALU | BPF_RSH | BPF_K: /* A >>= K; */ 428 if (K == 0) 429 break; 430 /* srl %r5,K */ 431 EMIT4_DISP(0x88500000, K); 432 break; 433 case BPF_ALU | BPF_NEG: /* A = -A */ 434 /* lcr %r5,%r5 */ 435 EMIT2(0x1355); 436 break; 437 case BPF_JMP | BPF_JA: /* ip += K */ 438 offset = addrs[i + K] + jit->start - jit->prg; 439 EMIT4_PCREL(0xa7f40000, offset); 440 break; 441 case BPF_JMP | BPF_JGT | BPF_K: /* ip += (A > K) ? jt : jf */ 442 mask = 0x200000; /* jh */ 443 goto kbranch; 444 case BPF_JMP | BPF_JGE | BPF_K: /* ip += (A >= K) ? jt : jf */ 445 mask = 0xa00000; /* jhe */ 446 goto kbranch; 447 case BPF_JMP | BPF_JEQ | BPF_K: /* ip += (A == K) ? jt : jf */ 448 mask = 0x800000; /* je */ 449 kbranch: /* Emit compare if the branch targets are different */ 450 if (filter->jt != filter->jf) { 451 if (test_facility(21)) 452 /* clfi %r5,<K> */ 453 EMIT6_IMM(0xc25f0000, K); 454 else 455 /* cl %r5,<d(K)>(%r13) */ 456 EMIT4_DISP(0x5550d000, EMIT_CONST(K)); 457 } 458 branch: if (filter->jt == filter->jf) { 459 if (filter->jt == 0) 460 break; 461 /* j <jt> */ 462 offset = addrs[i + filter->jt] + jit->start - jit->prg; 463 EMIT4_PCREL(0xa7f40000, offset); 464 break; 465 } 466 if (filter->jt != 0) { 467 /* brc <mask>,<jt> */ 468 offset = addrs[i + filter->jt] + jit->start - jit->prg; 469 EMIT4_PCREL(0xa7040000 | mask, offset); 470 } 471 if (filter->jf != 0) { 472 /* brc <mask^15>,<jf> */ 473 offset = addrs[i + filter->jf] + jit->start - jit->prg; 474 EMIT4_PCREL(0xa7040000 | (mask ^ 0xf00000), offset); 475 } 476 break; 477 case BPF_JMP | BPF_JSET | BPF_K: /* ip += (A & K) ? jt : jf */ 478 mask = 0x700000; /* jnz */ 479 /* Emit test if the branch targets are different */ 480 if (filter->jt != filter->jf) { 481 if (K > 65535) { 482 /* lr %r4,%r5 */ 483 EMIT2(0x1845); 484 /* n %r4,<d(K)>(%r13) */ 485 EMIT4_DISP(0x5440d000, EMIT_CONST(K)); 486 } else 487 /* tmll %r5,K */ 488 EMIT4_IMM(0xa7510000, K); 489 } 490 goto branch; 491 case BPF_JMP | BPF_JGT | BPF_X: /* ip += (A > X) ? jt : jf */ 492 mask = 0x200000; /* jh */ 493 goto xbranch; 494 case BPF_JMP | BPF_JGE | BPF_X: /* ip += (A >= X) ? jt : jf */ 495 mask = 0xa00000; /* jhe */ 496 goto xbranch; 497 case BPF_JMP | BPF_JEQ | BPF_X: /* ip += (A == X) ? jt : jf */ 498 mask = 0x800000; /* je */ 499 xbranch: /* Emit compare if the branch targets are different */ 500 if (filter->jt != filter->jf) { 501 jit->seen |= SEEN_XREG; 502 /* clr %r5,%r12 */ 503 EMIT2(0x155c); 504 } 505 goto branch; 506 case BPF_JMP | BPF_JSET | BPF_X: /* ip += (A & X) ? jt : jf */ 507 mask = 0x700000; /* jnz */ 508 /* Emit test if the branch targets are different */ 509 if (filter->jt != filter->jf) { 510 jit->seen |= SEEN_XREG; 511 /* lr %r4,%r5 */ 512 EMIT2(0x1845); 513 /* nr %r4,%r12 */ 514 EMIT2(0x144c); 515 } 516 goto branch; 517 case BPF_LD | BPF_W | BPF_ABS: /* A = *(u32 *) (skb->data+K) */ 518 jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_WORD; 519 offset = jit->off_load_word; 520 goto load_abs; 521 case BPF_LD | BPF_H | BPF_ABS: /* A = *(u16 *) (skb->data+K) */ 522 jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_HALF; 523 offset = jit->off_load_half; 524 goto load_abs; 525 case BPF_LD | BPF_B | BPF_ABS: /* A = *(u8 *) (skb->data+K) */ 526 jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_BYTE; 527 offset = jit->off_load_byte; 528 load_abs: if ((int) K < 0) 529 goto out; 530 call_fn: /* lg %r1,<d(function)>(%r13) */ 531 EMIT6_DISP(0xe310d000, 0x0004, offset); 532 /* l %r3,<d(K)>(%r13) */ 533 EMIT4_DISP(0x5830d000, EMIT_CONST(K)); 534 /* basr %r8,%r1 */ 535 EMIT2(0x0d81); 536 /* jnz <ret0> */ 537 EMIT4_PCREL(0xa7740000, (jit->ret0_ip - jit->prg)); 538 break; 539 case BPF_LD | BPF_W | BPF_IND: /* A = *(u32 *) (skb->data+K+X) */ 540 jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_IWORD; 541 offset = jit->off_load_iword; 542 goto call_fn; 543 case BPF_LD | BPF_H | BPF_IND: /* A = *(u16 *) (skb->data+K+X) */ 544 jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_IHALF; 545 offset = jit->off_load_ihalf; 546 goto call_fn; 547 case BPF_LD | BPF_B | BPF_IND: /* A = *(u8 *) (skb->data+K+X) */ 548 jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_IBYTE; 549 offset = jit->off_load_ibyte; 550 goto call_fn; 551 case BPF_LDX | BPF_B | BPF_MSH: 552 /* X = (*(u8 *)(skb->data+K) & 0xf) << 2 */ 553 jit->seen |= SEEN_RET0; 554 if ((int) K < 0) { 555 /* j <ret0> */ 556 EMIT4_PCREL(0xa7f40000, (jit->ret0_ip - jit->prg)); 557 break; 558 } 559 jit->seen |= SEEN_DATAREF | SEEN_LOAD_BMSH; 560 offset = jit->off_load_bmsh; 561 goto call_fn; 562 case BPF_LD | BPF_W | BPF_LEN: /* A = skb->len; */ 563 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, len) != 4); 564 /* l %r5,<d(len)>(%r2) */ 565 EMIT4_DISP(0x58502000, offsetof(struct sk_buff, len)); 566 break; 567 case BPF_LDX | BPF_W | BPF_LEN: /* X = skb->len; */ 568 jit->seen |= SEEN_XREG; 569 /* l %r12,<d(len)>(%r2) */ 570 EMIT4_DISP(0x58c02000, offsetof(struct sk_buff, len)); 571 break; 572 case BPF_LD | BPF_IMM: /* A = K */ 573 if (K <= 16383) 574 /* lhi %r5,K */ 575 EMIT4_IMM(0xa7580000, K); 576 else if (test_facility(21)) 577 /* llilf %r5,<K> */ 578 EMIT6_IMM(0xc05f0000, K); 579 else 580 /* l %r5,<d(K)>(%r13) */ 581 EMIT4_DISP(0x5850d000, EMIT_CONST(K)); 582 break; 583 case BPF_LDX | BPF_IMM: /* X = K */ 584 jit->seen |= SEEN_XREG; 585 if (K <= 16383) 586 /* lhi %r12,<K> */ 587 EMIT4_IMM(0xa7c80000, K); 588 else if (test_facility(21)) 589 /* llilf %r12,<K> */ 590 EMIT6_IMM(0xc0cf0000, K); 591 else 592 /* l %r12,<d(K)>(%r13) */ 593 EMIT4_DISP(0x58c0d000, EMIT_CONST(K)); 594 break; 595 case BPF_LD | BPF_MEM: /* A = mem[K] */ 596 jit->seen |= SEEN_MEM; 597 /* l %r5,<K>(%r15) */ 598 EMIT4_DISP(0x5850f000, 599 (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4); 600 break; 601 case BPF_LDX | BPF_MEM: /* X = mem[K] */ 602 jit->seen |= SEEN_XREG | SEEN_MEM; 603 /* l %r12,<K>(%r15) */ 604 EMIT4_DISP(0x58c0f000, 605 (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4); 606 break; 607 case BPF_MISC | BPF_TAX: /* X = A */ 608 jit->seen |= SEEN_XREG; 609 /* lr %r12,%r5 */ 610 EMIT2(0x18c5); 611 break; 612 case BPF_MISC | BPF_TXA: /* A = X */ 613 jit->seen |= SEEN_XREG; 614 /* lr %r5,%r12 */ 615 EMIT2(0x185c); 616 break; 617 case BPF_RET | BPF_K: 618 if (K == 0) { 619 jit->seen |= SEEN_RET0; 620 if (last) 621 break; 622 /* j <ret0> */ 623 EMIT4_PCREL(0xa7f40000, jit->ret0_ip - jit->prg); 624 } else { 625 if (K <= 16383) 626 /* lghi %r2,K */ 627 EMIT4_IMM(0xa7290000, K); 628 else 629 /* llgf %r2,<K>(%r13) */ 630 EMIT6_DISP(0xe320d000, 0x0016, EMIT_CONST(K)); 631 /* j <exit> */ 632 if (last && !(jit->seen & SEEN_RET0)) 633 break; 634 EMIT4_PCREL(0xa7f40000, jit->exit_ip - jit->prg); 635 } 636 break; 637 case BPF_RET | BPF_A: 638 /* llgfr %r2,%r5 */ 639 EMIT4(0xb9160025); 640 /* j <exit> */ 641 EMIT4_PCREL(0xa7f40000, jit->exit_ip - jit->prg); 642 break; 643 case BPF_ST: /* mem[K] = A */ 644 jit->seen |= SEEN_MEM; 645 /* st %r5,<K>(%r15) */ 646 EMIT4_DISP(0x5050f000, 647 (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4); 648 break; 649 case BPF_STX: /* mem[K] = X : mov %ebx,off8(%rbp) */ 650 jit->seen |= SEEN_XREG | SEEN_MEM; 651 /* st %r12,<K>(%r15) */ 652 EMIT4_DISP(0x50c0f000, 653 (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4); 654 break; 655 case BPF_ANC | SKF_AD_PROTOCOL: /* A = ntohs(skb->protocol); */ 656 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, protocol) != 2); 657 /* lhi %r5,0 */ 658 EMIT4(0xa7580000); 659 /* icm %r5,3,<d(protocol)>(%r2) */ 660 EMIT4_DISP(0xbf532000, offsetof(struct sk_buff, protocol)); 661 break; 662 case BPF_ANC | SKF_AD_IFINDEX: /* if (!skb->dev) return 0; 663 * A = skb->dev->ifindex */ 664 BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, ifindex) != 4); 665 jit->seen |= SEEN_RET0; 666 /* lg %r1,<d(dev)>(%r2) */ 667 EMIT6_DISP(0xe3102000, 0x0004, offsetof(struct sk_buff, dev)); 668 /* ltgr %r1,%r1 */ 669 EMIT4(0xb9020011); 670 /* jz <ret0> */ 671 EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg); 672 /* l %r5,<d(ifindex)>(%r1) */ 673 EMIT4_DISP(0x58501000, offsetof(struct net_device, ifindex)); 674 break; 675 case BPF_ANC | SKF_AD_MARK: /* A = skb->mark */ 676 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4); 677 /* l %r5,<d(mark)>(%r2) */ 678 EMIT4_DISP(0x58502000, offsetof(struct sk_buff, mark)); 679 break; 680 case BPF_ANC | SKF_AD_QUEUE: /* A = skb->queue_mapping */ 681 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, queue_mapping) != 2); 682 /* lhi %r5,0 */ 683 EMIT4(0xa7580000); 684 /* icm %r5,3,<d(queue_mapping)>(%r2) */ 685 EMIT4_DISP(0xbf532000, offsetof(struct sk_buff, queue_mapping)); 686 break; 687 case BPF_ANC | SKF_AD_HATYPE: /* if (!skb->dev) return 0; 688 * A = skb->dev->type */ 689 BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, type) != 2); 690 jit->seen |= SEEN_RET0; 691 /* lg %r1,<d(dev)>(%r2) */ 692 EMIT6_DISP(0xe3102000, 0x0004, offsetof(struct sk_buff, dev)); 693 /* ltgr %r1,%r1 */ 694 EMIT4(0xb9020011); 695 /* jz <ret0> */ 696 EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg); 697 /* lhi %r5,0 */ 698 EMIT4(0xa7580000); 699 /* icm %r5,3,<d(type)>(%r1) */ 700 EMIT4_DISP(0xbf531000, offsetof(struct net_device, type)); 701 break; 702 case BPF_ANC | SKF_AD_RXHASH: /* A = skb->hash */ 703 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, hash) != 4); 704 /* l %r5,<d(hash)>(%r2) */ 705 EMIT4_DISP(0x58502000, offsetof(struct sk_buff, hash)); 706 break; 707 case BPF_ANC | SKF_AD_VLAN_TAG: 708 case BPF_ANC | SKF_AD_VLAN_TAG_PRESENT: 709 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, vlan_tci) != 2); 710 BUILD_BUG_ON(VLAN_TAG_PRESENT != 0x1000); 711 /* lhi %r5,0 */ 712 EMIT4(0xa7580000); 713 /* icm %r5,3,<d(vlan_tci)>(%r2) */ 714 EMIT4_DISP(0xbf532000, offsetof(struct sk_buff, vlan_tci)); 715 if (code == (BPF_ANC | SKF_AD_VLAN_TAG)) { 716 /* nill %r5,0xefff */ 717 EMIT4_IMM(0xa5570000, ~VLAN_TAG_PRESENT); 718 } else { 719 /* nill %r5,0x1000 */ 720 EMIT4_IMM(0xa5570000, VLAN_TAG_PRESENT); 721 /* srl %r5,12 */ 722 EMIT4_DISP(0x88500000, 12); 723 } 724 break; 725 case BPF_ANC | SKF_AD_PKTTYPE: 726 /* lhi %r5,0 */ 727 EMIT4(0xa7580000); 728 /* ic %r5,<d(pkt_type_offset)>(%r2) */ 729 EMIT4_DISP(0x43502000, PKT_TYPE_OFFSET()); 730 /* srl %r5,5 */ 731 EMIT4_DISP(0x88500000, 5); 732 break; 733 case BPF_ANC | SKF_AD_CPU: /* A = smp_processor_id() */ 734 #ifdef CONFIG_SMP 735 /* l %r5,<d(cpu_nr)> */ 736 EMIT4_DISP(0x58500000, offsetof(struct _lowcore, cpu_nr)); 737 #else 738 /* lhi %r5,0 */ 739 EMIT4(0xa7580000); 740 #endif 741 break; 742 default: /* too complex, give up */ 743 goto out; 744 } 745 addrs[i] = jit->prg - jit->start; 746 return 0; 747 out: 748 return -1; 749 } 750 751 void bpf_jit_compile(struct bpf_prog *fp) 752 { 753 struct bpf_binary_header *header = NULL; 754 unsigned long size, prg_len, lit_len; 755 struct bpf_jit jit, cjit; 756 unsigned int *addrs; 757 int pass, i; 758 759 if (!bpf_jit_enable) 760 return; 761 addrs = kcalloc(fp->len, sizeof(*addrs), GFP_KERNEL); 762 if (addrs == NULL) 763 return; 764 memset(&jit, 0, sizeof(cjit)); 765 memset(&cjit, 0, sizeof(cjit)); 766 767 for (pass = 0; pass < 10; pass++) { 768 jit.prg = jit.start; 769 jit.lit = jit.mid; 770 771 bpf_jit_prologue(&jit); 772 bpf_jit_noleaks(&jit, fp->insns); 773 for (i = 0; i < fp->len; i++) { 774 if (bpf_jit_insn(&jit, fp->insns + i, addrs, i, 775 i == fp->len - 1)) 776 goto out; 777 } 778 bpf_jit_epilogue(&jit); 779 if (jit.start) { 780 WARN_ON(jit.prg > cjit.prg || jit.lit > cjit.lit); 781 if (memcmp(&jit, &cjit, sizeof(jit)) == 0) 782 break; 783 } else if (jit.prg == cjit.prg && jit.lit == cjit.lit) { 784 prg_len = jit.prg - jit.start; 785 lit_len = jit.lit - jit.mid; 786 size = prg_len + lit_len; 787 if (size >= BPF_SIZE_MAX) 788 goto out; 789 header = bpf_jit_binary_alloc(size, &jit.start, 790 2, bpf_jit_fill_hole); 791 if (!header) 792 goto out; 793 jit.prg = jit.mid = jit.start + prg_len; 794 jit.lit = jit.end = jit.start + prg_len + lit_len; 795 jit.base_ip += (unsigned long) jit.start; 796 jit.exit_ip += (unsigned long) jit.start; 797 jit.ret0_ip += (unsigned long) jit.start; 798 } 799 cjit = jit; 800 } 801 if (bpf_jit_enable > 1) { 802 bpf_jit_dump(fp->len, jit.end - jit.start, pass, jit.start); 803 if (jit.start) 804 print_fn_code(jit.start, jit.mid - jit.start); 805 } 806 if (jit.start) { 807 set_memory_ro((unsigned long)header, header->pages); 808 fp->bpf_func = (void *) jit.start; 809 fp->jited = true; 810 } 811 out: 812 kfree(addrs); 813 } 814 815 void bpf_jit_free(struct bpf_prog *fp) 816 { 817 unsigned long addr = (unsigned long)fp->bpf_func & PAGE_MASK; 818 struct bpf_binary_header *header = (void *)addr; 819 820 if (!fp->jited) 821 goto free_filter; 822 823 set_memory_rw(addr, header->pages); 824 bpf_jit_binary_free(header); 825 826 free_filter: 827 bpf_prog_unlock_free(fp); 828 } 829