1 /* 2 * BPF Jit compiler for s390. 3 * 4 * Copyright IBM Corp. 2012 5 * 6 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> 7 */ 8 #include <linux/netdevice.h> 9 #include <linux/if_vlan.h> 10 #include <linux/filter.h> 11 #include <linux/init.h> 12 #include <asm/cacheflush.h> 13 #include <asm/facility.h> 14 #include <asm/dis.h> 15 16 /* 17 * Conventions: 18 * %r2 = skb pointer 19 * %r3 = offset parameter 20 * %r4 = scratch register / length parameter 21 * %r5 = BPF A accumulator 22 * %r8 = return address 23 * %r9 = save register for skb pointer 24 * %r10 = skb->data 25 * %r11 = skb->len - skb->data_len (headlen) 26 * %r12 = BPF X accumulator 27 * %r13 = literal pool pointer 28 * 0(%r15) - 63(%r15) scratch memory array with BPF_MEMWORDS 29 */ 30 int bpf_jit_enable __read_mostly; 31 32 /* 33 * assembly code in arch/x86/net/bpf_jit.S 34 */ 35 extern u8 sk_load_word[], sk_load_half[], sk_load_byte[], sk_load_byte_msh[]; 36 extern u8 sk_load_word_ind[], sk_load_half_ind[], sk_load_byte_ind[]; 37 38 struct bpf_jit { 39 unsigned int seen; 40 u8 *start; 41 u8 *prg; 42 u8 *mid; 43 u8 *lit; 44 u8 *end; 45 u8 *base_ip; 46 u8 *ret0_ip; 47 u8 *exit_ip; 48 unsigned int off_load_word; 49 unsigned int off_load_half; 50 unsigned int off_load_byte; 51 unsigned int off_load_bmsh; 52 unsigned int off_load_iword; 53 unsigned int off_load_ihalf; 54 unsigned int off_load_ibyte; 55 }; 56 57 #define BPF_SIZE_MAX 4096 /* Max size for program */ 58 59 #define SEEN_DATAREF 1 /* might call external helpers */ 60 #define SEEN_XREG 2 /* ebx is used */ 61 #define SEEN_MEM 4 /* use mem[] for temporary storage */ 62 #define SEEN_RET0 8 /* pc_ret0 points to a valid return 0 */ 63 #define SEEN_LITERAL 16 /* code uses literals */ 64 #define SEEN_LOAD_WORD 32 /* code uses sk_load_word */ 65 #define SEEN_LOAD_HALF 64 /* code uses sk_load_half */ 66 #define SEEN_LOAD_BYTE 128 /* code uses sk_load_byte */ 67 #define SEEN_LOAD_BMSH 256 /* code uses sk_load_byte_msh */ 68 #define SEEN_LOAD_IWORD 512 /* code uses sk_load_word_ind */ 69 #define SEEN_LOAD_IHALF 1024 /* code uses sk_load_half_ind */ 70 #define SEEN_LOAD_IBYTE 2048 /* code uses sk_load_byte_ind */ 71 72 #define EMIT2(op) \ 73 ({ \ 74 if (jit->prg + 2 <= jit->mid) \ 75 *(u16 *) jit->prg = op; \ 76 jit->prg += 2; \ 77 }) 78 79 #define EMIT4(op) \ 80 ({ \ 81 if (jit->prg + 4 <= jit->mid) \ 82 *(u32 *) jit->prg = op; \ 83 jit->prg += 4; \ 84 }) 85 86 #define EMIT4_DISP(op, disp) \ 87 ({ \ 88 unsigned int __disp = (disp) & 0xfff; \ 89 EMIT4(op | __disp); \ 90 }) 91 92 #define EMIT4_IMM(op, imm) \ 93 ({ \ 94 unsigned int __imm = (imm) & 0xffff; \ 95 EMIT4(op | __imm); \ 96 }) 97 98 #define EMIT4_PCREL(op, pcrel) \ 99 ({ \ 100 long __pcrel = ((pcrel) >> 1) & 0xffff; \ 101 EMIT4(op | __pcrel); \ 102 }) 103 104 #define EMIT6(op1, op2) \ 105 ({ \ 106 if (jit->prg + 6 <= jit->mid) { \ 107 *(u32 *) jit->prg = op1; \ 108 *(u16 *) (jit->prg + 4) = op2; \ 109 } \ 110 jit->prg += 6; \ 111 }) 112 113 #define EMIT6_DISP(op1, op2, disp) \ 114 ({ \ 115 unsigned int __disp = (disp) & 0xfff; \ 116 EMIT6(op1 | __disp, op2); \ 117 }) 118 119 #define EMIT6_IMM(op, imm) \ 120 ({ \ 121 unsigned int __imm = (imm); \ 122 EMIT6(op | (__imm >> 16), __imm & 0xffff); \ 123 }) 124 125 #define EMIT_CONST(val) \ 126 ({ \ 127 unsigned int ret; \ 128 ret = (unsigned int) (jit->lit - jit->base_ip); \ 129 jit->seen |= SEEN_LITERAL; \ 130 if (jit->lit + 4 <= jit->end) \ 131 *(u32 *) jit->lit = val; \ 132 jit->lit += 4; \ 133 ret; \ 134 }) 135 136 #define EMIT_FN_CONST(bit, fn) \ 137 ({ \ 138 unsigned int ret; \ 139 ret = (unsigned int) (jit->lit - jit->base_ip); \ 140 if (jit->seen & bit) { \ 141 jit->seen |= SEEN_LITERAL; \ 142 if (jit->lit + 8 <= jit->end) \ 143 *(void **) jit->lit = fn; \ 144 jit->lit += 8; \ 145 } \ 146 ret; \ 147 }) 148 149 static void bpf_jit_fill_hole(void *area, unsigned int size) 150 { 151 /* Fill whole space with illegal instructions */ 152 memset(area, 0, size); 153 } 154 155 static void bpf_jit_prologue(struct bpf_jit *jit) 156 { 157 /* Save registers and create stack frame if necessary */ 158 if (jit->seen & SEEN_DATAREF) { 159 /* stmg %r8,%r15,88(%r15) */ 160 EMIT6(0xeb8ff058, 0x0024); 161 /* lgr %r14,%r15 */ 162 EMIT4(0xb90400ef); 163 /* aghi %r15,<offset> */ 164 EMIT4_IMM(0xa7fb0000, (jit->seen & SEEN_MEM) ? -112 : -80); 165 /* stg %r14,152(%r15) */ 166 EMIT6(0xe3e0f098, 0x0024); 167 } else if ((jit->seen & SEEN_XREG) && (jit->seen & SEEN_LITERAL)) 168 /* stmg %r12,%r13,120(%r15) */ 169 EMIT6(0xebcdf078, 0x0024); 170 else if (jit->seen & SEEN_XREG) 171 /* stg %r12,120(%r15) */ 172 EMIT6(0xe3c0f078, 0x0024); 173 else if (jit->seen & SEEN_LITERAL) 174 /* stg %r13,128(%r15) */ 175 EMIT6(0xe3d0f080, 0x0024); 176 177 /* Setup literal pool */ 178 if (jit->seen & SEEN_LITERAL) { 179 /* basr %r13,0 */ 180 EMIT2(0x0dd0); 181 jit->base_ip = jit->prg; 182 } 183 jit->off_load_word = EMIT_FN_CONST(SEEN_LOAD_WORD, sk_load_word); 184 jit->off_load_half = EMIT_FN_CONST(SEEN_LOAD_HALF, sk_load_half); 185 jit->off_load_byte = EMIT_FN_CONST(SEEN_LOAD_BYTE, sk_load_byte); 186 jit->off_load_bmsh = EMIT_FN_CONST(SEEN_LOAD_BMSH, sk_load_byte_msh); 187 jit->off_load_iword = EMIT_FN_CONST(SEEN_LOAD_IWORD, sk_load_word_ind); 188 jit->off_load_ihalf = EMIT_FN_CONST(SEEN_LOAD_IHALF, sk_load_half_ind); 189 jit->off_load_ibyte = EMIT_FN_CONST(SEEN_LOAD_IBYTE, sk_load_byte_ind); 190 191 /* Filter needs to access skb data */ 192 if (jit->seen & SEEN_DATAREF) { 193 /* l %r11,<len>(%r2) */ 194 EMIT4_DISP(0x58b02000, offsetof(struct sk_buff, len)); 195 /* s %r11,<data_len>(%r2) */ 196 EMIT4_DISP(0x5bb02000, offsetof(struct sk_buff, data_len)); 197 /* lg %r10,<data>(%r2) */ 198 EMIT6_DISP(0xe3a02000, 0x0004, 199 offsetof(struct sk_buff, data)); 200 } 201 } 202 203 static void bpf_jit_epilogue(struct bpf_jit *jit) 204 { 205 /* Return 0 */ 206 if (jit->seen & SEEN_RET0) { 207 jit->ret0_ip = jit->prg; 208 /* lghi %r2,0 */ 209 EMIT4(0xa7290000); 210 } 211 jit->exit_ip = jit->prg; 212 /* Restore registers */ 213 if (jit->seen & SEEN_DATAREF) 214 /* lmg %r8,%r15,<offset>(%r15) */ 215 EMIT6_DISP(0xeb8ff000, 0x0004, 216 (jit->seen & SEEN_MEM) ? 200 : 168); 217 else if ((jit->seen & SEEN_XREG) && (jit->seen & SEEN_LITERAL)) 218 /* lmg %r12,%r13,120(%r15) */ 219 EMIT6(0xebcdf078, 0x0004); 220 else if (jit->seen & SEEN_XREG) 221 /* lg %r12,120(%r15) */ 222 EMIT6(0xe3c0f078, 0x0004); 223 else if (jit->seen & SEEN_LITERAL) 224 /* lg %r13,128(%r15) */ 225 EMIT6(0xe3d0f080, 0x0004); 226 /* br %r14 */ 227 EMIT2(0x07fe); 228 } 229 230 /* 231 * make sure we dont leak kernel information to user 232 */ 233 static void bpf_jit_noleaks(struct bpf_jit *jit, struct sock_filter *filter) 234 { 235 /* Clear temporary memory if (seen & SEEN_MEM) */ 236 if (jit->seen & SEEN_MEM) 237 /* xc 0(64,%r15),0(%r15) */ 238 EMIT6(0xd73ff000, 0xf000); 239 /* Clear X if (seen & SEEN_XREG) */ 240 if (jit->seen & SEEN_XREG) 241 /* lhi %r12,0 */ 242 EMIT4(0xa7c80000); 243 /* Clear A if the first register does not set it. */ 244 switch (filter[0].code) { 245 case BPF_LD | BPF_W | BPF_ABS: 246 case BPF_LD | BPF_H | BPF_ABS: 247 case BPF_LD | BPF_B | BPF_ABS: 248 case BPF_LD | BPF_W | BPF_LEN: 249 case BPF_LD | BPF_W | BPF_IND: 250 case BPF_LD | BPF_H | BPF_IND: 251 case BPF_LD | BPF_B | BPF_IND: 252 case BPF_LD | BPF_IMM: 253 case BPF_LD | BPF_MEM: 254 case BPF_MISC | BPF_TXA: 255 case BPF_RET | BPF_K: 256 /* first instruction sets A register */ 257 break; 258 default: /* A = 0 */ 259 /* lhi %r5,0 */ 260 EMIT4(0xa7580000); 261 } 262 } 263 264 static int bpf_jit_insn(struct bpf_jit *jit, struct sock_filter *filter, 265 unsigned int *addrs, int i, int last) 266 { 267 unsigned int K; 268 int offset; 269 unsigned int mask; 270 u16 code; 271 272 K = filter->k; 273 code = bpf_anc_helper(filter); 274 275 switch (code) { 276 case BPF_ALU | BPF_ADD | BPF_X: /* A += X */ 277 jit->seen |= SEEN_XREG; 278 /* ar %r5,%r12 */ 279 EMIT2(0x1a5c); 280 break; 281 case BPF_ALU | BPF_ADD | BPF_K: /* A += K */ 282 if (!K) 283 break; 284 if (K <= 16383) 285 /* ahi %r5,<K> */ 286 EMIT4_IMM(0xa75a0000, K); 287 else if (test_facility(21)) 288 /* alfi %r5,<K> */ 289 EMIT6_IMM(0xc25b0000, K); 290 else 291 /* a %r5,<d(K)>(%r13) */ 292 EMIT4_DISP(0x5a50d000, EMIT_CONST(K)); 293 break; 294 case BPF_ALU | BPF_SUB | BPF_X: /* A -= X */ 295 jit->seen |= SEEN_XREG; 296 /* sr %r5,%r12 */ 297 EMIT2(0x1b5c); 298 break; 299 case BPF_ALU | BPF_SUB | BPF_K: /* A -= K */ 300 if (!K) 301 break; 302 if (K <= 16384) 303 /* ahi %r5,-K */ 304 EMIT4_IMM(0xa75a0000, -K); 305 else if (test_facility(21)) 306 /* alfi %r5,-K */ 307 EMIT6_IMM(0xc25b0000, -K); 308 else 309 /* s %r5,<d(K)>(%r13) */ 310 EMIT4_DISP(0x5b50d000, EMIT_CONST(K)); 311 break; 312 case BPF_ALU | BPF_MUL | BPF_X: /* A *= X */ 313 jit->seen |= SEEN_XREG; 314 /* msr %r5,%r12 */ 315 EMIT4(0xb252005c); 316 break; 317 case BPF_ALU | BPF_MUL | BPF_K: /* A *= K */ 318 if (K <= 16383) 319 /* mhi %r5,K */ 320 EMIT4_IMM(0xa75c0000, K); 321 else if (test_facility(34)) 322 /* msfi %r5,<K> */ 323 EMIT6_IMM(0xc2510000, K); 324 else 325 /* ms %r5,<d(K)>(%r13) */ 326 EMIT4_DISP(0x7150d000, EMIT_CONST(K)); 327 break; 328 case BPF_ALU | BPF_DIV | BPF_X: /* A /= X */ 329 jit->seen |= SEEN_XREG | SEEN_RET0; 330 /* ltr %r12,%r12 */ 331 EMIT2(0x12cc); 332 /* jz <ret0> */ 333 EMIT4_PCREL(0xa7840000, (jit->ret0_ip - jit->prg)); 334 /* lhi %r4,0 */ 335 EMIT4(0xa7480000); 336 /* dlr %r4,%r12 */ 337 EMIT4(0xb997004c); 338 break; 339 case BPF_ALU | BPF_DIV | BPF_K: /* A /= K */ 340 if (K == 1) 341 break; 342 /* lhi %r4,0 */ 343 EMIT4(0xa7480000); 344 /* dl %r4,<d(K)>(%r13) */ 345 EMIT6_DISP(0xe340d000, 0x0097, EMIT_CONST(K)); 346 break; 347 case BPF_ALU | BPF_MOD | BPF_X: /* A %= X */ 348 jit->seen |= SEEN_XREG | SEEN_RET0; 349 /* ltr %r12,%r12 */ 350 EMIT2(0x12cc); 351 /* jz <ret0> */ 352 EMIT4_PCREL(0xa7840000, (jit->ret0_ip - jit->prg)); 353 /* lhi %r4,0 */ 354 EMIT4(0xa7480000); 355 /* dlr %r4,%r12 */ 356 EMIT4(0xb997004c); 357 /* lr %r5,%r4 */ 358 EMIT2(0x1854); 359 break; 360 case BPF_ALU | BPF_MOD | BPF_K: /* A %= K */ 361 if (K == 1) { 362 /* lhi %r5,0 */ 363 EMIT4(0xa7580000); 364 break; 365 } 366 /* lhi %r4,0 */ 367 EMIT4(0xa7480000); 368 /* dl %r4,<d(K)>(%r13) */ 369 EMIT6_DISP(0xe340d000, 0x0097, EMIT_CONST(K)); 370 /* lr %r5,%r4 */ 371 EMIT2(0x1854); 372 break; 373 case BPF_ALU | BPF_AND | BPF_X: /* A &= X */ 374 jit->seen |= SEEN_XREG; 375 /* nr %r5,%r12 */ 376 EMIT2(0x145c); 377 break; 378 case BPF_ALU | BPF_AND | BPF_K: /* A &= K */ 379 if (test_facility(21)) 380 /* nilf %r5,<K> */ 381 EMIT6_IMM(0xc05b0000, K); 382 else 383 /* n %r5,<d(K)>(%r13) */ 384 EMIT4_DISP(0x5450d000, EMIT_CONST(K)); 385 break; 386 case BPF_ALU | BPF_OR | BPF_X: /* A |= X */ 387 jit->seen |= SEEN_XREG; 388 /* or %r5,%r12 */ 389 EMIT2(0x165c); 390 break; 391 case BPF_ALU | BPF_OR | BPF_K: /* A |= K */ 392 if (test_facility(21)) 393 /* oilf %r5,<K> */ 394 EMIT6_IMM(0xc05d0000, K); 395 else 396 /* o %r5,<d(K)>(%r13) */ 397 EMIT4_DISP(0x5650d000, EMIT_CONST(K)); 398 break; 399 case BPF_ANC | SKF_AD_ALU_XOR_X: /* A ^= X; */ 400 case BPF_ALU | BPF_XOR | BPF_X: 401 jit->seen |= SEEN_XREG; 402 /* xr %r5,%r12 */ 403 EMIT2(0x175c); 404 break; 405 case BPF_ALU | BPF_XOR | BPF_K: /* A ^= K */ 406 if (!K) 407 break; 408 /* x %r5,<d(K)>(%r13) */ 409 EMIT4_DISP(0x5750d000, EMIT_CONST(K)); 410 break; 411 case BPF_ALU | BPF_LSH | BPF_X: /* A <<= X; */ 412 jit->seen |= SEEN_XREG; 413 /* sll %r5,0(%r12) */ 414 EMIT4(0x8950c000); 415 break; 416 case BPF_ALU | BPF_LSH | BPF_K: /* A <<= K */ 417 if (K == 0) 418 break; 419 /* sll %r5,K */ 420 EMIT4_DISP(0x89500000, K); 421 break; 422 case BPF_ALU | BPF_RSH | BPF_X: /* A >>= X; */ 423 jit->seen |= SEEN_XREG; 424 /* srl %r5,0(%r12) */ 425 EMIT4(0x8850c000); 426 break; 427 case BPF_ALU | BPF_RSH | BPF_K: /* A >>= K; */ 428 if (K == 0) 429 break; 430 /* srl %r5,K */ 431 EMIT4_DISP(0x88500000, K); 432 break; 433 case BPF_ALU | BPF_NEG: /* A = -A */ 434 /* lnr %r5,%r5 */ 435 EMIT2(0x1155); 436 break; 437 case BPF_JMP | BPF_JA: /* ip += K */ 438 offset = addrs[i + K] + jit->start - jit->prg; 439 EMIT4_PCREL(0xa7f40000, offset); 440 break; 441 case BPF_JMP | BPF_JGT | BPF_K: /* ip += (A > K) ? jt : jf */ 442 mask = 0x200000; /* jh */ 443 goto kbranch; 444 case BPF_JMP | BPF_JGE | BPF_K: /* ip += (A >= K) ? jt : jf */ 445 mask = 0xa00000; /* jhe */ 446 goto kbranch; 447 case BPF_JMP | BPF_JEQ | BPF_K: /* ip += (A == K) ? jt : jf */ 448 mask = 0x800000; /* je */ 449 kbranch: /* Emit compare if the branch targets are different */ 450 if (filter->jt != filter->jf) { 451 if (K <= 16383) 452 /* chi %r5,<K> */ 453 EMIT4_IMM(0xa75e0000, K); 454 else if (test_facility(21)) 455 /* clfi %r5,<K> */ 456 EMIT6_IMM(0xc25f0000, K); 457 else 458 /* c %r5,<d(K)>(%r13) */ 459 EMIT4_DISP(0x5950d000, EMIT_CONST(K)); 460 } 461 branch: if (filter->jt == filter->jf) { 462 if (filter->jt == 0) 463 break; 464 /* j <jt> */ 465 offset = addrs[i + filter->jt] + jit->start - jit->prg; 466 EMIT4_PCREL(0xa7f40000, offset); 467 break; 468 } 469 if (filter->jt != 0) { 470 /* brc <mask>,<jt> */ 471 offset = addrs[i + filter->jt] + jit->start - jit->prg; 472 EMIT4_PCREL(0xa7040000 | mask, offset); 473 } 474 if (filter->jf != 0) { 475 /* brc <mask^15>,<jf> */ 476 offset = addrs[i + filter->jf] + jit->start - jit->prg; 477 EMIT4_PCREL(0xa7040000 | (mask ^ 0xf00000), offset); 478 } 479 break; 480 case BPF_JMP | BPF_JSET | BPF_K: /* ip += (A & K) ? jt : jf */ 481 mask = 0x700000; /* jnz */ 482 /* Emit test if the branch targets are different */ 483 if (filter->jt != filter->jf) { 484 if (K > 65535) { 485 /* lr %r4,%r5 */ 486 EMIT2(0x1845); 487 /* n %r4,<d(K)>(%r13) */ 488 EMIT4_DISP(0x5440d000, EMIT_CONST(K)); 489 } else 490 /* tmll %r5,K */ 491 EMIT4_IMM(0xa7510000, K); 492 } 493 goto branch; 494 case BPF_JMP | BPF_JGT | BPF_X: /* ip += (A > X) ? jt : jf */ 495 mask = 0x200000; /* jh */ 496 goto xbranch; 497 case BPF_JMP | BPF_JGE | BPF_X: /* ip += (A >= X) ? jt : jf */ 498 mask = 0xa00000; /* jhe */ 499 goto xbranch; 500 case BPF_JMP | BPF_JEQ | BPF_X: /* ip += (A == X) ? jt : jf */ 501 mask = 0x800000; /* je */ 502 xbranch: /* Emit compare if the branch targets are different */ 503 if (filter->jt != filter->jf) { 504 jit->seen |= SEEN_XREG; 505 /* cr %r5,%r12 */ 506 EMIT2(0x195c); 507 } 508 goto branch; 509 case BPF_JMP | BPF_JSET | BPF_X: /* ip += (A & X) ? jt : jf */ 510 mask = 0x700000; /* jnz */ 511 /* Emit test if the branch targets are different */ 512 if (filter->jt != filter->jf) { 513 jit->seen |= SEEN_XREG; 514 /* lr %r4,%r5 */ 515 EMIT2(0x1845); 516 /* nr %r4,%r12 */ 517 EMIT2(0x144c); 518 } 519 goto branch; 520 case BPF_LD | BPF_W | BPF_ABS: /* A = *(u32 *) (skb->data+K) */ 521 jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_WORD; 522 offset = jit->off_load_word; 523 goto load_abs; 524 case BPF_LD | BPF_H | BPF_ABS: /* A = *(u16 *) (skb->data+K) */ 525 jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_HALF; 526 offset = jit->off_load_half; 527 goto load_abs; 528 case BPF_LD | BPF_B | BPF_ABS: /* A = *(u8 *) (skb->data+K) */ 529 jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_BYTE; 530 offset = jit->off_load_byte; 531 load_abs: if ((int) K < 0) 532 goto out; 533 call_fn: /* lg %r1,<d(function)>(%r13) */ 534 EMIT6_DISP(0xe310d000, 0x0004, offset); 535 /* l %r3,<d(K)>(%r13) */ 536 EMIT4_DISP(0x5830d000, EMIT_CONST(K)); 537 /* basr %r8,%r1 */ 538 EMIT2(0x0d81); 539 /* jnz <ret0> */ 540 EMIT4_PCREL(0xa7740000, (jit->ret0_ip - jit->prg)); 541 break; 542 case BPF_LD | BPF_W | BPF_IND: /* A = *(u32 *) (skb->data+K+X) */ 543 jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_IWORD; 544 offset = jit->off_load_iword; 545 goto call_fn; 546 case BPF_LD | BPF_H | BPF_IND: /* A = *(u16 *) (skb->data+K+X) */ 547 jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_IHALF; 548 offset = jit->off_load_ihalf; 549 goto call_fn; 550 case BPF_LD | BPF_B | BPF_IND: /* A = *(u8 *) (skb->data+K+X) */ 551 jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_IBYTE; 552 offset = jit->off_load_ibyte; 553 goto call_fn; 554 case BPF_LDX | BPF_B | BPF_MSH: 555 /* X = (*(u8 *)(skb->data+K) & 0xf) << 2 */ 556 jit->seen |= SEEN_RET0; 557 if ((int) K < 0) { 558 /* j <ret0> */ 559 EMIT4_PCREL(0xa7f40000, (jit->ret0_ip - jit->prg)); 560 break; 561 } 562 jit->seen |= SEEN_DATAREF | SEEN_LOAD_BMSH; 563 offset = jit->off_load_bmsh; 564 goto call_fn; 565 case BPF_LD | BPF_W | BPF_LEN: /* A = skb->len; */ 566 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, len) != 4); 567 /* l %r5,<d(len)>(%r2) */ 568 EMIT4_DISP(0x58502000, offsetof(struct sk_buff, len)); 569 break; 570 case BPF_LDX | BPF_W | BPF_LEN: /* X = skb->len; */ 571 jit->seen |= SEEN_XREG; 572 /* l %r12,<d(len)>(%r2) */ 573 EMIT4_DISP(0x58c02000, offsetof(struct sk_buff, len)); 574 break; 575 case BPF_LD | BPF_IMM: /* A = K */ 576 if (K <= 16383) 577 /* lhi %r5,K */ 578 EMIT4_IMM(0xa7580000, K); 579 else if (test_facility(21)) 580 /* llilf %r5,<K> */ 581 EMIT6_IMM(0xc05f0000, K); 582 else 583 /* l %r5,<d(K)>(%r13) */ 584 EMIT4_DISP(0x5850d000, EMIT_CONST(K)); 585 break; 586 case BPF_LDX | BPF_IMM: /* X = K */ 587 jit->seen |= SEEN_XREG; 588 if (K <= 16383) 589 /* lhi %r12,<K> */ 590 EMIT4_IMM(0xa7c80000, K); 591 else if (test_facility(21)) 592 /* llilf %r12,<K> */ 593 EMIT6_IMM(0xc0cf0000, K); 594 else 595 /* l %r12,<d(K)>(%r13) */ 596 EMIT4_DISP(0x58c0d000, EMIT_CONST(K)); 597 break; 598 case BPF_LD | BPF_MEM: /* A = mem[K] */ 599 jit->seen |= SEEN_MEM; 600 /* l %r5,<K>(%r15) */ 601 EMIT4_DISP(0x5850f000, 602 (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4); 603 break; 604 case BPF_LDX | BPF_MEM: /* X = mem[K] */ 605 jit->seen |= SEEN_XREG | SEEN_MEM; 606 /* l %r12,<K>(%r15) */ 607 EMIT4_DISP(0x58c0f000, 608 (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4); 609 break; 610 case BPF_MISC | BPF_TAX: /* X = A */ 611 jit->seen |= SEEN_XREG; 612 /* lr %r12,%r5 */ 613 EMIT2(0x18c5); 614 break; 615 case BPF_MISC | BPF_TXA: /* A = X */ 616 jit->seen |= SEEN_XREG; 617 /* lr %r5,%r12 */ 618 EMIT2(0x185c); 619 break; 620 case BPF_RET | BPF_K: 621 if (K == 0) { 622 jit->seen |= SEEN_RET0; 623 if (last) 624 break; 625 /* j <ret0> */ 626 EMIT4_PCREL(0xa7f40000, jit->ret0_ip - jit->prg); 627 } else { 628 if (K <= 16383) 629 /* lghi %r2,K */ 630 EMIT4_IMM(0xa7290000, K); 631 else 632 /* llgf %r2,<K>(%r13) */ 633 EMIT6_DISP(0xe320d000, 0x0016, EMIT_CONST(K)); 634 /* j <exit> */ 635 if (last && !(jit->seen & SEEN_RET0)) 636 break; 637 EMIT4_PCREL(0xa7f40000, jit->exit_ip - jit->prg); 638 } 639 break; 640 case BPF_RET | BPF_A: 641 /* llgfr %r2,%r5 */ 642 EMIT4(0xb9160025); 643 /* j <exit> */ 644 EMIT4_PCREL(0xa7f40000, jit->exit_ip - jit->prg); 645 break; 646 case BPF_ST: /* mem[K] = A */ 647 jit->seen |= SEEN_MEM; 648 /* st %r5,<K>(%r15) */ 649 EMIT4_DISP(0x5050f000, 650 (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4); 651 break; 652 case BPF_STX: /* mem[K] = X : mov %ebx,off8(%rbp) */ 653 jit->seen |= SEEN_XREG | SEEN_MEM; 654 /* st %r12,<K>(%r15) */ 655 EMIT4_DISP(0x50c0f000, 656 (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4); 657 break; 658 case BPF_ANC | SKF_AD_PROTOCOL: /* A = ntohs(skb->protocol); */ 659 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, protocol) != 2); 660 /* lhi %r5,0 */ 661 EMIT4(0xa7580000); 662 /* icm %r5,3,<d(protocol)>(%r2) */ 663 EMIT4_DISP(0xbf532000, offsetof(struct sk_buff, protocol)); 664 break; 665 case BPF_ANC | SKF_AD_IFINDEX: /* if (!skb->dev) return 0; 666 * A = skb->dev->ifindex */ 667 BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, ifindex) != 4); 668 jit->seen |= SEEN_RET0; 669 /* lg %r1,<d(dev)>(%r2) */ 670 EMIT6_DISP(0xe3102000, 0x0004, offsetof(struct sk_buff, dev)); 671 /* ltgr %r1,%r1 */ 672 EMIT4(0xb9020011); 673 /* jz <ret0> */ 674 EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg); 675 /* l %r5,<d(ifindex)>(%r1) */ 676 EMIT4_DISP(0x58501000, offsetof(struct net_device, ifindex)); 677 break; 678 case BPF_ANC | SKF_AD_MARK: /* A = skb->mark */ 679 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4); 680 /* l %r5,<d(mark)>(%r2) */ 681 EMIT4_DISP(0x58502000, offsetof(struct sk_buff, mark)); 682 break; 683 case BPF_ANC | SKF_AD_QUEUE: /* A = skb->queue_mapping */ 684 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, queue_mapping) != 2); 685 /* lhi %r5,0 */ 686 EMIT4(0xa7580000); 687 /* icm %r5,3,<d(queue_mapping)>(%r2) */ 688 EMIT4_DISP(0xbf532000, offsetof(struct sk_buff, queue_mapping)); 689 break; 690 case BPF_ANC | SKF_AD_HATYPE: /* if (!skb->dev) return 0; 691 * A = skb->dev->type */ 692 BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, type) != 2); 693 jit->seen |= SEEN_RET0; 694 /* lg %r1,<d(dev)>(%r2) */ 695 EMIT6_DISP(0xe3102000, 0x0004, offsetof(struct sk_buff, dev)); 696 /* ltgr %r1,%r1 */ 697 EMIT4(0xb9020011); 698 /* jz <ret0> */ 699 EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg); 700 /* lhi %r5,0 */ 701 EMIT4(0xa7580000); 702 /* icm %r5,3,<d(type)>(%r1) */ 703 EMIT4_DISP(0xbf531000, offsetof(struct net_device, type)); 704 break; 705 case BPF_ANC | SKF_AD_RXHASH: /* A = skb->hash */ 706 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, hash) != 4); 707 /* l %r5,<d(hash)>(%r2) */ 708 EMIT4_DISP(0x58502000, offsetof(struct sk_buff, hash)); 709 break; 710 case BPF_ANC | SKF_AD_VLAN_TAG: 711 case BPF_ANC | SKF_AD_VLAN_TAG_PRESENT: 712 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, vlan_tci) != 2); 713 BUILD_BUG_ON(VLAN_TAG_PRESENT != 0x1000); 714 /* lhi %r5,0 */ 715 EMIT4(0xa7580000); 716 /* icm %r5,3,<d(vlan_tci)>(%r2) */ 717 EMIT4_DISP(0xbf532000, offsetof(struct sk_buff, vlan_tci)); 718 if (code == (BPF_ANC | SKF_AD_VLAN_TAG)) { 719 /* nill %r5,0xefff */ 720 EMIT4_IMM(0xa5570000, ~VLAN_TAG_PRESENT); 721 } else { 722 /* nill %r5,0x1000 */ 723 EMIT4_IMM(0xa5570000, VLAN_TAG_PRESENT); 724 /* srl %r5,12 */ 725 EMIT4_DISP(0x88500000, 12); 726 } 727 break; 728 case BPF_ANC | SKF_AD_PKTTYPE: 729 /* lhi %r5,0 */ 730 EMIT4(0xa7580000); 731 /* ic %r5,<d(pkt_type_offset)>(%r2) */ 732 EMIT4_DISP(0x43502000, PKT_TYPE_OFFSET()); 733 /* srl %r5,5 */ 734 EMIT4_DISP(0x88500000, 5); 735 break; 736 case BPF_ANC | SKF_AD_CPU: /* A = smp_processor_id() */ 737 #ifdef CONFIG_SMP 738 /* l %r5,<d(cpu_nr)> */ 739 EMIT4_DISP(0x58500000, offsetof(struct _lowcore, cpu_nr)); 740 #else 741 /* lhi %r5,0 */ 742 EMIT4(0xa7580000); 743 #endif 744 break; 745 default: /* too complex, give up */ 746 goto out; 747 } 748 addrs[i] = jit->prg - jit->start; 749 return 0; 750 out: 751 return -1; 752 } 753 754 void bpf_jit_compile(struct bpf_prog *fp) 755 { 756 struct bpf_binary_header *header = NULL; 757 unsigned long size, prg_len, lit_len; 758 struct bpf_jit jit, cjit; 759 unsigned int *addrs; 760 int pass, i; 761 762 if (!bpf_jit_enable) 763 return; 764 addrs = kcalloc(fp->len, sizeof(*addrs), GFP_KERNEL); 765 if (addrs == NULL) 766 return; 767 memset(&jit, 0, sizeof(cjit)); 768 memset(&cjit, 0, sizeof(cjit)); 769 770 for (pass = 0; pass < 10; pass++) { 771 jit.prg = jit.start; 772 jit.lit = jit.mid; 773 774 bpf_jit_prologue(&jit); 775 bpf_jit_noleaks(&jit, fp->insns); 776 for (i = 0; i < fp->len; i++) { 777 if (bpf_jit_insn(&jit, fp->insns + i, addrs, i, 778 i == fp->len - 1)) 779 goto out; 780 } 781 bpf_jit_epilogue(&jit); 782 if (jit.start) { 783 WARN_ON(jit.prg > cjit.prg || jit.lit > cjit.lit); 784 if (memcmp(&jit, &cjit, sizeof(jit)) == 0) 785 break; 786 } else if (jit.prg == cjit.prg && jit.lit == cjit.lit) { 787 prg_len = jit.prg - jit.start; 788 lit_len = jit.lit - jit.mid; 789 size = prg_len + lit_len; 790 if (size >= BPF_SIZE_MAX) 791 goto out; 792 header = bpf_jit_binary_alloc(size, &jit.start, 793 2, bpf_jit_fill_hole); 794 if (!header) 795 goto out; 796 jit.prg = jit.mid = jit.start + prg_len; 797 jit.lit = jit.end = jit.start + prg_len + lit_len; 798 jit.base_ip += (unsigned long) jit.start; 799 jit.exit_ip += (unsigned long) jit.start; 800 jit.ret0_ip += (unsigned long) jit.start; 801 } 802 cjit = jit; 803 } 804 if (bpf_jit_enable > 1) { 805 bpf_jit_dump(fp->len, jit.end - jit.start, pass, jit.start); 806 if (jit.start) 807 print_fn_code(jit.start, jit.mid - jit.start); 808 } 809 if (jit.start) { 810 set_memory_ro((unsigned long)header, header->pages); 811 fp->bpf_func = (void *) jit.start; 812 fp->jited = true; 813 } 814 out: 815 kfree(addrs); 816 } 817 818 void bpf_jit_free(struct bpf_prog *fp) 819 { 820 unsigned long addr = (unsigned long)fp->bpf_func & PAGE_MASK; 821 struct bpf_binary_header *header = (void *)addr; 822 823 if (!fp->jited) 824 goto free_filter; 825 826 set_memory_rw(addr, header->pages); 827 bpf_jit_binary_free(header); 828 829 free_filter: 830 bpf_prog_unlock_free(fp); 831 } 832