1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * bpf_jit_comp.c: BPF JIT compiler 4 * 5 * Copyright (C) 2011-2013 Eric Dumazet (eric.dumazet@gmail.com) 6 * Internal BPF Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 7 */ 8 #include <linux/netdevice.h> 9 #include <linux/filter.h> 10 #include <linux/if_vlan.h> 11 #include <linux/bpf.h> 12 13 #include <asm/set_memory.h> 14 #include <asm/nospec-branch.h> 15 16 static u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len) 17 { 18 if (len == 1) 19 *ptr = bytes; 20 else if (len == 2) 21 *(u16 *)ptr = bytes; 22 else { 23 *(u32 *)ptr = bytes; 24 barrier(); 25 } 26 return ptr + len; 27 } 28 29 #define EMIT(bytes, len) \ 30 do { prog = emit_code(prog, bytes, len); cnt += len; } while (0) 31 32 #define EMIT1(b1) EMIT(b1, 1) 33 #define EMIT2(b1, b2) EMIT((b1) + ((b2) << 8), 2) 34 #define EMIT3(b1, b2, b3) EMIT((b1) + ((b2) << 8) + ((b3) << 16), 3) 35 #define EMIT4(b1, b2, b3, b4) EMIT((b1) + ((b2) << 8) + ((b3) << 16) + ((b4) << 24), 4) 36 37 #define EMIT1_off32(b1, off) \ 38 do { EMIT1(b1); EMIT(off, 4); } while (0) 39 #define EMIT2_off32(b1, b2, off) \ 40 do { EMIT2(b1, b2); EMIT(off, 4); } while (0) 41 #define EMIT3_off32(b1, b2, b3, off) \ 42 do { EMIT3(b1, b2, b3); EMIT(off, 4); } while (0) 43 #define EMIT4_off32(b1, b2, b3, b4, off) \ 44 do { EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0) 45 46 static bool is_imm8(int value) 47 { 48 return value <= 127 && value >= -128; 49 } 50 51 static bool is_simm32(s64 value) 52 { 53 return value == (s64)(s32)value; 54 } 55 56 static bool is_uimm32(u64 value) 57 { 58 return value == (u64)(u32)value; 59 } 60 61 /* mov dst, src */ 62 #define EMIT_mov(DST, SRC) \ 63 do { \ 64 if (DST != SRC) \ 65 EMIT3(add_2mod(0x48, DST, SRC), 0x89, add_2reg(0xC0, DST, SRC)); \ 66 } while (0) 67 68 static int bpf_size_to_x86_bytes(int bpf_size) 69 { 70 if (bpf_size == BPF_W) 71 return 4; 72 else if (bpf_size == BPF_H) 73 return 2; 74 else if (bpf_size == BPF_B) 75 return 1; 76 else if (bpf_size == BPF_DW) 77 return 4; /* imm32 */ 78 else 79 return 0; 80 } 81 82 /* 83 * List of x86 cond jumps opcodes (. + s8) 84 * Add 0x10 (and an extra 0x0f) to generate far jumps (. + s32) 85 */ 86 #define X86_JB 0x72 87 #define X86_JAE 0x73 88 #define X86_JE 0x74 89 #define X86_JNE 0x75 90 #define X86_JBE 0x76 91 #define X86_JA 0x77 92 #define X86_JL 0x7C 93 #define X86_JGE 0x7D 94 #define X86_JLE 0x7E 95 #define X86_JG 0x7F 96 97 /* Pick a register outside of BPF range for JIT internal work */ 98 #define AUX_REG (MAX_BPF_JIT_REG + 1) 99 100 /* 101 * The following table maps BPF registers to x86-64 registers. 102 * 103 * x86-64 register R12 is unused, since if used as base address 104 * register in load/store instructions, it always needs an 105 * extra byte of encoding and is callee saved. 106 * 107 * Also x86-64 register R9 is unused. x86-64 register R10 is 108 * used for blinding (if enabled). 109 */ 110 static const int reg2hex[] = { 111 [BPF_REG_0] = 0, /* RAX */ 112 [BPF_REG_1] = 7, /* RDI */ 113 [BPF_REG_2] = 6, /* RSI */ 114 [BPF_REG_3] = 2, /* RDX */ 115 [BPF_REG_4] = 1, /* RCX */ 116 [BPF_REG_5] = 0, /* R8 */ 117 [BPF_REG_6] = 3, /* RBX callee saved */ 118 [BPF_REG_7] = 5, /* R13 callee saved */ 119 [BPF_REG_8] = 6, /* R14 callee saved */ 120 [BPF_REG_9] = 7, /* R15 callee saved */ 121 [BPF_REG_FP] = 5, /* RBP readonly */ 122 [BPF_REG_AX] = 2, /* R10 temp register */ 123 [AUX_REG] = 3, /* R11 temp register */ 124 }; 125 126 /* 127 * is_ereg() == true if BPF register 'reg' maps to x86-64 r8..r15 128 * which need extra byte of encoding. 129 * rax,rcx,...,rbp have simpler encoding 130 */ 131 static bool is_ereg(u32 reg) 132 { 133 return (1 << reg) & (BIT(BPF_REG_5) | 134 BIT(AUX_REG) | 135 BIT(BPF_REG_7) | 136 BIT(BPF_REG_8) | 137 BIT(BPF_REG_9) | 138 BIT(BPF_REG_AX)); 139 } 140 141 static bool is_axreg(u32 reg) 142 { 143 return reg == BPF_REG_0; 144 } 145 146 /* Add modifiers if 'reg' maps to x86-64 registers R8..R15 */ 147 static u8 add_1mod(u8 byte, u32 reg) 148 { 149 if (is_ereg(reg)) 150 byte |= 1; 151 return byte; 152 } 153 154 static u8 add_2mod(u8 byte, u32 r1, u32 r2) 155 { 156 if (is_ereg(r1)) 157 byte |= 1; 158 if (is_ereg(r2)) 159 byte |= 4; 160 return byte; 161 } 162 163 /* Encode 'dst_reg' register into x86-64 opcode 'byte' */ 164 static u8 add_1reg(u8 byte, u32 dst_reg) 165 { 166 return byte + reg2hex[dst_reg]; 167 } 168 169 /* Encode 'dst_reg' and 'src_reg' registers into x86-64 opcode 'byte' */ 170 static u8 add_2reg(u8 byte, u32 dst_reg, u32 src_reg) 171 { 172 return byte + reg2hex[dst_reg] + (reg2hex[src_reg] << 3); 173 } 174 175 static void jit_fill_hole(void *area, unsigned int size) 176 { 177 /* Fill whole space with INT3 instructions */ 178 memset(area, 0xcc, size); 179 } 180 181 struct jit_context { 182 int cleanup_addr; /* Epilogue code offset */ 183 }; 184 185 /* Maximum number of bytes emitted while JITing one eBPF insn */ 186 #define BPF_MAX_INSN_SIZE 128 187 #define BPF_INSN_SAFETY 64 188 189 #define AUX_STACK_SPACE 40 /* Space for RBX, R13, R14, R15, tailcnt */ 190 191 #define PROLOGUE_SIZE 37 192 193 /* 194 * Emit x86-64 prologue code for BPF program and check its size. 195 * bpf_tail_call helper will skip it while jumping into another program 196 */ 197 static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf) 198 { 199 u8 *prog = *pprog; 200 int cnt = 0; 201 202 /* push rbp */ 203 EMIT1(0x55); 204 205 /* mov rbp,rsp */ 206 EMIT3(0x48, 0x89, 0xE5); 207 208 /* sub rsp, rounded_stack_depth + AUX_STACK_SPACE */ 209 EMIT3_off32(0x48, 0x81, 0xEC, 210 round_up(stack_depth, 8) + AUX_STACK_SPACE); 211 212 /* sub rbp, AUX_STACK_SPACE */ 213 EMIT4(0x48, 0x83, 0xED, AUX_STACK_SPACE); 214 215 /* mov qword ptr [rbp+0],rbx */ 216 EMIT4(0x48, 0x89, 0x5D, 0); 217 /* mov qword ptr [rbp+8],r13 */ 218 EMIT4(0x4C, 0x89, 0x6D, 8); 219 /* mov qword ptr [rbp+16],r14 */ 220 EMIT4(0x4C, 0x89, 0x75, 16); 221 /* mov qword ptr [rbp+24],r15 */ 222 EMIT4(0x4C, 0x89, 0x7D, 24); 223 224 if (!ebpf_from_cbpf) { 225 /* 226 * Clear the tail call counter (tail_call_cnt): for eBPF tail 227 * calls we need to reset the counter to 0. It's done in two 228 * instructions, resetting RAX register to 0, and moving it 229 * to the counter location. 230 */ 231 232 /* xor eax, eax */ 233 EMIT2(0x31, 0xc0); 234 /* mov qword ptr [rbp+32], rax */ 235 EMIT4(0x48, 0x89, 0x45, 32); 236 237 BUILD_BUG_ON(cnt != PROLOGUE_SIZE); 238 } 239 240 *pprog = prog; 241 } 242 243 /* 244 * Generate the following code: 245 * 246 * ... bpf_tail_call(void *ctx, struct bpf_array *array, u64 index) ... 247 * if (index >= array->map.max_entries) 248 * goto out; 249 * if (++tail_call_cnt > MAX_TAIL_CALL_CNT) 250 * goto out; 251 * prog = array->ptrs[index]; 252 * if (prog == NULL) 253 * goto out; 254 * goto *(prog->bpf_func + prologue_size); 255 * out: 256 */ 257 static void emit_bpf_tail_call(u8 **pprog) 258 { 259 u8 *prog = *pprog; 260 int label1, label2, label3; 261 int cnt = 0; 262 263 /* 264 * rdi - pointer to ctx 265 * rsi - pointer to bpf_array 266 * rdx - index in bpf_array 267 */ 268 269 /* 270 * if (index >= array->map.max_entries) 271 * goto out; 272 */ 273 EMIT2(0x89, 0xD2); /* mov edx, edx */ 274 EMIT3(0x39, 0x56, /* cmp dword ptr [rsi + 16], edx */ 275 offsetof(struct bpf_array, map.max_entries)); 276 #define OFFSET1 (41 + RETPOLINE_RAX_BPF_JIT_SIZE) /* Number of bytes to jump */ 277 EMIT2(X86_JBE, OFFSET1); /* jbe out */ 278 label1 = cnt; 279 280 /* 281 * if (tail_call_cnt > MAX_TAIL_CALL_CNT) 282 * goto out; 283 */ 284 EMIT2_off32(0x8B, 0x85, 36); /* mov eax, dword ptr [rbp + 36] */ 285 EMIT3(0x83, 0xF8, MAX_TAIL_CALL_CNT); /* cmp eax, MAX_TAIL_CALL_CNT */ 286 #define OFFSET2 (30 + RETPOLINE_RAX_BPF_JIT_SIZE) 287 EMIT2(X86_JA, OFFSET2); /* ja out */ 288 label2 = cnt; 289 EMIT3(0x83, 0xC0, 0x01); /* add eax, 1 */ 290 EMIT2_off32(0x89, 0x85, 36); /* mov dword ptr [rbp + 36], eax */ 291 292 /* prog = array->ptrs[index]; */ 293 EMIT4_off32(0x48, 0x8B, 0x84, 0xD6, /* mov rax, [rsi + rdx * 8 + offsetof(...)] */ 294 offsetof(struct bpf_array, ptrs)); 295 296 /* 297 * if (prog == NULL) 298 * goto out; 299 */ 300 EMIT3(0x48, 0x85, 0xC0); /* test rax,rax */ 301 #define OFFSET3 (8 + RETPOLINE_RAX_BPF_JIT_SIZE) 302 EMIT2(X86_JE, OFFSET3); /* je out */ 303 label3 = cnt; 304 305 /* goto *(prog->bpf_func + prologue_size); */ 306 EMIT4(0x48, 0x8B, 0x40, /* mov rax, qword ptr [rax + 32] */ 307 offsetof(struct bpf_prog, bpf_func)); 308 EMIT4(0x48, 0x83, 0xC0, PROLOGUE_SIZE); /* add rax, prologue_size */ 309 310 /* 311 * Wow we're ready to jump into next BPF program 312 * rdi == ctx (1st arg) 313 * rax == prog->bpf_func + prologue_size 314 */ 315 RETPOLINE_RAX_BPF_JIT(); 316 317 /* out: */ 318 BUILD_BUG_ON(cnt - label1 != OFFSET1); 319 BUILD_BUG_ON(cnt - label2 != OFFSET2); 320 BUILD_BUG_ON(cnt - label3 != OFFSET3); 321 *pprog = prog; 322 } 323 324 static void emit_mov_imm32(u8 **pprog, bool sign_propagate, 325 u32 dst_reg, const u32 imm32) 326 { 327 u8 *prog = *pprog; 328 u8 b1, b2, b3; 329 int cnt = 0; 330 331 /* 332 * Optimization: if imm32 is positive, use 'mov %eax, imm32' 333 * (which zero-extends imm32) to save 2 bytes. 334 */ 335 if (sign_propagate && (s32)imm32 < 0) { 336 /* 'mov %rax, imm32' sign extends imm32 */ 337 b1 = add_1mod(0x48, dst_reg); 338 b2 = 0xC7; 339 b3 = 0xC0; 340 EMIT3_off32(b1, b2, add_1reg(b3, dst_reg), imm32); 341 goto done; 342 } 343 344 /* 345 * Optimization: if imm32 is zero, use 'xor %eax, %eax' 346 * to save 3 bytes. 347 */ 348 if (imm32 == 0) { 349 if (is_ereg(dst_reg)) 350 EMIT1(add_2mod(0x40, dst_reg, dst_reg)); 351 b2 = 0x31; /* xor */ 352 b3 = 0xC0; 353 EMIT2(b2, add_2reg(b3, dst_reg, dst_reg)); 354 goto done; 355 } 356 357 /* mov %eax, imm32 */ 358 if (is_ereg(dst_reg)) 359 EMIT1(add_1mod(0x40, dst_reg)); 360 EMIT1_off32(add_1reg(0xB8, dst_reg), imm32); 361 done: 362 *pprog = prog; 363 } 364 365 static void emit_mov_imm64(u8 **pprog, u32 dst_reg, 366 const u32 imm32_hi, const u32 imm32_lo) 367 { 368 u8 *prog = *pprog; 369 int cnt = 0; 370 371 if (is_uimm32(((u64)imm32_hi << 32) | (u32)imm32_lo)) { 372 /* 373 * For emitting plain u32, where sign bit must not be 374 * propagated LLVM tends to load imm64 over mov32 375 * directly, so save couple of bytes by just doing 376 * 'mov %eax, imm32' instead. 377 */ 378 emit_mov_imm32(&prog, false, dst_reg, imm32_lo); 379 } else { 380 /* movabsq %rax, imm64 */ 381 EMIT2(add_1mod(0x48, dst_reg), add_1reg(0xB8, dst_reg)); 382 EMIT(imm32_lo, 4); 383 EMIT(imm32_hi, 4); 384 } 385 386 *pprog = prog; 387 } 388 389 static void emit_mov_reg(u8 **pprog, bool is64, u32 dst_reg, u32 src_reg) 390 { 391 u8 *prog = *pprog; 392 int cnt = 0; 393 394 if (is64) { 395 /* mov dst, src */ 396 EMIT_mov(dst_reg, src_reg); 397 } else { 398 /* mov32 dst, src */ 399 if (is_ereg(dst_reg) || is_ereg(src_reg)) 400 EMIT1(add_2mod(0x40, dst_reg, src_reg)); 401 EMIT2(0x89, add_2reg(0xC0, dst_reg, src_reg)); 402 } 403 404 *pprog = prog; 405 } 406 407 static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, 408 int oldproglen, struct jit_context *ctx) 409 { 410 struct bpf_insn *insn = bpf_prog->insnsi; 411 int insn_cnt = bpf_prog->len; 412 bool seen_exit = false; 413 u8 temp[BPF_MAX_INSN_SIZE + BPF_INSN_SAFETY]; 414 int i, cnt = 0; 415 int proglen = 0; 416 u8 *prog = temp; 417 418 emit_prologue(&prog, bpf_prog->aux->stack_depth, 419 bpf_prog_was_classic(bpf_prog)); 420 421 for (i = 0; i < insn_cnt; i++, insn++) { 422 const s32 imm32 = insn->imm; 423 u32 dst_reg = insn->dst_reg; 424 u32 src_reg = insn->src_reg; 425 u8 b2 = 0, b3 = 0; 426 s64 jmp_offset; 427 u8 jmp_cond; 428 int ilen; 429 u8 *func; 430 431 switch (insn->code) { 432 /* ALU */ 433 case BPF_ALU | BPF_ADD | BPF_X: 434 case BPF_ALU | BPF_SUB | BPF_X: 435 case BPF_ALU | BPF_AND | BPF_X: 436 case BPF_ALU | BPF_OR | BPF_X: 437 case BPF_ALU | BPF_XOR | BPF_X: 438 case BPF_ALU64 | BPF_ADD | BPF_X: 439 case BPF_ALU64 | BPF_SUB | BPF_X: 440 case BPF_ALU64 | BPF_AND | BPF_X: 441 case BPF_ALU64 | BPF_OR | BPF_X: 442 case BPF_ALU64 | BPF_XOR | BPF_X: 443 switch (BPF_OP(insn->code)) { 444 case BPF_ADD: b2 = 0x01; break; 445 case BPF_SUB: b2 = 0x29; break; 446 case BPF_AND: b2 = 0x21; break; 447 case BPF_OR: b2 = 0x09; break; 448 case BPF_XOR: b2 = 0x31; break; 449 } 450 if (BPF_CLASS(insn->code) == BPF_ALU64) 451 EMIT1(add_2mod(0x48, dst_reg, src_reg)); 452 else if (is_ereg(dst_reg) || is_ereg(src_reg)) 453 EMIT1(add_2mod(0x40, dst_reg, src_reg)); 454 EMIT2(b2, add_2reg(0xC0, dst_reg, src_reg)); 455 break; 456 457 case BPF_ALU64 | BPF_MOV | BPF_X: 458 case BPF_ALU | BPF_MOV | BPF_X: 459 emit_mov_reg(&prog, 460 BPF_CLASS(insn->code) == BPF_ALU64, 461 dst_reg, src_reg); 462 break; 463 464 /* neg dst */ 465 case BPF_ALU | BPF_NEG: 466 case BPF_ALU64 | BPF_NEG: 467 if (BPF_CLASS(insn->code) == BPF_ALU64) 468 EMIT1(add_1mod(0x48, dst_reg)); 469 else if (is_ereg(dst_reg)) 470 EMIT1(add_1mod(0x40, dst_reg)); 471 EMIT2(0xF7, add_1reg(0xD8, dst_reg)); 472 break; 473 474 case BPF_ALU | BPF_ADD | BPF_K: 475 case BPF_ALU | BPF_SUB | BPF_K: 476 case BPF_ALU | BPF_AND | BPF_K: 477 case BPF_ALU | BPF_OR | BPF_K: 478 case BPF_ALU | BPF_XOR | BPF_K: 479 case BPF_ALU64 | BPF_ADD | BPF_K: 480 case BPF_ALU64 | BPF_SUB | BPF_K: 481 case BPF_ALU64 | BPF_AND | BPF_K: 482 case BPF_ALU64 | BPF_OR | BPF_K: 483 case BPF_ALU64 | BPF_XOR | BPF_K: 484 if (BPF_CLASS(insn->code) == BPF_ALU64) 485 EMIT1(add_1mod(0x48, dst_reg)); 486 else if (is_ereg(dst_reg)) 487 EMIT1(add_1mod(0x40, dst_reg)); 488 489 /* 490 * b3 holds 'normal' opcode, b2 short form only valid 491 * in case dst is eax/rax. 492 */ 493 switch (BPF_OP(insn->code)) { 494 case BPF_ADD: 495 b3 = 0xC0; 496 b2 = 0x05; 497 break; 498 case BPF_SUB: 499 b3 = 0xE8; 500 b2 = 0x2D; 501 break; 502 case BPF_AND: 503 b3 = 0xE0; 504 b2 = 0x25; 505 break; 506 case BPF_OR: 507 b3 = 0xC8; 508 b2 = 0x0D; 509 break; 510 case BPF_XOR: 511 b3 = 0xF0; 512 b2 = 0x35; 513 break; 514 } 515 516 if (is_imm8(imm32)) 517 EMIT3(0x83, add_1reg(b3, dst_reg), imm32); 518 else if (is_axreg(dst_reg)) 519 EMIT1_off32(b2, imm32); 520 else 521 EMIT2_off32(0x81, add_1reg(b3, dst_reg), imm32); 522 break; 523 524 case BPF_ALU64 | BPF_MOV | BPF_K: 525 case BPF_ALU | BPF_MOV | BPF_K: 526 emit_mov_imm32(&prog, BPF_CLASS(insn->code) == BPF_ALU64, 527 dst_reg, imm32); 528 break; 529 530 case BPF_LD | BPF_IMM | BPF_DW: 531 emit_mov_imm64(&prog, dst_reg, insn[1].imm, insn[0].imm); 532 insn++; 533 i++; 534 break; 535 536 /* dst %= src, dst /= src, dst %= imm32, dst /= imm32 */ 537 case BPF_ALU | BPF_MOD | BPF_X: 538 case BPF_ALU | BPF_DIV | BPF_X: 539 case BPF_ALU | BPF_MOD | BPF_K: 540 case BPF_ALU | BPF_DIV | BPF_K: 541 case BPF_ALU64 | BPF_MOD | BPF_X: 542 case BPF_ALU64 | BPF_DIV | BPF_X: 543 case BPF_ALU64 | BPF_MOD | BPF_K: 544 case BPF_ALU64 | BPF_DIV | BPF_K: 545 EMIT1(0x50); /* push rax */ 546 EMIT1(0x52); /* push rdx */ 547 548 if (BPF_SRC(insn->code) == BPF_X) 549 /* mov r11, src_reg */ 550 EMIT_mov(AUX_REG, src_reg); 551 else 552 /* mov r11, imm32 */ 553 EMIT3_off32(0x49, 0xC7, 0xC3, imm32); 554 555 /* mov rax, dst_reg */ 556 EMIT_mov(BPF_REG_0, dst_reg); 557 558 /* 559 * xor edx, edx 560 * equivalent to 'xor rdx, rdx', but one byte less 561 */ 562 EMIT2(0x31, 0xd2); 563 564 if (BPF_CLASS(insn->code) == BPF_ALU64) 565 /* div r11 */ 566 EMIT3(0x49, 0xF7, 0xF3); 567 else 568 /* div r11d */ 569 EMIT3(0x41, 0xF7, 0xF3); 570 571 if (BPF_OP(insn->code) == BPF_MOD) 572 /* mov r11, rdx */ 573 EMIT3(0x49, 0x89, 0xD3); 574 else 575 /* mov r11, rax */ 576 EMIT3(0x49, 0x89, 0xC3); 577 578 EMIT1(0x5A); /* pop rdx */ 579 EMIT1(0x58); /* pop rax */ 580 581 /* mov dst_reg, r11 */ 582 EMIT_mov(dst_reg, AUX_REG); 583 break; 584 585 case BPF_ALU | BPF_MUL | BPF_K: 586 case BPF_ALU | BPF_MUL | BPF_X: 587 case BPF_ALU64 | BPF_MUL | BPF_K: 588 case BPF_ALU64 | BPF_MUL | BPF_X: 589 { 590 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64; 591 592 if (dst_reg != BPF_REG_0) 593 EMIT1(0x50); /* push rax */ 594 if (dst_reg != BPF_REG_3) 595 EMIT1(0x52); /* push rdx */ 596 597 /* mov r11, dst_reg */ 598 EMIT_mov(AUX_REG, dst_reg); 599 600 if (BPF_SRC(insn->code) == BPF_X) 601 emit_mov_reg(&prog, is64, BPF_REG_0, src_reg); 602 else 603 emit_mov_imm32(&prog, is64, BPF_REG_0, imm32); 604 605 if (is64) 606 EMIT1(add_1mod(0x48, AUX_REG)); 607 else if (is_ereg(AUX_REG)) 608 EMIT1(add_1mod(0x40, AUX_REG)); 609 /* mul(q) r11 */ 610 EMIT2(0xF7, add_1reg(0xE0, AUX_REG)); 611 612 if (dst_reg != BPF_REG_3) 613 EMIT1(0x5A); /* pop rdx */ 614 if (dst_reg != BPF_REG_0) { 615 /* mov dst_reg, rax */ 616 EMIT_mov(dst_reg, BPF_REG_0); 617 EMIT1(0x58); /* pop rax */ 618 } 619 break; 620 } 621 /* Shifts */ 622 case BPF_ALU | BPF_LSH | BPF_K: 623 case BPF_ALU | BPF_RSH | BPF_K: 624 case BPF_ALU | BPF_ARSH | BPF_K: 625 case BPF_ALU64 | BPF_LSH | BPF_K: 626 case BPF_ALU64 | BPF_RSH | BPF_K: 627 case BPF_ALU64 | BPF_ARSH | BPF_K: 628 if (BPF_CLASS(insn->code) == BPF_ALU64) 629 EMIT1(add_1mod(0x48, dst_reg)); 630 else if (is_ereg(dst_reg)) 631 EMIT1(add_1mod(0x40, dst_reg)); 632 633 switch (BPF_OP(insn->code)) { 634 case BPF_LSH: b3 = 0xE0; break; 635 case BPF_RSH: b3 = 0xE8; break; 636 case BPF_ARSH: b3 = 0xF8; break; 637 } 638 639 if (imm32 == 1) 640 EMIT2(0xD1, add_1reg(b3, dst_reg)); 641 else 642 EMIT3(0xC1, add_1reg(b3, dst_reg), imm32); 643 break; 644 645 case BPF_ALU | BPF_LSH | BPF_X: 646 case BPF_ALU | BPF_RSH | BPF_X: 647 case BPF_ALU | BPF_ARSH | BPF_X: 648 case BPF_ALU64 | BPF_LSH | BPF_X: 649 case BPF_ALU64 | BPF_RSH | BPF_X: 650 case BPF_ALU64 | BPF_ARSH | BPF_X: 651 652 /* Check for bad case when dst_reg == rcx */ 653 if (dst_reg == BPF_REG_4) { 654 /* mov r11, dst_reg */ 655 EMIT_mov(AUX_REG, dst_reg); 656 dst_reg = AUX_REG; 657 } 658 659 if (src_reg != BPF_REG_4) { /* common case */ 660 EMIT1(0x51); /* push rcx */ 661 662 /* mov rcx, src_reg */ 663 EMIT_mov(BPF_REG_4, src_reg); 664 } 665 666 /* shl %rax, %cl | shr %rax, %cl | sar %rax, %cl */ 667 if (BPF_CLASS(insn->code) == BPF_ALU64) 668 EMIT1(add_1mod(0x48, dst_reg)); 669 else if (is_ereg(dst_reg)) 670 EMIT1(add_1mod(0x40, dst_reg)); 671 672 switch (BPF_OP(insn->code)) { 673 case BPF_LSH: b3 = 0xE0; break; 674 case BPF_RSH: b3 = 0xE8; break; 675 case BPF_ARSH: b3 = 0xF8; break; 676 } 677 EMIT2(0xD3, add_1reg(b3, dst_reg)); 678 679 if (src_reg != BPF_REG_4) 680 EMIT1(0x59); /* pop rcx */ 681 682 if (insn->dst_reg == BPF_REG_4) 683 /* mov dst_reg, r11 */ 684 EMIT_mov(insn->dst_reg, AUX_REG); 685 break; 686 687 case BPF_ALU | BPF_END | BPF_FROM_BE: 688 switch (imm32) { 689 case 16: 690 /* Emit 'ror %ax, 8' to swap lower 2 bytes */ 691 EMIT1(0x66); 692 if (is_ereg(dst_reg)) 693 EMIT1(0x41); 694 EMIT3(0xC1, add_1reg(0xC8, dst_reg), 8); 695 696 /* Emit 'movzwl eax, ax' */ 697 if (is_ereg(dst_reg)) 698 EMIT3(0x45, 0x0F, 0xB7); 699 else 700 EMIT2(0x0F, 0xB7); 701 EMIT1(add_2reg(0xC0, dst_reg, dst_reg)); 702 break; 703 case 32: 704 /* Emit 'bswap eax' to swap lower 4 bytes */ 705 if (is_ereg(dst_reg)) 706 EMIT2(0x41, 0x0F); 707 else 708 EMIT1(0x0F); 709 EMIT1(add_1reg(0xC8, dst_reg)); 710 break; 711 case 64: 712 /* Emit 'bswap rax' to swap 8 bytes */ 713 EMIT3(add_1mod(0x48, dst_reg), 0x0F, 714 add_1reg(0xC8, dst_reg)); 715 break; 716 } 717 break; 718 719 case BPF_ALU | BPF_END | BPF_FROM_LE: 720 switch (imm32) { 721 case 16: 722 /* 723 * Emit 'movzwl eax, ax' to zero extend 16-bit 724 * into 64 bit 725 */ 726 if (is_ereg(dst_reg)) 727 EMIT3(0x45, 0x0F, 0xB7); 728 else 729 EMIT2(0x0F, 0xB7); 730 EMIT1(add_2reg(0xC0, dst_reg, dst_reg)); 731 break; 732 case 32: 733 /* Emit 'mov eax, eax' to clear upper 32-bits */ 734 if (is_ereg(dst_reg)) 735 EMIT1(0x45); 736 EMIT2(0x89, add_2reg(0xC0, dst_reg, dst_reg)); 737 break; 738 case 64: 739 /* nop */ 740 break; 741 } 742 break; 743 744 /* ST: *(u8*)(dst_reg + off) = imm */ 745 case BPF_ST | BPF_MEM | BPF_B: 746 if (is_ereg(dst_reg)) 747 EMIT2(0x41, 0xC6); 748 else 749 EMIT1(0xC6); 750 goto st; 751 case BPF_ST | BPF_MEM | BPF_H: 752 if (is_ereg(dst_reg)) 753 EMIT3(0x66, 0x41, 0xC7); 754 else 755 EMIT2(0x66, 0xC7); 756 goto st; 757 case BPF_ST | BPF_MEM | BPF_W: 758 if (is_ereg(dst_reg)) 759 EMIT2(0x41, 0xC7); 760 else 761 EMIT1(0xC7); 762 goto st; 763 case BPF_ST | BPF_MEM | BPF_DW: 764 EMIT2(add_1mod(0x48, dst_reg), 0xC7); 765 766 st: if (is_imm8(insn->off)) 767 EMIT2(add_1reg(0x40, dst_reg), insn->off); 768 else 769 EMIT1_off32(add_1reg(0x80, dst_reg), insn->off); 770 771 EMIT(imm32, bpf_size_to_x86_bytes(BPF_SIZE(insn->code))); 772 break; 773 774 /* STX: *(u8*)(dst_reg + off) = src_reg */ 775 case BPF_STX | BPF_MEM | BPF_B: 776 /* Emit 'mov byte ptr [rax + off], al' */ 777 if (is_ereg(dst_reg) || is_ereg(src_reg) || 778 /* We have to add extra byte for x86 SIL, DIL regs */ 779 src_reg == BPF_REG_1 || src_reg == BPF_REG_2) 780 EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x88); 781 else 782 EMIT1(0x88); 783 goto stx; 784 case BPF_STX | BPF_MEM | BPF_H: 785 if (is_ereg(dst_reg) || is_ereg(src_reg)) 786 EMIT3(0x66, add_2mod(0x40, dst_reg, src_reg), 0x89); 787 else 788 EMIT2(0x66, 0x89); 789 goto stx; 790 case BPF_STX | BPF_MEM | BPF_W: 791 if (is_ereg(dst_reg) || is_ereg(src_reg)) 792 EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x89); 793 else 794 EMIT1(0x89); 795 goto stx; 796 case BPF_STX | BPF_MEM | BPF_DW: 797 EMIT2(add_2mod(0x48, dst_reg, src_reg), 0x89); 798 stx: if (is_imm8(insn->off)) 799 EMIT2(add_2reg(0x40, dst_reg, src_reg), insn->off); 800 else 801 EMIT1_off32(add_2reg(0x80, dst_reg, src_reg), 802 insn->off); 803 break; 804 805 /* LDX: dst_reg = *(u8*)(src_reg + off) */ 806 case BPF_LDX | BPF_MEM | BPF_B: 807 /* Emit 'movzx rax, byte ptr [rax + off]' */ 808 EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB6); 809 goto ldx; 810 case BPF_LDX | BPF_MEM | BPF_H: 811 /* Emit 'movzx rax, word ptr [rax + off]' */ 812 EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB7); 813 goto ldx; 814 case BPF_LDX | BPF_MEM | BPF_W: 815 /* Emit 'mov eax, dword ptr [rax+0x14]' */ 816 if (is_ereg(dst_reg) || is_ereg(src_reg)) 817 EMIT2(add_2mod(0x40, src_reg, dst_reg), 0x8B); 818 else 819 EMIT1(0x8B); 820 goto ldx; 821 case BPF_LDX | BPF_MEM | BPF_DW: 822 /* Emit 'mov rax, qword ptr [rax+0x14]' */ 823 EMIT2(add_2mod(0x48, src_reg, dst_reg), 0x8B); 824 ldx: /* 825 * If insn->off == 0 we can save one extra byte, but 826 * special case of x86 R13 which always needs an offset 827 * is not worth the hassle 828 */ 829 if (is_imm8(insn->off)) 830 EMIT2(add_2reg(0x40, src_reg, dst_reg), insn->off); 831 else 832 EMIT1_off32(add_2reg(0x80, src_reg, dst_reg), 833 insn->off); 834 break; 835 836 /* STX XADD: lock *(u32*)(dst_reg + off) += src_reg */ 837 case BPF_STX | BPF_XADD | BPF_W: 838 /* Emit 'lock add dword ptr [rax + off], eax' */ 839 if (is_ereg(dst_reg) || is_ereg(src_reg)) 840 EMIT3(0xF0, add_2mod(0x40, dst_reg, src_reg), 0x01); 841 else 842 EMIT2(0xF0, 0x01); 843 goto xadd; 844 case BPF_STX | BPF_XADD | BPF_DW: 845 EMIT3(0xF0, add_2mod(0x48, dst_reg, src_reg), 0x01); 846 xadd: if (is_imm8(insn->off)) 847 EMIT2(add_2reg(0x40, dst_reg, src_reg), insn->off); 848 else 849 EMIT1_off32(add_2reg(0x80, dst_reg, src_reg), 850 insn->off); 851 break; 852 853 /* call */ 854 case BPF_JMP | BPF_CALL: 855 func = (u8 *) __bpf_call_base + imm32; 856 jmp_offset = func - (image + addrs[i]); 857 if (!imm32 || !is_simm32(jmp_offset)) { 858 pr_err("unsupported BPF func %d addr %p image %p\n", 859 imm32, func, image); 860 return -EINVAL; 861 } 862 EMIT1_off32(0xE8, jmp_offset); 863 break; 864 865 case BPF_JMP | BPF_TAIL_CALL: 866 emit_bpf_tail_call(&prog); 867 break; 868 869 /* cond jump */ 870 case BPF_JMP | BPF_JEQ | BPF_X: 871 case BPF_JMP | BPF_JNE | BPF_X: 872 case BPF_JMP | BPF_JGT | BPF_X: 873 case BPF_JMP | BPF_JLT | BPF_X: 874 case BPF_JMP | BPF_JGE | BPF_X: 875 case BPF_JMP | BPF_JLE | BPF_X: 876 case BPF_JMP | BPF_JSGT | BPF_X: 877 case BPF_JMP | BPF_JSLT | BPF_X: 878 case BPF_JMP | BPF_JSGE | BPF_X: 879 case BPF_JMP | BPF_JSLE | BPF_X: 880 case BPF_JMP32 | BPF_JEQ | BPF_X: 881 case BPF_JMP32 | BPF_JNE | BPF_X: 882 case BPF_JMP32 | BPF_JGT | BPF_X: 883 case BPF_JMP32 | BPF_JLT | BPF_X: 884 case BPF_JMP32 | BPF_JGE | BPF_X: 885 case BPF_JMP32 | BPF_JLE | BPF_X: 886 case BPF_JMP32 | BPF_JSGT | BPF_X: 887 case BPF_JMP32 | BPF_JSLT | BPF_X: 888 case BPF_JMP32 | BPF_JSGE | BPF_X: 889 case BPF_JMP32 | BPF_JSLE | BPF_X: 890 /* cmp dst_reg, src_reg */ 891 if (BPF_CLASS(insn->code) == BPF_JMP) 892 EMIT1(add_2mod(0x48, dst_reg, src_reg)); 893 else if (is_ereg(dst_reg) || is_ereg(src_reg)) 894 EMIT1(add_2mod(0x40, dst_reg, src_reg)); 895 EMIT2(0x39, add_2reg(0xC0, dst_reg, src_reg)); 896 goto emit_cond_jmp; 897 898 case BPF_JMP | BPF_JSET | BPF_X: 899 case BPF_JMP32 | BPF_JSET | BPF_X: 900 /* test dst_reg, src_reg */ 901 if (BPF_CLASS(insn->code) == BPF_JMP) 902 EMIT1(add_2mod(0x48, dst_reg, src_reg)); 903 else if (is_ereg(dst_reg) || is_ereg(src_reg)) 904 EMIT1(add_2mod(0x40, dst_reg, src_reg)); 905 EMIT2(0x85, add_2reg(0xC0, dst_reg, src_reg)); 906 goto emit_cond_jmp; 907 908 case BPF_JMP | BPF_JSET | BPF_K: 909 case BPF_JMP32 | BPF_JSET | BPF_K: 910 /* test dst_reg, imm32 */ 911 if (BPF_CLASS(insn->code) == BPF_JMP) 912 EMIT1(add_1mod(0x48, dst_reg)); 913 else if (is_ereg(dst_reg)) 914 EMIT1(add_1mod(0x40, dst_reg)); 915 EMIT2_off32(0xF7, add_1reg(0xC0, dst_reg), imm32); 916 goto emit_cond_jmp; 917 918 case BPF_JMP | BPF_JEQ | BPF_K: 919 case BPF_JMP | BPF_JNE | BPF_K: 920 case BPF_JMP | BPF_JGT | BPF_K: 921 case BPF_JMP | BPF_JLT | BPF_K: 922 case BPF_JMP | BPF_JGE | BPF_K: 923 case BPF_JMP | BPF_JLE | BPF_K: 924 case BPF_JMP | BPF_JSGT | BPF_K: 925 case BPF_JMP | BPF_JSLT | BPF_K: 926 case BPF_JMP | BPF_JSGE | BPF_K: 927 case BPF_JMP | BPF_JSLE | BPF_K: 928 case BPF_JMP32 | BPF_JEQ | BPF_K: 929 case BPF_JMP32 | BPF_JNE | BPF_K: 930 case BPF_JMP32 | BPF_JGT | BPF_K: 931 case BPF_JMP32 | BPF_JLT | BPF_K: 932 case BPF_JMP32 | BPF_JGE | BPF_K: 933 case BPF_JMP32 | BPF_JLE | BPF_K: 934 case BPF_JMP32 | BPF_JSGT | BPF_K: 935 case BPF_JMP32 | BPF_JSLT | BPF_K: 936 case BPF_JMP32 | BPF_JSGE | BPF_K: 937 case BPF_JMP32 | BPF_JSLE | BPF_K: 938 /* cmp dst_reg, imm8/32 */ 939 if (BPF_CLASS(insn->code) == BPF_JMP) 940 EMIT1(add_1mod(0x48, dst_reg)); 941 else if (is_ereg(dst_reg)) 942 EMIT1(add_1mod(0x40, dst_reg)); 943 944 if (is_imm8(imm32)) 945 EMIT3(0x83, add_1reg(0xF8, dst_reg), imm32); 946 else 947 EMIT2_off32(0x81, add_1reg(0xF8, dst_reg), imm32); 948 949 emit_cond_jmp: /* Convert BPF opcode to x86 */ 950 switch (BPF_OP(insn->code)) { 951 case BPF_JEQ: 952 jmp_cond = X86_JE; 953 break; 954 case BPF_JSET: 955 case BPF_JNE: 956 jmp_cond = X86_JNE; 957 break; 958 case BPF_JGT: 959 /* GT is unsigned '>', JA in x86 */ 960 jmp_cond = X86_JA; 961 break; 962 case BPF_JLT: 963 /* LT is unsigned '<', JB in x86 */ 964 jmp_cond = X86_JB; 965 break; 966 case BPF_JGE: 967 /* GE is unsigned '>=', JAE in x86 */ 968 jmp_cond = X86_JAE; 969 break; 970 case BPF_JLE: 971 /* LE is unsigned '<=', JBE in x86 */ 972 jmp_cond = X86_JBE; 973 break; 974 case BPF_JSGT: 975 /* Signed '>', GT in x86 */ 976 jmp_cond = X86_JG; 977 break; 978 case BPF_JSLT: 979 /* Signed '<', LT in x86 */ 980 jmp_cond = X86_JL; 981 break; 982 case BPF_JSGE: 983 /* Signed '>=', GE in x86 */ 984 jmp_cond = X86_JGE; 985 break; 986 case BPF_JSLE: 987 /* Signed '<=', LE in x86 */ 988 jmp_cond = X86_JLE; 989 break; 990 default: /* to silence GCC warning */ 991 return -EFAULT; 992 } 993 jmp_offset = addrs[i + insn->off] - addrs[i]; 994 if (is_imm8(jmp_offset)) { 995 EMIT2(jmp_cond, jmp_offset); 996 } else if (is_simm32(jmp_offset)) { 997 EMIT2_off32(0x0F, jmp_cond + 0x10, jmp_offset); 998 } else { 999 pr_err("cond_jmp gen bug %llx\n", jmp_offset); 1000 return -EFAULT; 1001 } 1002 1003 break; 1004 1005 case BPF_JMP | BPF_JA: 1006 if (insn->off == -1) 1007 /* -1 jmp instructions will always jump 1008 * backwards two bytes. Explicitly handling 1009 * this case avoids wasting too many passes 1010 * when there are long sequences of replaced 1011 * dead code. 1012 */ 1013 jmp_offset = -2; 1014 else 1015 jmp_offset = addrs[i + insn->off] - addrs[i]; 1016 1017 if (!jmp_offset) 1018 /* Optimize out nop jumps */ 1019 break; 1020 emit_jmp: 1021 if (is_imm8(jmp_offset)) { 1022 EMIT2(0xEB, jmp_offset); 1023 } else if (is_simm32(jmp_offset)) { 1024 EMIT1_off32(0xE9, jmp_offset); 1025 } else { 1026 pr_err("jmp gen bug %llx\n", jmp_offset); 1027 return -EFAULT; 1028 } 1029 break; 1030 1031 case BPF_JMP | BPF_EXIT: 1032 if (seen_exit) { 1033 jmp_offset = ctx->cleanup_addr - addrs[i]; 1034 goto emit_jmp; 1035 } 1036 seen_exit = true; 1037 /* Update cleanup_addr */ 1038 ctx->cleanup_addr = proglen; 1039 /* mov rbx, qword ptr [rbp+0] */ 1040 EMIT4(0x48, 0x8B, 0x5D, 0); 1041 /* mov r13, qword ptr [rbp+8] */ 1042 EMIT4(0x4C, 0x8B, 0x6D, 8); 1043 /* mov r14, qword ptr [rbp+16] */ 1044 EMIT4(0x4C, 0x8B, 0x75, 16); 1045 /* mov r15, qword ptr [rbp+24] */ 1046 EMIT4(0x4C, 0x8B, 0x7D, 24); 1047 1048 /* add rbp, AUX_STACK_SPACE */ 1049 EMIT4(0x48, 0x83, 0xC5, AUX_STACK_SPACE); 1050 EMIT1(0xC9); /* leave */ 1051 EMIT1(0xC3); /* ret */ 1052 break; 1053 1054 default: 1055 /* 1056 * By design x86-64 JIT should support all BPF instructions. 1057 * This error will be seen if new instruction was added 1058 * to the interpreter, but not to the JIT, or if there is 1059 * junk in bpf_prog. 1060 */ 1061 pr_err("bpf_jit: unknown opcode %02x\n", insn->code); 1062 return -EINVAL; 1063 } 1064 1065 ilen = prog - temp; 1066 if (ilen > BPF_MAX_INSN_SIZE) { 1067 pr_err("bpf_jit: fatal insn size error\n"); 1068 return -EFAULT; 1069 } 1070 1071 if (image) { 1072 if (unlikely(proglen + ilen > oldproglen)) { 1073 pr_err("bpf_jit: fatal error\n"); 1074 return -EFAULT; 1075 } 1076 memcpy(image + proglen, temp, ilen); 1077 } 1078 proglen += ilen; 1079 addrs[i] = proglen; 1080 prog = temp; 1081 } 1082 return proglen; 1083 } 1084 1085 struct x64_jit_data { 1086 struct bpf_binary_header *header; 1087 int *addrs; 1088 u8 *image; 1089 int proglen; 1090 struct jit_context ctx; 1091 }; 1092 1093 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) 1094 { 1095 struct bpf_binary_header *header = NULL; 1096 struct bpf_prog *tmp, *orig_prog = prog; 1097 struct x64_jit_data *jit_data; 1098 int proglen, oldproglen = 0; 1099 struct jit_context ctx = {}; 1100 bool tmp_blinded = false; 1101 bool extra_pass = false; 1102 u8 *image = NULL; 1103 int *addrs; 1104 int pass; 1105 int i; 1106 1107 if (!prog->jit_requested) 1108 return orig_prog; 1109 1110 tmp = bpf_jit_blind_constants(prog); 1111 /* 1112 * If blinding was requested and we failed during blinding, 1113 * we must fall back to the interpreter. 1114 */ 1115 if (IS_ERR(tmp)) 1116 return orig_prog; 1117 if (tmp != prog) { 1118 tmp_blinded = true; 1119 prog = tmp; 1120 } 1121 1122 jit_data = prog->aux->jit_data; 1123 if (!jit_data) { 1124 jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL); 1125 if (!jit_data) { 1126 prog = orig_prog; 1127 goto out; 1128 } 1129 prog->aux->jit_data = jit_data; 1130 } 1131 addrs = jit_data->addrs; 1132 if (addrs) { 1133 ctx = jit_data->ctx; 1134 oldproglen = jit_data->proglen; 1135 image = jit_data->image; 1136 header = jit_data->header; 1137 extra_pass = true; 1138 goto skip_init_addrs; 1139 } 1140 addrs = kmalloc_array(prog->len, sizeof(*addrs), GFP_KERNEL); 1141 if (!addrs) { 1142 prog = orig_prog; 1143 goto out_addrs; 1144 } 1145 1146 /* 1147 * Before first pass, make a rough estimation of addrs[] 1148 * each BPF instruction is translated to less than 64 bytes 1149 */ 1150 for (proglen = 0, i = 0; i < prog->len; i++) { 1151 proglen += 64; 1152 addrs[i] = proglen; 1153 } 1154 ctx.cleanup_addr = proglen; 1155 skip_init_addrs: 1156 1157 /* 1158 * JITed image shrinks with every pass and the loop iterates 1159 * until the image stops shrinking. Very large BPF programs 1160 * may converge on the last pass. In such case do one more 1161 * pass to emit the final image. 1162 */ 1163 for (pass = 0; pass < 20 || image; pass++) { 1164 proglen = do_jit(prog, addrs, image, oldproglen, &ctx); 1165 if (proglen <= 0) { 1166 out_image: 1167 image = NULL; 1168 if (header) 1169 bpf_jit_binary_free(header); 1170 prog = orig_prog; 1171 goto out_addrs; 1172 } 1173 if (image) { 1174 if (proglen != oldproglen) { 1175 pr_err("bpf_jit: proglen=%d != oldproglen=%d\n", 1176 proglen, oldproglen); 1177 goto out_image; 1178 } 1179 break; 1180 } 1181 if (proglen == oldproglen) { 1182 header = bpf_jit_binary_alloc(proglen, &image, 1183 1, jit_fill_hole); 1184 if (!header) { 1185 prog = orig_prog; 1186 goto out_addrs; 1187 } 1188 } 1189 oldproglen = proglen; 1190 cond_resched(); 1191 } 1192 1193 if (bpf_jit_enable > 1) 1194 bpf_jit_dump(prog->len, proglen, pass + 1, image); 1195 1196 if (image) { 1197 if (!prog->is_func || extra_pass) { 1198 bpf_jit_binary_lock_ro(header); 1199 } else { 1200 jit_data->addrs = addrs; 1201 jit_data->ctx = ctx; 1202 jit_data->proglen = proglen; 1203 jit_data->image = image; 1204 jit_data->header = header; 1205 } 1206 prog->bpf_func = (void *)image; 1207 prog->jited = 1; 1208 prog->jited_len = proglen; 1209 } else { 1210 prog = orig_prog; 1211 } 1212 1213 if (!image || !prog->is_func || extra_pass) { 1214 if (image) 1215 bpf_prog_fill_jited_linfo(prog, addrs); 1216 out_addrs: 1217 kfree(addrs); 1218 kfree(jit_data); 1219 prog->aux->jit_data = NULL; 1220 } 1221 out: 1222 if (tmp_blinded) 1223 bpf_jit_prog_release_other(prog, prog == orig_prog ? 1224 tmp : orig_prog); 1225 return prog; 1226 } 1227