Lines Matching refs:prog
34 do { prog = emit_code(prog, bytes, len); } while (0)
260 u8 *prog = *pprog; in push_callee_regs() local
270 *pprog = prog; in push_callee_regs()
275 u8 *prog = *pprog; in pop_callee_regs() local
285 *pprog = prog; in pop_callee_regs()
296 u8 *prog = *pprog; in emit_prologue() local
302 memcpy(prog, x86_nops[5], X86_PATCH_SIZE); in emit_prologue()
303 prog += X86_PATCH_SIZE; in emit_prologue()
321 *pprog = prog; in emit_prologue()
326 u8 *prog = *pprog; in emit_patch() local
335 *pprog = prog; in emit_patch()
362 u8 *prog; in __bpf_arch_text_poke() local
367 prog = old_insn; in __bpf_arch_text_poke()
369 emit_call(&prog, old_addr, ip) : in __bpf_arch_text_poke()
370 emit_jump(&prog, old_addr, ip); in __bpf_arch_text_poke()
377 prog = new_insn; in __bpf_arch_text_poke()
379 emit_call(&prog, new_addr, ip) : in __bpf_arch_text_poke()
380 emit_jump(&prog, new_addr, ip); in __bpf_arch_text_poke()
421 u8 *prog = *pprog; in emit_indirect_jump() local
429 emit_jump(&prog, &__x86_indirect_jump_thunk_array[reg], ip); in emit_indirect_jump()
431 emit_jump(&prog, &__x86_indirect_thunk_array[reg], ip); in emit_indirect_jump()
438 *pprog = prog; in emit_indirect_jump()
443 u8 *prog = *pprog; in emit_return() local
446 emit_jump(&prog, x86_return_thunk, ip); in emit_return()
453 *pprog = prog; in emit_return()
475 u8 *prog = *pprog, *start = *pprog; in emit_bpf_tail_call_indirect() local
492 offset = ctx->tail_call_indirect_label - (prog + 2 - start); in emit_bpf_tail_call_indirect()
502 offset = ctx->tail_call_indirect_label - (prog + 2 - start); in emit_bpf_tail_call_indirect()
517 offset = ctx->tail_call_indirect_label - (prog + 2 - start); in emit_bpf_tail_call_indirect()
520 pop_callee_regs(&prog, callee_regs_used); in emit_bpf_tail_call_indirect()
537 emit_indirect_jump(&prog, 1 /* rcx */, ip + (prog - start)); in emit_bpf_tail_call_indirect()
540 ctx->tail_call_indirect_label = prog - start; in emit_bpf_tail_call_indirect()
541 *pprog = prog; in emit_bpf_tail_call_indirect()
550 u8 *prog = *pprog, *start = *pprog; in emit_bpf_tail_call_direct() local
560 offset = ctx->tail_call_direct_label - (prog + 2 - start); in emit_bpf_tail_call_direct()
565 poke->tailcall_bypass = ip + (prog - start); in emit_bpf_tail_call_direct()
570 emit_jump(&prog, (u8 *)poke->tailcall_target + X86_PATCH_SIZE, in emit_bpf_tail_call_direct()
573 pop_callee_regs(&prog, callee_regs_used); in emit_bpf_tail_call_direct()
578 memcpy(prog, x86_nops[5], X86_PATCH_SIZE); in emit_bpf_tail_call_direct()
579 prog += X86_PATCH_SIZE; in emit_bpf_tail_call_direct()
582 ctx->tail_call_direct_label = prog - start; in emit_bpf_tail_call_direct()
584 *pprog = prog; in emit_bpf_tail_call_direct()
587 static void bpf_tail_call_direct_fixup(struct bpf_prog *prog) in bpf_tail_call_direct_fixup() argument
594 for (i = 0; i < prog->aux->size_poke_tab; i++) { in bpf_tail_call_direct_fixup()
595 poke = &prog->aux->poke_tab[i]; in bpf_tail_call_direct_fixup()
596 if (poke->aux && poke->aux != prog->aux) in bpf_tail_call_direct_fixup()
627 u8 *prog = *pprog; in emit_mov_imm32() local
661 *pprog = prog; in emit_mov_imm32()
667 u8 *prog = *pprog; in emit_mov_imm64() local
676 emit_mov_imm32(&prog, false, dst_reg, imm32_lo); in emit_mov_imm64()
684 *pprog = prog; in emit_mov_imm64()
689 u8 *prog = *pprog; in emit_mov_reg() local
701 *pprog = prog; in emit_mov_reg()
707 u8 *prog = *pprog; in emit_movsx_reg() local
733 *pprog = prog; in emit_movsx_reg()
739 u8 *prog = *pprog; in emit_insn_suffix() local
753 *pprog = prog; in emit_insn_suffix()
761 u8 *prog = *pprog; in maybe_emit_mod() local
767 *pprog = prog; in maybe_emit_mod()
775 u8 *prog = *pprog; in maybe_emit_1mod() local
781 *pprog = prog; in maybe_emit_1mod()
787 u8 *prog = *pprog; in emit_ldx() local
810 emit_insn_suffix(&prog, src_reg, dst_reg, off); in emit_ldx()
811 *pprog = prog; in emit_ldx()
817 u8 *prog = *pprog; in emit_ldsx() local
833 emit_insn_suffix(&prog, src_reg, dst_reg, off); in emit_ldsx()
834 *pprog = prog; in emit_ldsx()
840 u8 *prog = *pprog; in emit_stx() local
867 emit_insn_suffix(&prog, dst_reg, src_reg, off); in emit_stx()
868 *pprog = prog; in emit_stx()
874 u8 *prog = *pprog; in emit_atomic() local
878 maybe_emit_mod(&prog, dst_reg, src_reg, bpf_size == BPF_DW); in emit_atomic()
906 emit_insn_suffix(&prog, dst_reg, src_reg, off); in emit_atomic()
908 *pprog = prog; in emit_atomic()
943 u8 *prog = *pprog; in emit_nops() local
957 *pprog = prog; in emit_nops()
974 u8 *prog = *pprog; in emit_3vex() local
1004 *pprog = prog; in emit_3vex()
1010 u8 *prog = *pprog; in emit_shiftx() local
1014 emit_3vex(&prog, r, false, r, m, is64, src_reg, false, op); in emit_shiftx()
1016 *pprog = prog; in emit_shiftx()
1019 #define INSN_SZ_DIFF (((addrs[i] - addrs[i - 1]) - (prog - temp)))
1037 u8 *prog = temp; in do_jit() local
1046 emit_prologue(&prog, bpf_prog->aux->stack_depth, in do_jit()
1049 push_callee_regs(&prog, callee_regs_used); in do_jit()
1051 ilen = prog - temp; in do_jit()
1056 prog = temp; in do_jit()
1082 maybe_emit_mod(&prog, dst_reg, src_reg, in do_jit()
1091 emit_mov_reg(&prog, in do_jit()
1095 emit_movsx_reg(&prog, insn->off, in do_jit()
1103 maybe_emit_1mod(&prog, dst_reg, in do_jit()
1118 maybe_emit_1mod(&prog, dst_reg, in do_jit()
1158 emit_mov_imm32(&prog, BPF_CLASS(insn->code) == BPF_ALU64, in do_jit()
1163 emit_mov_imm64(&prog, dst_reg, insn[1].imm, insn[0].imm); in do_jit()
1199 emit_mov_reg(&prog, is64, BPF_REG_0, dst_reg); in do_jit()
1209 maybe_emit_1mod(&prog, src_reg, is64); in do_jit()
1218 maybe_emit_1mod(&prog, src_reg, is64); in do_jit()
1225 emit_mov_reg(&prog, is64, dst_reg, BPF_REG_3); in do_jit()
1229 emit_mov_reg(&prog, is64, dst_reg, BPF_REG_0); in do_jit()
1240 maybe_emit_mod(&prog, dst_reg, dst_reg, in do_jit()
1256 maybe_emit_mod(&prog, src_reg, dst_reg, in do_jit()
1270 maybe_emit_1mod(&prog, dst_reg, in do_jit()
1304 emit_shiftx(&prog, dst_reg, src_reg, w, op); in do_jit()
1323 maybe_emit_1mod(&prog, dst_reg, in do_jit()
1437 emit_stx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off); in do_jit()
1470 emit_mov_imm64(&prog, BPF_REG_AX, (long)VSYSCALL_ADDR >> 32, in do_jit()
1478 maybe_emit_1mod(&prog, AUX_REG, true); in do_jit()
1483 maybe_emit_mod(&prog, AUX_REG, BPF_REG_AX, true); in do_jit()
1487 emit_mov_imm64(&prog, BPF_REG_AX, (long)limit >> 32, in do_jit()
1491 maybe_emit_mod(&prog, AUX_REG, BPF_REG_AX, true); in do_jit()
1496 end_of_jmp = prog; in do_jit()
1499 emit_mov_imm32(&prog, false, dst_reg, 0); in do_jit()
1504 start_of_ldx = prog; in do_jit()
1509 emit_ldsx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn_off); in do_jit()
1511 emit_ldx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn_off); in do_jit()
1519 start_of_ldx[-1] = prog - start_of_ldx; in do_jit()
1554 ex->fixup = (prog - start_of_ldx) | (reg2pt_regs[dst_reg] << 8); in do_jit()
1574 emit_mov_reg(&prog, true, BPF_REG_AX, BPF_REG_0); in do_jit()
1580 branch_target = prog; in do_jit()
1582 emit_ldx(&prog, BPF_SIZE(insn->code), in do_jit()
1588 emit_mov_reg(&prog, is64, AUX_REG, BPF_REG_0); in do_jit()
1589 maybe_emit_mod(&prog, AUX_REG, real_src_reg, is64); in do_jit()
1593 err = emit_atomic(&prog, BPF_CMPXCHG, in do_jit()
1603 EMIT2(X86_JNE, -(prog - branch_target) - 2); in do_jit()
1605 emit_mov_reg(&prog, is64, real_src_reg, BPF_REG_0); in do_jit()
1607 emit_mov_reg(&prog, true, BPF_REG_0, BPF_REG_AX); in do_jit()
1611 err = emit_atomic(&prog, insn->imm, dst_reg, src_reg, in do_jit()
1626 offs = 7 + x86_call_depth_emit_accounting(&prog, func); in do_jit()
1630 offs = x86_call_depth_emit_accounting(&prog, func); in do_jit()
1632 if (emit_call(&prog, func, image + addrs[i - 1] + offs)) in do_jit()
1640 &prog, image + addrs[i - 1], in do_jit()
1645 emit_bpf_tail_call_indirect(&prog, in do_jit()
1674 maybe_emit_mod(&prog, dst_reg, src_reg, in do_jit()
1682 maybe_emit_mod(&prog, dst_reg, src_reg, in do_jit()
1690 maybe_emit_1mod(&prog, dst_reg, in do_jit()
1717 maybe_emit_mod(&prog, dst_reg, dst_reg, in do_jit()
1724 maybe_emit_1mod(&prog, dst_reg, in do_jit()
1800 emit_nops(&prog, nops); in do_jit()
1854 emit_nops(&prog, nops); in do_jit()
1879 emit_nops(&prog, INSN_SZ_DIFF - 2); in do_jit()
1898 pop_callee_regs(&prog, callee_regs_used); in do_jit()
1900 emit_return(&prog, image + addrs[i - 1] + (prog - temp)); in do_jit()
1914 ilen = prog - temp; in do_jit()
1938 prog = temp; in do_jit()
1953 u8 *prog; in clean_stack_garbage() local
1985 prog = *pprog; in clean_stack_garbage()
1992 *pprog = prog; in clean_stack_garbage()
2013 static void save_args(const struct btf_func_model *m, u8 **prog, in save_args() argument
2056 emit_ldx(prog, BPF_DW, BPF_REG_0, BPF_REG_FP, in save_args()
2058 emit_stx(prog, BPF_DW, BPF_REG_FP, BPF_REG_0, in save_args()
2078 emit_stx(prog, BPF_DW, BPF_REG_FP, in save_args()
2087 clean_stack_garbage(m, prog, nr_stack_slots, first_off); in save_args()
2090 static void restore_regs(const struct btf_func_model *m, u8 **prog, in restore_regs() argument
2106 emit_ldx(prog, BPF_DW, in restore_regs()
2126 u8 *prog = *pprog; in invoke_bpf_prog() local
2129 struct bpf_prog *p = l->link.prog; in invoke_bpf_prog()
2133 emit_mov_imm64(&prog, BPF_REG_1, (long) cookie >> 32, (u32) (long) cookie); in invoke_bpf_prog()
2142 emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_1, -run_ctx_off + ctx_cookie_off); in invoke_bpf_prog()
2145 emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p); in invoke_bpf_prog()
2152 if (emit_rsb_call(&prog, bpf_trampoline_enter(p), prog)) in invoke_bpf_prog()
2155 emit_mov_reg(&prog, true, BPF_REG_6, BPF_REG_0); in invoke_bpf_prog()
2162 jmp_insn = prog; in invoke_bpf_prog()
2163 emit_nops(&prog, 2); in invoke_bpf_prog()
2172 emit_mov_imm64(&prog, BPF_REG_2, in invoke_bpf_prog()
2176 if (emit_rsb_call(&prog, p->bpf_func, prog)) in invoke_bpf_prog()
2188 emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8); in invoke_bpf_prog()
2192 jmp_insn[1] = prog - jmp_insn - 2; in invoke_bpf_prog()
2195 emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p); in invoke_bpf_prog()
2197 emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_6); in invoke_bpf_prog()
2203 if (emit_rsb_call(&prog, bpf_trampoline_exit(p), prog)) in invoke_bpf_prog()
2206 *pprog = prog; in invoke_bpf_prog()
2212 u8 *target, *prog = *pprog; in emit_align() local
2214 target = PTR_ALIGN(prog, align); in emit_align()
2215 if (target != prog) in emit_align()
2216 emit_nops(&prog, target - prog); in emit_align()
2218 *pprog = prog; in emit_align()
2223 u8 *prog = *pprog; in emit_cond_near_jump() local
2232 *pprog = prog; in emit_cond_near_jump()
2241 u8 *prog = *pprog; in invoke_bpf() local
2244 if (invoke_bpf_prog(m, &prog, tl->links[i], stack_size, in invoke_bpf()
2248 *pprog = prog; in invoke_bpf()
2256 u8 *prog = *pprog; in invoke_bpf_mod_ret() local
2262 emit_mov_imm32(&prog, false, BPF_REG_0, 0); in invoke_bpf_mod_ret()
2263 emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8); in invoke_bpf_mod_ret()
2265 if (invoke_bpf_prog(m, &prog, tl->links[i], stack_size, run_ctx_off, true)) in invoke_bpf_mod_ret()
2280 branches[i] = prog; in invoke_bpf_mod_ret()
2281 emit_nops(&prog, 4 + 2); in invoke_bpf_mod_ret()
2284 *pprog = prog; in invoke_bpf_mod_ret()
2360 u8 *prog; in arch_prepare_bpf_trampoline() local
2447 prog = image; in arch_prepare_bpf_trampoline()
2454 x86_call_depth_emit_accounting(&prog, NULL); in arch_prepare_bpf_trampoline()
2466 emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_6, -rbx_off); in arch_prepare_bpf_trampoline()
2472 emit_mov_imm64(&prog, BPF_REG_0, 0, (u32) nr_regs); in arch_prepare_bpf_trampoline()
2473 emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -nregs_off); in arch_prepare_bpf_trampoline()
2480 emit_mov_imm64(&prog, BPF_REG_0, (long) func_addr >> 32, (u32) (long) func_addr); in arch_prepare_bpf_trampoline()
2481 emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -ip_off); in arch_prepare_bpf_trampoline()
2484 save_args(m, &prog, regs_off, false); in arch_prepare_bpf_trampoline()
2488 emit_mov_imm64(&prog, BPF_REG_1, (long) im >> 32, (u32) (long) im); in arch_prepare_bpf_trampoline()
2489 if (emit_rsb_call(&prog, __bpf_tramp_enter, prog)) { in arch_prepare_bpf_trampoline()
2496 if (invoke_bpf(m, &prog, fentry, regs_off, run_ctx_off, in arch_prepare_bpf_trampoline()
2506 if (invoke_bpf_mod_ret(m, &prog, fmod_ret, regs_off, in arch_prepare_bpf_trampoline()
2514 restore_regs(m, &prog, regs_off); in arch_prepare_bpf_trampoline()
2515 save_args(m, &prog, arg_stack_off, true); in arch_prepare_bpf_trampoline()
2524 emit_ldx(&prog, BPF_DW, BPF_REG_6, BPF_REG_FP, 8); in arch_prepare_bpf_trampoline()
2528 if (emit_rsb_call(&prog, orig_call, prog)) { in arch_prepare_bpf_trampoline()
2534 emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8); in arch_prepare_bpf_trampoline()
2535 im->ip_after_call = prog; in arch_prepare_bpf_trampoline()
2536 memcpy(prog, x86_nops[5], X86_PATCH_SIZE); in arch_prepare_bpf_trampoline()
2537 prog += X86_PATCH_SIZE; in arch_prepare_bpf_trampoline()
2546 emit_align(&prog, 16); in arch_prepare_bpf_trampoline()
2551 emit_cond_near_jump(&branches[i], prog, branches[i], in arch_prepare_bpf_trampoline()
2556 if (invoke_bpf(m, &prog, fexit, regs_off, run_ctx_off, false)) { in arch_prepare_bpf_trampoline()
2562 restore_regs(m, &prog, regs_off); in arch_prepare_bpf_trampoline()
2569 im->ip_epilogue = prog; in arch_prepare_bpf_trampoline()
2571 emit_mov_imm64(&prog, BPF_REG_1, (long) im >> 32, (u32) (long) im); in arch_prepare_bpf_trampoline()
2572 if (emit_rsb_call(&prog, __bpf_tramp_exit, prog)) { in arch_prepare_bpf_trampoline()
2584 emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, -8); in arch_prepare_bpf_trampoline()
2586 emit_ldx(&prog, BPF_DW, BPF_REG_6, BPF_REG_FP, -rbx_off); in arch_prepare_bpf_trampoline()
2591 emit_return(&prog, prog); in arch_prepare_bpf_trampoline()
2593 if (WARN_ON_ONCE(prog > (u8 *)image_end - BPF_INSN_SAFETY)) { in arch_prepare_bpf_trampoline()
2597 ret = prog - (u8 *)image; in arch_prepare_bpf_trampoline()
2606 u8 *jg_reloc, *prog = *pprog; in emit_bpf_dispatcher() local
2619 err = emit_cond_near_jump(&prog, /* je func */ in emit_bpf_dispatcher()
2620 (void *)progs[a], image + (prog - buf), in emit_bpf_dispatcher()
2625 emit_indirect_jump(&prog, 2 /* rdx */, image + (prog - buf)); in emit_bpf_dispatcher()
2627 *pprog = prog; in emit_bpf_dispatcher()
2647 jg_reloc = prog; in emit_bpf_dispatcher()
2649 err = emit_bpf_dispatcher(&prog, a, a + pivot, /* emit lower_part */ in emit_bpf_dispatcher()
2659 emit_align(&prog, 16); in emit_bpf_dispatcher()
2660 jg_offset = prog - jg_reloc; in emit_bpf_dispatcher()
2663 err = emit_bpf_dispatcher(&prog, a + pivot + 1, /* emit upper_part */ in emit_bpf_dispatcher()
2668 *pprog = prog; in emit_bpf_dispatcher()
2686 u8 *prog = buf; in arch_prepare_bpf_dispatcher() local
2689 return emit_bpf_dispatcher(&prog, 0, num_funcs - 1, funcs, image, buf); in arch_prepare_bpf_dispatcher()
2704 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) in bpf_int_jit_compile() argument
2708 struct bpf_prog *tmp, *orig_prog = prog; in bpf_int_jit_compile()
2721 if (!prog->jit_requested) in bpf_int_jit_compile()
2724 tmp = bpf_jit_blind_constants(prog); in bpf_int_jit_compile()
2731 if (tmp != prog) { in bpf_int_jit_compile()
2733 prog = tmp; in bpf_int_jit_compile()
2736 jit_data = prog->aux->jit_data; in bpf_int_jit_compile()
2740 prog = orig_prog; in bpf_int_jit_compile()
2743 prog->aux->jit_data = jit_data; in bpf_int_jit_compile()
2757 addrs = kvmalloc_array(prog->len + 1, sizeof(*addrs), GFP_KERNEL); in bpf_int_jit_compile()
2759 prog = orig_prog; in bpf_int_jit_compile()
2767 for (proglen = 0, i = 0; i <= prog->len; i++) { in bpf_int_jit_compile()
2783 proglen = do_jit(prog, addrs, image, rw_image, oldproglen, &ctx, padding); in bpf_int_jit_compile()
2793 prog = orig_prog; in bpf_int_jit_compile()
2795 prog->bpf_func = NULL; in bpf_int_jit_compile()
2796 prog->jited = 0; in bpf_int_jit_compile()
2797 prog->jited_len = 0; in bpf_int_jit_compile()
2817 u32 extable_size = prog->aux->num_exentries * in bpf_int_jit_compile()
2825 prog = orig_prog; in bpf_int_jit_compile()
2828 prog->aux->extable = (void *) image + roundup(proglen, align); in bpf_int_jit_compile()
2835 bpf_jit_dump(prog->len, proglen, pass + 1, rw_image); in bpf_int_jit_compile()
2838 if (!prog->is_func || extra_pass) { in bpf_int_jit_compile()
2846 if (WARN_ON(bpf_jit_binary_pack_finalize(prog, header, rw_header))) { in bpf_int_jit_compile()
2852 bpf_tail_call_direct_fixup(prog); in bpf_int_jit_compile()
2861 prog->bpf_func = (void *)image; in bpf_int_jit_compile()
2862 prog->jited = 1; in bpf_int_jit_compile()
2863 prog->jited_len = proglen; in bpf_int_jit_compile()
2865 prog = orig_prog; in bpf_int_jit_compile()
2868 if (!image || !prog->is_func || extra_pass) { in bpf_int_jit_compile()
2870 bpf_prog_fill_jited_linfo(prog, addrs + 1); in bpf_int_jit_compile()
2874 prog->aux->jit_data = NULL; in bpf_int_jit_compile()
2878 bpf_jit_prog_release_other(prog, prog == orig_prog ? in bpf_int_jit_compile()
2880 return prog; in bpf_int_jit_compile()
2901 void bpf_jit_free(struct bpf_prog *prog) in bpf_jit_free() argument
2903 if (prog->jited) { in bpf_jit_free()
2904 struct x64_jit_data *jit_data = prog->aux->jit_data; in bpf_jit_free()
2913 bpf_jit_binary_pack_finalize(prog, jit_data->header, in bpf_jit_free()
2918 hdr = bpf_jit_binary_pack_hdr(prog); in bpf_jit_free()
2920 WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog)); in bpf_jit_free()
2923 bpf_prog_unlock_free(prog); in bpf_jit_free()