308 static void push_callee_regs(u8 **pprog, bool *callee_regs_used)  in push_callee_regs()  argument
310 	u8 *prog = *pprog;  in push_callee_regs()
320 	*pprog = prog;  in push_callee_regs()
323 static void pop_callee_regs(u8 **pprog, bool *callee_regs_used)  in pop_callee_regs()  argument
325 	u8 *prog = *pprog;  in pop_callee_regs()
335 	*pprog = prog;  in pop_callee_regs()
343 static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf,  in emit_prologue()  argument
346 	u8 *prog = *pprog;  in emit_prologue()
371 	*pprog = prog;  in emit_prologue()
374 static int emit_patch(u8 **pprog, void *func, void *ip, u8 opcode)  in emit_patch()  argument
376 	u8 *prog = *pprog;  in emit_patch()
385 	*pprog = prog;  in emit_patch()
389 static int emit_call(u8 **pprog, void *func, void *ip)  in emit_call()  argument
391 	return emit_patch(pprog, func, ip, 0xE8);  in emit_call()
394 static int emit_rsb_call(u8 **pprog, void *func, void *ip)  in emit_rsb_call()  argument
397 	ip += x86_call_depth_emit_accounting(pprog, func);  in emit_rsb_call()
398 	return emit_patch(pprog, func, ip, 0xE8);  in emit_rsb_call()
401 static int emit_jump(u8 **pprog, void *func, void *ip)  in emit_jump()  argument
403 	return emit_patch(pprog, func, ip, 0xE9);  in emit_jump()
469 static void emit_indirect_jump(u8 **pprog, int reg, u8 *ip)  in emit_indirect_jump()  argument
471 	u8 *prog = *pprog;  in emit_indirect_jump()
488 	*pprog = prog;  in emit_indirect_jump()
491 static void emit_return(u8 **pprog, u8 *ip)  in emit_return()  argument
493 	u8 *prog = *pprog;  in emit_return()
503 	*pprog = prog;  in emit_return()
520 static void emit_bpf_tail_call_indirect(u8 **pprog, bool *callee_regs_used,  in emit_bpf_tail_call_indirect()  argument
525 	u8 *prog = *pprog, *start = *pprog;  in emit_bpf_tail_call_indirect()
591 	*pprog = prog;  in emit_bpf_tail_call_indirect()
595 				      u8 **pprog, u8 *ip,  in emit_bpf_tail_call_direct()  argument
600 	u8 *prog = *pprog, *start = *pprog;  in emit_bpf_tail_call_direct()
634 	*pprog = prog;  in emit_bpf_tail_call_direct()
674 static void emit_mov_imm32(u8 **pprog, bool sign_propagate,  in emit_mov_imm32()  argument
677 	u8 *prog = *pprog;  in emit_mov_imm32()
711 	*pprog = prog;  in emit_mov_imm32()
714 static void emit_mov_imm64(u8 **pprog, u32 dst_reg,  in emit_mov_imm64()  argument
717 	u8 *prog = *pprog;  in emit_mov_imm64()
734 	*pprog = prog;  in emit_mov_imm64()
737 static void emit_mov_reg(u8 **pprog, bool is64, u32 dst_reg, u32 src_reg)  in emit_mov_reg()  argument
739 	u8 *prog = *pprog;  in emit_mov_reg()
751 	*pprog = prog;  in emit_mov_reg()
754 static void emit_movsx_reg(u8 **pprog, int num_bits, bool is64, u32 dst_reg,  in emit_movsx_reg()  argument
757 	u8 *prog = *pprog;  in emit_movsx_reg()
783 	*pprog = prog;  in emit_movsx_reg()
787 static void emit_insn_suffix(u8 **pprog, u32 ptr_reg, u32 val_reg, int off)  in emit_insn_suffix()  argument
789 	u8 *prog = *pprog;  in emit_insn_suffix()
803 	*pprog = prog;  in emit_insn_suffix()
809 static void maybe_emit_mod(u8 **pprog, u32 dst_reg, u32 src_reg, bool is64)  in maybe_emit_mod()  argument
811 	u8 *prog = *pprog;  in maybe_emit_mod()
817 	*pprog = prog;  in maybe_emit_mod()
823 static void maybe_emit_1mod(u8 **pprog, u32 reg, bool is64)  in maybe_emit_1mod()  argument
825 	u8 *prog = *pprog;  in maybe_emit_1mod()
831 	*pprog = prog;  in maybe_emit_1mod()
835 static void emit_ldx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)  in emit_ldx()  argument
837 	u8 *prog = *pprog;  in emit_ldx()
861 	*pprog = prog;  in emit_ldx()
865 static void emit_ldsx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)  in emit_ldsx()  argument
867 	u8 *prog = *pprog;  in emit_ldsx()
884 	*pprog = prog;  in emit_ldsx()
888 static void emit_stx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)  in emit_stx()  argument
890 	u8 *prog = *pprog;  in emit_stx()
918 	*pprog = prog;  in emit_stx()
921 static int emit_atomic(u8 **pprog, u8 atomic_op,  in emit_atomic()  argument
924 	u8 *prog = *pprog;  in emit_atomic()
958 	*pprog = prog;  in emit_atomic()
991 static void emit_nops(u8 **pprog, int len)  in emit_nops()  argument
993 	u8 *prog = *pprog;  in emit_nops()
1007 	*pprog = prog;  in emit_nops()
1021 static void emit_3vex(u8 **pprog, bool r, bool x, bool b, u8 m,  in emit_3vex()  argument
1024 	u8 *prog = *pprog;  in emit_3vex()
1054 	*pprog = prog;  in emit_3vex()
1058 static void emit_shiftx(u8 **pprog, u32 dst_reg, u8 src_reg, bool is64, u8 op)  in emit_shiftx()  argument
1060 	u8 *prog = *pprog;  in emit_shiftx()
1066 	*pprog = prog;  in emit_shiftx()
1999 				u8 **pprog, int nr_stack_slots,  in clean_stack_garbage()  argument
2035 		prog = *pprog;  in clean_stack_garbage()
2042 		*pprog = prog;  in clean_stack_garbage()
2172 static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,  in invoke_bpf_prog()  argument
2176 	u8 *prog = *pprog;  in invoke_bpf_prog()
2256 	*pprog = prog;  in invoke_bpf_prog()
2260 static void emit_align(u8 **pprog, u32 align)  in emit_align()  argument
2262 	u8 *target, *prog = *pprog;  in emit_align()
2268 	*pprog = prog;  in emit_align()
2271 static int emit_cond_near_jump(u8 **pprog, void *func, void *ip, u8 jmp_cond)  in emit_cond_near_jump()  argument
2273 	u8 *prog = *pprog;  in emit_cond_near_jump()
2282 	*pprog = prog;  in emit_cond_near_jump()
2286 static int invoke_bpf(const struct btf_func_model *m, u8 **pprog,  in invoke_bpf()  argument
2291 	u8 *prog = *pprog;  in invoke_bpf()
2298 	*pprog = prog;  in invoke_bpf()
2302 static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog,  in invoke_bpf_mod_ret()  argument
2306 	u8 *prog = *pprog;  in invoke_bpf_mod_ret()
2334 	*pprog = prog;  in invoke_bpf_mod_ret()
2654 static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs, u8 *image, u8 *buf)  in emit_bpf_dispatcher()  argument
2656 	u8 *jg_reloc, *prog = *pprog;  in emit_bpf_dispatcher()
2677 		*pprog = prog;  in emit_bpf_dispatcher()
2718 	*pprog = prog;  in emit_bpf_dispatcher()