259 static u8 add_2reg(u8 byte, u32 dst_reg, u32 src_reg)  in add_2reg()  argument
261 	return byte + reg2hex[dst_reg] + (reg2hex[src_reg] << 3);  in add_2reg()
737 static void emit_mov_reg(u8 **pprog, bool is64, u32 dst_reg, u32 src_reg)  in emit_mov_reg()  argument
743 		EMIT_mov(dst_reg, src_reg);  in emit_mov_reg()
746 		if (is_ereg(dst_reg) || is_ereg(src_reg))  in emit_mov_reg()
747 			EMIT1(add_2mod(0x40, dst_reg, src_reg));  in emit_mov_reg()
748 		EMIT2(0x89, add_2reg(0xC0, dst_reg, src_reg));  in emit_mov_reg()
755 			   u32 src_reg)  in emit_movsx_reg()  argument
762 			EMIT4(add_2mod(0x48, src_reg, dst_reg), 0x0f, 0xbe,  in emit_movsx_reg()
763 			      add_2reg(0xC0, src_reg, dst_reg));  in emit_movsx_reg()
765 			EMIT4(add_2mod(0x48, src_reg, dst_reg), 0x0f, 0xbf,  in emit_movsx_reg()
766 			      add_2reg(0xC0, src_reg, dst_reg));  in emit_movsx_reg()
768 			EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x63,  in emit_movsx_reg()
769 			      add_2reg(0xC0, src_reg, dst_reg));  in emit_movsx_reg()
773 			EMIT4(add_2mod(0x40, src_reg, dst_reg), 0x0f, 0xbe,  in emit_movsx_reg()
774 			      add_2reg(0xC0, src_reg, dst_reg));  in emit_movsx_reg()
776 			if (is_ereg(dst_reg) || is_ereg(src_reg))  in emit_movsx_reg()
777 				EMIT1(add_2mod(0x40, src_reg, dst_reg));  in emit_movsx_reg()
778 			EMIT3(add_2mod(0x0f, src_reg, dst_reg), 0xbf,  in emit_movsx_reg()
779 			      add_2reg(0xC0, src_reg, dst_reg));  in emit_movsx_reg()
809 static void maybe_emit_mod(u8 **pprog, u32 dst_reg, u32 src_reg, bool is64)  in maybe_emit_mod()  argument
814 		EMIT1(add_2mod(0x48, dst_reg, src_reg));  in maybe_emit_mod()
815 	else if (is_ereg(dst_reg) || is_ereg(src_reg))  in maybe_emit_mod()
816 		EMIT1(add_2mod(0x40, dst_reg, src_reg));  in maybe_emit_mod()
835 static void emit_ldx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)  in emit_ldx()  argument
842 		EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB6);  in emit_ldx()
846 		EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB7);  in emit_ldx()
850 		if (is_ereg(dst_reg) || is_ereg(src_reg))  in emit_ldx()
851 			EMIT2(add_2mod(0x40, src_reg, dst_reg), 0x8B);  in emit_ldx()
857 		EMIT2(add_2mod(0x48, src_reg, dst_reg), 0x8B);  in emit_ldx()
860 	emit_insn_suffix(&prog, src_reg, dst_reg, off);  in emit_ldx()
865 static void emit_ldsx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)  in emit_ldsx()  argument
872 		EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xBE);  in emit_ldsx()
876 		EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xBF);  in emit_ldsx()
880 		EMIT2(add_2mod(0x48, src_reg, dst_reg), 0x63);  in emit_ldsx()
883 	emit_insn_suffix(&prog, src_reg, dst_reg, off);  in emit_ldsx()
888 static void emit_stx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)  in emit_stx()  argument
895 		if (is_ereg(dst_reg) || is_ereg_8l(src_reg))  in emit_stx()
897 			EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x88);  in emit_stx()
902 		if (is_ereg(dst_reg) || is_ereg(src_reg))  in emit_stx()
903 			EMIT3(0x66, add_2mod(0x40, dst_reg, src_reg), 0x89);  in emit_stx()
908 		if (is_ereg(dst_reg) || is_ereg(src_reg))  in emit_stx()
909 			EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x89);  in emit_stx()
914 		EMIT2(add_2mod(0x48, dst_reg, src_reg), 0x89);  in emit_stx()
917 	emit_insn_suffix(&prog, dst_reg, src_reg, off);  in emit_stx()
922 		       u32 dst_reg, u32 src_reg, s16 off, u8 bpf_size)  in emit_atomic()  argument
928 	maybe_emit_mod(&prog, dst_reg, src_reg, bpf_size == BPF_DW);  in emit_atomic()
956 	emit_insn_suffix(&prog, dst_reg, src_reg, off);  in emit_atomic()
980 		if (insn->dst_reg == BPF_REG_6 || insn->src_reg == BPF_REG_6)  in detect_reg_usage()
982 		if (insn->dst_reg == BPF_REG_7 || insn->src_reg == BPF_REG_7)  in detect_reg_usage()
984 		if (insn->dst_reg == BPF_REG_8 || insn->src_reg == BPF_REG_8)  in detect_reg_usage()
986 		if (insn->dst_reg == BPF_REG_9 || insn->src_reg == BPF_REG_9)  in detect_reg_usage()
1058 static void emit_shiftx(u8 **pprog, u32 dst_reg, u8 src_reg, bool is64, u8 op)  in emit_shiftx()  argument
1064 	emit_3vex(&prog, r, false, r, m, is64, src_reg, false, op);  in emit_shiftx()
1111 		u32 src_reg = insn->src_reg;  in do_jit()  local
1132 			maybe_emit_mod(&prog, dst_reg, src_reg,  in do_jit()
1135 			EMIT2(b2, add_2reg(0xC0, dst_reg, src_reg));  in do_jit()
1143 					     dst_reg, src_reg);  in do_jit()
1147 					       dst_reg, src_reg);  in do_jit()
1235 				if (src_reg == BPF_REG_0 ||  in do_jit()
1236 				    src_reg == BPF_REG_3) {  in do_jit()
1238 					EMIT_mov(AUX_REG, src_reg);  in do_jit()
1239 					src_reg = AUX_REG;  in do_jit()
1244 				src_reg = AUX_REG;  in do_jit()
1259 				maybe_emit_1mod(&prog, src_reg, is64);  in do_jit()
1260 				EMIT2(0xF7, add_1reg(0xF0, src_reg));  in do_jit()
1268 				maybe_emit_1mod(&prog, src_reg, is64);  in do_jit()
1269 				EMIT2(0xF7, add_1reg(0xF8, src_reg));  in do_jit()
1306 			maybe_emit_mod(&prog, src_reg, dst_reg,  in do_jit()
1310 			EMIT3(0x0F, 0xAF, add_2reg(0xC0, src_reg, dst_reg));  in do_jit()
1337 			if (boot_cpu_has(X86_FEATURE_BMI2) && src_reg != BPF_REG_4) {  in do_jit()
1354 				emit_shiftx(&prog, dst_reg, src_reg, w, op);  in do_jit()
1359 			if (src_reg != BPF_REG_4) { /* common case */  in do_jit()
1369 				EMIT_mov(BPF_REG_4, src_reg);  in do_jit()
1379 			if (src_reg != BPF_REG_4) {  in do_jit()
1487 			emit_stx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off);  in do_jit()
1524 				EMIT_mov(AUX_REG, src_reg);  in do_jit()
1559 				emit_ldsx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn_off);  in do_jit()
1561 				emit_ldx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn_off);  in do_jit()
1614 				u32 real_src_reg = src_reg;  in do_jit()
1625 				if (src_reg == BPF_REG_0)  in do_jit()
1661 			err = emit_atomic(&prog, insn->imm, dst_reg, src_reg,  in do_jit()
1724 			maybe_emit_mod(&prog, dst_reg, src_reg,  in do_jit()
1726 			EMIT2(0x39, add_2reg(0xC0, dst_reg, src_reg));  in do_jit()
1732 			maybe_emit_mod(&prog, dst_reg, src_reg,  in do_jit()
1734 			EMIT2(0x85, add_2reg(0xC0, dst_reg, src_reg));  in do_jit()