243  * Emit an instruction that will be executed unconditionally.
245 static inline void emit(u32 inst, struct jit_ctx *ctx)  in emit()  function
429 	emit(ARM_LDR_I(rd, ARM_PC, imm_offset(val, ctx)), ctx);  in emit_mov_i_no8m()
431 	emit(ARM_MOVW(rd, val & 0xffff), ctx);  in emit_mov_i_no8m()
433 		emit(ARM_MOVT(rd, val >> 16), ctx);  in emit_mov_i_no8m()
442 		emit(ARM_MOV_I(rd, imm12), ctx);  in emit_mov_i()
450 		emit(ARM_BX(tgt_reg), ctx);  in emit_bx_r()
452 		emit(ARM_MOV_R(ARM_PC, tgt_reg), ctx);  in emit_bx_r()
458 	emit(ARM_MOV_R(ARM_LR, ARM_PC), ctx);  in emit_blx_r()
461 	emit(ARM_BLX_R(tgt_reg), ctx);  in emit_blx_r()
485 			emit(ARM_UDIV(rd, rm, rn), ctx);  in emit_udivmod()
487 			emit(ARM_UDIV(ARM_IP, rm, rn), ctx);  in emit_udivmod()
488 			emit(ARM_MLS(rd, rn, ARM_IP, rm), ctx);  in emit_udivmod()
503 		emit(ARM_MOV_R(tmp[0], ARM_R1), ctx);  in emit_udivmod()
504 		emit(ARM_MOV_R(ARM_R1, rn), ctx);  in emit_udivmod()
507 		emit(ARM_MOV_R(tmp[1], ARM_R0), ctx);  in emit_udivmod()
508 		emit(ARM_MOV_R(ARM_R0, rm), ctx);  in emit_udivmod()
512 	emit(ARM_PUSH(CALLER_MASK & ~exclude_mask), ctx);  in emit_udivmod()
520 	emit(ARM_POP(CALLER_MASK & ~exclude_mask), ctx);  in emit_udivmod()
524 		emit(ARM_MOV_R(rd, ARM_R0), ctx);  in emit_udivmod()
528 		emit(ARM_MOV_R(ARM_R1, tmp[0]), ctx);  in emit_udivmod()
530 		emit(ARM_MOV_R(ARM_R0, tmp[1]), ctx);  in emit_udivmod()
546 		emit(ARM_LDR_I(tmp, ARM_FP, EBPF_SCRATCH_TO_ARM_FP(reg)), ctx);  in arm_bpf_get_reg32()
558 			emit(ARM_LDRD_I(tmp[1], ARM_FP,  in arm_bpf_get_reg64()
561 			emit(ARM_LDR_I(tmp[1], ARM_FP,  in arm_bpf_get_reg64()
563 			emit(ARM_LDR_I(tmp[0], ARM_FP,  in arm_bpf_get_reg64()
578 		emit(ARM_STR_I(src, ARM_FP, EBPF_SCRATCH_TO_ARM_FP(reg)), ctx);  in arm_bpf_put_reg32()
580 		emit(ARM_MOV_R(reg, src), ctx);  in arm_bpf_put_reg32()
589 			emit(ARM_STRD_I(src[1], ARM_FP,  in arm_bpf_put_reg64()
592 			emit(ARM_STR_I(src[1], ARM_FP,  in arm_bpf_put_reg64()
594 			emit(ARM_STR_I(src[0], ARM_FP,  in arm_bpf_put_reg64()
599 			emit(ARM_MOV_R(reg[1], src[1]), ctx);  in arm_bpf_put_reg64()
601 			emit(ARM_MOV_R(reg[0], src[0]), ctx);  in arm_bpf_put_reg64()
649 		emit(ARM_ADDS_R(dst, dst, src), ctx);  in emit_a32_add_r()
651 		emit(ARM_ADC_R(dst, dst, src), ctx);  in emit_a32_add_r()
653 		emit(ARM_ADD_R(dst, dst, src), ctx);  in emit_a32_add_r()
666 		emit(ARM_SUBS_R(dst, dst, src), ctx);  in emit_a32_sub_r()
668 		emit(ARM_SBC_R(dst, dst, src), ctx);  in emit_a32_sub_r()
670 		emit(ARM_SUB_R(dst, dst, src), ctx);  in emit_a32_sub_r()
686 		emit(ARM_ORR_R(dst, dst, src), ctx);  in emit_alu_r()
690 		emit(ARM_AND_R(dst, dst, src), ctx);  in emit_alu_r()
694 		emit(ARM_EOR_R(dst, dst, src), ctx);  in emit_alu_r()
698 		emit(ARM_MUL(dst, dst, src), ctx);  in emit_alu_r()
702 		emit(ARM_LSL_R(dst, dst, src), ctx);  in emit_alu_r()
706 		emit(ARM_LSR_R(dst, dst, src), ctx);  in emit_alu_r()
710 		emit(ARM_MOV_SR(dst, dst, SRTYPE_ASR, src), ctx);  in emit_alu_r()
773 		emit(ARM_LDRD_I(tmp[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(src_lo)), ctx);  in emit_a32_mov_r64()
774 		emit(ARM_STRD_I(tmp[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(dst_lo)), ctx);  in emit_a32_mov_r64()
776 		emit(ARM_LDRD_I(dst[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(src_lo)), ctx);  in emit_a32_mov_r64()
778 		emit(ARM_STRD_I(src[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(dst_lo)), ctx);  in emit_a32_mov_r64()
780 		emit(ARM_MOV_R(dst[0], src[0]), ctx);  in emit_a32_mov_r64()
781 		emit(ARM_MOV_R(dst[1], src[1]), ctx);  in emit_a32_mov_r64()
796 		emit(ARM_LSL_I(rd, rd, val), ctx);  in emit_a32_alu_i()
799 		emit(ARM_LSR_I(rd, rd, val), ctx);  in emit_a32_alu_i()
802 		emit(ARM_ASR_I(rd, rd, val), ctx);  in emit_a32_alu_i()
805 		emit(ARM_RSB_I(rd, rd, val), ctx);  in emit_a32_alu_i()
822 	emit(ARM_RSBS_I(rd[1], rd[1], 0), ctx);  in emit_a32_neg64()
823 	emit(ARM_RSC_I(rd[0], rd[0], 0), ctx);  in emit_a32_neg64()
841 	emit(ARM_SUB_I(ARM_IP, rt, 32), ctx);  in emit_a32_lsh_r64()
842 	emit(ARM_RSB_I(tmp2[0], rt, 32), ctx);  in emit_a32_lsh_r64()
843 	emit(ARM_MOV_SR(ARM_LR, rd[0], SRTYPE_ASL, rt), ctx);  in emit_a32_lsh_r64()
844 	emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[1], SRTYPE_ASL, ARM_IP), ctx);  in emit_a32_lsh_r64()
845 	emit(ARM_ORR_SR(ARM_IP, ARM_LR, rd[1], SRTYPE_LSR, tmp2[0]), ctx);  in emit_a32_lsh_r64()
846 	emit(ARM_MOV_SR(ARM_LR, rd[1], SRTYPE_ASL, rt), ctx);  in emit_a32_lsh_r64()
865 	emit(ARM_RSB_I(ARM_IP, rt, 32), ctx);  in emit_a32_arsh_r64()
866 	emit(ARM_SUBS_I(tmp2[0], rt, 32), ctx);  in emit_a32_arsh_r64()
867 	emit(ARM_MOV_SR(ARM_LR, rd[1], SRTYPE_LSR, rt), ctx);  in emit_a32_arsh_r64()
868 	emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_ASL, ARM_IP), ctx);  in emit_a32_arsh_r64()
871 	emit(ARM_MOV_SR(ARM_IP, rd[0], SRTYPE_ASR, rt), ctx);  in emit_a32_arsh_r64()
890 	emit(ARM_RSB_I(ARM_IP, rt, 32), ctx);  in emit_a32_rsh_r64()
891 	emit(ARM_SUBS_I(tmp2[0], rt, 32), ctx);  in emit_a32_rsh_r64()
892 	emit(ARM_MOV_SR(ARM_LR, rd[1], SRTYPE_LSR, rt), ctx);  in emit_a32_rsh_r64()
893 	emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_ASL, ARM_IP), ctx);  in emit_a32_rsh_r64()
894 	emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_LSR, tmp2[0]), ctx);  in emit_a32_rsh_r64()
895 	emit(ARM_MOV_SR(ARM_IP, rd[0], SRTYPE_LSR, rt), ctx);  in emit_a32_rsh_r64()
913 		emit(ARM_MOV_SI(tmp2[0], rd[0], SRTYPE_ASL, val), ctx);  in emit_a32_lsh_i64()
914 		emit(ARM_ORR_SI(rd[0], tmp2[0], rd[1], SRTYPE_LSR, 32 - val), ctx);  in emit_a32_lsh_i64()
915 		emit(ARM_MOV_SI(rd[1], rd[1], SRTYPE_ASL, val), ctx);  in emit_a32_lsh_i64()
918 			emit(ARM_MOV_R(rd[0], rd[1]), ctx);  in emit_a32_lsh_i64()
920 			emit(ARM_MOV_SI(rd[0], rd[1], SRTYPE_ASL, val - 32), ctx);  in emit_a32_lsh_i64()
921 		emit(ARM_EOR_R(rd[1], rd[1], rd[1]), ctx);  in emit_a32_lsh_i64()
943 		emit(ARM_MOV_SI(tmp2[1], rd[1], SRTYPE_LSR, val), ctx);  in emit_a32_rsh_i64()
944 		emit(ARM_ORR_SI(rd[1], tmp2[1], rd[0], SRTYPE_ASL, 32 - val), ctx);  in emit_a32_rsh_i64()
945 		emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_LSR, val), ctx);  in emit_a32_rsh_i64()
947 		emit(ARM_MOV_R(rd[1], rd[0]), ctx);  in emit_a32_rsh_i64()
948 		emit(ARM_MOV_I(rd[0], 0), ctx);  in emit_a32_rsh_i64()
950 		emit(ARM_MOV_SI(rd[1], rd[0], SRTYPE_LSR, val - 32), ctx);  in emit_a32_rsh_i64()
951 		emit(ARM_MOV_I(rd[0], 0), ctx);  in emit_a32_rsh_i64()
973 		emit(ARM_MOV_SI(tmp2[1], rd[1], SRTYPE_LSR, val), ctx);  in emit_a32_arsh_i64()
974 		emit(ARM_ORR_SI(rd[1], tmp2[1], rd[0], SRTYPE_ASL, 32 - val), ctx);  in emit_a32_arsh_i64()
975 		emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_ASR, val), ctx);  in emit_a32_arsh_i64()
977 		emit(ARM_MOV_R(rd[1], rd[0]), ctx);  in emit_a32_arsh_i64()
978 		emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_ASR, 31), ctx);  in emit_a32_arsh_i64()
980 		emit(ARM_MOV_SI(rd[1], rd[0], SRTYPE_ASR, val - 32), ctx);  in emit_a32_arsh_i64()
981 		emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_ASR, 31), ctx);  in emit_a32_arsh_i64()
998 	emit(ARM_MUL(ARM_IP, rd[1], rt[0]), ctx);  in emit_a32_mul_r64()
999 	emit(ARM_MUL(ARM_LR, rd[0], rt[1]), ctx);  in emit_a32_mul_r64()
1000 	emit(ARM_ADD_R(ARM_LR, ARM_IP, ARM_LR), ctx);  in emit_a32_mul_r64()
1002 	emit(ARM_UMULL(ARM_IP, rd[0], rd[1], rt[1]), ctx);  in emit_a32_mul_r64()
1003 	emit(ARM_ADD_R(rd[0], ARM_LR, rd[0]), ctx);  in emit_a32_mul_r64()
1039 		emit(ARM_ADD_R(tmp[0], tmp[0], rd), ctx);  in emit_str_r()
1046 		emit(ARM_STRB_I(src_lo, rd, off), ctx);  in emit_str_r()
1050 		emit(ARM_STRH_I(src_lo, rd, off), ctx);  in emit_str_r()
1054 		emit(ARM_STR_I(src_lo, rd, off), ctx);  in emit_str_r()
1058 		emit(ARM_STR_I(src_lo, rd, off), ctx);  in emit_str_r()
1059 		emit(ARM_STR_I(src_hi, rd, off + 4), ctx);  in emit_str_r()
1073 		emit(ARM_ADD_R(tmp[0], tmp[0], src), ctx);  in emit_ldx_r()
1077 		emit(ARM_MOV_R(tmp[0], rm), ctx);  in emit_ldx_r()
1083 		emit(ARM_LDRB_I(rd[1], rm, off), ctx);  in emit_ldx_r()
1089 		emit(ARM_LDRH_I(rd[1], rm, off), ctx);  in emit_ldx_r()
1095 		emit(ARM_LDR_I(rd[1], rm, off), ctx);  in emit_ldx_r()
1101 		emit(ARM_LDR_I(rd[1], rm, off), ctx);  in emit_ldx_r()
1102 		emit(ARM_LDR_I(rd[0], rm, off + 4), ctx);  in emit_ldx_r()
1115 			emit(ARM_AND_R(ARM_IP, rt, rn), ctx);  in emit_ar_r()
1116 			emit(ARM_AND_R(ARM_LR, rd, rm), ctx);  in emit_ar_r()
1117 			emit(ARM_ORRS_R(ARM_IP, ARM_LR, ARM_IP), ctx);  in emit_ar_r()
1119 			emit(ARM_ANDS_R(ARM_IP, rt, rn), ctx);  in emit_ar_r()
1129 			emit(ARM_CMP_R(rd, rm), ctx);  in emit_ar_r()
1133 			emit(ARM_CMP_R(rt, rn), ctx);  in emit_ar_r()
1138 		emit(ARM_CMP_R(rn, rt), ctx);  in emit_ar_r()
1140 			emit(ARM_SBCS_R(ARM_IP, rm, rd), ctx);  in emit_ar_r()
1144 		emit(ARM_CMP_R(rt, rn), ctx);  in emit_ar_r()
1146 			emit(ARM_SBCS_R(ARM_IP, rd, rm), ctx);  in emit_ar_r()
1179 	emit(ARM_LDR_I(tmp[1], r_array, off), ctx);  in emit_bpf_tail_call()
1181 	emit(ARM_CMP_R(r_index, tmp[1]), ctx);  in emit_bpf_tail_call()
1194 	emit(ARM_CMP_I(tc[0], hi), ctx);  in emit_bpf_tail_call()
1197 	emit(ARM_ADDS_I(tc[1], tc[1], 1), ctx);  in emit_bpf_tail_call()
1198 	emit(ARM_ADC_I(tc[0], tc[0], 0), ctx);  in emit_bpf_tail_call()
1207 	emit(ARM_ADD_I(tmp[1], r_array, off), ctx);  in emit_bpf_tail_call()
1208 	emit(ARM_LDR_R_SI(tmp[1], tmp[1], r_index, SRTYPE_ASL, 2), ctx);  in emit_bpf_tail_call()
1209 	emit(ARM_CMP_I(tmp[1], 0), ctx);  in emit_bpf_tail_call()
1216 	emit(ARM_LDR_I(tmp[1], tmp[1], off), ctx);  in emit_bpf_tail_call()
1217 	emit(ARM_ADD_I(tmp[1], tmp[1], ctx->prologue_bytes), ctx);  in emit_bpf_tail_call()
1239 	emit(ARM_AND_I(tmp2[1], rn, 0xff), ctx);  in emit_rev16()
1240 	emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 8), ctx);  in emit_rev16()
1241 	emit(ARM_AND_I(tmp2[0], tmp2[0], 0xff), ctx);  in emit_rev16()
1242 	emit(ARM_ORR_SI(rd, tmp2[0], tmp2[1], SRTYPE_LSL, 8), ctx);  in emit_rev16()
1244 	emit(ARM_REV16(rd, rn), ctx);  in emit_rev16()
1254 	emit(ARM_AND_I(tmp2[1], rn, 0xff), ctx);  in emit_rev32()
1255 	emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 24), ctx);  in emit_rev32()
1256 	emit(ARM_ORR_SI(ARM_IP, tmp2[0], tmp2[1], SRTYPE_LSL, 24), ctx);  in emit_rev32()
1258 	emit(ARM_MOV_SI(tmp2[1], rn, SRTYPE_LSR, 8), ctx);  in emit_rev32()
1259 	emit(ARM_AND_I(tmp2[1], tmp2[1], 0xff), ctx);  in emit_rev32()
1260 	emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 16), ctx);  in emit_rev32()
1261 	emit(ARM_AND_I(tmp2[0], tmp2[0], 0xff), ctx);  in emit_rev32()
1262 	emit(ARM_MOV_SI(tmp2[0], tmp2[0], SRTYPE_LSL, 8), ctx);  in emit_rev32()
1263 	emit(ARM_ORR_SI(tmp2[0], tmp2[0], tmp2[1], SRTYPE_LSL, 16), ctx);  in emit_rev32()
1264 	emit(ARM_ORR_R(rd, ARM_IP, tmp2[0]), ctx);  in emit_rev32()
1267 	emit(ARM_REV(rd, rn), ctx);  in emit_rev32()
1281 	emit(ARM_PUSH(reg_set), ctx);  in emit_push_r64()
1294 	emit(ARM_MOV_R(ARM_IP, ARM_SP), ctx);  in build_prologue()
1295 	emit(ARM_PUSH(reg_set), ctx);  in build_prologue()
1296 	emit(ARM_SUB_I(ARM_FP, ARM_IP, 4), ctx);  in build_prologue()
1298 	emit(ARM_PUSH(CALLEE_PUSH_MASK), ctx);  in build_prologue()
1299 	emit(ARM_MOV_R(ARM_FP, ARM_SP), ctx);  in build_prologue()
1303 	emit(ARM_MOV_I(bpf_r1[0], 0), ctx);  in build_prologue()
1304 	emit(ARM_SUB_I(bpf_r1[1], ARM_SP, SCRATCH_SIZE), ctx);  in build_prologue()
1309 	emit(ARM_SUB_I(ARM_SP, ARM_SP, ctx->stack_size), ctx);  in build_prologue()
1315 	emit(ARM_MOV_I(bpf_r1[1], 0), ctx);  in build_prologue()
1319 	emit(ARM_MOV_R(bpf_r1[1], arm_r0), ctx);  in build_prologue()
1331 	emit(ARM_SUB_I(ARM_SP, ARM_FP, hweight16(reg_set) * 4), ctx);  in build_epilogue()
1332 	emit(ARM_LDM(ARM_SP, reg_set), ctx);  in build_epilogue()
1335 	emit(ARM_MOV_R(ARM_SP, ARM_FP), ctx);  in build_epilogue()
1336 	emit(ARM_POP(CALLEE_POP_MASK), ctx);  in build_epilogue()
1563 			emit(ARM_MOV_R(rd[0], ARM_LR), ctx);  in build_insn()
1573 			emit(ARM_AND_R(rd[1], rd[1], tmp2[1]), ctx);  in build_insn()
1575 			emit(ARM_UXTH(rd[1], rd[1]), ctx);  in build_insn()
1578 				emit(ARM_EOR_R(rd[0], rd[0], rd[0]), ctx);  in build_insn()
1583 				emit(ARM_EOR_R(rd[0], rd[0], rd[0]), ctx);  in build_insn()
1769 		emit(ARM_B(jmp_offset), ctx);  in build_insn()
1797 		emit(ARM_ADD_I(ARM_SP, ARM_SP, imm8m(24)), ctx); // callee clean  in build_insn()
1809 		emit(ARM_B(jmp_offset), ctx);  in build_insn()