Lines Matching +full:se +full:- +full:neg
21 #include "exec/exec-all.h"
23 #include "translate-a64.h"
75 #include "decode-sme-fa64.c.inc"
76 #include "decode-a64.c.inc"
78 /* Table based decoder typedefs - used when the relevant bits for decode
124 ARMMMUIdx useridx = s->mmu_idx; in get_a64_user_mem_index()
126 if (unpriv && s->unpriv) { in get_a64_user_mem_index()
128 * We have pre-computed the condition for AccType_UNPRIV. in get_a64_user_mem_index()
156 /* BTYPE is a 2-bit field, and 0 should be done with reset_btype. */ in set_btype()
159 s->btype = -1; in set_btype()
164 if (s->btype != 0) { in reset_btype()
166 s->btype = 0; in reset_btype()
172 assert(s->pc_save != -1); in gen_pc_plus_diff()
173 if (tb_cflags(s->base.tb) & CF_PCREL) { in gen_pc_plus_diff()
174 tcg_gen_addi_i64(dest, cpu_pc, (s->pc_curr - s->pc_save) + diff); in gen_pc_plus_diff()
176 tcg_gen_movi_i64(dest, s->pc_curr + diff); in gen_pc_plus_diff()
183 s->pc_save = s->pc_curr + diff; in gen_a64_update_pc()
191 * then the address is zero-extended, clearing bits [63:56]
195 * the address is sign-extended from bit 55 into bits [63:56]
205 } else if (!regime_has_2_ranges(s->mmu_idx)) { in gen_top_byte_ignore()
209 /* Sign-extend from bit 55. */ in gen_top_byte_ignore()
236 gen_top_byte_ignore(s, cpu_pc, src, s->tbii); in gen_a64_set_pc()
237 s->pc_save = -1; in gen_a64_set_pc()
244 * for the tag to be present in the FAR_ELx register. But for user-only
249 * of the write-back address.
256 gen_top_byte_ignore(s, clean, addr, s->tbid); in clean_data_tbi()
281 * is not relevant to MTE, per se, but watchpoints do require the size,
289 if (tag_checked && s->mte_active[is_unpriv]) { in gen_mte_check1_mmuidx()
294 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid); in gen_mte_check1_mmuidx()
295 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma); in gen_mte_check1_mmuidx()
298 desc = FIELD_DP32(desc, MTEDESC, SIZEM1, memop_size(memop) - 1); in gen_mte_check1_mmuidx()
321 if (tag_checked && s->mte_active[0]) { in gen_mte_checkN()
326 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid); in gen_mte_checkN()
327 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma); in gen_mte_checkN()
330 desc = FIELD_DP32(desc, MTEDESC, SIZEM1, total_size - 1); in gen_mte_checkN()
343 * naturally aligned, but it must not cross a 16-byte boundary.
386 * If size == MO_128, this is a LDXP, and the operation is single-copy in check_atomic_align()
417 } else if (!s->naa) { in check_ordered_align()
435 * Sign-extend the 32-bit value so that the GE/LT comparisons work in a64_test_cc()
438 c64->cond = c32.cond; in a64_test_cc()
439 c64->value = tcg_temp_new_i64(); in a64_test_cc()
440 tcg_gen_ext_i32_i64(c64->value, c32.value); in a64_test_cc()
445 gen_helper_rebuild_hflags_a64(tcg_env, tcg_constant_i32(s->current_el)); in gen_rebuild_hflags()
458 s->base.is_jmp = DISAS_NORETURN; in gen_exception_internal_insn()
465 s->base.is_jmp = DISAS_NORETURN; in gen_exception_bkpt_insn()
470 /* We just completed step of an insn. Move from Active-not-pending in gen_step_complete_exception()
471 * to Active-pending, and then also take the swstep exception. in gen_step_complete_exception()
480 gen_swstep_exception(s, 1, s->is_ldex); in gen_step_complete_exception()
481 s->base.is_jmp = DISAS_NORETURN; in gen_step_complete_exception()
486 if (s->ss_active) { in use_goto_tb()
489 return translator_use_goto_tb(&s->base, dest); in use_goto_tb()
494 if (use_goto_tb(s, s->pc_curr + diff)) { in gen_goto_tb()
496 * For pcrel, the pc must always be up-to-date on entry to in gen_goto_tb()
503 if (tb_cflags(s->base.tb) & CF_PCREL) { in gen_goto_tb()
510 tcg_gen_exit_tb(s->base.tb, n); in gen_goto_tb()
511 s->base.is_jmp = DISAS_NORETURN; in gen_goto_tb()
514 if (s->ss_active) { in gen_goto_tb()
518 s->base.is_jmp = DISAS_NORETURN; in gen_goto_tb()
556 * representing the register contents. This TCGv is an auto-freed
631 /* Clear the bits above an N-bit vector, for N = (is_q ? 128 : 64).
659 /* Expand a 2-operand AdvSIMD vector operation using an expander function. */
667 /* Expand a 2-operand + immediate AdvSIMD vector operation using
677 /* Expand a 3-operand AdvSIMD vector operation using an expander function. */
685 /* Expand a 4-operand AdvSIMD vector operation using an expander function. */
694 /* Expand a 2-operand operation using an out-of-line helper. */
703 /* Expand a 3-operand operation using an out-of-line helper. */
713 /* Expand a 3-operand + fpstatus pointer + simd data value operation using
714 * an out-of-line helper.
727 /* Expand a 4-operand operation using an out-of-line helper. */
739 * Expand a 4-operand operation using an out-of-line helper that takes
755 * Expand a 4-operand + fpstatus pointer + simd data value operation using
756 * an out-of-line helper.
841 /* dest = T0 - T1; compute C, N, V and Z flags */
1051 /* This always zero-extends and writes to a full 128 bit wide vector */ in do_fp_ld()
1211 * unallocated-encoding checks (otherwise the syndrome information
1216 if (s->fp_excp_el) { in fp_access_check_only()
1217 assert(!s->fp_access_checked); in fp_access_check_only()
1218 s->fp_access_checked = -1; in fp_access_check_only()
1222 s->fp_excp_el); in fp_access_check_only()
1225 s->fp_access_checked = 1; in fp_access_check_only()
1234 if (s->sme_trap_nonstreaming && s->is_nonstreaming) { in fp_access_check()
1249 if (s->pstate_sm || !dc_isar_feature(aa64_sve, s)) { in sve_access_check()
1254 s->sve_access_checked = (ret ? 1 : -1); in sve_access_check()
1257 if (s->sve_excp_el) { in sve_access_check()
1259 assert(!s->sve_access_checked); in sve_access_check()
1261 syn_sve_access_trap(), s->sve_excp_el); in sve_access_check()
1262 s->sve_access_checked = -1; in sve_access_check()
1265 s->sve_access_checked = 1; in sve_access_check()
1276 if (s->sme_excp_el) { in sme_access_check()
1279 s->sme_excp_el); in sme_access_check()
1293 if (!s->fp_excp_el || s->sme_excp_el < s->fp_excp_el) { in sme_enabled_check()
1295 s->fp_access_checked = (ret ? 1 : -1); in sme_enabled_check()
1307 if (FIELD_EX64(req, SVCR, SM) && !s->pstate_sm) { in sme_enabled_check_with_svcr()
1312 if (FIELD_EX64(req, SVCR, ZA) && !s->pstate_za) { in sme_enabled_check_with_svcr()
1327 if (!a->q && a->esz == MO_64) { in do_gvec_op2_ool()
1331 gen_gvec_op2_ool(s, a->q, a->rd, a->rn, data, fn); in do_gvec_op2_ool()
1339 if (!a->q && a->esz == MO_64) { in do_gvec_op3_ool()
1343 gen_gvec_op3_ool(s, a->q, a->rd, a->rn, a->rm, data, fn); in do_gvec_op3_ool()
1350 if (!a->q && a->esz == MO_64) { in do_gvec_fn3()
1354 gen_gvec_fn3(s, a->q, a->rd, a->rn, a->rm, fn, a->esz); in do_gvec_fn3()
1361 if (a->esz == MO_64) { in do_gvec_fn3_no64()
1365 gen_gvec_fn3(s, a->q, a->rd, a->rn, a->rm, fn, a->esz); in do_gvec_fn3_no64()
1372 if (a->esz == MO_8) { in do_gvec_fn3_no8_no64()
1380 if (!a->q && a->esz == MO_64) { in do_gvec_fn4()
1384 gen_gvec_fn4(s, a->q, a->rd, a->rn, a->rm, a->ra, fn, a->esz); in do_gvec_fn4()
1407 * or SCTLR bits) there is a check that SP is 16-aligned on every in gen_check_sp_alignment()
1408 * SP-relative load or store (with an exception generated if it is not). in gen_check_sp_alignment()
1433 while (tptr->mask) { in lookup_disas_fn()
1434 if ((insn & tptr->mask) == tptr->pattern) { in lookup_disas_fn()
1435 return tptr->disas_fn; in lookup_disas_fn()
1453 gen_goto_tb(s, 0, a->imm); in trans_B()
1461 gen_goto_tb(s, 0, a->imm); in trans_BL()
1471 tcg_cmp = read_cpu_reg(s, a->rt, a->sf); in trans_CBZ()
1475 tcg_gen_brcondi_i64(a->nz ? TCG_COND_NE : TCG_COND_EQ, in trans_CBZ()
1479 gen_goto_tb(s, 1, a->imm); in trans_CBZ()
1489 tcg_gen_andi_i64(tcg_cmp, cpu_reg(s, a->rt), 1ULL << a->bitpos); in trans_TBZ()
1494 tcg_gen_brcondi_i64(a->nz ? TCG_COND_NE : TCG_COND_EQ, in trans_TBZ()
1498 gen_goto_tb(s, 1, a->imm); in trans_TBZ()
1505 if (a->c && !dc_isar_feature(aa64_hbc, s)) { in trans_B_cond()
1509 if (a->cond < 0x0e) { in trans_B_cond()
1512 arm_gen_test_cc(a->cond, match.label); in trans_B_cond()
1515 gen_goto_tb(s, 1, a->imm); in trans_B_cond()
1518 gen_goto_tb(s, 0, a->imm); in trans_B_cond()
1526 /* BR to {x16,x17} or !guard -> 1, else 3. */ in set_btype_for_br()
1533 s->btype = -1; in set_btype_for_br()
1548 set_btype_for_br(s, a->rn); in trans_BR()
1549 gen_a64_set_pc(s, cpu_reg(s, a->rn)); in trans_BR()
1550 s->base.is_jmp = DISAS_JUMP; in trans_BR()
1556 TCGv_i64 dst = cpu_reg(s, a->rn); in trans_BLR()
1566 s->base.is_jmp = DISAS_JUMP; in trans_BLR()
1572 gen_a64_set_pc(s, cpu_reg(s, a->rn)); in trans_RET()
1573 s->base.is_jmp = DISAS_JUMP; in trans_RET()
1586 if (!s->pauth_active) { in auth_branch_target()
1607 dst = auth_branch_target(s, cpu_reg(s, a->rn), tcg_constant_i64(0), !a->m); in trans_BRAZ()
1608 set_btype_for_br(s, a->rn); in trans_BRAZ()
1610 s->base.is_jmp = DISAS_JUMP; in trans_BRAZ()
1622 dst = auth_branch_target(s, cpu_reg(s, a->rn), tcg_constant_i64(0), !a->m); in trans_BLRAZ()
1632 s->base.is_jmp = DISAS_JUMP; in trans_BLRAZ()
1640 dst = auth_branch_target(s, cpu_reg(s, 30), cpu_X[31], !a->m); in trans_RETA()
1642 s->base.is_jmp = DISAS_JUMP; in trans_RETA()
1653 dst = auth_branch_target(s, cpu_reg(s,a->rn), cpu_reg_sp(s, a->rm), !a->m); in trans_BRA()
1655 set_btype_for_br(s, a->rn); in trans_BRA()
1656 s->base.is_jmp = DISAS_JUMP; in trans_BRA()
1667 dst = auth_branch_target(s, cpu_reg(s, a->rn), cpu_reg_sp(s, a->rm), !a->m); in trans_BLRA()
1677 s->base.is_jmp = DISAS_JUMP; in trans_BLRA()
1685 if (s->current_el == 0) { in trans_ERET()
1688 if (s->trap_eret) { in trans_ERET()
1694 offsetof(CPUARMState, elr_el[s->current_el])); in trans_ERET()
1696 translator_io_start(&s->base); in trans_ERET()
1699 /* Must exit loop to check un-masked IRQs */ in trans_ERET()
1700 s->base.is_jmp = DISAS_EXIT; in trans_ERET()
1711 if (s->current_el == 0) { in trans_ERETA()
1715 if (s->trap_eret) { in trans_ERETA()
1716 gen_exception_insn_el(s, 0, EXCP_UDEF, syn_erettrap(a->m ? 3 : 2), 2); in trans_ERETA()
1721 offsetof(CPUARMState, elr_el[s->current_el])); in trans_ERETA()
1723 dst = auth_branch_target(s, dst, cpu_X[31], !a->m); in trans_ERETA()
1725 translator_io_start(&s->base); in trans_ERETA()
1728 /* Must exit loop to check un-masked IRQs */ in trans_ERETA()
1729 s->base.is_jmp = DISAS_EXIT; in trans_ERETA()
1746 if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) { in trans_YIELD()
1747 s->base.is_jmp = DISAS_YIELD; in trans_YIELD()
1754 s->base.is_jmp = DISAS_WFI; in trans_WFI()
1766 if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) { in trans_WFE()
1767 s->base.is_jmp = DISAS_WFE; in trans_WFE()
1782 * check ss_active so that single-stepping a WFIT doesn't halt. in trans_WFIT()
1784 if (s->ss_active) { in trans_WFIT()
1790 gen_helper_wfit(tcg_env, cpu_reg(s, a->rd)); in trans_WFIT()
1792 s->base.is_jmp = DISAS_EXIT; in trans_WFIT()
1807 if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) { in trans_WFET()
1808 s->base.is_jmp = DISAS_WFE; in trans_WFET()
1815 if (s->pauth_active) { in trans_XPACLRI()
1823 if (s->pauth_active) { in trans_PACIA1716()
1831 if (s->pauth_active) { in trans_PACIB1716()
1839 if (s->pauth_active) { in trans_AUTIA1716()
1847 if (s->pauth_active) { in trans_AUTIB1716()
1866 if (s->current_el <= 1 && arm_dc_feature(s, ARM_FEATURE_EL2)) { in trans_ESB()
1875 if (s->pauth_active) { in trans_PACIAZ()
1883 if (s->pauth_active) { in trans_PACIASP()
1891 if (s->pauth_active) { in trans_PACIBZ()
1899 if (s->pauth_active) { in trans_PACIBSP()
1907 if (s->pauth_active) { in trans_AUTIAZ()
1915 if (s->pauth_active) { in trans_AUTIASP()
1923 if (s->pauth_active) { in trans_AUTIBZ()
1931 if (s->pauth_active) { in trans_AUTIBSP()
1939 tcg_gen_movi_i64(cpu_exclusive_addr, -1); in trans_CLREX()
1948 switch (a->types) { in trans_DSB_DMB()
1967 * self-modifying code correctly and also to take in trans_ISB()
2014 * ~-(C | Z) in trans_XAFLAG()
2015 * (C | Z) - 1 in trans_XAFLAG()
2024 /* (!C & Z) << 31 -> -(Z & ~C) */ in trans_XAFLAG()
2040 tcg_gen_sari_i32(cpu_VF, cpu_VF, 31); /* V ? -1 : 0 */ in trans_AXFLAG()
2043 /* !(Z | V) -> !(!ZF | V) -> ZF & !V -> ZF & ~VF */ in trans_AXFLAG()
2054 if (!dc_isar_feature(aa64_uao, s) || s->current_el == 0) { in trans_MSR_i_UAO()
2057 if (a->imm & 1) { in trans_MSR_i_UAO()
2063 s->base.is_jmp = DISAS_TOO_MANY; in trans_MSR_i_UAO()
2069 if (!dc_isar_feature(aa64_pan, s) || s->current_el == 0) { in trans_MSR_i_PAN()
2072 if (a->imm & 1) { in trans_MSR_i_PAN()
2078 s->base.is_jmp = DISAS_TOO_MANY; in trans_MSR_i_PAN()
2084 if (s->current_el == 0) { in trans_MSR_i_SPSEL()
2087 gen_helper_msr_i_spsel(tcg_env, tcg_constant_i32(a->imm & PSTATE_SP)); in trans_MSR_i_SPSEL()
2088 s->base.is_jmp = DISAS_TOO_MANY; in trans_MSR_i_SPSEL()
2097 if (a->imm & 1) { in trans_MSR_i_SBSS()
2103 s->base.is_jmp = DISAS_TOO_MANY; in trans_MSR_i_SBSS()
2112 if (a->imm & 1) { in trans_MSR_i_DIT()
2118 s->base.is_jmp = DISAS_TOO_MANY; in trans_MSR_i_DIT()
2125 /* Full MTE is enabled -- set the TCO bit as directed. */ in trans_MSR_i_TCO()
2126 if (a->imm & 1) { in trans_MSR_i_TCO()
2133 s->base.is_jmp = DISAS_UPDATE_NOCHAIN; in trans_MSR_i_TCO()
2136 /* Only "instructions accessible at EL0" -- PSTATE.TCO is WI. */ in trans_MSR_i_TCO()
2146 gen_helper_msr_i_daifset(tcg_env, tcg_constant_i32(a->imm)); in trans_MSR_i_DAIFSET()
2147 s->base.is_jmp = DISAS_TOO_MANY; in trans_MSR_i_DAIFSET()
2153 gen_helper_msr_i_daifclear(tcg_env, tcg_constant_i32(a->imm)); in trans_MSR_i_DAIFCLEAR()
2154 /* Exit the cpu loop to re-evaluate pending IRQs. */ in trans_MSR_i_DAIFCLEAR()
2155 s->base.is_jmp = DISAS_UPDATE_EXIT; in trans_MSR_i_DAIFCLEAR()
2161 if (!dc_isar_feature(aa64_nmi, s) || s->current_el == 0) { in trans_MSR_i_ALLINT()
2165 if (a->imm == 0) { in trans_MSR_i_ALLINT()
2167 } else if (s->current_el > 1) { in trans_MSR_i_ALLINT()
2173 /* Exit the cpu loop to re-evaluate pending IRQs. */ in trans_MSR_i_ALLINT()
2174 s->base.is_jmp = DISAS_UPDATE_EXIT; in trans_MSR_i_ALLINT()
2180 if (!dc_isar_feature(aa64_sme, s) || a->mask == 0) { in trans_MSR_i_SVCR()
2184 int old = s->pstate_sm | (s->pstate_za << 1); in trans_MSR_i_SVCR()
2185 int new = a->imm * 3; in trans_MSR_i_SVCR()
2187 if ((old ^ new) & a->mask) { in trans_MSR_i_SVCR()
2190 tcg_constant_i32(a->mask)); in trans_MSR_i_SVCR()
2191 s->base.is_jmp = DISAS_TOO_MANY; in trans_MSR_i_SVCR()
2259 /* MRS - move from system register
2260 * MSR (register) - move to system register
2272 const ARMCPRegInfo *ri = get_arm_cp_reginfo(s->cp_regs, key); in handle_sys()
2287 switch (s->current_el) { in handle_sys()
2310 if (s->nv2 && ri->nv2_redirect_offset) { in handle_sys()
2316 if (ri->nv2_redirect_offset & NV2_REDIR_NV1) { in handle_sys()
2317 nv2_mem_redirect = s->nv1; in handle_sys()
2318 } else if (ri->nv2_redirect_offset & NV2_REDIR_NO_NV1) { in handle_sys()
2319 nv2_mem_redirect = !s->nv1; in handle_sys()
2326 if (!cp_access_ok(s->current_el, ri, isread)) { in handle_sys()
2333 if (s->nv2 && (ri->type & ARM_CP_NV2_REDIRECT)) { in handle_sys()
2343 * NV2 redirect-to-memory takes precedence over trap to EL2 or in handle_sys()
2346 } else if (s->nv && arm_cpreg_traps_in_nv(ri)) { in handle_sys()
2353 * over the trap-to-EL2. Most trapped-by-FEAT_NV registers have in handle_sys()
2355 * the trap-to-EL3 controls which would apply to that register in handle_sys()
2356 * at EL2 don't take priority over the FEAT_NV trap-to-EL2.) in handle_sys()
2365 if (ri->accessfn || (ri->fgt && s->fgt_active)) { in handle_sys()
2375 } else if (ri->type & ARM_CP_RAISES_EXC) { in handle_sys()
2384 if ((ri->type & ARM_CP_FPU) && !fp_access_check_only(s)) { in handle_sys()
2386 } else if ((ri->type & ARM_CP_SVE) && !sve_access_check(s)) { in handle_sys()
2388 } else if ((ri->type & ARM_CP_SME) && !sme_access_check(s)) { in handle_sys()
2405 * fine-grained-traps on EL1 also do not apply here. in handle_sys()
2409 ri = get_arm_cp_reginfo(s->cp_regs, key); in handle_sys()
2411 assert(cp_access_ok(s->current_el, ri, isread)); in handle_sys()
2416 assert(!(ri->type & ARM_CP_RAISES_EXC)); in handle_sys()
2425 * The access is 64-bit single copy atomic, guaranteed aligned because in handle_sys()
2434 ARMMMUIdx armmemidx = s->nv2_mem_e20 ? ARMMMUIdx_E20_2 : ARMMMUIdx_E2; in handle_sys()
2438 mop |= (s->nv2_mem_be ? MO_BE : MO_LE); in handle_sys()
2442 (ri->nv2_redirect_offset & ~NV2_REDIR_FLAG_MASK)); in handle_sys()
2456 switch (ri->type & ARM_CP_SPECIAL_MASK) { in handle_sys()
2476 int el = s->nv ? 2 : s->current_el; in handle_sys()
2483 if (s->mte_active[0]) { in handle_sys()
2487 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid); in handle_sys()
2488 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma); in handle_sys()
2510 if (s->ata[0]) { in handle_sys()
2527 if (s->ata[0]) { in handle_sys()
2539 if (ri->type & ARM_CP_IO) { in handle_sys()
2541 need_exit_tb = translator_io_start(&s->base); in handle_sys()
2547 if (ri->type & ARM_CP_CONST) { in handle_sys()
2548 tcg_gen_movi_i64(tcg_rt, ri->resetvalue); in handle_sys()
2549 } else if (ri->readfn) { in handle_sys()
2555 tcg_gen_ld_i64(tcg_rt, tcg_env, ri->fieldoffset); in handle_sys()
2558 if (ri->type & ARM_CP_CONST) { in handle_sys()
2561 } else if (ri->writefn) { in handle_sys()
2567 tcg_gen_st_i64(tcg_rt, tcg_env, ri->fieldoffset); in handle_sys()
2571 if (!isread && !(ri->type & ARM_CP_SUPPRESS_TB_END)) { in handle_sys()
2585 s->base.is_jmp = DISAS_UPDATE_EXIT; in handle_sys()
2591 handle_sys(s, a->l, a->op0, a->op1, a->op2, a->crn, a->crm, a->rt); in trans_SYS()
2598 * For SVC, HVC and SMC we advance the single-step state in trans_SVC()
2600 * mandated, to ensure that single-stepping a system call in trans_SVC()
2603 uint32_t syndrome = syn_aa64_svc(a->imm); in trans_SVC()
2604 if (s->fgt_svc) { in trans_SVC()
2615 int target_el = s->current_el == 3 ? 3 : 2; in trans_HVC()
2617 if (s->current_el == 0) { in trans_HVC()
2629 gen_exception_insn_el(s, 4, EXCP_HVC, syn_aa64_hvc(a->imm), target_el); in trans_HVC()
2635 if (s->current_el == 0) { in trans_SMC()
2640 gen_helper_pre_smc(tcg_env, tcg_constant_i32(syn_aa64_smc(a->imm))); in trans_SMC()
2643 gen_exception_insn_el(s, 4, EXCP_SMC, syn_aa64_smc(a->imm), 3); in trans_SMC()
2649 gen_exception_bkpt_insn(s, syn_aa64_bkpt(a->imm)); in trans_BRK()
2662 if (semihosting_enabled(s->current_el == 0) && a->imm == 0xf000) { in trans_HLT()
2678 * races in multi-threaded linux-user and when MTTCG softmmu is
2688 s->is_ldex = true; in gen_load_exclusive()
2697 if (s->be_data == MO_LE) { in gen_load_exclusive()
2709 if (s->be_data == MO_LE) { in gen_load_exclusive()
2729 /* if (env->exclusive_addr == addr && env->exclusive_val == [addr] in gen_store_exclusive()
2730 * && (!is_pair || env->exclusive_high == [addr + datasize])) { in gen_store_exclusive()
2739 * env->exclusive_addr = -1; in gen_store_exclusive()
2765 * It is possible to trigger an alignment fault pre-LSE2, e.g. with an in gen_store_exclusive()
2766 * unaligned 4-byte write within the range of an aligned 8-byte load. in gen_store_exclusive()
2767 * With LSE2, the store would need to cross a 16-byte boundary when the in gen_store_exclusive()
2787 if (s->be_data == MO_LE) { in gen_store_exclusive()
2801 if (s->be_data == MO_LE) { in gen_store_exclusive()
2816 if (s->be_data == MO_LE) { in gen_store_exclusive()
2839 tcg_gen_movi_i64(cpu_exclusive_addr, -1); in gen_store_exclusive()
2883 if (s->be_data == MO_LE) { in gen_compare_and_swap_pair()
2893 if (s->be_data == MO_LE) { in gen_compare_and_swap_pair()
2902 if (s->be_data == MO_LE) { in gen_compare_and_swap_pair()
2912 if (s->be_data == MO_LE) { in gen_compare_and_swap_pair()
2923 * is accessing a 32-bit or 64-bit register. This logic is derived
2932 * do a zero-extend from 32 to 64 after the load. in ldst_iss_sf()
2944 if (a->rn == 31) { in trans_STXR()
2947 if (a->lasr) { in trans_STXR()
2950 gen_store_exclusive(s, a->rs, a->rt, a->rt2, a->rn, a->sz, false); in trans_STXR()
2956 if (a->rn == 31) { in trans_LDXR()
2959 gen_load_exclusive(s, a->rt, a->rt2, a->rn, a->sz, false); in trans_LDXR()
2960 if (a->lasr) { in trans_LDXR()
2970 bool iss_sf = ldst_iss_sf(a->sz, false, false); in trans_STLR()
2973 * StoreLORelease is the same as Store-Release for QEMU, but in trans_STLR()
2974 * needs the feature-test. in trans_STLR()
2976 if (!a->lasr && !dc_isar_feature(aa64_lor, s)) { in trans_STLR()
2979 /* Generate ISS for non-exclusive accesses including LASR. */ in trans_STLR()
2980 if (a->rn == 31) { in trans_STLR()
2984 memop = check_ordered_align(s, a->rn, 0, true, a->sz); in trans_STLR()
2985 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn), in trans_STLR()
2986 true, a->rn != 31, memop); in trans_STLR()
2987 do_gpr_st(s, cpu_reg(s, a->rt), clean_addr, memop, true, a->rt, in trans_STLR()
2988 iss_sf, a->lasr); in trans_STLR()
2996 bool iss_sf = ldst_iss_sf(a->sz, false, false); in trans_LDAR()
2998 /* LoadLOAcquire is the same as Load-Acquire for QEMU. */ in trans_LDAR()
2999 if (!a->lasr && !dc_isar_feature(aa64_lor, s)) { in trans_LDAR()
3002 /* Generate ISS for non-exclusive accesses including LASR. */ in trans_LDAR()
3003 if (a->rn == 31) { in trans_LDAR()
3006 memop = check_ordered_align(s, a->rn, 0, false, a->sz); in trans_LDAR()
3007 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn), in trans_LDAR()
3008 false, a->rn != 31, memop); in trans_LDAR()
3009 do_gpr_ld(s, cpu_reg(s, a->rt), clean_addr, memop, false, true, in trans_LDAR()
3010 a->rt, iss_sf, a->lasr); in trans_LDAR()
3017 if (a->rn == 31) { in trans_STXP()
3020 if (a->lasr) { in trans_STXP()
3023 gen_store_exclusive(s, a->rs, a->rt, a->rt2, a->rn, a->sz, true); in trans_STXP()
3029 if (a->rn == 31) { in trans_LDXP()
3032 gen_load_exclusive(s, a->rt, a->rt2, a->rn, a->sz, true); in trans_LDXP()
3033 if (a->lasr) { in trans_LDXP()
3044 if (((a->rt | a->rs) & 1) != 0) { in trans_CASP()
3048 gen_compare_and_swap_pair(s, a->rs, a->rt, a->rn, a->sz); in trans_CASP()
3057 gen_compare_and_swap(s, a->rs, a->rt, a->rn, a->sz); in trans_CAS()
3063 bool iss_sf = ldst_iss_sf(a->sz, a->sign, false); in trans_LD_lit()
3064 TCGv_i64 tcg_rt = cpu_reg(s, a->rt); in trans_LD_lit()
3066 MemOp memop = finalize_memop(s, a->sz + a->sign * MO_SIGN); in trans_LD_lit()
3068 gen_pc_plus_diff(s, clean_addr, a->imm); in trans_LD_lit()
3070 false, true, a->rt, iss_sf, false); in trans_LD_lit()
3083 memop = finalize_memop_asimd(s, a->sz); in trans_LD_lit_v()
3085 gen_pc_plus_diff(s, clean_addr, a->imm); in trans_LD_lit_v()
3086 do_fp_ld(s, a->rt, clean_addr, memop); in trans_LD_lit_v()
3094 if (a->rn == 31) { in op_addr_ldstpair_pre()
3098 *dirty_addr = read_cpu_reg_sp(s, a->rn, 1); in op_addr_ldstpair_pre()
3099 if (!a->p) { in op_addr_ldstpair_pre()
3104 (a->w || a->rn != 31), 2 << a->sz, mop); in op_addr_ldstpair_pre()
3110 if (a->w) { in op_addr_ldstpair_post()
3111 if (a->p) { in op_addr_ldstpair_post()
3114 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), dirty_addr); in op_addr_ldstpair_post()
3120 uint64_t offset = a->imm << a->sz; in trans_STP()
3122 MemOp mop = finalize_memop(s, a->sz); in trans_STP()
3125 tcg_rt = cpu_reg(s, a->rt); in trans_STP()
3126 tcg_rt2 = cpu_reg(s, a->rt2); in trans_STP()
3128 * We built mop above for the single logical access -- rebuild it in trans_STP()
3131 * With LSE2, non-sign-extending pairs are treated atomically if in trans_STP()
3133 * within a 16-byte block and that element will be atomic. in trans_STP()
3137 mop = a->sz + 1; in trans_STP()
3138 if (s->align_mem) { in trans_STP()
3139 mop |= (a->sz == 2 ? MO_ALIGN_4 : MO_ALIGN_8); in trans_STP()
3142 if (a->sz == 2) { in trans_STP()
3145 if (s->be_data == MO_LE) { in trans_STP()
3154 if (s->be_data == MO_LE) { in trans_STP()
3167 uint64_t offset = a->imm << a->sz; in trans_LDP()
3169 MemOp mop = finalize_memop(s, a->sz); in trans_LDP()
3172 tcg_rt = cpu_reg(s, a->rt); in trans_LDP()
3173 tcg_rt2 = cpu_reg(s, a->rt2); in trans_LDP()
3176 * We built mop above for the single logical access -- rebuild it in trans_LDP()
3179 * With LSE2, non-sign-extending pairs are treated atomically if in trans_LDP()
3181 * within a 16-byte block and that element will be atomic. in trans_LDP()
3185 * This treats sign-extending loads like zero-extending loads, in trans_LDP()
3188 mop = a->sz + 1; in trans_LDP()
3189 if (s->align_mem) { in trans_LDP()
3190 mop |= (a->sz == 2 ? MO_ALIGN_4 : MO_ALIGN_8); in trans_LDP()
3193 if (a->sz == 2) { in trans_LDP()
3194 int o2 = s->be_data == MO_LE ? 32 : 0; in trans_LDP()
3198 if (a->sign) { in trans_LDP()
3209 if (s->be_data == MO_LE) { in trans_LDP()
3221 uint64_t offset = a->imm << a->sz; in trans_STP_v()
3230 mop = finalize_memop_asimd(s, a->sz); in trans_STP_v()
3232 do_fp_st(s, a->rt, clean_addr, mop); in trans_STP_v()
3233 tcg_gen_addi_i64(clean_addr, clean_addr, 1 << a->sz); in trans_STP_v()
3234 do_fp_st(s, a->rt2, clean_addr, mop); in trans_STP_v()
3241 uint64_t offset = a->imm << a->sz; in trans_LDP_v()
3250 mop = finalize_memop_asimd(s, a->sz); in trans_LDP_v()
3252 do_fp_ld(s, a->rt, clean_addr, mop); in trans_LDP_v()
3253 tcg_gen_addi_i64(clean_addr, clean_addr, 1 << a->sz); in trans_LDP_v()
3254 do_fp_ld(s, a->rt2, clean_addr, mop); in trans_LDP_v()
3262 uint64_t offset = a->imm << LOG2_TAG_GRANULE; in trans_STGP()
3267 tcg_debug_assert(a->sz == MO_64); in trans_STGP()
3273 if (a->rn == 31) { in trans_STGP()
3277 dirty_addr = read_cpu_reg_sp(s, a->rn, 1); in trans_STGP()
3278 if (!a->p) { in trans_STGP()
3283 tcg_rt = cpu_reg(s, a->rt); in trans_STGP()
3284 tcg_rt2 = cpu_reg(s, a->rt2); in trans_STGP()
3287 * STGP is defined as two 8-byte memory operations, aligned to TAG_GRANULE, in trans_STGP()
3288 * and one tag operation. We implement it as one single aligned 16-byte in trans_STGP()
3290 * MO_ATOM_IFALIGN_PAIR produces 8-byte atomicity for the memory store. in trans_STGP()
3295 if (s->be_data == MO_LE) { in trans_STGP()
3303 if (s->ata[0]) { in trans_STGP()
3304 if (tb_cflags(s->base.tb) & CF_PARALLEL) { in trans_STGP()
3321 if (a->rn == 31) { in op_addr_ldst_imm_pre()
3325 *dirty_addr = read_cpu_reg_sp(s, a->rn, 1); in op_addr_ldst_imm_pre()
3326 if (!a->p) { in op_addr_ldst_imm_pre()
3329 memidx = get_a64_user_mem_index(s, a->unpriv); in op_addr_ldst_imm_pre()
3331 a->w || a->rn != 31, in op_addr_ldst_imm_pre()
3332 mop, a->unpriv, memidx); in op_addr_ldst_imm_pre()
3338 if (a->w) { in op_addr_ldst_imm_post()
3339 if (a->p) { in op_addr_ldst_imm_post()
3342 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), dirty_addr); in op_addr_ldst_imm_post()
3348 bool iss_sf, iss_valid = !a->w; in trans_STR_i()
3350 int memidx = get_a64_user_mem_index(s, a->unpriv); in trans_STR_i()
3351 MemOp mop = finalize_memop(s, a->sz + a->sign * MO_SIGN); in trans_STR_i()
3353 op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, true, mop); in trans_STR_i()
3355 tcg_rt = cpu_reg(s, a->rt); in trans_STR_i()
3356 iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext); in trans_STR_i()
3359 iss_valid, a->rt, iss_sf, false); in trans_STR_i()
3360 op_addr_ldst_imm_post(s, a, dirty_addr, a->imm); in trans_STR_i()
3366 bool iss_sf, iss_valid = !a->w; in trans_LDR_i()
3368 int memidx = get_a64_user_mem_index(s, a->unpriv); in trans_LDR_i()
3369 MemOp mop = finalize_memop(s, a->sz + a->sign * MO_SIGN); in trans_LDR_i()
3371 op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, false, mop); in trans_LDR_i()
3373 tcg_rt = cpu_reg(s, a->rt); in trans_LDR_i()
3374 iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext); in trans_LDR_i()
3377 a->ext, memidx, iss_valid, a->rt, iss_sf, false); in trans_LDR_i()
3378 op_addr_ldst_imm_post(s, a, dirty_addr, a->imm); in trans_LDR_i()
3390 mop = finalize_memop_asimd(s, a->sz); in trans_STR_v_i()
3391 op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, true, mop); in trans_STR_v_i()
3392 do_fp_st(s, a->rt, clean_addr, mop); in trans_STR_v_i()
3393 op_addr_ldst_imm_post(s, a, dirty_addr, a->imm); in trans_STR_v_i()
3405 mop = finalize_memop_asimd(s, a->sz); in trans_LDR_v_i()
3406 op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, false, mop); in trans_LDR_v_i()
3407 do_fp_ld(s, a->rt, clean_addr, mop); in trans_LDR_v_i()
3408 op_addr_ldst_imm_post(s, a, dirty_addr, a->imm); in trans_LDR_v_i()
3418 if (a->rn == 31) { in op_addr_ldst_pre()
3421 *dirty_addr = read_cpu_reg_sp(s, a->rn, 1); in op_addr_ldst_pre()
3423 tcg_rm = read_cpu_reg(s, a->rm, 1); in op_addr_ldst_pre()
3424 ext_and_shift_reg(tcg_rm, tcg_rm, a->opt, a->s ? a->sz : 0); in op_addr_ldst_pre()
3433 bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext); in trans_LDR()
3436 if (extract32(a->opt, 1, 1) == 0) { in trans_LDR()
3440 memop = finalize_memop(s, a->sz + a->sign * MO_SIGN); in trans_LDR()
3442 tcg_rt = cpu_reg(s, a->rt); in trans_LDR()
3444 a->ext, true, a->rt, iss_sf, false); in trans_LDR()
3451 bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext); in trans_STR()
3454 if (extract32(a->opt, 1, 1) == 0) { in trans_STR()
3458 memop = finalize_memop(s, a->sz); in trans_STR()
3460 tcg_rt = cpu_reg(s, a->rt); in trans_STR()
3461 do_gpr_st(s, tcg_rt, clean_addr, memop, true, a->rt, iss_sf, false); in trans_STR()
3470 if (extract32(a->opt, 1, 1) == 0) { in trans_LDR_v()
3478 memop = finalize_memop_asimd(s, a->sz); in trans_LDR_v()
3480 do_fp_ld(s, a->rt, clean_addr, memop); in trans_LDR_v()
3489 if (extract32(a->opt, 1, 1) == 0) { in trans_STR_v()
3497 memop = finalize_memop_asimd(s, a->sz); in trans_STR_v()
3499 do_fp_st(s, a->rt, clean_addr, memop); in trans_STR_v()
3507 MemOp mop = a->sz | sign; in do_atomic_ld()
3510 if (a->rn == 31) { in do_atomic_ld()
3513 mop = check_atomic_align(s, a->rn, mop); in do_atomic_ld()
3514 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn), false, in do_atomic_ld()
3515 a->rn != 31, mop); in do_atomic_ld()
3516 tcg_rs = read_cpu_reg(s, a->rs, true); in do_atomic_ld()
3517 tcg_rt = cpu_reg(s, a->rt); in do_atomic_ld()
3528 switch (a->sz) { in do_atomic_ld()
3559 bool iss_sf = ldst_iss_sf(a->sz, false, false); in TRANS_FEAT()
3567 if (a->rn == 31) { in TRANS_FEAT()
3570 mop = check_ordered_align(s, a->rn, 0, false, a->sz); in TRANS_FEAT()
3571 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn), false, in TRANS_FEAT()
3572 a->rn != 31, mop); in TRANS_FEAT()
3575 * fetch-and-do-something op. in TRANS_FEAT()
3577 * full load-acquire (we only need "load-acquire processor consistent"), in TRANS_FEAT()
3580 do_gpr_ld(s, cpu_reg(s, a->rt), clean_addr, mop, false, in TRANS_FEAT()
3581 true, a->rt, iss_sf, true); in TRANS_FEAT()
3596 if (a->rn == 31) { in trans_LDRA()
3599 dirty_addr = read_cpu_reg_sp(s, a->rn, 1); in trans_LDRA()
3601 if (s->pauth_active) { in trans_LDRA()
3602 if (!a->m) { in trans_LDRA()
3611 tcg_gen_addi_i64(dirty_addr, dirty_addr, a->imm); in trans_LDRA()
3617 a->w || a->rn != 31, memop); in trans_LDRA()
3619 tcg_rt = cpu_reg(s, a->rt); in trans_LDRA()
3621 /* extend */ false, /* iss_valid */ !a->w, in trans_LDRA()
3622 /* iss_srt */ a->rt, /* iss_sf */ true, /* iss_ar */ false); in trans_LDRA()
3624 if (a->w) { in trans_LDRA()
3625 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), dirty_addr); in trans_LDRA()
3633 MemOp mop = a->sz | (a->sign ? MO_SIGN : 0); in trans_LDAPR_i()
3634 bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext); in trans_LDAPR_i()
3640 if (a->rn == 31) { in trans_LDAPR_i()
3644 mop = check_ordered_align(s, a->rn, a->imm, false, mop); in trans_LDAPR_i()
3645 dirty_addr = read_cpu_reg_sp(s, a->rn, 1); in trans_LDAPR_i()
3646 tcg_gen_addi_i64(dirty_addr, dirty_addr, a->imm); in trans_LDAPR_i()
3650 * Load-AcquirePC semantics; we implement as the slightly more in trans_LDAPR_i()
3651 * restrictive Load-Acquire. in trans_LDAPR_i()
3653 do_gpr_ld(s, cpu_reg(s, a->rt), clean_addr, mop, a->ext, true, in trans_LDAPR_i()
3654 a->rt, iss_sf, true); in trans_LDAPR_i()
3662 MemOp mop = a->sz; in trans_STLR_i()
3663 bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext); in trans_STLR_i()
3669 /* TODO: ARMv8.4-LSE SCTLR.nAA */ in trans_STLR_i()
3671 if (a->rn == 31) { in trans_STLR_i()
3675 mop = check_ordered_align(s, a->rn, a->imm, true, mop); in trans_STLR_i()
3676 dirty_addr = read_cpu_reg_sp(s, a->rn, 1); in trans_STLR_i()
3677 tcg_gen_addi_i64(dirty_addr, dirty_addr, a->imm); in trans_STLR_i()
3680 /* Store-Release semantics */ in trans_STLR_i()
3682 do_gpr_st(s, cpu_reg(s, a->rt), clean_addr, mop, true, a->rt, iss_sf, true); in trans_STLR_i()
3694 int size = a->sz; in trans_LD_mult()
3696 if (!a->p && a->rm != 0) { in trans_LD_mult()
3697 /* For non-postindexed accesses the Rm field must be 0 */ in trans_LD_mult()
3700 if (size == 3 && !a->q && a->selem != 1) { in trans_LD_mult()
3707 if (a->rn == 31) { in trans_LD_mult()
3711 /* For our purposes, bytes are always little-endian. */ in trans_LD_mult()
3712 endian = s->be_data; in trans_LD_mult()
3717 total = a->rpt * a->selem * (a->q ? 16 : 8); in trans_LD_mult()
3718 tcg_rn = cpu_reg_sp(s, a->rn); in trans_LD_mult()
3722 * promote consecutive little-endian elements below. in trans_LD_mult()
3724 clean_addr = gen_mte_checkN(s, tcg_rn, false, a->p || a->rn != 31, total, in trans_LD_mult()
3728 * Consecutive little-endian elements from a single register in trans_LD_mult()
3729 * can be promoted to a larger little-endian operation. in trans_LD_mult()
3732 if (a->selem == 1 && endian == MO_LE) { in trans_LD_mult()
3736 if (!s->align_mem) { in trans_LD_mult()
3741 elements = (a->q ? 16 : 8) >> size; in trans_LD_mult()
3743 for (r = 0; r < a->rpt; r++) { in trans_LD_mult()
3747 for (xs = 0; xs < a->selem; xs++) { in trans_LD_mult()
3748 int tt = (a->rt + r + xs) % 32; in trans_LD_mult()
3756 * For non-quad operations, setting a slice of the low 64 bits of in trans_LD_mult()
3762 for (r = 0; r < a->rpt * a->selem; r++) { in trans_LD_mult()
3763 int tt = (a->rt + r) % 32; in trans_LD_mult()
3764 clear_vec_high(s, a->q, tt); in trans_LD_mult()
3767 if (a->p) { in trans_LD_mult()
3768 if (a->rm == 31) { in trans_LD_mult()
3771 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm)); in trans_LD_mult()
3785 int size = a->sz; in trans_ST_mult()
3787 if (!a->p && a->rm != 0) { in trans_ST_mult()
3788 /* For non-postindexed accesses the Rm field must be 0 */ in trans_ST_mult()
3791 if (size == 3 && !a->q && a->selem != 1) { in trans_ST_mult()
3798 if (a->rn == 31) { in trans_ST_mult()
3802 /* For our purposes, bytes are always little-endian. */ in trans_ST_mult()
3803 endian = s->be_data; in trans_ST_mult()
3808 total = a->rpt * a->selem * (a->q ? 16 : 8); in trans_ST_mult()
3809 tcg_rn = cpu_reg_sp(s, a->rn); in trans_ST_mult()
3813 * promote consecutive little-endian elements below. in trans_ST_mult()
3815 clean_addr = gen_mte_checkN(s, tcg_rn, true, a->p || a->rn != 31, total, in trans_ST_mult()
3819 * Consecutive little-endian elements from a single register in trans_ST_mult()
3820 * can be promoted to a larger little-endian operation. in trans_ST_mult()
3823 if (a->selem == 1 && endian == MO_LE) { in trans_ST_mult()
3827 if (!s->align_mem) { in trans_ST_mult()
3832 elements = (a->q ? 16 : 8) >> size; in trans_ST_mult()
3834 for (r = 0; r < a->rpt; r++) { in trans_ST_mult()
3838 for (xs = 0; xs < a->selem; xs++) { in trans_ST_mult()
3839 int tt = (a->rt + r + xs) % 32; in trans_ST_mult()
3846 if (a->p) { in trans_ST_mult()
3847 if (a->rm == 31) { in trans_ST_mult()
3850 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm)); in trans_ST_mult()
3862 if (!a->p && a->rm != 0) { in trans_ST_single()
3869 if (a->rn == 31) { in trans_ST_single()
3873 total = a->selem << a->scale; in trans_ST_single()
3874 tcg_rn = cpu_reg_sp(s, a->rn); in trans_ST_single()
3876 mop = finalize_memop_asimd(s, a->scale); in trans_ST_single()
3877 clean_addr = gen_mte_checkN(s, tcg_rn, true, a->p || a->rn != 31, in trans_ST_single()
3880 tcg_ebytes = tcg_constant_i64(1 << a->scale); in trans_ST_single()
3881 for (xs = 0, rt = a->rt; xs < a->selem; xs++, rt = (rt + 1) % 32) { in trans_ST_single()
3882 do_vec_st(s, rt, a->index, clean_addr, mop); in trans_ST_single()
3886 if (a->p) { in trans_ST_single()
3887 if (a->rm == 31) { in trans_ST_single()
3890 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm)); in trans_ST_single()
3902 if (!a->p && a->rm != 0) { in trans_LD_single()
3909 if (a->rn == 31) { in trans_LD_single()
3913 total = a->selem << a->scale; in trans_LD_single()
3914 tcg_rn = cpu_reg_sp(s, a->rn); in trans_LD_single()
3916 mop = finalize_memop_asimd(s, a->scale); in trans_LD_single()
3917 clean_addr = gen_mte_checkN(s, tcg_rn, false, a->p || a->rn != 31, in trans_LD_single()
3920 tcg_ebytes = tcg_constant_i64(1 << a->scale); in trans_LD_single()
3921 for (xs = 0, rt = a->rt; xs < a->selem; xs++, rt = (rt + 1) % 32) { in trans_LD_single()
3922 do_vec_ld(s, rt, a->index, clean_addr, mop); in trans_LD_single()
3926 if (a->p) { in trans_LD_single()
3927 if (a->rm == 31) { in trans_LD_single()
3930 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm)); in trans_LD_single()
3942 if (!a->p && a->rm != 0) { in trans_LD_single_repl()
3949 if (a->rn == 31) { in trans_LD_single_repl()
3953 total = a->selem << a->scale; in trans_LD_single_repl()
3954 tcg_rn = cpu_reg_sp(s, a->rn); in trans_LD_single_repl()
3956 mop = finalize_memop_asimd(s, a->scale); in trans_LD_single_repl()
3957 clean_addr = gen_mte_checkN(s, tcg_rn, false, a->p || a->rn != 31, in trans_LD_single_repl()
3960 tcg_ebytes = tcg_constant_i64(1 << a->scale); in trans_LD_single_repl()
3961 for (xs = 0, rt = a->rt; xs < a->selem; xs++, rt = (rt + 1) % 32) { in trans_LD_single_repl()
3966 tcg_gen_gvec_dup_i64(a->scale, vec_full_reg_offset(s, rt), in trans_LD_single_repl()
3967 (a->q + 1) * 8, vec_full_reg_size(s), tcg_tmp); in trans_LD_single_repl()
3971 if (a->p) { in trans_LD_single_repl()
3972 if (a->rm == 31) { in trans_LD_single_repl()
3975 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm)); in trans_LD_single_repl()
3984 int size = 4 << s->dcz_blocksize; in trans_STZGM()
3989 if (s->current_el == 0) { in trans_STZGM()
3993 if (a->rn == 31) { in trans_STZGM()
3997 addr = read_cpu_reg_sp(s, a->rn, true); in trans_STZGM()
3998 tcg_gen_addi_i64(addr, addr, a->imm); in trans_STZGM()
3999 tcg_rt = cpu_reg(s, a->rt); in trans_STZGM()
4001 if (s->ata[0]) { in trans_STZGM()
4005 * The non-tags portion of STZGM is mostly like DC_ZVA, in trans_STZGM()
4009 tcg_gen_andi_i64(clean_addr, clean_addr, -size); in trans_STZGM()
4021 if (s->current_el == 0) { in trans_STGM()
4025 if (a->rn == 31) { in trans_STGM()
4029 addr = read_cpu_reg_sp(s, a->rn, true); in trans_STGM()
4030 tcg_gen_addi_i64(addr, addr, a->imm); in trans_STGM()
4031 tcg_rt = cpu_reg(s, a->rt); in trans_STGM()
4033 if (s->ata[0]) { in trans_STGM()
4037 int size = 4 << s->gm_blocksize; in trans_STGM()
4040 tcg_gen_andi_i64(clean_addr, clean_addr, -size); in trans_STGM()
4053 if (s->current_el == 0) { in trans_LDGM()
4057 if (a->rn == 31) { in trans_LDGM()
4061 addr = read_cpu_reg_sp(s, a->rn, true); in trans_LDGM()
4062 tcg_gen_addi_i64(addr, addr, a->imm); in trans_LDGM()
4063 tcg_rt = cpu_reg(s, a->rt); in trans_LDGM()
4065 if (s->ata[0]) { in trans_LDGM()
4069 int size = 4 << s->gm_blocksize; in trans_LDGM()
4072 tcg_gen_andi_i64(clean_addr, clean_addr, -size); in trans_LDGM()
4088 if (a->rn == 31) { in trans_LDG()
4092 addr = read_cpu_reg_sp(s, a->rn, true); in trans_LDG()
4093 if (!a->p) { in trans_LDG()
4094 /* pre-index or signed offset */ in trans_LDG()
4095 tcg_gen_addi_i64(addr, addr, a->imm); in trans_LDG()
4098 tcg_gen_andi_i64(addr, addr, -TAG_GRANULE); in trans_LDG()
4099 tcg_rt = cpu_reg(s, a->rt); in trans_LDG()
4100 if (s->ata[0]) { in trans_LDG()
4112 if (a->w) { in trans_LDG()
4113 /* pre-index or post-index */ in trans_LDG()
4114 if (a->p) { in trans_LDG()
4115 /* post-index */ in trans_LDG()
4116 tcg_gen_addi_i64(addr, addr, a->imm); in trans_LDG()
4118 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), addr); in trans_LDG()
4127 if (a->rn == 31) { in do_STG()
4131 addr = read_cpu_reg_sp(s, a->rn, true); in do_STG()
4132 if (!a->p) { in do_STG()
4133 /* pre-index or signed offset */ in do_STG()
4134 tcg_gen_addi_i64(addr, addr, a->imm); in do_STG()
4136 tcg_rt = cpu_reg_sp(s, a->rt); in do_STG()
4137 if (!s->ata[0]) { in do_STG()
4141 * at least for system mode; user-only won't enforce alignment. in do_STG()
4148 } else if (tb_cflags(s->base.tb) & CF_PARALLEL) { in do_STG()
4171 /* This is 1 or 2 atomic 16-byte operations. */ in do_STG()
4179 if (a->w) { in do_STG()
4180 /* pre-index or post-index */ in do_STG()
4181 if (a->p) { in do_STG()
4182 /* post-index */ in do_STG()
4183 tcg_gen_addi_i64(addr, addr, a->imm); in do_STG()
4185 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), addr); in do_STG()
4212 if (a->rs == a->rn || a->rs == a->rd || a->rn == a->rd || in do_SET()
4213 a->rd == 31 || a->rn == 31) { in do_SET()
4217 memidx = get_a64_user_mem_index(s, a->unpriv); in do_SET()
4223 syndrome = syn_mop(true, is_setg, (a->nontemp << 1) | a->unpriv, in do_SET()
4224 is_epilogue, false, true, a->rd, a->rs, a->rn); in do_SET()
4226 if (is_setg ? s->ata[a->unpriv] : s->mte_active[a->unpriv]) { in do_SET()
4228 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid); in do_SET()
4229 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma); in do_SET()
4258 bool wunpriv = extract32(a->options, 0, 1); in do_CPY()
4259 bool runpriv = extract32(a->options, 1, 1); in do_CPY()
4266 if (a->rs == a->rn || a->rs == a->rd || a->rn == a->rd || in do_CPY()
4267 a->rd == 31 || a->rs == 31 || a->rn == 31) { in do_CPY()
4278 syndrome = syn_mop(false, false, a->options, is_epilogue, in do_CPY()
4279 false, true, a->rd, a->rs, a->rn); in do_CPY()
4282 if (s->mte_active[runpriv]) { in do_CPY()
4283 rdesc = FIELD_DP32(rdesc, MTEDESC, TBI, s->tbid); in do_CPY()
4284 rdesc = FIELD_DP32(rdesc, MTEDESC, TCMA, s->tcma); in do_CPY()
4286 if (s->mte_active[wunpriv]) { in do_CPY()
4287 wdesc = FIELD_DP32(wdesc, MTEDESC, TBI, s->tbid); in do_CPY()
4288 wdesc = FIELD_DP32(wdesc, MTEDESC, TCMA, s->tcma); in do_CPY()
4317 TCGv_i64 tcg_rn = rn_sp ? cpu_reg_sp(s, a->rn) : cpu_reg(s, a->rn); in gen_rri()
4318 TCGv_i64 tcg_rd = rd_sp ? cpu_reg_sp(s, a->rd) : cpu_reg(s, a->rd); in gen_rri()
4319 TCGv_i64 tcg_imm = tcg_constant_i64(a->imm); in gen_rri()
4322 if (!a->sf) { in gen_rri()
4329 * PC-rel. addressing
4334 gen_pc_plus_diff(s, cpu_reg(s, a->rd), a->imm); in trans_ADR()
4340 int64_t offset = (int64_t)a->imm << 12; in trans_ADRP()
4343 offset -= s->pc_curr & 0xfff; in trans_ADRP()
4344 gen_pc_plus_diff(s, cpu_reg(s, a->rd), offset); in trans_ADRP()
4353 TRANS(ADDS_i, gen_rri, a, 0, 1, a->sf ? gen_add64_CC : gen_add32_CC)
4354 TRANS(SUBS_i, gen_rri, a, 0, 1, a->sf ? gen_sub64_CC : gen_sub32_CC)
4366 imm = a->uimm6 << LOG2_TAG_GRANULE; in gen_add_sub_imm_with_tags()
4368 imm = -imm; in gen_add_sub_imm_with_tags()
4371 tcg_rn = cpu_reg_sp(s, a->rn); in gen_add_sub_imm_with_tags()
4372 tcg_rd = cpu_reg_sp(s, a->rd); in gen_add_sub_imm_with_tags()
4374 if (s->ata[0]) { in gen_add_sub_imm_with_tags()
4377 tcg_constant_i32(a->uimm4)); in gen_add_sub_imm_with_tags()
4424 * of between 1 and e-1 non-zero bits, rotated within the in logic_imm_decode_wmask()
4425 * element by between 0 and e-1 bits. in logic_imm_decode_wmask()
4429 * 64 bit elements: immn = 1, imms = <length of run - 1> in logic_imm_decode_wmask()
4430 * 32 bit elements: immn = 0, imms = 0 : <length of run - 1> in logic_imm_decode_wmask()
4431 * 16 bit elements: immn = 0, imms = 10 : <length of run - 1> in logic_imm_decode_wmask()
4432 * 8 bit elements: immn = 0, imms = 110 : <length of run - 1> in logic_imm_decode_wmask()
4433 * 4 bit elements: immn = 0, imms = 1110 : <length of run - 1> in logic_imm_decode_wmask()
4434 * 2 bit elements: immn = 0, imms = 11110 : <length of run - 1> in logic_imm_decode_wmask()
4437 * Further, <length of run - 1> all-ones is a reserved pattern. in logic_imm_decode_wmask()
4443 len = 31 - clz32((immn << 6) | (~imms & 0x3f)); in logic_imm_decode_wmask()
4450 levels = e - 1; in logic_imm_decode_wmask()
4455 /* <length of run - 1> mustn't be all-ones. */ in logic_imm_decode_wmask()
4464 mask = (mask >> r) | (mask << (e - r)); in logic_imm_decode_wmask()
4480 if (!logic_imm_decode_wmask(&imm, extract32(a->dbm, 12, 1), in gen_rri_log()
4481 extract32(a->dbm, 0, 6), in gen_rri_log()
4482 extract32(a->dbm, 6, 6))) { in gen_rri_log()
4485 if (!a->sf) { in gen_rri_log()
4489 tcg_rd = set_cc ? cpu_reg(s, a->rd) : cpu_reg_sp(s, a->rd); in gen_rri_log()
4490 tcg_rn = cpu_reg(s, a->rn); in gen_rri_log()
4494 gen_logic_CC(a->sf, tcg_rd); in gen_rri_log()
4496 if (!a->sf) { in gen_rri_log()
4513 int pos = a->hw << 4; in TRANS()
4514 tcg_gen_movi_i64(cpu_reg(s, a->rd), (uint64_t)a->imm << pos); in TRANS()
4520 int pos = a->hw << 4; in trans_MOVN()
4521 uint64_t imm = a->imm; in trans_MOVN()
4524 if (!a->sf) { in trans_MOVN()
4527 tcg_gen_movi_i64(cpu_reg(s, a->rd), imm); in trans_MOVN()
4533 int pos = a->hw << 4; in trans_MOVK()
4536 tcg_rd = cpu_reg(s, a->rd); in trans_MOVK()
4537 tcg_im = tcg_constant_i64(a->imm); in trans_MOVK()
4539 if (!a->sf) { in trans_MOVK()
4551 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); in trans_SBFM()
4552 TCGv_i64 tcg_tmp = read_cpu_reg(s, a->rn, 1); in trans_SBFM()
4553 unsigned int bitsize = a->sf ? 64 : 32; in trans_SBFM()
4554 unsigned int ri = a->immr; in trans_SBFM()
4555 unsigned int si = a->imms; in trans_SBFM()
4559 /* Wd<s-r:0> = Wn<s:r> */ in trans_SBFM()
4560 len = (si - ri) + 1; in trans_SBFM()
4562 if (!a->sf) { in trans_SBFM()
4566 /* Wd<32+s-r,32-r> = Wn<s:0> */ in trans_SBFM()
4568 pos = (bitsize - ri) & (bitsize - 1); in trans_SBFM()
4582 * bitsize, therefore no final zero-extension is unneeded for !sf. in trans_SBFM()
4591 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); in trans_UBFM()
4592 TCGv_i64 tcg_tmp = read_cpu_reg(s, a->rn, 1); in trans_UBFM()
4593 unsigned int bitsize = a->sf ? 64 : 32; in trans_UBFM()
4594 unsigned int ri = a->immr; in trans_UBFM()
4595 unsigned int si = a->imms; in trans_UBFM()
4598 tcg_rd = cpu_reg(s, a->rd); in trans_UBFM()
4599 tcg_tmp = read_cpu_reg(s, a->rn, 1); in trans_UBFM()
4602 /* Wd<s-r:0> = Wn<s:r> */ in trans_UBFM()
4603 len = (si - ri) + 1; in trans_UBFM()
4606 /* Wd<32+s-r,32-r> = Wn<s:0> */ in trans_UBFM()
4608 pos = (bitsize - ri) & (bitsize - 1); in trans_UBFM()
4616 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); in trans_BFM()
4617 TCGv_i64 tcg_tmp = read_cpu_reg(s, a->rn, 1); in trans_BFM()
4618 unsigned int bitsize = a->sf ? 64 : 32; in trans_BFM()
4619 unsigned int ri = a->immr; in trans_BFM()
4620 unsigned int si = a->imms; in trans_BFM()
4623 tcg_rd = cpu_reg(s, a->rd); in trans_BFM()
4624 tcg_tmp = read_cpu_reg(s, a->rn, 1); in trans_BFM()
4627 /* Wd<s-r:0> = Wn<s:r> */ in trans_BFM()
4629 len = (si - ri) + 1; in trans_BFM()
4632 /* Wd<32+s-r,32-r> = Wn<s:0> */ in trans_BFM()
4634 pos = (bitsize - ri) & (bitsize - 1); in trans_BFM()
4638 if (!a->sf) { in trans_BFM()
4648 tcg_rd = cpu_reg(s, a->rd); in trans_EXTR()
4650 if (unlikely(a->imm == 0)) { in trans_EXTR()
4655 if (a->sf) { in trans_EXTR()
4656 tcg_gen_mov_i64(tcg_rd, cpu_reg(s, a->rm)); in trans_EXTR()
4658 tcg_gen_ext32u_i64(tcg_rd, cpu_reg(s, a->rm)); in trans_EXTR()
4661 tcg_rm = cpu_reg(s, a->rm); in trans_EXTR()
4662 tcg_rn = cpu_reg(s, a->rn); in trans_EXTR()
4664 if (a->sf) { in trans_EXTR()
4666 tcg_gen_extract2_i64(tcg_rd, tcg_rm, tcg_rn, a->imm); in trans_EXTR()
4671 if (a->rm == a->rn) { in trans_EXTR()
4672 tcg_gen_rotri_i32(t0, t0, a->imm); in trans_EXTR()
4676 tcg_gen_extract2_i32(t0, t0, t1, a->imm); in trans_EXTR()
4687 int len = (a->len + 1) * 16; in trans_TBL_TBX()
4689 tcg_gen_gvec_2_ptr(vec_full_reg_offset(s, a->rd), in trans_TBL_TBX()
4690 vec_full_reg_offset(s, a->rm), tcg_env, in trans_TBL_TBX()
4691 a->q ? 16 : 8, vec_full_reg_size(s), in trans_TBL_TBX()
4692 (len << 6) | (a->tbx << 5) | a->rn, in trans_TBL_TBX()
4703 MemOp esz = a->esz; in do_simd_permute()
4704 int datasize = a->q ? 16 : 8; in do_simd_permute()
4708 if (esz == MO_64 && !a->q) { in do_simd_permute()
4716 tcg_res[1] = a->q ? tcg_temp_new_i64() : NULL; in do_simd_permute()
4723 read_vec_element(s, tcg_ele, (idx & elements ? a->rm : a->rn), in do_simd_permute()
4724 idx & (elements - 1), esz); in do_simd_permute()
4735 for (int i = a->q; i >= 0; --i) { in do_simd_permute()
4736 write_vec_element(s, tcg_res[i], a->rd, i, MO_64); in do_simd_permute()
4738 clear_vec_high(s, a->q, a->rd); in do_simd_permute()
4814 read_vec_element_i32(s, tcg_op1, a->rn, 3, MO_32); in TRANS_FEAT()
4815 read_vec_element_i32(s, tcg_op2, a->rm, 3, MO_32); in TRANS_FEAT()
4816 read_vec_element_i32(s, tcg_op3, a->ra, 3, MO_32); in TRANS_FEAT()
4825 dofs = vec_full_reg_offset(s, a->rd); in TRANS_FEAT()
4827 write_vec_element_i32(s, tcg_res, a->rd, 3, MO_32); in TRANS_FEAT()
4835 gen_gvec_op3_ool(s, true, a->rd, a->rn, a->rm, a->imm, fn); in do_crypto3i()
4850 gen_gvec_xar(MO_64, vec_full_reg_offset(s, a->rd), in TRANS_FEAT()
4851 vec_full_reg_offset(s, a->rn), in TRANS_FEAT()
4852 vec_full_reg_offset(s, a->rm), a->imm, 16, in TRANS_FEAT()
4878 if (!decode_esz_idx(a->imm, &esz, &idx)) { in trans_DUP_element_s()
4884 * zero-extends it into the bottom of the destination register. in trans_DUP_element_s()
4887 read_vec_element(s, tmp, a->rn, idx, esz); in trans_DUP_element_s()
4888 write_fp_dreg(s, a->rd, tmp); in trans_DUP_element_s()
4898 if (!decode_esz_idx(a->imm, &esz, &idx)) { in trans_DUP_element_v()
4901 if (esz == MO_64 && !a->q) { in trans_DUP_element_v()
4905 tcg_gen_gvec_dup_mem(esz, vec_full_reg_offset(s, a->rd), in trans_DUP_element_v()
4906 vec_reg_offset(s, a->rn, idx, esz), in trans_DUP_element_v()
4907 a->q ? 16 : 8, vec_full_reg_size(s)); in trans_DUP_element_v()
4917 if (!decode_esz_idx(a->imm, &esz, &idx)) { in trans_DUP_general()
4920 if (esz == MO_64 && !a->q) { in trans_DUP_general()
4924 tcg_gen_gvec_dup_i64(esz, vec_full_reg_offset(s, a->rd), in trans_DUP_general()
4925 a->q ? 16 : 8, vec_full_reg_size(s), in trans_DUP_general()
4926 cpu_reg(s, a->rn)); in trans_DUP_general()
4936 if (!decode_esz_idx(a->imm, &esz, &idx)) { in do_smov_umov()
4940 if (esz == MO_64 || (esz == MO_32 && !a->q)) { in do_smov_umov()
4944 if (esz == MO_64 ? !a->q : a->q) { in do_smov_umov()
4949 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); in do_smov_umov()
4950 read_vec_element(s, tcg_rd, a->rn, idx, esz | is_signed); in do_smov_umov()
4951 if (is_signed && !a->q) { in do_smov_umov()
4966 if (!decode_esz_idx(a->imm, &esz, &idx)) { in TRANS()
4970 write_vec_element(s, cpu_reg(s, a->rn), a->rd, idx, esz); in TRANS()
4971 clear_vec_high(s, true, a->rd); in TRANS()
4981 if (!decode_esz_idx(a->di, &esz, &didx)) { in trans_INS_element()
4984 sidx = a->si >> esz; in trans_INS_element()
4988 read_vec_element(s, tmp, a->rn, sidx, esz); in trans_INS_element()
4989 write_vec_element(s, tmp, a->rd, didx, esz); in trans_INS_element()
4991 /* INS is considered a 128-bit write for SVE. */ in trans_INS_element()
4992 clear_vec_high(s, true, a->rd); in trans_INS_element()
5009 switch (a->esz) { in do_fp3_scalar()
5012 TCGv_i64 t0 = read_fp_dreg(s, a->rn); in do_fp3_scalar()
5013 TCGv_i64 t1 = read_fp_dreg(s, a->rm); in do_fp3_scalar()
5014 f->gen_d(t0, t0, t1, fpstatus_ptr(FPST_FPCR)); in do_fp3_scalar()
5015 write_fp_dreg(s, a->rd, t0); in do_fp3_scalar()
5020 TCGv_i32 t0 = read_fp_sreg(s, a->rn); in do_fp3_scalar()
5021 TCGv_i32 t1 = read_fp_sreg(s, a->rm); in do_fp3_scalar()
5022 f->gen_s(t0, t0, t1, fpstatus_ptr(FPST_FPCR)); in do_fp3_scalar()
5023 write_fp_sreg(s, a->rd, t0); in do_fp3_scalar()
5031 TCGv_i32 t0 = read_fp_hreg(s, a->rn); in do_fp3_scalar()
5032 TCGv_i32 t1 = read_fp_hreg(s, a->rm); in do_fp3_scalar()
5033 f->gen_h(t0, t0, t1, fpstatus_ptr(FPST_FPCR_F16)); in do_fp3_scalar()
5034 write_fp_sreg(s, a->rd, t0); in do_fp3_scalar()
5211 MemOp esz = a->esz; in do_satacc_s()
5221 read_vec_element(s, t1, a->rn, 0, esz | sgn_n); in do_satacc_s()
5222 read_vec_element(s, t2, a->rm, 0, esz | sgn_m); in do_satacc_s()
5232 write_fp_dreg(s, a->rd, t0); in do_satacc_s()
5251 read_vec_element(s, t0, a->rn, 0, MO_64); in TRANS()
5252 read_vec_element(s, t1, a->rm, 0, MO_64); in TRANS()
5254 write_fp_dreg(s, a->rd, t0); in TRANS()
5276 if (a->esz == MO_64) { in do_env_scalar2()
5277 TCGv_i64 t0 = read_fp_dreg(s, a->rn); in do_env_scalar2()
5278 TCGv_i64 t1 = read_fp_dreg(s, a->rm); in do_env_scalar2()
5279 f->gen_d(t0, tcg_env, t0, t1); in do_env_scalar2()
5280 write_fp_dreg(s, a->rd, t0); in do_env_scalar2()
5285 read_vec_element_i32(s, t0, a->rn, 0, a->esz); in do_env_scalar2()
5286 read_vec_element_i32(s, t1, a->rm, 0, a->esz); in do_env_scalar2()
5287 f->gen_bhs[a->esz](t0, tcg_env, t0, t1); in do_env_scalar2()
5288 write_fp_sreg(s, a->rd, t0); in do_env_scalar2()
5328 if (a->esz == MO_16 || a->esz == MO_32) { in do_env_scalar2_hs()
5353 if (a->esz != MO_16 && a->esz != MO_32) { in do_env_scalar3_hs()
5363 read_vec_element_i32(s, t0, a->rn, 0, a->esz); in do_env_scalar3_hs()
5364 read_vec_element_i32(s, t1, a->rm, 0, a->esz); in do_env_scalar3_hs()
5365 read_vec_element_i32(s, t2, a->rd, 0, a->esz); in do_env_scalar3_hs()
5366 f->gen_hs[a->esz - 1](t0, tcg_env, t0, t1, t2); in do_env_scalar3_hs()
5367 write_fp_sreg(s, a->rd, t0); in do_env_scalar3_hs()
5384 TCGv_i64 t0 = read_fp_dreg(s, a->rn); in do_cmop_d()
5385 TCGv_i64 t1 = read_fp_dreg(s, a->rm); in do_cmop_d()
5387 write_fp_dreg(s, a->rd, t0); in do_cmop_d()
5402 MemOp esz = a->esz; in TRANS()
5406 if (!a->q) { in TRANS()
5421 gen_gvec_op3_fpst(s, a->q, a->rd, a->rn, a->rm, in TRANS()
5422 esz == MO_16, data, fns[esz - 1]); in TRANS()
5599 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, a->rd), in do_fmlal()
5600 vec_full_reg_offset(s, a->rn), in do_fmlal()
5601 vec_full_reg_offset(s, a->rm), tcg_env, in do_fmlal()
5602 a->q ? 16 : 8, vec_full_reg_size(s), in do_fmlal()
5633 TRANS(BSL_v, do_bitsel, a->q, a->rd, a->rd, a->rn, a->rm)
5634 TRANS(BIT_v, do_bitsel, a->q, a->rd, a->rm, a->rn, a->rd)
5635 TRANS(BIF_v, do_bitsel, a->q, a->rd, a->rm, a->rd, a->rn)
5676 if (a->esz == MO_64 && !a->q) { in TRANS()
5680 tcg_gen_gvec_cmp(cond, a->esz, in TRANS()
5681 vec_full_reg_offset(s, a->rd), in TRANS()
5682 vec_full_reg_offset(s, a->rn), in TRANS()
5683 vec_full_reg_offset(s, a->rm), in TRANS()
5684 a->q ? 16 : 8, vec_full_reg_size(s)); in TRANS()
5705 gen_gvec_op4_ool(s, a->q, a->rd, a->rn, a->rm, a->rd, 0, fn); in TRANS()
5714 gen_gvec_op4_env(s, a->q, a->rd, a->rn, a->rm, a->rd, 0, fn); in do_dot_vector_env()
5735 gen_gvec_op4_fpst(s, true, a->rd, a->rn, a->rm, a->rd, false, a->q, in TRANS_FEAT()
5756 switch (a->esz) { in trans_FCMLA_v()
5758 if (!a->q) { in trans_FCMLA_v()
5776 gen_gvec_op4_fpst(s, a->q, a->rd, a->rn, a->rm, a->rd, in trans_FCMLA_v()
5777 a->esz == MO_16, a->rot, fn); in trans_FCMLA_v()
5785 * These read from the top or bottom half of a 128-bit vector.
5786 * After widening, optionally accumulate with a 128-bit vector.
5807 /* There are no 64x64->128 bit operations. */ in do_3op_widening()
5825 top_swap = top ? 0 : half - 1; in do_3op_widening()
5860 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
5863 a->esz, a->q, a->rd, a->rn, a->rm, -1,
5866 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
5869 a->esz, a->q, a->rd, a->rn, a->rm, -1,
5872 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
5875 a->esz, a->q, a->rd, a->rn, a->rm, -1,
5879 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
5882 a->esz, a->q, a->rd, a->rn, a->rm, a->idx,
5885 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
5888 a->esz, a->q, a->rd, a->rn, a->rm, a->idx,
5891 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
5894 a->esz, a->q, a->rd, a->rn, a->rm, a->idx,
5932 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
5935 a->esz, a->q, a->rd, a->rn, a->rm, -1,
5938 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
5941 a->esz, a->q, a->rd, a->rn, a->rm, -1,
5944 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
5947 a->esz, a->q, a->rd, a->rn, a->rm, -1,
5950 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
5953 a->esz, a->q, a->rd, a->rn, a->rm, -1,
6007 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6008 a->esz == MO_16 ? gen_sqdmull_h : gen_sqdmull_s, false)
6010 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6011 a->esz == MO_16 ? gen_sqdmlal_h : gen_sqdmlal_s, true)
6013 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6014 a->esz == MO_16 ? gen_sqdmlsl_h : gen_sqdmlsl_s, true)
6017 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
6018 a->esz == MO_16 ? gen_sqdmull_h : gen_sqdmull_s, false)
6020 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
6021 a->esz == MO_16 ? gen_sqdmlal_h : gen_sqdmlal_s, true)
6023 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
6024 a->esz == MO_16 ? gen_sqdmlsl_h : gen_sqdmlsl_s, true)
6030 MemOp esz = a->esz; in do_addsub_wide()
6032 bool top = a->q; in do_addsub_wide()
6033 int top_swap = top ? 0 : half - 1; in do_addsub_wide()
6036 /* There are no 64x64->128 bit operations. */ in do_addsub_wide()
6049 read_vec_element(s, tcg_op1, a->rm, elt + top_half, esz | sign); in do_addsub_wide()
6050 read_vec_element(s, tcg_op0, a->rn, elt, esz + 1); in do_addsub_wide()
6056 write_vec_element(s, tcg_op0, a->rd, elt, esz + 1); in do_addsub_wide()
6058 clear_vec_high(s, 1, a->rd); in do_addsub_wide()
6071 MemOp esz = a->esz; in TRANS()
6073 bool top = a->q; in TRANS()
6075 uint64_t rbit = 1ull << (ebits - 1); in TRANS()
6078 /* There are no 128x128->64 bit operations. */ in TRANS()
6093 top_swap = top ? half - 1 : 0; in TRANS()
6099 read_vec_element(s, tcg_op1, a->rm, elt, esz + 1); in TRANS()
6100 read_vec_element(s, tcg_op0, a->rn, elt, esz + 1); in TRANS()
6110 write_vec_element(s, tcg_op0, a->rd, elt + top_half, esz); in TRANS()
6112 clear_vec_high(s, top, a->rd); in TRANS()
6125 gen_gvec_op3_ool(s, true, a->rd, a->rn, a->rm, a->q, fn); in TRANS()
6139 switch (a->esz) { in TRANS()
6142 TCGv_i64 t0 = read_fp_dreg(s, a->rn); in TRANS()
6145 read_vec_element(s, t1, a->rm, a->idx, MO_64); in TRANS()
6146 f->gen_d(t0, t0, t1, fpstatus_ptr(FPST_FPCR)); in TRANS()
6147 write_fp_dreg(s, a->rd, t0); in TRANS()
6152 TCGv_i32 t0 = read_fp_sreg(s, a->rn); in TRANS()
6155 read_vec_element_i32(s, t1, a->rm, a->idx, MO_32); in TRANS()
6156 f->gen_s(t0, t0, t1, fpstatus_ptr(FPST_FPCR)); in TRANS()
6157 write_fp_sreg(s, a->rd, t0); in TRANS()
6165 TCGv_i32 t0 = read_fp_hreg(s, a->rn); in TRANS()
6168 read_vec_element_i32(s, t1, a->rm, a->idx, MO_16); in TRANS()
6169 f->gen_h(t0, t0, t1, fpstatus_ptr(FPST_FPCR_F16)); in TRANS()
6170 write_fp_sreg(s, a->rd, t0); in TRANS()
6182 static bool do_fmla_scalar_idx(DisasContext *s, arg_rrx_e *a, bool neg) in do_fmla_scalar_idx() argument
6184 switch (a->esz) { in do_fmla_scalar_idx()
6187 TCGv_i64 t0 = read_fp_dreg(s, a->rd); in do_fmla_scalar_idx()
6188 TCGv_i64 t1 = read_fp_dreg(s, a->rn); in do_fmla_scalar_idx()
6191 read_vec_element(s, t2, a->rm, a->idx, MO_64); in do_fmla_scalar_idx()
6192 if (neg) { in do_fmla_scalar_idx()
6196 write_fp_dreg(s, a->rd, t0); in do_fmla_scalar_idx()
6201 TCGv_i32 t0 = read_fp_sreg(s, a->rd); in do_fmla_scalar_idx()
6202 TCGv_i32 t1 = read_fp_sreg(s, a->rn); in do_fmla_scalar_idx()
6205 read_vec_element_i32(s, t2, a->rm, a->idx, MO_32); in do_fmla_scalar_idx()
6206 if (neg) { in do_fmla_scalar_idx()
6210 write_fp_sreg(s, a->rd, t0); in do_fmla_scalar_idx()
6218 TCGv_i32 t0 = read_fp_hreg(s, a->rd); in do_fmla_scalar_idx()
6219 TCGv_i32 t1 = read_fp_hreg(s, a->rn); in do_fmla_scalar_idx()
6222 read_vec_element_i32(s, t2, a->rm, a->idx, MO_16); in do_fmla_scalar_idx()
6223 if (neg) { in do_fmla_scalar_idx()
6228 write_fp_sreg(s, a->rd, t0); in do_fmla_scalar_idx()
6243 if (a->esz < MO_16 || a->esz > MO_32) { in TRANS()
6250 read_vec_element_i32(s, t0, a->rn, 0, a->esz); in TRANS()
6251 read_vec_element_i32(s, t1, a->rm, a->idx, a->esz); in TRANS()
6252 f->gen_bhs[a->esz](t0, tcg_env, t0, t1); in TRANS()
6253 write_fp_sreg(s, a->rd, t0); in TRANS()
6264 if (a->esz < MO_16 || a->esz > MO_32) { in do_env_scalar3_idx_hs()
6272 read_vec_element_i32(s, t0, a->rn, 0, a->esz); in do_env_scalar3_idx_hs()
6273 read_vec_element_i32(s, t1, a->rm, a->idx, a->esz); in do_env_scalar3_idx_hs()
6274 read_vec_element_i32(s, t2, a->rd, 0, a->esz); in do_env_scalar3_idx_hs()
6275 f->gen_hs[a->esz - 1](t0, tcg_env, t0, t1, t2); in do_env_scalar3_idx_hs()
6276 write_fp_sreg(s, a->rd, t0); in do_env_scalar3_idx_hs()
6294 read_vec_element(s, t0, a->rd, 0, a->esz + 1); in do_scalar_muladd_widening_idx()
6296 read_vec_element(s, t1, a->rn, 0, a->esz | MO_SIGN); in do_scalar_muladd_widening_idx()
6297 read_vec_element(s, t2, a->rm, a->idx, a->esz | MO_SIGN); in do_scalar_muladd_widening_idx()
6302 dofs = vec_full_reg_offset(s, a->rd); in do_scalar_muladd_widening_idx()
6304 write_vec_element(s, t0, a->rd, 0, a->esz + 1); in do_scalar_muladd_widening_idx()
6310 a->esz == MO_16 ? gen_sqdmull_h : gen_sqdmull_s, false)
6312 a->esz == MO_16 ? gen_sqdmlal_h : gen_sqdmlal_s, true)
6314 a->esz == MO_16 ? gen_sqdmlsl_h : gen_sqdmlsl_s, true)
6319 MemOp esz = a->esz; in do_fp3_vector_idx()
6323 if (!a->q) { in do_fp3_vector_idx()
6338 gen_gvec_op3_fpst(s, a->q, a->rd, a->rn, a->rm, in do_fp3_vector_idx()
6339 esz == MO_16, a->idx, fns[esz - 1]); in do_fp3_vector_idx()
6358 static bool do_fmla_vector_idx(DisasContext *s, arg_qrrx_e *a, bool neg) in TRANS()
6365 MemOp esz = a->esz; in TRANS()
6369 if (!a->q) { in TRANS()
6384 gen_gvec_op4_fpst(s, a->q, a->rd, a->rn, a->rm, a->rd, in TRANS()
6385 esz == MO_16, (a->idx << 1) | neg, in TRANS()
6386 fns[esz - 1]); in TRANS()
6397 int data = (a->idx << 2) | (is_2 << 1) | is_s; in TRANS()
6398 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, a->rd), in TRANS()
6399 vec_full_reg_offset(s, a->rn), in TRANS()
6400 vec_full_reg_offset(s, a->rm), tcg_env, in TRANS()
6401 a->q ? 16 : 8, vec_full_reg_size(s), in TRANS()
6415 assert(a->esz == MO_16 || a->esz == MO_32); in TRANS_FEAT()
6417 gen_gvec_op3_ool(s, a->q, a->rd, a->rn, a->rm, a->idx, fns[a->esz - 1]); in TRANS_FEAT()
6435 assert(a->esz == MO_16 || a->esz == MO_32); in TRANS()
6437 gen_gvec_op4_ool(s, a->q, a->rd, a->rn, a->rm, a->rd, in TRANS()
6438 a->idx, fns[a->esz - 1][sub]); in TRANS()
6449 assert(a->esz == MO_16 || a->esz == MO_32); in TRANS()
6451 tcg_gen_gvec_4_ool(vec_full_reg_offset(s, a->rd), in TRANS()
6452 vec_full_reg_offset(s, a->rn), in TRANS()
6453 vec_full_reg_offset(s, a->rm), in TRANS()
6455 a->q ? 16 : 8, vec_full_reg_size(s), in TRANS()
6456 a->idx, fns[a->esz - 1]); in TRANS()
6491 gen_gvec_op4_ool(s, a->q, a->rd, a->rn, a->rm, a->rd, a->idx, fn); in TRANS_FEAT()
6500 gen_gvec_op4_env(s, a->q, a->rd, a->rn, a->rm, a->rd, a->idx, fn); in do_dot_vector_idx_env()
6521 gen_gvec_op4_fpst(s, true, a->rd, a->rn, a->rm, a->rd, 0, in TRANS_FEAT()
6522 (a->idx << 1) | a->q, in TRANS_FEAT()
6535 switch (a->esz) { in trans_FCMLA_vi()
6549 gen_gvec_op4_fpst(s, a->q, a->rd, a->rn, a->rm, a->rd, in trans_FCMLA_vi()
6550 a->esz == MO_16, (a->idx << 2) | a->rot, fn); in trans_FCMLA_vi()
6561 switch (a->esz) { in do_fp3_scalar_pair()
6567 read_vec_element(s, t0, a->rn, 0, MO_64); in do_fp3_scalar_pair()
6568 read_vec_element(s, t1, a->rn, 1, MO_64); in do_fp3_scalar_pair()
6569 f->gen_d(t0, t0, t1, fpstatus_ptr(FPST_FPCR)); in do_fp3_scalar_pair()
6570 write_fp_dreg(s, a->rd, t0); in do_fp3_scalar_pair()
6578 read_vec_element_i32(s, t0, a->rn, 0, MO_32); in do_fp3_scalar_pair()
6579 read_vec_element_i32(s, t1, a->rn, 1, MO_32); in do_fp3_scalar_pair()
6580 f->gen_s(t0, t0, t1, fpstatus_ptr(FPST_FPCR)); in do_fp3_scalar_pair()
6581 write_fp_sreg(s, a->rd, t0); in do_fp3_scalar_pair()
6592 read_vec_element_i32(s, t0, a->rn, 0, MO_16); in do_fp3_scalar_pair()
6593 read_vec_element_i32(s, t1, a->rn, 1, MO_16); in do_fp3_scalar_pair()
6594 f->gen_h(t0, t0, t1, fpstatus_ptr(FPST_FPCR_F16)); in do_fp3_scalar_pair()
6595 write_fp_sreg(s, a->rd, t0); in do_fp3_scalar_pair()
6616 read_vec_element(s, t0, a->rn, 0, MO_64); in trans_ADDP_s()
6617 read_vec_element(s, t1, a->rn, 1, MO_64); in trans_ADDP_s()
6619 write_fp_dreg(s, a->rd, t0); in trans_ADDP_s()
6625 * Floating-point conditional select
6633 switch (a->esz) { in trans_FCSEL()
6653 read_vec_element(s, t_true, a->rn, 0, a->esz); in trans_FCSEL()
6654 read_vec_element(s, t_false, a->rm, 0, a->esz); in trans_FCSEL()
6656 a64_test_cc(&c, a->cond); in trans_FCSEL()
6662 * and we've already done the zero-extension. in trans_FCSEL()
6664 write_fp_dreg(s, a->rd, t_true); in trans_FCSEL()
6675 TCGv_i64 lo = read_fp_dreg(s, a->rn); in trans_EXT_d()
6676 if (a->imm != 0) { in trans_EXT_d()
6677 TCGv_i64 hi = read_fp_dreg(s, a->rm); in trans_EXT_d()
6678 tcg_gen_extract2_i64(lo, lo, hi, a->imm * 8); in trans_EXT_d()
6680 write_fp_dreg(s, a->rd, lo); in trans_EXT_d()
6688 int pos = (a->imm & 7) * 8; in trans_EXT_q()
6689 int elt = a->imm >> 3; in trans_EXT_q()
6698 read_vec_element(s, lo, a->rn, elt, MO_64); in trans_EXT_q()
6700 read_vec_element(s, hi, elt & 2 ? a->rm : a->rn, elt & 1, MO_64); in trans_EXT_q()
6706 read_vec_element(s, hh, a->rm, elt & 1, MO_64); in trans_EXT_q()
6710 write_vec_element(s, lo, a->rd, 0, MO_64); in trans_EXT_q()
6711 write_vec_element(s, hi, a->rd, 1, MO_64); in trans_EXT_q()
6712 clear_vec_high(s, true, a->rd); in trans_EXT_q()
6717 * Floating-point data-processing (3 source)
6725 * These are fused multiply-add. Note that doing the negations here in do_fmadd()
6727 * its sign bit flipped if it is a negated-input. in do_fmadd()
6729 switch (a->esz) { in do_fmadd()
6732 TCGv_i64 tn = read_fp_dreg(s, a->rn); in do_fmadd()
6733 TCGv_i64 tm = read_fp_dreg(s, a->rm); in do_fmadd()
6734 TCGv_i64 ta = read_fp_dreg(s, a->ra); in do_fmadd()
6744 write_fp_dreg(s, a->rd, ta); in do_fmadd()
6750 TCGv_i32 tn = read_fp_sreg(s, a->rn); in do_fmadd()
6751 TCGv_i32 tm = read_fp_sreg(s, a->rm); in do_fmadd()
6752 TCGv_i32 ta = read_fp_sreg(s, a->ra); in do_fmadd()
6762 write_fp_sreg(s, a->rd, ta); in do_fmadd()
6771 TCGv_i32 tn = read_fp_hreg(s, a->rn); in do_fmadd()
6772 TCGv_i32 tm = read_fp_hreg(s, a->rm); in do_fmadd()
6773 TCGv_i32 ta = read_fp_hreg(s, a->ra); in do_fmadd()
6783 write_fp_sreg(s, a->rd, ta); in do_fmadd()
6806 MemOp src_mop = a->esz | src_sign; in TRANS()
6807 int elements = (a->q ? 16 : 8) >> a->esz; in TRANS()
6820 read_vec_element(s, tcg_res, a->rn, 0, src_mop); in TRANS()
6822 read_vec_element(s, tcg_elt, a->rn, i, src_mop); in TRANS()
6826 tcg_gen_ext_i64(tcg_res, tcg_res, a->esz + widen); in TRANS()
6827 write_fp_dreg(s, a->rd, tcg_res); in TRANS()
6873 MemOp esz = a->esz; in do_fp_reduction()
6874 int elts = (a->q ? 16 : 8) >> esz; in do_fp_reduction()
6876 TCGv_i32 res = do_reduction_op(s, a->rn, esz, 0, elts, fpst, fn); in do_fp_reduction()
6877 write_fp_sreg(s, a->rd, res); in do_fp_reduction()
6893 * Floating-point Immediate in TRANS_FEAT()
6898 switch (a->esz) { in TRANS_FEAT()
6911 uint64_t imm = vfp_expand_imm(a->esz, a->imm); in TRANS_FEAT()
6912 write_fp_dreg(s, a->rd, tcg_constant_i64(imm)); in TRANS_FEAT()
6927 tcg_gen_gvec_dup_imm(MO_16, vec_full_reg_offset(s, a->rd), in trans_FMOVI_v_h()
6928 a->q ? 16 : 8, vec_full_reg_size(s), in trans_FMOVI_v_h()
6929 vfp_expand_imm(MO_16, a->abcdefgh)); in trans_FMOVI_v_h()
6945 if ((a->cmode & 1) && a->cmode < 12) { in trans_Vimm()
6947 fn = a->op ? tcg_gen_gvec_andi : tcg_gen_gvec_ori; in trans_Vimm()
6950 if (a->cmode == 15 && a->op == 1 && a->q == 0) { in trans_Vimm()
6957 uint64_t imm = asimd_imm_const(a->abcdefgh, a->cmode, a->op); in trans_Vimm()
6958 gen_gvec_fn2i(s, a->q, a->rd, a->rd, imm, fn, MO_64); in trans_Vimm()
6970 gen_gvec_fn2i(s, a->q, a->rd, a->rn, a->imm, fn, a->esz); in do_vec_shift_imm()
6993 int esz = a->esz; in TRANS()
7007 read_vec_element(s, tcg_rn, a->rn, a->q, MO_64); in TRANS()
7016 tcg_gen_shli_i64(tcg_rd, tcg_rd, a->imm); in TRANS()
7017 write_vec_element(s, tcg_rd, a->rd, i, esz + 1); in TRANS()
7019 clear_vec_high(s, true, a->rd); in TRANS()
7058 TCGv_i64 rnd = tcg_constant_i64(1ull << (shift - 1)); in gen_srshr_bhs()
7070 TCGv_i64 rnd = tcg_constant_i64(1ull << (shift - 1)); in gen_urshr_bhs()
7084 /* Extension of sign bit (0,-1) plus sign bit (0,1) is zero. */ in gen_srshr_d()
7088 tcg_gen_extract_i64(rnd, src, shift - 1, 1); in gen_srshr_d()
7104 tcg_gen_extract_i64(rnd, src, shift - 1, 1); in gen_urshr_d()
7127 tcg_gen_deposit_i64(dst, dst, src, 0, 64 - shift); in gen_sri_d()
7133 tcg_gen_deposit_i64(dst, dst, src, shift, 64 - shift); in gen_sli_d()
7140 int esz = a->esz; in do_vec_shift_imm_narrow()
7157 read_vec_element(s, tcg_rn, a->rn, i, (esz + 1) | sign); in do_vec_shift_imm_narrow()
7158 fn(tcg_rn, tcg_rn, a->imm); in do_vec_shift_imm_narrow()
7162 write_vec_element(s, tcg_rd, a->rd, a->q, MO_64); in do_vec_shift_imm_narrow()
7163 clear_vec_high(s, a->q, a->rd); in do_vec_shift_imm_narrow()
7351 read_vec_element(s, rn, a->rn, 0, a->esz | sign); in TRANS()
7353 read_vec_element(s, rd, a->rd, 0, a->esz | sign); in TRANS()
7355 fn(rd, rn, a->imm); in TRANS()
7356 write_fp_dreg(s, a->rd, rd); in TRANS()
7455 /* Note that the helpers sign-extend their inputs, so don't do it here. */
7456 TRANS(SQSHL_si, do_scalar_shift_imm, a, f_scalar_sqshli[a->esz], false, 0)
7457 TRANS(UQSHL_si, do_scalar_shift_imm, a, f_scalar_uqshli[a->esz], false, 0)
7458 TRANS(SQSHLU_si, do_scalar_shift_imm, a, f_scalar_sqshlui[a->esz], false, 0)
7464 MemOp esz = a->esz; in do_scalar_shift_imm_narrow()
7472 read_vec_element(s, rn, a->rn, 0, (esz + 1) | sign); in do_scalar_shift_imm_narrow()
7473 fns[esz](rd, rn, a->imm); in do_scalar_shift_imm_narrow()
7477 write_fp_dreg(s, a->rd, rd); in do_scalar_shift_imm_narrow()
7551 * +----+-----+-----------+-------+---+------+--------+------+------+
7553 * +----+-----+-----------+-------+---+------+--------+------+------+
7578 * register-register MOV and MVN, so it is worth special casing. in disas_logic_reg()
7643 * +--+--+--+-----------+-----+--+-------+------+------+----+----+
7645 * +--+--+--+-----------+-----+--+-------+------+------+----+----+
7647 * sf: 0 -> 32bit, 1 -> 64bit
7648 * op: 0 -> add , 1 -> sub
7649 * S: 1 -> set flags
7677 /* non-flag setting ops may use SP */ in disas_add_sub_ext_reg()
7715 * +--+--+--+-----------+-----+--+-------+---------+------+------+
7717 * +--+--+--+-----------+-----+--+-------+---------+------+------+
7719 * sf: 0 -> 32bit, 1 -> 64bit
7720 * op: 0 -> add , 1 -> sub
7721 * S: 1 -> set flags
7722 * shift: 00 -> LSL, 01 -> LSR, 10 -> ASR, 11 -> RESERVED
7773 /* Data-processing (3 source)
7776 * +--+------+-----------+------+------+----+------+------+------+
7778 * +--+------+-----------+------+------+----+------+------+------+
7849 /* Special-case MADD with rA == XZR; it is the standard MUL alias */ in disas_data_proc_3src()
7867 * +--+--+--+------------------------+------+-------------+------+-----+
7869 * +--+--+--+------------------------+------+-------------+------+-----+
7904 * +--+--+--+-----------------+--------+-----------+------+--+------+
7906 * +--+--+--+-----------------+--------+-----------+------+--+------+
7930 tcg_gen_shli_i32(cpu_NF, nzcv, 31 - 3); in disas_rotate_right_into_flags()
7940 tcg_gen_shli_i32(cpu_VF, nzcv, 31 - 0); in disas_rotate_right_into_flags()
7947 * +--+--+--+-----------------+---------+----+---------+------+--+------+
7949 * +--+--+--+-----------------+---------+----+---------+------+--+------+
7971 tcg_gen_shli_i32(cpu_VF, tmp, shift - 1); in disas_evaluate_into_flags()
7978 * +--+--+--+------------------------+--------+------+----+--+------+--+-----+
7980 * +--+--+--+------------------------+--------+------+----+--+------+--+-----+
8029 * to help with this: T1 = (COND ? 0 : -1), T2 = (COND ? -1 : 0). in disas_cc()
8078 * +----+----+---+-----------------+------+------+-----+------+------+
8080 * +----+----+---+-----------------+------+------+-----+------+------+
8228 /* Data-processing (1 source)
8230 * +----+---+---+-----------------+---------+--------+------+------+
8232 * +----+---+---+-----------------+---------+--------+------+------+
8277 if (s->pauth_active) { in disas_data_proc_1src()
8285 if (s->pauth_active) { in disas_data_proc_1src()
8293 if (s->pauth_active) { in disas_data_proc_1src()
8301 if (s->pauth_active) { in disas_data_proc_1src()
8309 if (s->pauth_active) { in disas_data_proc_1src()
8317 if (s->pauth_active) { in disas_data_proc_1src()
8325 if (s->pauth_active) { in disas_data_proc_1src()
8333 if (s->pauth_active) { in disas_data_proc_1src()
8343 } else if (s->pauth_active) { in disas_data_proc_1src()
8351 } else if (s->pauth_active) { in disas_data_proc_1src()
8359 } else if (s->pauth_active) { in disas_data_proc_1src()
8367 } else if (s->pauth_active) { in disas_data_proc_1src()
8375 } else if (s->pauth_active) { in disas_data_proc_1src()
8383 } else if (s->pauth_active) { in disas_data_proc_1src()
8391 } else if (s->pauth_active) { in disas_data_proc_1src()
8399 } else if (s->pauth_active) { in disas_data_proc_1src()
8407 } else if (s->pauth_active) { in disas_data_proc_1src()
8415 } else if (s->pauth_active) { in disas_data_proc_1src()
8515 /* Data-processing (2 source)
8517 * +----+---+---+-----------------+------+--------+------+------+
8519 * +----+---+---+-----------------+------+--------+------+------+
8566 if (s->ata[0]) { in disas_data_proc_2src()
8626 * Data processing - register
8628 * +--+---+--+---+-------+-----+-------+-------+---------+
8630 * +--+---+--+---+-------+-----+-------+-------+---------+
8687 case 0x6: /* Data-processing */ in disas_data_proc_reg()
8762 * +---+---+---+-----------+------+---+------+-----+---------+------+-------+
8764 * +---+---+---+-----------+------+---+------+-----+---------+------+-------+
8811 * +---+---+---+-----------+------+---+------+------+-----+------+----+------+
8813 * +---+---+---+-----------+------+---+------+------+-----+------+----+------+
8873 /* Floating-point data-processing (1 source) - half precision */
8923 /* Floating-point data-processing (1 source) - single precision */
8929 int rmode = -1; in handle_fp_1src_single()
8995 /* Floating-point data-processing (1 source) - double precision */
9001 int rmode = -1; in handle_fp_1src_double()
9132 /* Floating point data-processing (1 source)
9134 * +---+---+---+-----------+------+---+--------+-----------+------+------+
9136 * +---+---+---+-----------+------+---+--------+-----------+------+------+
9174 /* 32-to-32 and 64-to-64 ops */ in disas_fp_1src()
9241 tcg_shift = tcg_constant_i32(64 - scale); in handle_fpfpcvt()
9389 /* Floating point <-> fixed point conversions
9391 * +----+---+---+-----------+------+---+-------+--------+-------+------+------+
9393 * +----+---+---+-----------+------+---+-------+--------+-------+------+------+
9522 /* Floating point <-> integer conversions
9524 * +----+---+---+-----------+------+---+-------+-----+-------------+----+----+
9526 * +----+---+---+-----------+------+---+-------+-----+-------------+----+----+
9576 case 0b01100110: /* FMOV half <-> 32-bit int */ in disas_fp_int_conv()
9578 case 0b11100110: /* FMOV half <-> 64-bit int */ in disas_fp_int_conv()
9584 case 0b00000110: /* FMOV 32-bit */ in disas_fp_int_conv()
9586 case 0b10100110: /* FMOV 64-bit */ in disas_fp_int_conv()
9588 case 0b11001110: /* FMOV top half of 128-bit */ in disas_fp_int_conv()
9614 /* FP-specific subcases of table C3-6 (SIMD and FP data processing)
9616 * +---+---+---+---------+-----------------------------+
9618 * +---+---+---+---------+-----------------------------+
9634 /* Floating point data-processing (2 source) */ in disas_data_proc_fp()
9652 /* Floating point data-processing (1 source) */ in disas_data_proc_fp()
9659 /* Floating point <-> integer conversions */ in disas_data_proc_fp()
9760 /* UCVTF/SCVTF - Integer to FP conversion */
9795 fracbits = (16 << size) - immhb; in handle_simd_shift_intfp_conv()
9804 /* FCVTZS, FVCVTZU - FP to fixedpoint conversion */
9843 fracbits = (16 << size) - immhb; in handle_simd_shift_fpint_conv()
9908 * +-----+---+-------------+------+------+--------+---+------+------+
9910 * +-----+---+-------------+------+------+--------+---+------+------+
9958 /* Handle 64->64 opcodes which are shared between the scalar and in handle_2misc_64()
9959 * vector 2-reg-misc groups. We cover every integer opcode where size == 3 in handle_2misc_64()
9960 * is valid in either group and also the double-precision fp ops. in handle_2misc_64()
9990 /* 64 bit integer comparison against zero, result is test ? -1 : 0. */ in handle_2misc_64()
9999 case 0xb: /* ABS, NEG */ in handle_2misc_64()
10259 /* Handle 2-reg-misc ops which are narrowing (so each 2*size element in handle_2misc_narrow()
10374 * +-----+---+-----------+------+-----------+--------+-----+------+------+
10376 * +-----+---+-----------+------+-----------+--------+-----+------+------+
10401 case 0xb: /* ABS, NEG */ in disas_simd_scalar_two_reg_misc()
10482 handle_2misc_narrow(s, true, opcode, u, false, size - 1, rn, rd); in disas_simd_scalar_two_reg_misc()
10562 * +---+---+---+-------------+------+------+--------+---+------+------+
10564 * +---+---+---+-------------+------+------+--------+---+------+------+
10611 /* Handle 2-reg-misc ops which are widening (so each size element in handle_2misc_widening()
10618 /* 32 -> 64 bit fp conversion */ in handle_2misc_widening()
10633 /* 16 -> 32 bit fp conversion */ in handle_2misc_widening()
10657 int grp_size = 3 - opsz; in handle_rev()
10695 int revmask = (1 << grp_size) - 1; in handle_rev()
10725 /* Implement the pairwise operations from 2-misc: in handle_2misc_pairwise()
10728 * double-width result element in the output (possibly accumulating). in handle_2misc_pairwise()
10736 /* 32 + 32 -> 64 op */ in handle_2misc_pairwise()
10818 * +---+---+---+-----------+------+-----------+--------+-----+------+------+
10820 * +---+---+---+-----------+------+-----------+--------+-----+------+------+
10831 int rmode = -1; in disas_simd_two_reg_misc()
10900 case 0xb: /* ABS, NEG */ in disas_simd_two_reg_misc()
11009 /* handle_2misc_narrow does a 2*size -> size operation, but these in disas_simd_two_reg_misc()
11015 handle_2misc_narrow(s, false, opcode, 0, is_q, size - 1, rn, rd); in disas_simd_two_reg_misc()
11025 handle_2misc_narrow(s, false, opcode, 0, is_q, size - 1, rn, rd); in disas_simd_two_reg_misc()
11125 if (u) { /* ABS, NEG */ in disas_simd_two_reg_misc()
11134 /* All 64-bit element operations can be shared with scalar 2misc */ in disas_simd_two_reg_misc()
11285 * +---+---+---+---+---------+---+-------------+--------+-----+------+------+
11287 * +---+---+---+---+---------+---+-------------+--------+-----+------+------+
11293 * in the vector form and are un-allocated for the scalar decode. Also
11308 int rmode = -1; in disas_simd_two_reg_misc_fp16()
11550 /* C3.6 Data processing - SIMD, inc Crypto
11567 /* Note that this is called with all non-FP cases from in disas_data_proc_simd()
11568 * table C3-6 so it must UNDEF for entries not specifically in disas_data_proc_simd()
11579 /* C3.6 Data processing - SIMD and floating point */
11597 s->is_nonstreaming = true; in trans_FAIL()
11605 * @btype: PSTATE.BTYPE, and is non-zero
11609 * - branch target identifiers,
11610 * - paciasp, pacibsp,
11611 * - BRK insn
11612 * - HLT insn
11658 case 0xd: /* Data processing - register */ in disas_a64_legacy()
11662 case 0xf: /* Data processing - SIMD and floating point */ in disas_a64_legacy()
11677 CPUARMTBFlags tb_flags = arm_tbflags_from_tb(dc->base.tb); in aarch64_tr_init_disas_context()
11680 dc->isar = &arm_cpu->isar; in aarch64_tr_init_disas_context()
11681 dc->condjmp = 0; in aarch64_tr_init_disas_context()
11682 dc->pc_save = dc->base.pc_first; in aarch64_tr_init_disas_context()
11683 dc->aarch64 = true; in aarch64_tr_init_disas_context()
11684 dc->thumb = false; in aarch64_tr_init_disas_context()
11685 dc->sctlr_b = 0; in aarch64_tr_init_disas_context()
11686 dc->be_data = EX_TBFLAG_ANY(tb_flags, BE_DATA) ? MO_BE : MO_LE; in aarch64_tr_init_disas_context()
11687 dc->condexec_mask = 0; in aarch64_tr_init_disas_context()
11688 dc->condexec_cond = 0; in aarch64_tr_init_disas_context()
11690 dc->mmu_idx = core_to_aa64_mmu_idx(core_mmu_idx); in aarch64_tr_init_disas_context()
11691 dc->tbii = EX_TBFLAG_A64(tb_flags, TBII); in aarch64_tr_init_disas_context()
11692 dc->tbid = EX_TBFLAG_A64(tb_flags, TBID); in aarch64_tr_init_disas_context()
11693 dc->tcma = EX_TBFLAG_A64(tb_flags, TCMA); in aarch64_tr_init_disas_context()
11694 dc->current_el = arm_mmu_idx_to_el(dc->mmu_idx); in aarch64_tr_init_disas_context()
11696 dc->user = (dc->current_el == 0); in aarch64_tr_init_disas_context()
11698 dc->fp_excp_el = EX_TBFLAG_ANY(tb_flags, FPEXC_EL); in aarch64_tr_init_disas_context()
11699 dc->align_mem = EX_TBFLAG_ANY(tb_flags, ALIGN_MEM); in aarch64_tr_init_disas_context()
11700 dc->pstate_il = EX_TBFLAG_ANY(tb_flags, PSTATE__IL); in aarch64_tr_init_disas_context()
11701 dc->fgt_active = EX_TBFLAG_ANY(tb_flags, FGT_ACTIVE); in aarch64_tr_init_disas_context()
11702 dc->fgt_svc = EX_TBFLAG_ANY(tb_flags, FGT_SVC); in aarch64_tr_init_disas_context()
11703 dc->trap_eret = EX_TBFLAG_A64(tb_flags, TRAP_ERET); in aarch64_tr_init_disas_context()
11704 dc->sve_excp_el = EX_TBFLAG_A64(tb_flags, SVEEXC_EL); in aarch64_tr_init_disas_context()
11705 dc->sme_excp_el = EX_TBFLAG_A64(tb_flags, SMEEXC_EL); in aarch64_tr_init_disas_context()
11706 dc->vl = (EX_TBFLAG_A64(tb_flags, VL) + 1) * 16; in aarch64_tr_init_disas_context()
11707 dc->svl = (EX_TBFLAG_A64(tb_flags, SVL) + 1) * 16; in aarch64_tr_init_disas_context()
11708 dc->pauth_active = EX_TBFLAG_A64(tb_flags, PAUTH_ACTIVE); in aarch64_tr_init_disas_context()
11709 dc->bt = EX_TBFLAG_A64(tb_flags, BT); in aarch64_tr_init_disas_context()
11710 dc->btype = EX_TBFLAG_A64(tb_flags, BTYPE); in aarch64_tr_init_disas_context()
11711 dc->unpriv = EX_TBFLAG_A64(tb_flags, UNPRIV); in aarch64_tr_init_disas_context()
11712 dc->ata[0] = EX_TBFLAG_A64(tb_flags, ATA); in aarch64_tr_init_disas_context()
11713 dc->ata[1] = EX_TBFLAG_A64(tb_flags, ATA0); in aarch64_tr_init_disas_context()
11714 dc->mte_active[0] = EX_TBFLAG_A64(tb_flags, MTE_ACTIVE); in aarch64_tr_init_disas_context()
11715 dc->mte_active[1] = EX_TBFLAG_A64(tb_flags, MTE0_ACTIVE); in aarch64_tr_init_disas_context()
11716 dc->pstate_sm = EX_TBFLAG_A64(tb_flags, PSTATE_SM); in aarch64_tr_init_disas_context()
11717 dc->pstate_za = EX_TBFLAG_A64(tb_flags, PSTATE_ZA); in aarch64_tr_init_disas_context()
11718 dc->sme_trap_nonstreaming = EX_TBFLAG_A64(tb_flags, SME_TRAP_NONSTREAMING); in aarch64_tr_init_disas_context()
11719 dc->naa = EX_TBFLAG_A64(tb_flags, NAA); in aarch64_tr_init_disas_context()
11720 dc->nv = EX_TBFLAG_A64(tb_flags, NV); in aarch64_tr_init_disas_context()
11721 dc->nv1 = EX_TBFLAG_A64(tb_flags, NV1); in aarch64_tr_init_disas_context()
11722 dc->nv2 = EX_TBFLAG_A64(tb_flags, NV2); in aarch64_tr_init_disas_context()
11723 dc->nv2_mem_e20 = EX_TBFLAG_A64(tb_flags, NV2_MEM_E20); in aarch64_tr_init_disas_context()
11724 dc->nv2_mem_be = EX_TBFLAG_A64(tb_flags, NV2_MEM_BE); in aarch64_tr_init_disas_context()
11725 dc->vec_len = 0; in aarch64_tr_init_disas_context()
11726 dc->vec_stride = 0; in aarch64_tr_init_disas_context()
11727 dc->cp_regs = arm_cpu->cp_regs; in aarch64_tr_init_disas_context()
11728 dc->features = env->features; in aarch64_tr_init_disas_context()
11729 dc->dcz_blocksize = arm_cpu->dcz_blocksize; in aarch64_tr_init_disas_context()
11730 dc->gm_blocksize = arm_cpu->gm_blocksize; in aarch64_tr_init_disas_context()
11734 tcg_debug_assert(dc->tbid & 1); in aarch64_tr_init_disas_context()
11737 dc->lse2 = dc_isar_feature(aa64_lse2, dc); in aarch64_tr_init_disas_context()
11739 /* Single step state. The code-generation logic here is: in aarch64_tr_init_disas_context()
11741 * generate code with no special handling for single-stepping (except in aarch64_tr_init_disas_context()
11745 * SS_ACTIVE == 1, PSTATE.SS == 1: (active-not-pending) in aarch64_tr_init_disas_context()
11750 * SS_ACTIVE == 1, PSTATE.SS == 0: (active-pending) in aarch64_tr_init_disas_context()
11754 dc->ss_active = EX_TBFLAG_ANY(tb_flags, SS_ACTIVE); in aarch64_tr_init_disas_context()
11755 dc->pstate_ss = EX_TBFLAG_ANY(tb_flags, PSTATE__SS); in aarch64_tr_init_disas_context()
11756 dc->is_ldex = false; in aarch64_tr_init_disas_context()
11759 bound = -(dc->base.pc_first | TARGET_PAGE_MASK) / 4; in aarch64_tr_init_disas_context()
11762 if (dc->ss_active) { in aarch64_tr_init_disas_context()
11765 dc->base.max_insns = MIN(dc->base.max_insns, bound); in aarch64_tr_init_disas_context()
11775 target_ulong pc_arg = dc->base.pc_next; in aarch64_tr_insn_start()
11777 if (tb_cflags(dcbase->tb) & CF_PCREL) { in aarch64_tr_insn_start()
11781 dc->insn_start_updated = false; in aarch64_tr_insn_start()
11788 uint64_t pc = s->base.pc_next; in aarch64_tr_translate_insn()
11792 if (s->ss_active && !s->pstate_ss) { in aarch64_tr_translate_insn()
11793 /* Singlestep state is Active-pending. in aarch64_tr_translate_insn()
11803 assert(s->base.num_insns == 1); in aarch64_tr_translate_insn()
11805 s->base.is_jmp = DISAS_NORETURN; in aarch64_tr_translate_insn()
11806 s->base.pc_next = pc + 4; in aarch64_tr_translate_insn()
11817 assert(s->base.num_insns == 1); in aarch64_tr_translate_insn()
11819 s->base.is_jmp = DISAS_NORETURN; in aarch64_tr_translate_insn()
11820 s->base.pc_next = QEMU_ALIGN_UP(pc, 4); in aarch64_tr_translate_insn()
11824 s->pc_curr = pc; in aarch64_tr_translate_insn()
11825 insn = arm_ldl_code(env, &s->base, pc, s->sctlr_b); in aarch64_tr_translate_insn()
11826 s->insn = insn; in aarch64_tr_translate_insn()
11827 s->base.pc_next = pc + 4; in aarch64_tr_translate_insn()
11829 s->fp_access_checked = 0; in aarch64_tr_translate_insn()
11830 s->sve_access_checked = 0; in aarch64_tr_translate_insn()
11832 if (s->pstate_il) { in aarch64_tr_translate_insn()
11842 if (s->base.num_insns == 1) { in aarch64_tr_translate_insn()
11843 /* First insn can have btype set to non-zero. */ in aarch64_tr_translate_insn()
11844 tcg_debug_assert(s->btype >= 0); in aarch64_tr_translate_insn()
11848 * priority -- below debugging exceptions but above most in aarch64_tr_translate_insn()
11855 if (s->btype != 0 in aarch64_tr_translate_insn()
11856 && !btype_destination_ok(insn, s->bt, s->btype)) { in aarch64_tr_translate_insn()
11861 tcg_debug_assert(s->btype == 0); in aarch64_tr_translate_insn()
11865 s->is_nonstreaming = false; in aarch64_tr_translate_insn()
11866 if (s->sme_trap_nonstreaming) { in aarch64_tr_translate_insn()
11878 * Note that we set btype == -1 when the insn sets btype. in aarch64_tr_translate_insn()
11880 if (s->btype > 0 && s->base.is_jmp != DISAS_NORETURN) { in aarch64_tr_translate_insn()
11889 if (unlikely(dc->ss_active)) { in aarch64_tr_tb_stop()
11893 * (and thus a tb-jump is not possible when singlestepping). in aarch64_tr_tb_stop()
11895 switch (dc->base.is_jmp) { in aarch64_tr_tb_stop()
11907 switch (dc->base.is_jmp) { in aarch64_tr_tb_stop()