Lines Matching +full:build +full:- +full:some +full:- +full:softmmu
21 #include "exec/exec-all.h"
23 #include "translate-a64.h"
75 #include "decode-sme-fa64.c.inc"
76 #include "decode-a64.c.inc"
78 /* Table based decoder typedefs - used when the relevant bits for decode
124 ARMMMUIdx useridx = s->mmu_idx; in get_a64_user_mem_index()
126 if (unpriv && s->unpriv) { in get_a64_user_mem_index()
128 * We have pre-computed the condition for AccType_UNPRIV. in get_a64_user_mem_index()
156 /* BTYPE is a 2-bit field, and 0 should be done with reset_btype. */ in set_btype()
159 s->btype = -1; in set_btype()
164 if (s->btype != 0) { in reset_btype()
166 s->btype = 0; in reset_btype()
172 assert(s->pc_save != -1); in gen_pc_plus_diff()
173 if (tb_cflags(s->base.tb) & CF_PCREL) { in gen_pc_plus_diff()
174 tcg_gen_addi_i64(dest, cpu_pc, (s->pc_curr - s->pc_save) + diff); in gen_pc_plus_diff()
176 tcg_gen_movi_i64(dest, s->pc_curr + diff); in gen_pc_plus_diff()
183 s->pc_save = s->pc_curr + diff; in gen_a64_update_pc()
191 * then the address is zero-extended, clearing bits [63:56]
195 * the address is sign-extended from bit 55 into bits [63:56]
205 } else if (!regime_has_2_ranges(s->mmu_idx)) { in gen_top_byte_ignore()
209 /* Sign-extend from bit 55. */ in gen_top_byte_ignore()
236 gen_top_byte_ignore(s, cpu_pc, src, s->tbii); in gen_a64_set_pc()
237 s->pc_save = -1; in gen_a64_set_pc()
244 * for the tag to be present in the FAR_ELx register. But for user-only
249 * of the write-back address.
256 gen_top_byte_ignore(s, clean, addr, s->tbid); in clean_data_tbi()
289 if (tag_checked && s->mte_active[is_unpriv]) { in gen_mte_check1_mmuidx()
294 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid); in gen_mte_check1_mmuidx()
295 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma); in gen_mte_check1_mmuidx()
298 desc = FIELD_DP32(desc, MTEDESC, SIZEM1, memop_size(memop) - 1); in gen_mte_check1_mmuidx()
321 if (tag_checked && s->mte_active[0]) { in gen_mte_checkN()
326 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid); in gen_mte_checkN()
327 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma); in gen_mte_checkN()
330 desc = FIELD_DP32(desc, MTEDESC, SIZEM1, total_size - 1); in gen_mte_checkN()
343 * naturally aligned, but it must not cross a 16-byte boundary.
386 * If size == MO_128, this is a LDXP, and the operation is single-copy in check_atomic_align()
417 } else if (!s->naa) { in check_ordered_align()
435 * Sign-extend the 32-bit value so that the GE/LT comparisons work in a64_test_cc()
438 c64->cond = c32.cond; in a64_test_cc()
439 c64->value = tcg_temp_new_i64(); in a64_test_cc()
440 tcg_gen_ext_i32_i64(c64->value, c32.value); in a64_test_cc()
445 gen_helper_rebuild_hflags_a64(tcg_env, tcg_constant_i32(s->current_el)); in gen_rebuild_hflags()
458 s->base.is_jmp = DISAS_NORETURN; in gen_exception_internal_insn()
465 s->base.is_jmp = DISAS_NORETURN; in gen_exception_bkpt_insn()
470 /* We just completed step of an insn. Move from Active-not-pending in gen_step_complete_exception()
471 * to Active-pending, and then also take the swstep exception. in gen_step_complete_exception()
480 gen_swstep_exception(s, 1, s->is_ldex); in gen_step_complete_exception()
481 s->base.is_jmp = DISAS_NORETURN; in gen_step_complete_exception()
486 if (s->ss_active) { in use_goto_tb()
489 return translator_use_goto_tb(&s->base, dest); in use_goto_tb()
494 if (use_goto_tb(s, s->pc_curr + diff)) { in gen_goto_tb()
496 * For pcrel, the pc must always be up-to-date on entry to in gen_goto_tb()
503 if (tb_cflags(s->base.tb) & CF_PCREL) { in gen_goto_tb()
510 tcg_gen_exit_tb(s->base.tb, n); in gen_goto_tb()
511 s->base.is_jmp = DISAS_NORETURN; in gen_goto_tb()
514 if (s->ss_active) { in gen_goto_tb()
518 s->base.is_jmp = DISAS_NORETURN; in gen_goto_tb()
556 * representing the register contents. This TCGv is an auto-freed
631 /* Clear the bits above an N-bit vector, for N = (is_q ? 128 : 64).
659 /* Expand a 2-operand AdvSIMD vector operation using an expander function. */
667 /* Expand a 2-operand + immediate AdvSIMD vector operation using
677 /* Expand a 3-operand AdvSIMD vector operation using an expander function. */
685 /* Expand a 4-operand AdvSIMD vector operation using an expander function. */
694 /* Expand a 2-operand operation using an out-of-line helper. */
703 /* Expand a 3-operand operation using an out-of-line helper. */
713 /* Expand a 3-operand + fpstatus pointer + simd data value operation using
714 * an out-of-line helper.
727 /* Expand a 4-operand operation using an out-of-line helper. */
739 * Expand a 4-operand operation using an out-of-line helper that takes
755 * Expand a 4-operand + fpstatus pointer + simd data value operation using
756 * an out-of-line helper.
841 /* dest = T0 - T1; compute C, N, V and Z flags */
1051 /* This always zero-extends and writes to a full 128 bit wide vector */ in do_fp_ld()
1211 * unallocated-encoding checks (otherwise the syndrome information
1216 if (s->fp_excp_el) { in fp_access_check_only()
1217 assert(!s->fp_access_checked); in fp_access_check_only()
1218 s->fp_access_checked = true; in fp_access_check_only()
1222 s->fp_excp_el); in fp_access_check_only()
1225 s->fp_access_checked = true; in fp_access_check_only()
1234 if (s->sme_trap_nonstreaming && s->is_nonstreaming) { in fp_access_check()
1249 if (s->pstate_sm || !dc_isar_feature(aa64_sve, s)) { in sve_access_check()
1254 } else if (s->sve_excp_el) { in sve_access_check()
1256 syn_sve_access_trap(), s->sve_excp_el); in sve_access_check()
1259 s->sve_access_checked = true; in sve_access_check()
1264 assert(!s->sve_access_checked); in sve_access_check()
1265 s->sve_access_checked = true; in sve_access_check()
1276 if (s->sme_excp_el) { in sme_access_check()
1279 s->sme_excp_el); in sme_access_check()
1293 if (!s->fp_excp_el || s->sme_excp_el < s->fp_excp_el) { in sme_enabled_check()
1294 s->fp_access_checked = true; in sme_enabled_check()
1306 if (FIELD_EX64(req, SVCR, SM) && !s->pstate_sm) { in sme_enabled_check_with_svcr()
1311 if (FIELD_EX64(req, SVCR, ZA) && !s->pstate_za) { in sme_enabled_check_with_svcr()
1326 if (!a->q && a->esz == MO_64) { in do_gvec_op2_ool()
1330 gen_gvec_op2_ool(s, a->q, a->rd, a->rn, data, fn); in do_gvec_op2_ool()
1338 if (!a->q && a->esz == MO_64) { in do_gvec_op3_ool()
1342 gen_gvec_op3_ool(s, a->q, a->rd, a->rn, a->rm, data, fn); in do_gvec_op3_ool()
1349 if (!a->q && a->esz == MO_64) { in do_gvec_fn3()
1353 gen_gvec_fn3(s, a->q, a->rd, a->rn, a->rm, fn, a->esz); in do_gvec_fn3()
1360 if (a->esz == MO_64) { in do_gvec_fn3_no64()
1364 gen_gvec_fn3(s, a->q, a->rd, a->rn, a->rm, fn, a->esz); in do_gvec_fn3_no64()
1371 if (a->esz == MO_8) { in do_gvec_fn3_no8_no64()
1379 if (!a->q && a->esz == MO_64) { in do_gvec_fn4()
1383 gen_gvec_fn4(s, a->q, a->rd, a->rn, a->rm, a->ra, fn, a->esz); in do_gvec_fn4()
1406 * or SCTLR bits) there is a check that SP is 16-aligned on every in gen_check_sp_alignment()
1407 * SP-relative load or store (with an exception generated if it is not). in gen_check_sp_alignment()
1432 while (tptr->mask) { in lookup_disas_fn()
1433 if ((insn & tptr->mask) == tptr->pattern) { in lookup_disas_fn()
1434 return tptr->disas_fn; in lookup_disas_fn()
1452 gen_goto_tb(s, 0, a->imm); in trans_B()
1460 gen_goto_tb(s, 0, a->imm); in trans_BL()
1470 tcg_cmp = read_cpu_reg(s, a->rt, a->sf); in trans_CBZ()
1474 tcg_gen_brcondi_i64(a->nz ? TCG_COND_NE : TCG_COND_EQ, in trans_CBZ()
1478 gen_goto_tb(s, 1, a->imm); in trans_CBZ()
1488 tcg_gen_andi_i64(tcg_cmp, cpu_reg(s, a->rt), 1ULL << a->bitpos); in trans_TBZ()
1493 tcg_gen_brcondi_i64(a->nz ? TCG_COND_NE : TCG_COND_EQ, in trans_TBZ()
1497 gen_goto_tb(s, 1, a->imm); in trans_TBZ()
1504 if (a->c && !dc_isar_feature(aa64_hbc, s)) { in trans_B_cond()
1508 if (a->cond < 0x0e) { in trans_B_cond()
1511 arm_gen_test_cc(a->cond, match.label); in trans_B_cond()
1514 gen_goto_tb(s, 1, a->imm); in trans_B_cond()
1517 gen_goto_tb(s, 0, a->imm); in trans_B_cond()
1525 /* BR to {x16,x17} or !guard -> 1, else 3. */ in set_btype_for_br()
1532 s->btype = -1; in set_btype_for_br()
1547 set_btype_for_br(s, a->rn); in trans_BR()
1548 gen_a64_set_pc(s, cpu_reg(s, a->rn)); in trans_BR()
1549 s->base.is_jmp = DISAS_JUMP; in trans_BR()
1555 TCGv_i64 dst = cpu_reg(s, a->rn); in trans_BLR()
1565 s->base.is_jmp = DISAS_JUMP; in trans_BLR()
1571 gen_a64_set_pc(s, cpu_reg(s, a->rn)); in trans_RET()
1572 s->base.is_jmp = DISAS_JUMP; in trans_RET()
1585 if (!s->pauth_active) { in auth_branch_target()
1606 dst = auth_branch_target(s, cpu_reg(s, a->rn), tcg_constant_i64(0), !a->m); in trans_BRAZ()
1607 set_btype_for_br(s, a->rn); in trans_BRAZ()
1609 s->base.is_jmp = DISAS_JUMP; in trans_BRAZ()
1621 dst = auth_branch_target(s, cpu_reg(s, a->rn), tcg_constant_i64(0), !a->m); in trans_BLRAZ()
1631 s->base.is_jmp = DISAS_JUMP; in trans_BLRAZ()
1639 dst = auth_branch_target(s, cpu_reg(s, 30), cpu_X[31], !a->m); in trans_RETA()
1641 s->base.is_jmp = DISAS_JUMP; in trans_RETA()
1652 dst = auth_branch_target(s, cpu_reg(s,a->rn), cpu_reg_sp(s, a->rm), !a->m); in trans_BRA()
1654 set_btype_for_br(s, a->rn); in trans_BRA()
1655 s->base.is_jmp = DISAS_JUMP; in trans_BRA()
1666 dst = auth_branch_target(s, cpu_reg(s, a->rn), cpu_reg_sp(s, a->rm), !a->m); in trans_BLRA()
1676 s->base.is_jmp = DISAS_JUMP; in trans_BLRA()
1684 if (s->current_el == 0) { in trans_ERET()
1687 if (s->trap_eret) { in trans_ERET()
1693 offsetof(CPUARMState, elr_el[s->current_el])); in trans_ERET()
1695 translator_io_start(&s->base); in trans_ERET()
1698 /* Must exit loop to check un-masked IRQs */ in trans_ERET()
1699 s->base.is_jmp = DISAS_EXIT; in trans_ERET()
1710 if (s->current_el == 0) { in trans_ERETA()
1714 if (s->trap_eret) { in trans_ERETA()
1715 gen_exception_insn_el(s, 0, EXCP_UDEF, syn_erettrap(a->m ? 3 : 2), 2); in trans_ERETA()
1720 offsetof(CPUARMState, elr_el[s->current_el])); in trans_ERETA()
1722 dst = auth_branch_target(s, dst, cpu_X[31], !a->m); in trans_ERETA()
1724 translator_io_start(&s->base); in trans_ERETA()
1727 /* Must exit loop to check un-masked IRQs */ in trans_ERETA()
1728 s->base.is_jmp = DISAS_EXIT; in trans_ERETA()
1745 if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) { in trans_YIELD()
1746 s->base.is_jmp = DISAS_YIELD; in trans_YIELD()
1753 s->base.is_jmp = DISAS_WFI; in trans_WFI()
1765 if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) { in trans_WFE()
1766 s->base.is_jmp = DISAS_WFE; in trans_WFE()
1781 * check ss_active so that single-stepping a WFIT doesn't halt. in trans_WFIT()
1783 if (s->ss_active) { in trans_WFIT()
1789 gen_helper_wfit(tcg_env, cpu_reg(s, a->rd)); in trans_WFIT()
1791 s->base.is_jmp = DISAS_EXIT; in trans_WFIT()
1806 if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) { in trans_WFET()
1807 s->base.is_jmp = DISAS_WFE; in trans_WFET()
1814 if (s->pauth_active) { in trans_XPACLRI()
1822 if (s->pauth_active) { in trans_PACIA1716()
1830 if (s->pauth_active) { in trans_PACIB1716()
1838 if (s->pauth_active) { in trans_AUTIA1716()
1846 if (s->pauth_active) { in trans_AUTIB1716()
1865 if (s->current_el <= 1 && arm_dc_feature(s, ARM_FEATURE_EL2)) { in trans_ESB()
1874 if (s->pauth_active) { in trans_PACIAZ()
1882 if (s->pauth_active) { in trans_PACIASP()
1890 if (s->pauth_active) { in trans_PACIBZ()
1898 if (s->pauth_active) { in trans_PACIBSP()
1906 if (s->pauth_active) { in trans_AUTIAZ()
1914 if (s->pauth_active) { in trans_AUTIASP()
1922 if (s->pauth_active) { in trans_AUTIBZ()
1930 if (s->pauth_active) { in trans_AUTIBSP()
1938 tcg_gen_movi_i64(cpu_exclusive_addr, -1); in trans_CLREX()
1947 switch (a->types) { in trans_DSB_DMB()
1966 * self-modifying code correctly and also to take in trans_ISB()
2013 * ~-(C | Z) in trans_XAFLAG()
2014 * (C | Z) - 1 in trans_XAFLAG()
2023 /* (!C & Z) << 31 -> -(Z & ~C) */ in trans_XAFLAG()
2039 tcg_gen_sari_i32(cpu_VF, cpu_VF, 31); /* V ? -1 : 0 */ in trans_AXFLAG()
2042 /* !(Z | V) -> !(!ZF | V) -> ZF & !V -> ZF & ~VF */ in trans_AXFLAG()
2053 if (!dc_isar_feature(aa64_uao, s) || s->current_el == 0) { in trans_MSR_i_UAO()
2056 if (a->imm & 1) { in trans_MSR_i_UAO()
2062 s->base.is_jmp = DISAS_TOO_MANY; in trans_MSR_i_UAO()
2068 if (!dc_isar_feature(aa64_pan, s) || s->current_el == 0) { in trans_MSR_i_PAN()
2071 if (a->imm & 1) { in trans_MSR_i_PAN()
2077 s->base.is_jmp = DISAS_TOO_MANY; in trans_MSR_i_PAN()
2083 if (s->current_el == 0) { in trans_MSR_i_SPSEL()
2086 gen_helper_msr_i_spsel(tcg_env, tcg_constant_i32(a->imm & PSTATE_SP)); in trans_MSR_i_SPSEL()
2087 s->base.is_jmp = DISAS_TOO_MANY; in trans_MSR_i_SPSEL()
2096 if (a->imm & 1) { in trans_MSR_i_SBSS()
2102 s->base.is_jmp = DISAS_TOO_MANY; in trans_MSR_i_SBSS()
2111 if (a->imm & 1) { in trans_MSR_i_DIT()
2117 s->base.is_jmp = DISAS_TOO_MANY; in trans_MSR_i_DIT()
2124 /* Full MTE is enabled -- set the TCO bit as directed. */ in trans_MSR_i_TCO()
2125 if (a->imm & 1) { in trans_MSR_i_TCO()
2132 s->base.is_jmp = DISAS_UPDATE_NOCHAIN; in trans_MSR_i_TCO()
2135 /* Only "instructions accessible at EL0" -- PSTATE.TCO is WI. */ in trans_MSR_i_TCO()
2145 gen_helper_msr_i_daifset(tcg_env, tcg_constant_i32(a->imm)); in trans_MSR_i_DAIFSET()
2146 s->base.is_jmp = DISAS_TOO_MANY; in trans_MSR_i_DAIFSET()
2152 gen_helper_msr_i_daifclear(tcg_env, tcg_constant_i32(a->imm)); in trans_MSR_i_DAIFCLEAR()
2153 /* Exit the cpu loop to re-evaluate pending IRQs. */ in trans_MSR_i_DAIFCLEAR()
2154 s->base.is_jmp = DISAS_UPDATE_EXIT; in trans_MSR_i_DAIFCLEAR()
2160 if (!dc_isar_feature(aa64_nmi, s) || s->current_el == 0) { in trans_MSR_i_ALLINT()
2164 if (a->imm == 0) { in trans_MSR_i_ALLINT()
2166 } else if (s->current_el > 1) { in trans_MSR_i_ALLINT()
2172 /* Exit the cpu loop to re-evaluate pending IRQs. */ in trans_MSR_i_ALLINT()
2173 s->base.is_jmp = DISAS_UPDATE_EXIT; in trans_MSR_i_ALLINT()
2179 if (!dc_isar_feature(aa64_sme, s) || a->mask == 0) { in trans_MSR_i_SVCR()
2183 int old = s->pstate_sm | (s->pstate_za << 1); in trans_MSR_i_SVCR()
2184 int new = a->imm * 3; in trans_MSR_i_SVCR()
2186 if ((old ^ new) & a->mask) { in trans_MSR_i_SVCR()
2189 tcg_constant_i32(a->mask)); in trans_MSR_i_SVCR()
2190 s->base.is_jmp = DISAS_TOO_MANY; in trans_MSR_i_SVCR()
2201 /* build bit 31, N */ in gen_get_nzcv()
2203 /* build bit 30, Z */ in gen_get_nzcv()
2206 /* build bit 29, C */ in gen_get_nzcv()
2208 /* build bit 28, V */ in gen_get_nzcv()
2258 /* MRS - move from system register
2259 * MSR (register) - move to system register
2271 const ARMCPRegInfo *ri = get_arm_cp_reginfo(s->cp_regs, key); in handle_sys()
2286 switch (s->current_el) { in handle_sys()
2309 if (s->nv2 && ri->nv2_redirect_offset) { in handle_sys()
2311 * Some registers always redirect to memory; some only do so if in handle_sys()
2312 * HCR_EL2.NV1 is 0, and some only if NV1 is 1 (these come in in handle_sys()
2315 if (ri->nv2_redirect_offset & NV2_REDIR_NV1) { in handle_sys()
2316 nv2_mem_redirect = s->nv1; in handle_sys()
2317 } else if (ri->nv2_redirect_offset & NV2_REDIR_NO_NV1) { in handle_sys()
2318 nv2_mem_redirect = !s->nv1; in handle_sys()
2325 if (!cp_access_ok(s->current_el, ri, isread)) { in handle_sys()
2332 if (s->nv2 && (ri->type & ARM_CP_NV2_REDIRECT)) { in handle_sys()
2342 * NV2 redirect-to-memory takes precedence over trap to EL2 or in handle_sys()
2345 } else if (s->nv && arm_cpreg_traps_in_nv(ri)) { in handle_sys()
2352 * over the trap-to-EL2. Most trapped-by-FEAT_NV registers have in handle_sys()
2354 * the trap-to-EL3 controls which would apply to that register in handle_sys()
2355 * at EL2 don't take priority over the FEAT_NV trap-to-EL2.) in handle_sys()
2364 if (ri->accessfn || (ri->fgt && s->fgt_active)) { in handle_sys()
2374 } else if (ri->type & ARM_CP_RAISES_EXC) { in handle_sys()
2383 if ((ri->type & ARM_CP_FPU) && !fp_access_check_only(s)) { in handle_sys()
2385 } else if ((ri->type & ARM_CP_SVE) && !sve_access_check(s)) { in handle_sys()
2387 } else if ((ri->type & ARM_CP_SME) && !sme_access_check(s)) { in handle_sys()
2404 * fine-grained-traps on EL1 also do not apply here. in handle_sys()
2408 ri = get_arm_cp_reginfo(s->cp_regs, key); in handle_sys()
2410 assert(cp_access_ok(s->current_el, ri, isread)); in handle_sys()
2415 assert(!(ri->type & ARM_CP_RAISES_EXC)); in handle_sys()
2424 * The access is 64-bit single copy atomic, guaranteed aligned because in handle_sys()
2433 ARMMMUIdx armmemidx = s->nv2_mem_e20 ? ARMMMUIdx_E20_2 : ARMMMUIdx_E2; in handle_sys()
2437 mop |= (s->nv2_mem_be ? MO_BE : MO_LE); in handle_sys()
2441 (ri->nv2_redirect_offset & ~NV2_REDIR_FLAG_MASK)); in handle_sys()
2455 switch (ri->type & ARM_CP_SPECIAL_MASK) { in handle_sys()
2475 int el = s->nv ? 2 : s->current_el; in handle_sys()
2482 if (s->mte_active[0]) { in handle_sys()
2486 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid); in handle_sys()
2487 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma); in handle_sys()
2509 if (s->ata[0]) { in handle_sys()
2526 if (s->ata[0]) { in handle_sys()
2538 if (ri->type & ARM_CP_IO) { in handle_sys()
2540 need_exit_tb = translator_io_start(&s->base); in handle_sys()
2546 if (ri->type & ARM_CP_CONST) { in handle_sys()
2547 tcg_gen_movi_i64(tcg_rt, ri->resetvalue); in handle_sys()
2548 } else if (ri->readfn) { in handle_sys()
2554 tcg_gen_ld_i64(tcg_rt, tcg_env, ri->fieldoffset); in handle_sys()
2557 if (ri->type & ARM_CP_CONST) { in handle_sys()
2560 } else if (ri->writefn) { in handle_sys()
2566 tcg_gen_st_i64(tcg_rt, tcg_env, ri->fieldoffset); in handle_sys()
2570 if (!isread && !(ri->type & ARM_CP_SUPPRESS_TB_END)) { in handle_sys()
2584 s->base.is_jmp = DISAS_UPDATE_EXIT; in handle_sys()
2590 handle_sys(s, a->l, a->op0, a->op1, a->op2, a->crn, a->crm, a->rt); in trans_SYS()
2597 * For SVC, HVC and SMC we advance the single-step state in trans_SVC()
2599 * mandated, to ensure that single-stepping a system call in trans_SVC()
2602 uint32_t syndrome = syn_aa64_svc(a->imm); in trans_SVC()
2603 if (s->fgt_svc) { in trans_SVC()
2614 int target_el = s->current_el == 3 ? 3 : 2; in trans_HVC()
2616 if (s->current_el == 0) { in trans_HVC()
2628 gen_exception_insn_el(s, 4, EXCP_HVC, syn_aa64_hvc(a->imm), target_el); in trans_HVC()
2634 if (s->current_el == 0) { in trans_SMC()
2639 gen_helper_pre_smc(tcg_env, tcg_constant_i32(syn_aa64_smc(a->imm))); in trans_SMC()
2642 gen_exception_insn_el(s, 4, EXCP_SMC, syn_aa64_smc(a->imm), 3); in trans_SMC()
2648 gen_exception_bkpt_insn(s, syn_aa64_bkpt(a->imm)); in trans_BRK()
2661 if (semihosting_enabled(s->current_el == 0) && a->imm == 0xf000) { in trans_HLT()
2677 * races in multi-threaded linux-user and when MTTCG softmmu is
2687 s->is_ldex = true; in gen_load_exclusive()
2696 if (s->be_data == MO_LE) { in gen_load_exclusive()
2708 if (s->be_data == MO_LE) { in gen_load_exclusive()
2728 /* if (env->exclusive_addr == addr && env->exclusive_val == [addr] in gen_store_exclusive()
2729 * && (!is_pair || env->exclusive_high == [addr + datasize])) { in gen_store_exclusive()
2738 * env->exclusive_addr = -1; in gen_store_exclusive()
2764 * It is possible to trigger an alignment fault pre-LSE2, e.g. with an in gen_store_exclusive()
2765 * unaligned 4-byte write within the range of an aligned 8-byte load. in gen_store_exclusive()
2766 * With LSE2, the store would need to cross a 16-byte boundary when the in gen_store_exclusive()
2786 if (s->be_data == MO_LE) { in gen_store_exclusive()
2800 if (s->be_data == MO_LE) { in gen_store_exclusive()
2815 if (s->be_data == MO_LE) { in gen_store_exclusive()
2838 tcg_gen_movi_i64(cpu_exclusive_addr, -1); in gen_store_exclusive()
2882 if (s->be_data == MO_LE) { in gen_compare_and_swap_pair()
2892 if (s->be_data == MO_LE) { in gen_compare_and_swap_pair()
2901 if (s->be_data == MO_LE) { in gen_compare_and_swap_pair()
2911 if (s->be_data == MO_LE) { in gen_compare_and_swap_pair()
2922 * is accessing a 32-bit or 64-bit register. This logic is derived
2931 * do a zero-extend from 32 to 64 after the load. in ldst_iss_sf()
2943 if (a->rn == 31) { in trans_STXR()
2946 if (a->lasr) { in trans_STXR()
2949 gen_store_exclusive(s, a->rs, a->rt, a->rt2, a->rn, a->sz, false); in trans_STXR()
2955 if (a->rn == 31) { in trans_LDXR()
2958 gen_load_exclusive(s, a->rt, a->rt2, a->rn, a->sz, false); in trans_LDXR()
2959 if (a->lasr) { in trans_LDXR()
2969 bool iss_sf = ldst_iss_sf(a->sz, false, false); in trans_STLR()
2972 * StoreLORelease is the same as Store-Release for QEMU, but in trans_STLR()
2973 * needs the feature-test. in trans_STLR()
2975 if (!a->lasr && !dc_isar_feature(aa64_lor, s)) { in trans_STLR()
2978 /* Generate ISS for non-exclusive accesses including LASR. */ in trans_STLR()
2979 if (a->rn == 31) { in trans_STLR()
2983 memop = check_ordered_align(s, a->rn, 0, true, a->sz); in trans_STLR()
2984 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn), in trans_STLR()
2985 true, a->rn != 31, memop); in trans_STLR()
2986 do_gpr_st(s, cpu_reg(s, a->rt), clean_addr, memop, true, a->rt, in trans_STLR()
2987 iss_sf, a->lasr); in trans_STLR()
2995 bool iss_sf = ldst_iss_sf(a->sz, false, false); in trans_LDAR()
2997 /* LoadLOAcquire is the same as Load-Acquire for QEMU. */ in trans_LDAR()
2998 if (!a->lasr && !dc_isar_feature(aa64_lor, s)) { in trans_LDAR()
3001 /* Generate ISS for non-exclusive accesses including LASR. */ in trans_LDAR()
3002 if (a->rn == 31) { in trans_LDAR()
3005 memop = check_ordered_align(s, a->rn, 0, false, a->sz); in trans_LDAR()
3006 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn), in trans_LDAR()
3007 false, a->rn != 31, memop); in trans_LDAR()
3008 do_gpr_ld(s, cpu_reg(s, a->rt), clean_addr, memop, false, true, in trans_LDAR()
3009 a->rt, iss_sf, a->lasr); in trans_LDAR()
3016 if (a->rn == 31) { in trans_STXP()
3019 if (a->lasr) { in trans_STXP()
3022 gen_store_exclusive(s, a->rs, a->rt, a->rt2, a->rn, a->sz, true); in trans_STXP()
3028 if (a->rn == 31) { in trans_LDXP()
3031 gen_load_exclusive(s, a->rt, a->rt2, a->rn, a->sz, true); in trans_LDXP()
3032 if (a->lasr) { in trans_LDXP()
3043 if (((a->rt | a->rs) & 1) != 0) { in trans_CASP()
3047 gen_compare_and_swap_pair(s, a->rs, a->rt, a->rn, a->sz); in trans_CASP()
3056 gen_compare_and_swap(s, a->rs, a->rt, a->rn, a->sz); in trans_CAS()
3062 bool iss_sf = ldst_iss_sf(a->sz, a->sign, false); in trans_LD_lit()
3063 TCGv_i64 tcg_rt = cpu_reg(s, a->rt); in trans_LD_lit()
3065 MemOp memop = finalize_memop(s, a->sz + a->sign * MO_SIGN); in trans_LD_lit()
3067 gen_pc_plus_diff(s, clean_addr, a->imm); in trans_LD_lit()
3069 false, true, a->rt, iss_sf, false); in trans_LD_lit()
3082 memop = finalize_memop_asimd(s, a->sz); in trans_LD_lit_v()
3084 gen_pc_plus_diff(s, clean_addr, a->imm); in trans_LD_lit_v()
3085 do_fp_ld(s, a->rt, clean_addr, memop); in trans_LD_lit_v()
3093 if (a->rn == 31) { in op_addr_ldstpair_pre()
3097 *dirty_addr = read_cpu_reg_sp(s, a->rn, 1); in op_addr_ldstpair_pre()
3098 if (!a->p) { in op_addr_ldstpair_pre()
3103 (a->w || a->rn != 31), 2 << a->sz, mop); in op_addr_ldstpair_pre()
3109 if (a->w) { in op_addr_ldstpair_post()
3110 if (a->p) { in op_addr_ldstpair_post()
3113 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), dirty_addr); in op_addr_ldstpair_post()
3119 uint64_t offset = a->imm << a->sz; in trans_STP()
3121 MemOp mop = finalize_memop(s, a->sz); in trans_STP()
3124 tcg_rt = cpu_reg(s, a->rt); in trans_STP()
3125 tcg_rt2 = cpu_reg(s, a->rt2); in trans_STP()
3127 * We built mop above for the single logical access -- rebuild it in trans_STP()
3130 * With LSE2, non-sign-extending pairs are treated atomically if in trans_STP()
3132 * within a 16-byte block and that element will be atomic. in trans_STP()
3136 mop = a->sz + 1; in trans_STP()
3137 if (s->align_mem) { in trans_STP()
3138 mop |= (a->sz == 2 ? MO_ALIGN_4 : MO_ALIGN_8); in trans_STP()
3141 if (a->sz == 2) { in trans_STP()
3144 if (s->be_data == MO_LE) { in trans_STP()
3153 if (s->be_data == MO_LE) { in trans_STP()
3166 uint64_t offset = a->imm << a->sz; in trans_LDP()
3168 MemOp mop = finalize_memop(s, a->sz); in trans_LDP()
3171 tcg_rt = cpu_reg(s, a->rt); in trans_LDP()
3172 tcg_rt2 = cpu_reg(s, a->rt2); in trans_LDP()
3175 * We built mop above for the single logical access -- rebuild it in trans_LDP()
3178 * With LSE2, non-sign-extending pairs are treated atomically if in trans_LDP()
3180 * within a 16-byte block and that element will be atomic. in trans_LDP()
3184 * This treats sign-extending loads like zero-extending loads, in trans_LDP()
3187 mop = a->sz + 1; in trans_LDP()
3188 if (s->align_mem) { in trans_LDP()
3189 mop |= (a->sz == 2 ? MO_ALIGN_4 : MO_ALIGN_8); in trans_LDP()
3192 if (a->sz == 2) { in trans_LDP()
3193 int o2 = s->be_data == MO_LE ? 32 : 0; in trans_LDP()
3197 if (a->sign) { in trans_LDP()
3208 if (s->be_data == MO_LE) { in trans_LDP()
3220 uint64_t offset = a->imm << a->sz; in trans_STP_v()
3229 mop = finalize_memop_asimd(s, a->sz); in trans_STP_v()
3231 do_fp_st(s, a->rt, clean_addr, mop); in trans_STP_v()
3232 tcg_gen_addi_i64(clean_addr, clean_addr, 1 << a->sz); in trans_STP_v()
3233 do_fp_st(s, a->rt2, clean_addr, mop); in trans_STP_v()
3240 uint64_t offset = a->imm << a->sz; in trans_LDP_v()
3249 mop = finalize_memop_asimd(s, a->sz); in trans_LDP_v()
3251 do_fp_ld(s, a->rt, clean_addr, mop); in trans_LDP_v()
3252 tcg_gen_addi_i64(clean_addr, clean_addr, 1 << a->sz); in trans_LDP_v()
3253 do_fp_ld(s, a->rt2, clean_addr, mop); in trans_LDP_v()
3261 uint64_t offset = a->imm << LOG2_TAG_GRANULE; in trans_STGP()
3266 tcg_debug_assert(a->sz == MO_64); in trans_STGP()
3272 if (a->rn == 31) { in trans_STGP()
3276 dirty_addr = read_cpu_reg_sp(s, a->rn, 1); in trans_STGP()
3277 if (!a->p) { in trans_STGP()
3282 tcg_rt = cpu_reg(s, a->rt); in trans_STGP()
3283 tcg_rt2 = cpu_reg(s, a->rt2); in trans_STGP()
3286 * STGP is defined as two 8-byte memory operations, aligned to TAG_GRANULE, in trans_STGP()
3287 * and one tag operation. We implement it as one single aligned 16-byte in trans_STGP()
3289 * MO_ATOM_IFALIGN_PAIR produces 8-byte atomicity for the memory store. in trans_STGP()
3294 if (s->be_data == MO_LE) { in trans_STGP()
3302 if (s->ata[0]) { in trans_STGP()
3303 if (tb_cflags(s->base.tb) & CF_PARALLEL) { in trans_STGP()
3320 if (a->rn == 31) { in op_addr_ldst_imm_pre()
3324 *dirty_addr = read_cpu_reg_sp(s, a->rn, 1); in op_addr_ldst_imm_pre()
3325 if (!a->p) { in op_addr_ldst_imm_pre()
3328 memidx = get_a64_user_mem_index(s, a->unpriv); in op_addr_ldst_imm_pre()
3330 a->w || a->rn != 31, in op_addr_ldst_imm_pre()
3331 mop, a->unpriv, memidx); in op_addr_ldst_imm_pre()
3337 if (a->w) { in op_addr_ldst_imm_post()
3338 if (a->p) { in op_addr_ldst_imm_post()
3341 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), dirty_addr); in op_addr_ldst_imm_post()
3347 bool iss_sf, iss_valid = !a->w; in trans_STR_i()
3349 int memidx = get_a64_user_mem_index(s, a->unpriv); in trans_STR_i()
3350 MemOp mop = finalize_memop(s, a->sz + a->sign * MO_SIGN); in trans_STR_i()
3352 op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, true, mop); in trans_STR_i()
3354 tcg_rt = cpu_reg(s, a->rt); in trans_STR_i()
3355 iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext); in trans_STR_i()
3358 iss_valid, a->rt, iss_sf, false); in trans_STR_i()
3359 op_addr_ldst_imm_post(s, a, dirty_addr, a->imm); in trans_STR_i()
3365 bool iss_sf, iss_valid = !a->w; in trans_LDR_i()
3367 int memidx = get_a64_user_mem_index(s, a->unpriv); in trans_LDR_i()
3368 MemOp mop = finalize_memop(s, a->sz + a->sign * MO_SIGN); in trans_LDR_i()
3370 op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, false, mop); in trans_LDR_i()
3372 tcg_rt = cpu_reg(s, a->rt); in trans_LDR_i()
3373 iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext); in trans_LDR_i()
3376 a->ext, memidx, iss_valid, a->rt, iss_sf, false); in trans_LDR_i()
3377 op_addr_ldst_imm_post(s, a, dirty_addr, a->imm); in trans_LDR_i()
3389 mop = finalize_memop_asimd(s, a->sz); in trans_STR_v_i()
3390 op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, true, mop); in trans_STR_v_i()
3391 do_fp_st(s, a->rt, clean_addr, mop); in trans_STR_v_i()
3392 op_addr_ldst_imm_post(s, a, dirty_addr, a->imm); in trans_STR_v_i()
3404 mop = finalize_memop_asimd(s, a->sz); in trans_LDR_v_i()
3405 op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, false, mop); in trans_LDR_v_i()
3406 do_fp_ld(s, a->rt, clean_addr, mop); in trans_LDR_v_i()
3407 op_addr_ldst_imm_post(s, a, dirty_addr, a->imm); in trans_LDR_v_i()
3417 if (a->rn == 31) { in op_addr_ldst_pre()
3420 *dirty_addr = read_cpu_reg_sp(s, a->rn, 1); in op_addr_ldst_pre()
3422 tcg_rm = read_cpu_reg(s, a->rm, 1); in op_addr_ldst_pre()
3423 ext_and_shift_reg(tcg_rm, tcg_rm, a->opt, a->s ? a->sz : 0); in op_addr_ldst_pre()
3432 bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext); in trans_LDR()
3435 if (extract32(a->opt, 1, 1) == 0) { in trans_LDR()
3439 memop = finalize_memop(s, a->sz + a->sign * MO_SIGN); in trans_LDR()
3441 tcg_rt = cpu_reg(s, a->rt); in trans_LDR()
3443 a->ext, true, a->rt, iss_sf, false); in trans_LDR()
3450 bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext); in trans_STR()
3453 if (extract32(a->opt, 1, 1) == 0) { in trans_STR()
3457 memop = finalize_memop(s, a->sz); in trans_STR()
3459 tcg_rt = cpu_reg(s, a->rt); in trans_STR()
3460 do_gpr_st(s, tcg_rt, clean_addr, memop, true, a->rt, iss_sf, false); in trans_STR()
3469 if (extract32(a->opt, 1, 1) == 0) { in trans_LDR_v()
3477 memop = finalize_memop_asimd(s, a->sz); in trans_LDR_v()
3479 do_fp_ld(s, a->rt, clean_addr, memop); in trans_LDR_v()
3488 if (extract32(a->opt, 1, 1) == 0) { in trans_STR_v()
3496 memop = finalize_memop_asimd(s, a->sz); in trans_STR_v()
3498 do_fp_st(s, a->rt, clean_addr, memop); in trans_STR_v()
3506 MemOp mop = a->sz | sign; in do_atomic_ld()
3509 if (a->rn == 31) { in do_atomic_ld()
3512 mop = check_atomic_align(s, a->rn, mop); in do_atomic_ld()
3513 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn), false, in do_atomic_ld()
3514 a->rn != 31, mop); in do_atomic_ld()
3515 tcg_rs = read_cpu_reg(s, a->rs, true); in do_atomic_ld()
3516 tcg_rt = cpu_reg(s, a->rt); in do_atomic_ld()
3527 switch (a->sz) { in do_atomic_ld()
3558 bool iss_sf = ldst_iss_sf(a->sz, false, false); in TRANS_FEAT()
3566 if (a->rn == 31) { in TRANS_FEAT()
3569 mop = check_ordered_align(s, a->rn, 0, false, a->sz); in TRANS_FEAT()
3570 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn), false, in TRANS_FEAT()
3571 a->rn != 31, mop); in TRANS_FEAT()
3574 * fetch-and-do-something op. in TRANS_FEAT()
3576 * full load-acquire (we only need "load-acquire processor consistent"), in TRANS_FEAT()
3579 do_gpr_ld(s, cpu_reg(s, a->rt), clean_addr, mop, false, in TRANS_FEAT()
3580 true, a->rt, iss_sf, true); in TRANS_FEAT()
3595 if (a->rn == 31) { in trans_LDRA()
3598 dirty_addr = read_cpu_reg_sp(s, a->rn, 1); in trans_LDRA()
3600 if (s->pauth_active) { in trans_LDRA()
3601 if (!a->m) { in trans_LDRA()
3610 tcg_gen_addi_i64(dirty_addr, dirty_addr, a->imm); in trans_LDRA()
3616 a->w || a->rn != 31, memop); in trans_LDRA()
3618 tcg_rt = cpu_reg(s, a->rt); in trans_LDRA()
3620 /* extend */ false, /* iss_valid */ !a->w, in trans_LDRA()
3621 /* iss_srt */ a->rt, /* iss_sf */ true, /* iss_ar */ false); in trans_LDRA()
3623 if (a->w) { in trans_LDRA()
3624 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), dirty_addr); in trans_LDRA()
3632 MemOp mop = a->sz | (a->sign ? MO_SIGN : 0); in trans_LDAPR_i()
3633 bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext); in trans_LDAPR_i()
3639 if (a->rn == 31) { in trans_LDAPR_i()
3643 mop = check_ordered_align(s, a->rn, a->imm, false, mop); in trans_LDAPR_i()
3644 dirty_addr = read_cpu_reg_sp(s, a->rn, 1); in trans_LDAPR_i()
3645 tcg_gen_addi_i64(dirty_addr, dirty_addr, a->imm); in trans_LDAPR_i()
3649 * Load-AcquirePC semantics; we implement as the slightly more in trans_LDAPR_i()
3650 * restrictive Load-Acquire. in trans_LDAPR_i()
3652 do_gpr_ld(s, cpu_reg(s, a->rt), clean_addr, mop, a->ext, true, in trans_LDAPR_i()
3653 a->rt, iss_sf, true); in trans_LDAPR_i()
3661 MemOp mop = a->sz; in trans_STLR_i()
3662 bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext); in trans_STLR_i()
3668 /* TODO: ARMv8.4-LSE SCTLR.nAA */ in trans_STLR_i()
3670 if (a->rn == 31) { in trans_STLR_i()
3674 mop = check_ordered_align(s, a->rn, a->imm, true, mop); in trans_STLR_i()
3675 dirty_addr = read_cpu_reg_sp(s, a->rn, 1); in trans_STLR_i()
3676 tcg_gen_addi_i64(dirty_addr, dirty_addr, a->imm); in trans_STLR_i()
3679 /* Store-Release semantics */ in trans_STLR_i()
3681 do_gpr_st(s, cpu_reg(s, a->rt), clean_addr, mop, true, a->rt, iss_sf, true); in trans_STLR_i()
3693 int size = a->sz; in trans_LD_mult()
3695 if (!a->p && a->rm != 0) { in trans_LD_mult()
3696 /* For non-postindexed accesses the Rm field must be 0 */ in trans_LD_mult()
3699 if (size == 3 && !a->q && a->selem != 1) { in trans_LD_mult()
3706 if (a->rn == 31) { in trans_LD_mult()
3710 /* For our purposes, bytes are always little-endian. */ in trans_LD_mult()
3711 endian = s->be_data; in trans_LD_mult()
3716 total = a->rpt * a->selem * (a->q ? 16 : 8); in trans_LD_mult()
3717 tcg_rn = cpu_reg_sp(s, a->rn); in trans_LD_mult()
3721 * promote consecutive little-endian elements below. in trans_LD_mult()
3723 clean_addr = gen_mte_checkN(s, tcg_rn, false, a->p || a->rn != 31, total, in trans_LD_mult()
3727 * Consecutive little-endian elements from a single register in trans_LD_mult()
3728 * can be promoted to a larger little-endian operation. in trans_LD_mult()
3731 if (a->selem == 1 && endian == MO_LE) { in trans_LD_mult()
3735 if (!s->align_mem) { in trans_LD_mult()
3740 elements = (a->q ? 16 : 8) >> size; in trans_LD_mult()
3742 for (r = 0; r < a->rpt; r++) { in trans_LD_mult()
3746 for (xs = 0; xs < a->selem; xs++) { in trans_LD_mult()
3747 int tt = (a->rt + r + xs) % 32; in trans_LD_mult()
3755 * For non-quad operations, setting a slice of the low 64 bits of in trans_LD_mult()
3761 for (r = 0; r < a->rpt * a->selem; r++) { in trans_LD_mult()
3762 int tt = (a->rt + r) % 32; in trans_LD_mult()
3763 clear_vec_high(s, a->q, tt); in trans_LD_mult()
3766 if (a->p) { in trans_LD_mult()
3767 if (a->rm == 31) { in trans_LD_mult()
3770 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm)); in trans_LD_mult()
3784 int size = a->sz; in trans_ST_mult()
3786 if (!a->p && a->rm != 0) { in trans_ST_mult()
3787 /* For non-postindexed accesses the Rm field must be 0 */ in trans_ST_mult()
3790 if (size == 3 && !a->q && a->selem != 1) { in trans_ST_mult()
3797 if (a->rn == 31) { in trans_ST_mult()
3801 /* For our purposes, bytes are always little-endian. */ in trans_ST_mult()
3802 endian = s->be_data; in trans_ST_mult()
3807 total = a->rpt * a->selem * (a->q ? 16 : 8); in trans_ST_mult()
3808 tcg_rn = cpu_reg_sp(s, a->rn); in trans_ST_mult()
3812 * promote consecutive little-endian elements below. in trans_ST_mult()
3814 clean_addr = gen_mte_checkN(s, tcg_rn, true, a->p || a->rn != 31, total, in trans_ST_mult()
3818 * Consecutive little-endian elements from a single register in trans_ST_mult()
3819 * can be promoted to a larger little-endian operation. in trans_ST_mult()
3822 if (a->selem == 1 && endian == MO_LE) { in trans_ST_mult()
3826 if (!s->align_mem) { in trans_ST_mult()
3831 elements = (a->q ? 16 : 8) >> size; in trans_ST_mult()
3833 for (r = 0; r < a->rpt; r++) { in trans_ST_mult()
3837 for (xs = 0; xs < a->selem; xs++) { in trans_ST_mult()
3838 int tt = (a->rt + r + xs) % 32; in trans_ST_mult()
3845 if (a->p) { in trans_ST_mult()
3846 if (a->rm == 31) { in trans_ST_mult()
3849 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm)); in trans_ST_mult()
3861 if (!a->p && a->rm != 0) { in trans_ST_single()
3868 if (a->rn == 31) { in trans_ST_single()
3872 total = a->selem << a->scale; in trans_ST_single()
3873 tcg_rn = cpu_reg_sp(s, a->rn); in trans_ST_single()
3875 mop = finalize_memop_asimd(s, a->scale); in trans_ST_single()
3876 clean_addr = gen_mte_checkN(s, tcg_rn, true, a->p || a->rn != 31, in trans_ST_single()
3879 tcg_ebytes = tcg_constant_i64(1 << a->scale); in trans_ST_single()
3880 for (xs = 0, rt = a->rt; xs < a->selem; xs++, rt = (rt + 1) % 32) { in trans_ST_single()
3881 do_vec_st(s, rt, a->index, clean_addr, mop); in trans_ST_single()
3885 if (a->p) { in trans_ST_single()
3886 if (a->rm == 31) { in trans_ST_single()
3889 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm)); in trans_ST_single()
3901 if (!a->p && a->rm != 0) { in trans_LD_single()
3908 if (a->rn == 31) { in trans_LD_single()
3912 total = a->selem << a->scale; in trans_LD_single()
3913 tcg_rn = cpu_reg_sp(s, a->rn); in trans_LD_single()
3915 mop = finalize_memop_asimd(s, a->scale); in trans_LD_single()
3916 clean_addr = gen_mte_checkN(s, tcg_rn, false, a->p || a->rn != 31, in trans_LD_single()
3919 tcg_ebytes = tcg_constant_i64(1 << a->scale); in trans_LD_single()
3920 for (xs = 0, rt = a->rt; xs < a->selem; xs++, rt = (rt + 1) % 32) { in trans_LD_single()
3921 do_vec_ld(s, rt, a->index, clean_addr, mop); in trans_LD_single()
3925 if (a->p) { in trans_LD_single()
3926 if (a->rm == 31) { in trans_LD_single()
3929 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm)); in trans_LD_single()
3941 if (!a->p && a->rm != 0) { in trans_LD_single_repl()
3948 if (a->rn == 31) { in trans_LD_single_repl()
3952 total = a->selem << a->scale; in trans_LD_single_repl()
3953 tcg_rn = cpu_reg_sp(s, a->rn); in trans_LD_single_repl()
3955 mop = finalize_memop_asimd(s, a->scale); in trans_LD_single_repl()
3956 clean_addr = gen_mte_checkN(s, tcg_rn, false, a->p || a->rn != 31, in trans_LD_single_repl()
3959 tcg_ebytes = tcg_constant_i64(1 << a->scale); in trans_LD_single_repl()
3960 for (xs = 0, rt = a->rt; xs < a->selem; xs++, rt = (rt + 1) % 32) { in trans_LD_single_repl()
3965 tcg_gen_gvec_dup_i64(a->scale, vec_full_reg_offset(s, rt), in trans_LD_single_repl()
3966 (a->q + 1) * 8, vec_full_reg_size(s), tcg_tmp); in trans_LD_single_repl()
3970 if (a->p) { in trans_LD_single_repl()
3971 if (a->rm == 31) { in trans_LD_single_repl()
3974 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm)); in trans_LD_single_repl()
3983 int size = 4 << s->dcz_blocksize; in trans_STZGM()
3988 if (s->current_el == 0) { in trans_STZGM()
3992 if (a->rn == 31) { in trans_STZGM()
3996 addr = read_cpu_reg_sp(s, a->rn, true); in trans_STZGM()
3997 tcg_gen_addi_i64(addr, addr, a->imm); in trans_STZGM()
3998 tcg_rt = cpu_reg(s, a->rt); in trans_STZGM()
4000 if (s->ata[0]) { in trans_STZGM()
4004 * The non-tags portion of STZGM is mostly like DC_ZVA, in trans_STZGM()
4008 tcg_gen_andi_i64(clean_addr, clean_addr, -size); in trans_STZGM()
4020 if (s->current_el == 0) { in trans_STGM()
4024 if (a->rn == 31) { in trans_STGM()
4028 addr = read_cpu_reg_sp(s, a->rn, true); in trans_STGM()
4029 tcg_gen_addi_i64(addr, addr, a->imm); in trans_STGM()
4030 tcg_rt = cpu_reg(s, a->rt); in trans_STGM()
4032 if (s->ata[0]) { in trans_STGM()
4036 int size = 4 << s->gm_blocksize; in trans_STGM()
4039 tcg_gen_andi_i64(clean_addr, clean_addr, -size); in trans_STGM()
4052 if (s->current_el == 0) { in trans_LDGM()
4056 if (a->rn == 31) { in trans_LDGM()
4060 addr = read_cpu_reg_sp(s, a->rn, true); in trans_LDGM()
4061 tcg_gen_addi_i64(addr, addr, a->imm); in trans_LDGM()
4062 tcg_rt = cpu_reg(s, a->rt); in trans_LDGM()
4064 if (s->ata[0]) { in trans_LDGM()
4068 int size = 4 << s->gm_blocksize; in trans_LDGM()
4071 tcg_gen_andi_i64(clean_addr, clean_addr, -size); in trans_LDGM()
4087 if (a->rn == 31) { in trans_LDG()
4091 addr = read_cpu_reg_sp(s, a->rn, true); in trans_LDG()
4092 if (!a->p) { in trans_LDG()
4093 /* pre-index or signed offset */ in trans_LDG()
4094 tcg_gen_addi_i64(addr, addr, a->imm); in trans_LDG()
4097 tcg_gen_andi_i64(addr, addr, -TAG_GRANULE); in trans_LDG()
4098 tcg_rt = cpu_reg(s, a->rt); in trans_LDG()
4099 if (s->ata[0]) { in trans_LDG()
4111 if (a->w) { in trans_LDG()
4112 /* pre-index or post-index */ in trans_LDG()
4113 if (a->p) { in trans_LDG()
4114 /* post-index */ in trans_LDG()
4115 tcg_gen_addi_i64(addr, addr, a->imm); in trans_LDG()
4117 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), addr); in trans_LDG()
4126 if (a->rn == 31) { in do_STG()
4130 addr = read_cpu_reg_sp(s, a->rn, true); in do_STG()
4131 if (!a->p) { in do_STG()
4132 /* pre-index or signed offset */ in do_STG()
4133 tcg_gen_addi_i64(addr, addr, a->imm); in do_STG()
4135 tcg_rt = cpu_reg_sp(s, a->rt); in do_STG()
4136 if (!s->ata[0]) { in do_STG()
4140 * at least for system mode; user-only won't enforce alignment. in do_STG()
4147 } else if (tb_cflags(s->base.tb) & CF_PARALLEL) { in do_STG()
4170 /* This is 1 or 2 atomic 16-byte operations. */ in do_STG()
4178 if (a->w) { in do_STG()
4179 /* pre-index or post-index */ in do_STG()
4180 if (a->p) { in do_STG()
4181 /* post-index */ in do_STG()
4182 tcg_gen_addi_i64(addr, addr, a->imm); in do_STG()
4184 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), addr); in do_STG()
4211 if (a->rs == a->rn || a->rs == a->rd || a->rn == a->rd || in do_SET()
4212 a->rd == 31 || a->rn == 31) { in do_SET()
4216 memidx = get_a64_user_mem_index(s, a->unpriv); in do_SET()
4222 syndrome = syn_mop(true, is_setg, (a->nontemp << 1) | a->unpriv, in do_SET()
4223 is_epilogue, false, true, a->rd, a->rs, a->rn); in do_SET()
4225 if (is_setg ? s->ata[a->unpriv] : s->mte_active[a->unpriv]) { in do_SET()
4227 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid); in do_SET()
4228 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma); in do_SET()
4257 bool wunpriv = extract32(a->options, 0, 1); in do_CPY()
4258 bool runpriv = extract32(a->options, 1, 1); in do_CPY()
4265 if (a->rs == a->rn || a->rs == a->rd || a->rn == a->rd || in do_CPY()
4266 a->rd == 31 || a->rs == 31 || a->rn == 31) { in do_CPY()
4277 syndrome = syn_mop(false, false, a->options, is_epilogue, in do_CPY()
4278 false, true, a->rd, a->rs, a->rn); in do_CPY()
4281 if (s->mte_active[runpriv]) { in do_CPY()
4282 rdesc = FIELD_DP32(rdesc, MTEDESC, TBI, s->tbid); in do_CPY()
4283 rdesc = FIELD_DP32(rdesc, MTEDESC, TCMA, s->tcma); in do_CPY()
4285 if (s->mte_active[wunpriv]) { in do_CPY()
4286 wdesc = FIELD_DP32(wdesc, MTEDESC, TBI, s->tbid); in do_CPY()
4287 wdesc = FIELD_DP32(wdesc, MTEDESC, TCMA, s->tcma); in do_CPY()
4316 TCGv_i64 tcg_rn = rn_sp ? cpu_reg_sp(s, a->rn) : cpu_reg(s, a->rn); in gen_rri()
4317 TCGv_i64 tcg_rd = rd_sp ? cpu_reg_sp(s, a->rd) : cpu_reg(s, a->rd); in gen_rri()
4318 TCGv_i64 tcg_imm = tcg_constant_i64(a->imm); in gen_rri()
4321 if (!a->sf) { in gen_rri()
4328 * PC-rel. addressing
4333 gen_pc_plus_diff(s, cpu_reg(s, a->rd), a->imm); in trans_ADR()
4339 int64_t offset = (int64_t)a->imm << 12; in trans_ADRP()
4342 offset -= s->pc_curr & 0xfff; in trans_ADRP()
4343 gen_pc_plus_diff(s, cpu_reg(s, a->rd), offset); in trans_ADRP()
4352 TRANS(ADDS_i, gen_rri, a, 0, 1, a->sf ? gen_add64_CC : gen_add32_CC)
4353 TRANS(SUBS_i, gen_rri, a, 0, 1, a->sf ? gen_sub64_CC : gen_sub32_CC)
4365 imm = a->uimm6 << LOG2_TAG_GRANULE; in gen_add_sub_imm_with_tags()
4367 imm = -imm; in gen_add_sub_imm_with_tags()
4370 tcg_rn = cpu_reg_sp(s, a->rn); in gen_add_sub_imm_with_tags()
4371 tcg_rd = cpu_reg_sp(s, a->rd); in gen_add_sub_imm_with_tags()
4373 if (s->ata[0]) { in gen_add_sub_imm_with_tags()
4376 tcg_constant_i32(a->uimm4)); in gen_add_sub_imm_with_tags()
4423 * of between 1 and e-1 non-zero bits, rotated within the in logic_imm_decode_wmask()
4424 * element by between 0 and e-1 bits. in logic_imm_decode_wmask()
4428 * 64 bit elements: immn = 1, imms = <length of run - 1> in logic_imm_decode_wmask()
4429 * 32 bit elements: immn = 0, imms = 0 : <length of run - 1> in logic_imm_decode_wmask()
4430 * 16 bit elements: immn = 0, imms = 10 : <length of run - 1> in logic_imm_decode_wmask()
4431 * 8 bit elements: immn = 0, imms = 110 : <length of run - 1> in logic_imm_decode_wmask()
4432 * 4 bit elements: immn = 0, imms = 1110 : <length of run - 1> in logic_imm_decode_wmask()
4433 * 2 bit elements: immn = 0, imms = 11110 : <length of run - 1> in logic_imm_decode_wmask()
4436 * Further, <length of run - 1> all-ones is a reserved pattern. in logic_imm_decode_wmask()
4442 len = 31 - clz32((immn << 6) | (~imms & 0x3f)); in logic_imm_decode_wmask()
4449 levels = e - 1; in logic_imm_decode_wmask()
4454 /* <length of run - 1> mustn't be all-ones. */ in logic_imm_decode_wmask()
4463 mask = (mask >> r) | (mask << (e - r)); in logic_imm_decode_wmask()
4478 /* Some immediate field values are reserved. */ in gen_rri_log()
4479 if (!logic_imm_decode_wmask(&imm, extract32(a->dbm, 12, 1), in gen_rri_log()
4480 extract32(a->dbm, 0, 6), in gen_rri_log()
4481 extract32(a->dbm, 6, 6))) { in gen_rri_log()
4484 if (!a->sf) { in gen_rri_log()
4488 tcg_rd = set_cc ? cpu_reg(s, a->rd) : cpu_reg_sp(s, a->rd); in gen_rri_log()
4489 tcg_rn = cpu_reg(s, a->rn); in gen_rri_log()
4493 gen_logic_CC(a->sf, tcg_rd); in gen_rri_log()
4495 if (!a->sf) { in gen_rri_log()
4512 int pos = a->hw << 4; in TRANS()
4513 tcg_gen_movi_i64(cpu_reg(s, a->rd), (uint64_t)a->imm << pos); in TRANS()
4519 int pos = a->hw << 4; in trans_MOVN()
4520 uint64_t imm = a->imm; in trans_MOVN()
4523 if (!a->sf) { in trans_MOVN()
4526 tcg_gen_movi_i64(cpu_reg(s, a->rd), imm); in trans_MOVN()
4532 int pos = a->hw << 4; in trans_MOVK()
4535 tcg_rd = cpu_reg(s, a->rd); in trans_MOVK()
4536 tcg_im = tcg_constant_i64(a->imm); in trans_MOVK()
4538 if (!a->sf) { in trans_MOVK()
4550 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); in trans_SBFM()
4551 TCGv_i64 tcg_tmp = read_cpu_reg(s, a->rn, 1); in trans_SBFM()
4552 unsigned int bitsize = a->sf ? 64 : 32; in trans_SBFM()
4553 unsigned int ri = a->immr; in trans_SBFM()
4554 unsigned int si = a->imms; in trans_SBFM()
4558 /* Wd<s-r:0> = Wn<s:r> */ in trans_SBFM()
4559 len = (si - ri) + 1; in trans_SBFM()
4561 if (!a->sf) { in trans_SBFM()
4565 /* Wd<32+s-r,32-r> = Wn<s:0> */ in trans_SBFM()
4567 pos = (bitsize - ri) & (bitsize - 1); in trans_SBFM()
4581 * bitsize, therefore no final zero-extension is unneeded for !sf. in trans_SBFM()
4590 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); in trans_UBFM()
4591 TCGv_i64 tcg_tmp = read_cpu_reg(s, a->rn, 1); in trans_UBFM()
4592 unsigned int bitsize = a->sf ? 64 : 32; in trans_UBFM()
4593 unsigned int ri = a->immr; in trans_UBFM()
4594 unsigned int si = a->imms; in trans_UBFM()
4597 tcg_rd = cpu_reg(s, a->rd); in trans_UBFM()
4598 tcg_tmp = read_cpu_reg(s, a->rn, 1); in trans_UBFM()
4601 /* Wd<s-r:0> = Wn<s:r> */ in trans_UBFM()
4602 len = (si - ri) + 1; in trans_UBFM()
4605 /* Wd<32+s-r,32-r> = Wn<s:0> */ in trans_UBFM()
4607 pos = (bitsize - ri) & (bitsize - 1); in trans_UBFM()
4615 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); in trans_BFM()
4616 TCGv_i64 tcg_tmp = read_cpu_reg(s, a->rn, 1); in trans_BFM()
4617 unsigned int bitsize = a->sf ? 64 : 32; in trans_BFM()
4618 unsigned int ri = a->immr; in trans_BFM()
4619 unsigned int si = a->imms; in trans_BFM()
4622 tcg_rd = cpu_reg(s, a->rd); in trans_BFM()
4623 tcg_tmp = read_cpu_reg(s, a->rn, 1); in trans_BFM()
4626 /* Wd<s-r:0> = Wn<s:r> */ in trans_BFM()
4628 len = (si - ri) + 1; in trans_BFM()
4631 /* Wd<32+s-r,32-r> = Wn<s:0> */ in trans_BFM()
4633 pos = (bitsize - ri) & (bitsize - 1); in trans_BFM()
4637 if (!a->sf) { in trans_BFM()
4647 tcg_rd = cpu_reg(s, a->rd); in trans_EXTR()
4649 if (unlikely(a->imm == 0)) { in trans_EXTR()
4654 if (a->sf) { in trans_EXTR()
4655 tcg_gen_mov_i64(tcg_rd, cpu_reg(s, a->rm)); in trans_EXTR()
4657 tcg_gen_ext32u_i64(tcg_rd, cpu_reg(s, a->rm)); in trans_EXTR()
4660 tcg_rm = cpu_reg(s, a->rm); in trans_EXTR()
4661 tcg_rn = cpu_reg(s, a->rn); in trans_EXTR()
4663 if (a->sf) { in trans_EXTR()
4665 tcg_gen_extract2_i64(tcg_rd, tcg_rm, tcg_rn, a->imm); in trans_EXTR()
4670 if (a->rm == a->rn) { in trans_EXTR()
4671 tcg_gen_rotri_i32(t0, t0, a->imm); in trans_EXTR()
4675 tcg_gen_extract2_i32(t0, t0, t1, a->imm); in trans_EXTR()
4686 int len = (a->len + 1) * 16; in trans_TBL_TBX()
4688 tcg_gen_gvec_2_ptr(vec_full_reg_offset(s, a->rd), in trans_TBL_TBX()
4689 vec_full_reg_offset(s, a->rm), tcg_env, in trans_TBL_TBX()
4690 a->q ? 16 : 8, vec_full_reg_size(s), in trans_TBL_TBX()
4691 (len << 6) | (a->tbx << 5) | a->rn, in trans_TBL_TBX()
4702 MemOp esz = a->esz; in do_simd_permute()
4703 int datasize = a->q ? 16 : 8; in do_simd_permute()
4707 if (esz == MO_64 && !a->q) { in do_simd_permute()
4715 tcg_res[1] = a->q ? tcg_temp_new_i64() : NULL; in do_simd_permute()
4722 read_vec_element(s, tcg_ele, (idx & elements ? a->rm : a->rn), in do_simd_permute()
4723 idx & (elements - 1), esz); in do_simd_permute()
4734 for (int i = a->q; i >= 0; --i) { in do_simd_permute()
4735 write_vec_element(s, tcg_res[i], a->rd, i, MO_64); in do_simd_permute()
4737 clear_vec_high(s, a->q, a->rd); in do_simd_permute()
4813 read_vec_element_i32(s, tcg_op1, a->rn, 3, MO_32); in TRANS_FEAT()
4814 read_vec_element_i32(s, tcg_op2, a->rm, 3, MO_32); in TRANS_FEAT()
4815 read_vec_element_i32(s, tcg_op3, a->ra, 3, MO_32); in TRANS_FEAT()
4824 dofs = vec_full_reg_offset(s, a->rd); in TRANS_FEAT()
4826 write_vec_element_i32(s, tcg_res, a->rd, 3, MO_32); in TRANS_FEAT()
4834 gen_gvec_op3_ool(s, true, a->rd, a->rn, a->rm, a->imm, fn); in do_crypto3i()
4849 gen_gvec_xar(MO_64, vec_full_reg_offset(s, a->rd), in TRANS_FEAT()
4850 vec_full_reg_offset(s, a->rn), in TRANS_FEAT()
4851 vec_full_reg_offset(s, a->rm), a->imm, 16, in TRANS_FEAT()
4877 if (!decode_esz_idx(a->imm, &esz, &idx)) { in trans_DUP_element_s()
4883 * zero-extends it into the bottom of the destination register. in trans_DUP_element_s()
4886 read_vec_element(s, tmp, a->rn, idx, esz); in trans_DUP_element_s()
4887 write_fp_dreg(s, a->rd, tmp); in trans_DUP_element_s()
4897 if (!decode_esz_idx(a->imm, &esz, &idx)) { in trans_DUP_element_v()
4900 if (esz == MO_64 && !a->q) { in trans_DUP_element_v()
4904 tcg_gen_gvec_dup_mem(esz, vec_full_reg_offset(s, a->rd), in trans_DUP_element_v()
4905 vec_reg_offset(s, a->rn, idx, esz), in trans_DUP_element_v()
4906 a->q ? 16 : 8, vec_full_reg_size(s)); in trans_DUP_element_v()
4916 if (!decode_esz_idx(a->imm, &esz, &idx)) { in trans_DUP_general()
4919 if (esz == MO_64 && !a->q) { in trans_DUP_general()
4923 tcg_gen_gvec_dup_i64(esz, vec_full_reg_offset(s, a->rd), in trans_DUP_general()
4924 a->q ? 16 : 8, vec_full_reg_size(s), in trans_DUP_general()
4925 cpu_reg(s, a->rn)); in trans_DUP_general()
4935 if (!decode_esz_idx(a->imm, &esz, &idx)) { in do_smov_umov()
4939 if (esz == MO_64 || (esz == MO_32 && !a->q)) { in do_smov_umov()
4943 if (esz == MO_64 ? !a->q : a->q) { in do_smov_umov()
4948 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); in do_smov_umov()
4949 read_vec_element(s, tcg_rd, a->rn, idx, esz | is_signed); in do_smov_umov()
4950 if (is_signed && !a->q) { in do_smov_umov()
4965 if (!decode_esz_idx(a->imm, &esz, &idx)) { in TRANS()
4969 write_vec_element(s, cpu_reg(s, a->rn), a->rd, idx, esz); in TRANS()
4970 clear_vec_high(s, true, a->rd); in TRANS()
4980 if (!decode_esz_idx(a->di, &esz, &didx)) { in trans_INS_element()
4983 sidx = a->si >> esz; in trans_INS_element()
4987 read_vec_element(s, tmp, a->rn, sidx, esz); in trans_INS_element()
4988 write_vec_element(s, tmp, a->rd, didx, esz); in trans_INS_element()
4990 /* INS is considered a 128-bit write for SVE. */ in trans_INS_element()
4991 clear_vec_high(s, true, a->rd); in trans_INS_element()
5008 switch (a->esz) { in do_fp3_scalar()
5011 TCGv_i64 t0 = read_fp_dreg(s, a->rn); in do_fp3_scalar()
5012 TCGv_i64 t1 = read_fp_dreg(s, a->rm); in do_fp3_scalar()
5013 f->gen_d(t0, t0, t1, fpstatus_ptr(FPST_FPCR)); in do_fp3_scalar()
5014 write_fp_dreg(s, a->rd, t0); in do_fp3_scalar()
5019 TCGv_i32 t0 = read_fp_sreg(s, a->rn); in do_fp3_scalar()
5020 TCGv_i32 t1 = read_fp_sreg(s, a->rm); in do_fp3_scalar()
5021 f->gen_s(t0, t0, t1, fpstatus_ptr(FPST_FPCR)); in do_fp3_scalar()
5022 write_fp_sreg(s, a->rd, t0); in do_fp3_scalar()
5030 TCGv_i32 t0 = read_fp_hreg(s, a->rn); in do_fp3_scalar()
5031 TCGv_i32 t1 = read_fp_hreg(s, a->rm); in do_fp3_scalar()
5032 f->gen_h(t0, t0, t1, fpstatus_ptr(FPST_FPCR_F16)); in do_fp3_scalar()
5033 write_fp_sreg(s, a->rd, t0); in do_fp3_scalar()
5210 MemOp esz = a->esz; in do_satacc_s()
5220 read_vec_element(s, t1, a->rn, 0, esz | sgn_n); in do_satacc_s()
5221 read_vec_element(s, t2, a->rm, 0, esz | sgn_m); in do_satacc_s()
5231 write_fp_dreg(s, a->rd, t0); in do_satacc_s()
5250 read_vec_element(s, t0, a->rn, 0, MO_64); in TRANS()
5251 read_vec_element(s, t1, a->rm, 0, MO_64); in TRANS()
5253 write_fp_dreg(s, a->rd, t0); in TRANS()
5275 if (a->esz == MO_64) { in do_env_scalar2()
5276 TCGv_i64 t0 = read_fp_dreg(s, a->rn); in do_env_scalar2()
5277 TCGv_i64 t1 = read_fp_dreg(s, a->rm); in do_env_scalar2()
5278 f->gen_d(t0, tcg_env, t0, t1); in do_env_scalar2()
5279 write_fp_dreg(s, a->rd, t0); in do_env_scalar2()
5284 read_vec_element_i32(s, t0, a->rn, 0, a->esz); in do_env_scalar2()
5285 read_vec_element_i32(s, t1, a->rm, 0, a->esz); in do_env_scalar2()
5286 f->gen_bhs[a->esz](t0, tcg_env, t0, t1); in do_env_scalar2()
5287 write_fp_sreg(s, a->rd, t0); in do_env_scalar2()
5327 if (a->esz == MO_16 || a->esz == MO_32) { in do_env_scalar2_hs()
5352 if (a->esz != MO_16 && a->esz != MO_32) { in do_env_scalar3_hs()
5362 read_vec_element_i32(s, t0, a->rn, 0, a->esz); in do_env_scalar3_hs()
5363 read_vec_element_i32(s, t1, a->rm, 0, a->esz); in do_env_scalar3_hs()
5364 read_vec_element_i32(s, t2, a->rd, 0, a->esz); in do_env_scalar3_hs()
5365 f->gen_hs[a->esz - 1](t0, tcg_env, t0, t1, t2); in do_env_scalar3_hs()
5366 write_fp_sreg(s, a->rd, t0); in do_env_scalar3_hs()
5383 TCGv_i64 t0 = read_fp_dreg(s, a->rn); in do_cmop_d()
5384 TCGv_i64 t1 = read_fp_dreg(s, a->rm); in do_cmop_d()
5386 write_fp_dreg(s, a->rd, t0); in do_cmop_d()
5401 MemOp esz = a->esz; in TRANS()
5405 if (!a->q) { in TRANS()
5420 gen_gvec_op3_fpst(s, a->q, a->rd, a->rn, a->rm, in TRANS()
5421 esz == MO_16, data, fns[esz - 1]); in TRANS()
5598 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, a->rd), in do_fmlal()
5599 vec_full_reg_offset(s, a->rn), in do_fmlal()
5600 vec_full_reg_offset(s, a->rm), tcg_env, in do_fmlal()
5601 a->q ? 16 : 8, vec_full_reg_size(s), in do_fmlal()
5632 TRANS(BSL_v, do_bitsel, a->q, a->rd, a->rd, a->rn, a->rm)
5633 TRANS(BIT_v, do_bitsel, a->q, a->rd, a->rm, a->rn, a->rd)
5634 TRANS(BIF_v, do_bitsel, a->q, a->rd, a->rm, a->rd, a->rn)
5675 if (a->esz == MO_64 && !a->q) { in TRANS()
5679 tcg_gen_gvec_cmp(cond, a->esz, in TRANS()
5680 vec_full_reg_offset(s, a->rd), in TRANS()
5681 vec_full_reg_offset(s, a->rn), in TRANS()
5682 vec_full_reg_offset(s, a->rm), in TRANS()
5683 a->q ? 16 : 8, vec_full_reg_size(s)); in TRANS()
5704 gen_gvec_op4_ool(s, a->q, a->rd, a->rn, a->rm, a->rd, 0, fn); in TRANS()
5713 gen_gvec_op4_env(s, a->q, a->rd, a->rn, a->rm, a->rd, 0, fn); in do_dot_vector_env()
5734 gen_gvec_op4_fpst(s, true, a->rd, a->rn, a->rm, a->rd, false, a->q, in TRANS_FEAT()
5755 switch (a->esz) { in trans_FCMLA_v()
5757 if (!a->q) { in trans_FCMLA_v()
5775 gen_gvec_op4_fpst(s, a->q, a->rd, a->rn, a->rm, a->rd, in trans_FCMLA_v()
5776 a->esz == MO_16, a->rot, fn); in trans_FCMLA_v()
5784 * These read from the top or bottom half of a 128-bit vector.
5785 * After widening, optionally accumulate with a 128-bit vector.
5806 /* There are no 64x64->128 bit operations. */ in do_3op_widening()
5824 top_swap = top ? 0 : half - 1; in do_3op_widening()
5859 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
5862 a->esz, a->q, a->rd, a->rn, a->rm, -1,
5865 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
5868 a->esz, a->q, a->rd, a->rn, a->rm, -1,
5871 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
5874 a->esz, a->q, a->rd, a->rn, a->rm, -1,
5878 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
5881 a->esz, a->q, a->rd, a->rn, a->rm, a->idx,
5884 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
5887 a->esz, a->q, a->rd, a->rn, a->rm, a->idx,
5890 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
5893 a->esz, a->q, a->rd, a->rn, a->rm, a->idx,
5931 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
5934 a->esz, a->q, a->rd, a->rn, a->rm, -1,
5937 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
5940 a->esz, a->q, a->rd, a->rn, a->rm, -1,
5943 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
5946 a->esz, a->q, a->rd, a->rn, a->rm, -1,
5949 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
5952 a->esz, a->q, a->rd, a->rn, a->rm, -1,
6006 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6007 a->esz == MO_16 ? gen_sqdmull_h : gen_sqdmull_s, false)
6009 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6010 a->esz == MO_16 ? gen_sqdmlal_h : gen_sqdmlal_s, true)
6012 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6013 a->esz == MO_16 ? gen_sqdmlsl_h : gen_sqdmlsl_s, true)
6016 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
6017 a->esz == MO_16 ? gen_sqdmull_h : gen_sqdmull_s, false)
6019 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
6020 a->esz == MO_16 ? gen_sqdmlal_h : gen_sqdmlal_s, true)
6022 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
6023 a->esz == MO_16 ? gen_sqdmlsl_h : gen_sqdmlsl_s, true)
6029 MemOp esz = a->esz; in do_addsub_wide()
6031 bool top = a->q; in do_addsub_wide()
6032 int top_swap = top ? 0 : half - 1; in do_addsub_wide()
6035 /* There are no 64x64->128 bit operations. */ in do_addsub_wide()
6048 read_vec_element(s, tcg_op1, a->rm, elt + top_half, esz | sign); in do_addsub_wide()
6049 read_vec_element(s, tcg_op0, a->rn, elt, esz + 1); in do_addsub_wide()
6055 write_vec_element(s, tcg_op0, a->rd, elt, esz + 1); in do_addsub_wide()
6057 clear_vec_high(s, 1, a->rd); in do_addsub_wide()
6070 MemOp esz = a->esz; in TRANS()
6072 bool top = a->q; in TRANS()
6074 uint64_t rbit = 1ull << (ebits - 1); in TRANS()
6077 /* There are no 128x128->64 bit operations. */ in TRANS()
6092 top_swap = top ? half - 1 : 0; in TRANS()
6098 read_vec_element(s, tcg_op1, a->rm, elt, esz + 1); in TRANS()
6099 read_vec_element(s, tcg_op0, a->rn, elt, esz + 1); in TRANS()
6109 write_vec_element(s, tcg_op0, a->rd, elt + top_half, esz); in TRANS()
6111 clear_vec_high(s, top, a->rd); in TRANS()
6124 gen_gvec_op3_ool(s, true, a->rd, a->rn, a->rm, a->q, fn); in TRANS()
6138 switch (a->esz) { in TRANS()
6141 TCGv_i64 t0 = read_fp_dreg(s, a->rn); in TRANS()
6144 read_vec_element(s, t1, a->rm, a->idx, MO_64); in TRANS()
6145 f->gen_d(t0, t0, t1, fpstatus_ptr(FPST_FPCR)); in TRANS()
6146 write_fp_dreg(s, a->rd, t0); in TRANS()
6151 TCGv_i32 t0 = read_fp_sreg(s, a->rn); in TRANS()
6154 read_vec_element_i32(s, t1, a->rm, a->idx, MO_32); in TRANS()
6155 f->gen_s(t0, t0, t1, fpstatus_ptr(FPST_FPCR)); in TRANS()
6156 write_fp_sreg(s, a->rd, t0); in TRANS()
6164 TCGv_i32 t0 = read_fp_hreg(s, a->rn); in TRANS()
6167 read_vec_element_i32(s, t1, a->rm, a->idx, MO_16); in TRANS()
6168 f->gen_h(t0, t0, t1, fpstatus_ptr(FPST_FPCR_F16)); in TRANS()
6169 write_fp_sreg(s, a->rd, t0); in TRANS()
6183 switch (a->esz) { in do_fmla_scalar_idx()
6186 TCGv_i64 t0 = read_fp_dreg(s, a->rd); in do_fmla_scalar_idx()
6187 TCGv_i64 t1 = read_fp_dreg(s, a->rn); in do_fmla_scalar_idx()
6190 read_vec_element(s, t2, a->rm, a->idx, MO_64); in do_fmla_scalar_idx()
6195 write_fp_dreg(s, a->rd, t0); in do_fmla_scalar_idx()
6200 TCGv_i32 t0 = read_fp_sreg(s, a->rd); in do_fmla_scalar_idx()
6201 TCGv_i32 t1 = read_fp_sreg(s, a->rn); in do_fmla_scalar_idx()
6204 read_vec_element_i32(s, t2, a->rm, a->idx, MO_32); in do_fmla_scalar_idx()
6209 write_fp_sreg(s, a->rd, t0); in do_fmla_scalar_idx()
6217 TCGv_i32 t0 = read_fp_hreg(s, a->rd); in do_fmla_scalar_idx()
6218 TCGv_i32 t1 = read_fp_hreg(s, a->rn); in do_fmla_scalar_idx()
6221 read_vec_element_i32(s, t2, a->rm, a->idx, MO_16); in do_fmla_scalar_idx()
6227 write_fp_sreg(s, a->rd, t0); in do_fmla_scalar_idx()
6242 if (a->esz < MO_16 || a->esz > MO_32) { in TRANS()
6249 read_vec_element_i32(s, t0, a->rn, 0, a->esz); in TRANS()
6250 read_vec_element_i32(s, t1, a->rm, a->idx, a->esz); in TRANS()
6251 f->gen_bhs[a->esz](t0, tcg_env, t0, t1); in TRANS()
6252 write_fp_sreg(s, a->rd, t0); in TRANS()
6263 if (a->esz < MO_16 || a->esz > MO_32) { in do_env_scalar3_idx_hs()
6271 read_vec_element_i32(s, t0, a->rn, 0, a->esz); in do_env_scalar3_idx_hs()
6272 read_vec_element_i32(s, t1, a->rm, a->idx, a->esz); in do_env_scalar3_idx_hs()
6273 read_vec_element_i32(s, t2, a->rd, 0, a->esz); in do_env_scalar3_idx_hs()
6274 f->gen_hs[a->esz - 1](t0, tcg_env, t0, t1, t2); in do_env_scalar3_idx_hs()
6275 write_fp_sreg(s, a->rd, t0); in do_env_scalar3_idx_hs()
6293 read_vec_element(s, t0, a->rd, 0, a->esz + 1); in do_scalar_muladd_widening_idx()
6295 read_vec_element(s, t1, a->rn, 0, a->esz | MO_SIGN); in do_scalar_muladd_widening_idx()
6296 read_vec_element(s, t2, a->rm, a->idx, a->esz | MO_SIGN); in do_scalar_muladd_widening_idx()
6301 dofs = vec_full_reg_offset(s, a->rd); in do_scalar_muladd_widening_idx()
6303 write_vec_element(s, t0, a->rd, 0, a->esz + 1); in do_scalar_muladd_widening_idx()
6309 a->esz == MO_16 ? gen_sqdmull_h : gen_sqdmull_s, false)
6311 a->esz == MO_16 ? gen_sqdmlal_h : gen_sqdmlal_s, true)
6313 a->esz == MO_16 ? gen_sqdmlsl_h : gen_sqdmlsl_s, true)
6318 MemOp esz = a->esz; in do_fp3_vector_idx()
6322 if (!a->q) { in do_fp3_vector_idx()
6337 gen_gvec_op3_fpst(s, a->q, a->rd, a->rn, a->rm, in do_fp3_vector_idx()
6338 esz == MO_16, a->idx, fns[esz - 1]); in do_fp3_vector_idx()
6364 MemOp esz = a->esz; in TRANS()
6368 if (!a->q) { in TRANS()
6383 gen_gvec_op4_fpst(s, a->q, a->rd, a->rn, a->rm, a->rd, in TRANS()
6384 esz == MO_16, (a->idx << 1) | neg, in TRANS()
6385 fns[esz - 1]); in TRANS()
6396 int data = (a->idx << 2) | (is_2 << 1) | is_s; in TRANS()
6397 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, a->rd), in TRANS()
6398 vec_full_reg_offset(s, a->rn), in TRANS()
6399 vec_full_reg_offset(s, a->rm), tcg_env, in TRANS()
6400 a->q ? 16 : 8, vec_full_reg_size(s), in TRANS()
6414 assert(a->esz == MO_16 || a->esz == MO_32); in TRANS_FEAT()
6416 gen_gvec_op3_ool(s, a->q, a->rd, a->rn, a->rm, a->idx, fns[a->esz - 1]); in TRANS_FEAT()
6434 assert(a->esz == MO_16 || a->esz == MO_32); in TRANS()
6436 gen_gvec_op4_ool(s, a->q, a->rd, a->rn, a->rm, a->rd, in TRANS()
6437 a->idx, fns[a->esz - 1][sub]); in TRANS()
6448 assert(a->esz == MO_16 || a->esz == MO_32); in TRANS()
6450 tcg_gen_gvec_4_ool(vec_full_reg_offset(s, a->rd), in TRANS()
6451 vec_full_reg_offset(s, a->rn), in TRANS()
6452 vec_full_reg_offset(s, a->rm), in TRANS()
6454 a->q ? 16 : 8, vec_full_reg_size(s), in TRANS()
6455 a->idx, fns[a->esz - 1]); in TRANS()
6490 gen_gvec_op4_ool(s, a->q, a->rd, a->rn, a->rm, a->rd, a->idx, fn); in TRANS_FEAT()
6499 gen_gvec_op4_env(s, a->q, a->rd, a->rn, a->rm, a->rd, a->idx, fn); in do_dot_vector_idx_env()
6520 gen_gvec_op4_fpst(s, true, a->rd, a->rn, a->rm, a->rd, 0, in TRANS_FEAT()
6521 (a->idx << 1) | a->q, in TRANS_FEAT()
6534 switch (a->esz) { in trans_FCMLA_vi()
6548 gen_gvec_op4_fpst(s, a->q, a->rd, a->rn, a->rm, a->rd, in trans_FCMLA_vi()
6549 a->esz == MO_16, (a->idx << 2) | a->rot, fn); in trans_FCMLA_vi()
6560 switch (a->esz) { in do_fp3_scalar_pair()
6566 read_vec_element(s, t0, a->rn, 0, MO_64); in do_fp3_scalar_pair()
6567 read_vec_element(s, t1, a->rn, 1, MO_64); in do_fp3_scalar_pair()
6568 f->gen_d(t0, t0, t1, fpstatus_ptr(FPST_FPCR)); in do_fp3_scalar_pair()
6569 write_fp_dreg(s, a->rd, t0); in do_fp3_scalar_pair()
6577 read_vec_element_i32(s, t0, a->rn, 0, MO_32); in do_fp3_scalar_pair()
6578 read_vec_element_i32(s, t1, a->rn, 1, MO_32); in do_fp3_scalar_pair()
6579 f->gen_s(t0, t0, t1, fpstatus_ptr(FPST_FPCR)); in do_fp3_scalar_pair()
6580 write_fp_sreg(s, a->rd, t0); in do_fp3_scalar_pair()
6591 read_vec_element_i32(s, t0, a->rn, 0, MO_16); in do_fp3_scalar_pair()
6592 read_vec_element_i32(s, t1, a->rn, 1, MO_16); in do_fp3_scalar_pair()
6593 f->gen_h(t0, t0, t1, fpstatus_ptr(FPST_FPCR_F16)); in do_fp3_scalar_pair()
6594 write_fp_sreg(s, a->rd, t0); in do_fp3_scalar_pair()
6615 read_vec_element(s, t0, a->rn, 0, MO_64); in trans_ADDP_s()
6616 read_vec_element(s, t1, a->rn, 1, MO_64); in trans_ADDP_s()
6618 write_fp_dreg(s, a->rd, t0); in trans_ADDP_s()
6624 * Floating-point conditional select
6632 switch (a->esz) { in trans_FCSEL()
6652 read_vec_element(s, t_true, a->rn, 0, a->esz); in trans_FCSEL()
6653 read_vec_element(s, t_false, a->rm, 0, a->esz); in trans_FCSEL()
6655 a64_test_cc(&c, a->cond); in trans_FCSEL()
6661 * and we've already done the zero-extension. in trans_FCSEL()
6663 write_fp_dreg(s, a->rd, t_true); in trans_FCSEL()
6674 TCGv_i64 lo = read_fp_dreg(s, a->rn); in trans_EXT_d()
6675 if (a->imm != 0) { in trans_EXT_d()
6676 TCGv_i64 hi = read_fp_dreg(s, a->rm); in trans_EXT_d()
6677 tcg_gen_extract2_i64(lo, lo, hi, a->imm * 8); in trans_EXT_d()
6679 write_fp_dreg(s, a->rd, lo); in trans_EXT_d()
6687 int pos = (a->imm & 7) * 8; in trans_EXT_q()
6688 int elt = a->imm >> 3; in trans_EXT_q()
6697 read_vec_element(s, lo, a->rn, elt, MO_64); in trans_EXT_q()
6699 read_vec_element(s, hi, elt & 2 ? a->rm : a->rn, elt & 1, MO_64); in trans_EXT_q()
6705 read_vec_element(s, hh, a->rm, elt & 1, MO_64); in trans_EXT_q()
6709 write_vec_element(s, lo, a->rd, 0, MO_64); in trans_EXT_q()
6710 write_vec_element(s, hi, a->rd, 1, MO_64); in trans_EXT_q()
6711 clear_vec_high(s, true, a->rd); in trans_EXT_q()
6716 * Floating-point data-processing (3 source)
6724 * These are fused multiply-add. Note that doing the negations here in do_fmadd()
6726 * its sign bit flipped if it is a negated-input. in do_fmadd()
6728 switch (a->esz) { in do_fmadd()
6731 TCGv_i64 tn = read_fp_dreg(s, a->rn); in do_fmadd()
6732 TCGv_i64 tm = read_fp_dreg(s, a->rm); in do_fmadd()
6733 TCGv_i64 ta = read_fp_dreg(s, a->ra); in do_fmadd()
6743 write_fp_dreg(s, a->rd, ta); in do_fmadd()
6749 TCGv_i32 tn = read_fp_sreg(s, a->rn); in do_fmadd()
6750 TCGv_i32 tm = read_fp_sreg(s, a->rm); in do_fmadd()
6751 TCGv_i32 ta = read_fp_sreg(s, a->ra); in do_fmadd()
6761 write_fp_sreg(s, a->rd, ta); in do_fmadd()
6770 TCGv_i32 tn = read_fp_hreg(s, a->rn); in do_fmadd()
6771 TCGv_i32 tm = read_fp_hreg(s, a->rm); in do_fmadd()
6772 TCGv_i32 ta = read_fp_hreg(s, a->ra); in do_fmadd()
6782 write_fp_sreg(s, a->rd, ta); in do_fmadd()
6805 MemOp src_mop = a->esz | src_sign; in TRANS()
6806 int elements = (a->q ? 16 : 8) >> a->esz; in TRANS()
6819 read_vec_element(s, tcg_res, a->rn, 0, src_mop); in TRANS()
6821 read_vec_element(s, tcg_elt, a->rn, i, src_mop); in TRANS()
6825 tcg_gen_ext_i64(tcg_res, tcg_res, a->esz + widen); in TRANS()
6826 write_fp_dreg(s, a->rd, tcg_res); in TRANS()
6872 MemOp esz = a->esz; in do_fp_reduction()
6873 int elts = (a->q ? 16 : 8) >> esz; in do_fp_reduction()
6875 TCGv_i32 res = do_reduction_op(s, a->rn, esz, 0, elts, fpst, fn); in do_fp_reduction()
6876 write_fp_sreg(s, a->rd, res); in do_fp_reduction()
6892 * Floating-point Immediate in TRANS_FEAT()
6897 switch (a->esz) { in TRANS_FEAT()
6910 uint64_t imm = vfp_expand_imm(a->esz, a->imm); in TRANS_FEAT()
6911 write_fp_dreg(s, a->rd, tcg_constant_i64(imm)); in TRANS_FEAT()
6926 tcg_gen_gvec_dup_imm(MO_16, vec_full_reg_offset(s, a->rd), in trans_FMOVI_v_h()
6927 a->q ? 16 : 8, vec_full_reg_size(s), in trans_FMOVI_v_h()
6928 vfp_expand_imm(MO_16, a->abcdefgh)); in trans_FMOVI_v_h()
6944 if ((a->cmode & 1) && a->cmode < 12) { in trans_Vimm()
6946 fn = a->op ? tcg_gen_gvec_andi : tcg_gen_gvec_ori; in trans_Vimm()
6949 if (a->cmode == 15 && a->op == 1 && a->q == 0) { in trans_Vimm()
6956 uint64_t imm = asimd_imm_const(a->abcdefgh, a->cmode, a->op); in trans_Vimm()
6957 gen_gvec_fn2i(s, a->q, a->rd, a->rd, imm, fn, MO_64); in trans_Vimm()
6969 gen_gvec_fn2i(s, a->q, a->rd, a->rn, a->imm, fn, a->esz); in do_vec_shift_imm()
6992 int esz = a->esz; in TRANS()
7006 read_vec_element(s, tcg_rn, a->rn, a->q, MO_64); in TRANS()
7015 tcg_gen_shli_i64(tcg_rd, tcg_rd, a->imm); in TRANS()
7016 write_vec_element(s, tcg_rd, a->rd, i, esz + 1); in TRANS()
7018 clear_vec_high(s, true, a->rd); in TRANS()
7057 TCGv_i64 rnd = tcg_constant_i64(1ull << (shift - 1)); in gen_srshr_bhs()
7069 TCGv_i64 rnd = tcg_constant_i64(1ull << (shift - 1)); in gen_urshr_bhs()
7083 /* Extension of sign bit (0,-1) plus sign bit (0,1) is zero. */ in gen_srshr_d()
7087 tcg_gen_extract_i64(rnd, src, shift - 1, 1); in gen_srshr_d()
7103 tcg_gen_extract_i64(rnd, src, shift - 1, 1); in gen_urshr_d()
7126 tcg_gen_deposit_i64(dst, dst, src, 0, 64 - shift); in gen_sri_d()
7132 tcg_gen_deposit_i64(dst, dst, src, shift, 64 - shift); in gen_sli_d()
7139 int esz = a->esz; in do_vec_shift_imm_narrow()
7156 read_vec_element(s, tcg_rn, a->rn, i, (esz + 1) | sign); in do_vec_shift_imm_narrow()
7157 fn(tcg_rn, tcg_rn, a->imm); in do_vec_shift_imm_narrow()
7161 write_vec_element(s, tcg_rd, a->rd, a->q, MO_64); in do_vec_shift_imm_narrow()
7162 clear_vec_high(s, a->q, a->rd); in do_vec_shift_imm_narrow()
7350 read_vec_element(s, rn, a->rn, 0, a->esz | sign); in TRANS()
7352 read_vec_element(s, rd, a->rd, 0, a->esz | sign); in TRANS()
7354 fn(rd, rn, a->imm); in TRANS()
7355 write_fp_dreg(s, a->rd, rd); in TRANS()
7454 /* Note that the helpers sign-extend their inputs, so don't do it here. */
7455 TRANS(SQSHL_si, do_scalar_shift_imm, a, f_scalar_sqshli[a->esz], false, 0)
7456 TRANS(UQSHL_si, do_scalar_shift_imm, a, f_scalar_uqshli[a->esz], false, 0)
7457 TRANS(SQSHLU_si, do_scalar_shift_imm, a, f_scalar_sqshlui[a->esz], false, 0)
7463 MemOp esz = a->esz; in do_scalar_shift_imm_narrow()
7471 read_vec_element(s, rn, a->rn, 0, (esz + 1) | sign); in do_scalar_shift_imm_narrow()
7472 fns[esz](rd, rn, a->imm); in do_scalar_shift_imm_narrow()
7476 write_fp_dreg(s, a->rd, rd); in do_scalar_shift_imm_narrow()
7550 * +----+-----+-----------+-------+---+------+--------+------+------+
7552 * +----+-----+-----------+-------+---+------+--------+------+------+
7577 * register-register MOV and MVN, so it is worth special casing. in disas_logic_reg()
7642 * +--+--+--+-----------+-----+--+-------+------+------+----+----+
7644 * +--+--+--+-----------+-----+--+-------+------+------+----+----+
7646 * sf: 0 -> 32bit, 1 -> 64bit
7647 * op: 0 -> add , 1 -> sub
7648 * S: 1 -> set flags
7676 /* non-flag setting ops may use SP */ in disas_add_sub_ext_reg()
7714 * +--+--+--+-----------+-----+--+-------+---------+------+------+
7716 * +--+--+--+-----------+-----+--+-------+---------+------+------+
7718 * sf: 0 -> 32bit, 1 -> 64bit
7719 * op: 0 -> add , 1 -> sub
7720 * S: 1 -> set flags
7721 * shift: 00 -> LSL, 01 -> LSR, 10 -> ASR, 11 -> RESERVED
7772 /* Data-processing (3 source)
7775 * +--+------+-----------+------+------+----+------+------+------+
7777 * +--+------+-----------+------+------+----+------+------+------+
7848 /* Special-case MADD with rA == XZR; it is the standard MUL alias */ in disas_data_proc_3src()
7866 * +--+--+--+------------------------+------+-------------+------+-----+
7868 * +--+--+--+------------------------+------+-------------+------+-----+
7903 * +--+--+--+-----------------+--------+-----------+------+--+------+
7905 * +--+--+--+-----------------+--------+-----------+------+--+------+
7929 tcg_gen_shli_i32(cpu_NF, nzcv, 31 - 3); in disas_rotate_right_into_flags()
7939 tcg_gen_shli_i32(cpu_VF, nzcv, 31 - 0); in disas_rotate_right_into_flags()
7946 * +--+--+--+-----------------+---------+----+---------+------+--+------+
7948 * +--+--+--+-----------------+---------+----+---------+------+--+------+
7970 tcg_gen_shli_i32(cpu_VF, tmp, shift - 1); in disas_evaluate_into_flags()
7977 * +--+--+--+------------------------+--------+------+----+--+------+--+-----+
7979 * +--+--+--+------------------------+--------+------+----+--+------+--+-----+
8028 * to help with this: T1 = (COND ? 0 : -1), T2 = (COND ? -1 : 0). in disas_cc()
8077 * +----+----+---+-----------------+------+------+-----+------+------+
8079 * +----+----+---+-----------------+------+------+-----+------+------+
8227 /* Data-processing (1 source)
8229 * +----+---+---+-----------------+---------+--------+------+------+
8231 * +----+---+---+-----------------+---------+--------+------+------+
8276 if (s->pauth_active) { in disas_data_proc_1src()
8284 if (s->pauth_active) { in disas_data_proc_1src()
8292 if (s->pauth_active) { in disas_data_proc_1src()
8300 if (s->pauth_active) { in disas_data_proc_1src()
8308 if (s->pauth_active) { in disas_data_proc_1src()
8316 if (s->pauth_active) { in disas_data_proc_1src()
8324 if (s->pauth_active) { in disas_data_proc_1src()
8332 if (s->pauth_active) { in disas_data_proc_1src()
8342 } else if (s->pauth_active) { in disas_data_proc_1src()
8350 } else if (s->pauth_active) { in disas_data_proc_1src()
8358 } else if (s->pauth_active) { in disas_data_proc_1src()
8366 } else if (s->pauth_active) { in disas_data_proc_1src()
8374 } else if (s->pauth_active) { in disas_data_proc_1src()
8382 } else if (s->pauth_active) { in disas_data_proc_1src()
8390 } else if (s->pauth_active) { in disas_data_proc_1src()
8398 } else if (s->pauth_active) { in disas_data_proc_1src()
8406 } else if (s->pauth_active) { in disas_data_proc_1src()
8414 } else if (s->pauth_active) { in disas_data_proc_1src()
8514 /* Data-processing (2 source)
8516 * +----+---+---+-----------------+------+--------+------+------+
8518 * +----+---+---+-----------------+------+--------+------+------+
8565 if (s->ata[0]) { in disas_data_proc_2src()
8625 * Data processing - register
8627 * +--+---+--+---+-------+-----+-------+-------+---------+
8629 * +--+---+--+---+-------+-----+-------+-------+---------+
8686 case 0x6: /* Data-processing */ in disas_data_proc_reg()
8761 * +---+---+---+-----------+------+---+------+-----+---------+------+-------+
8763 * +---+---+---+-----------+------+---+------+-----+---------+------+-------+
8810 * +---+---+---+-----------+------+---+------+------+-----+------+----+------+
8812 * +---+---+---+-----------+------+---+------+------+-----+------+----+------+
8872 /* Floating-point data-processing (1 source) - half precision */
8922 /* Floating-point data-processing (1 source) - single precision */
8928 int rmode = -1; in handle_fp_1src_single()
8994 /* Floating-point data-processing (1 source) - double precision */
9000 int rmode = -1; in handle_fp_1src_double()
9131 /* Floating point data-processing (1 source)
9133 * +---+---+---+-----------+------+---+--------+-----------+------+------+
9135 * +---+---+---+-----------+------+---+--------+-----------+------+------+
9173 /* 32-to-32 and 64-to-64 ops */ in disas_fp_1src()
9240 tcg_shift = tcg_constant_i32(64 - scale); in handle_fpfpcvt()
9388 /* Floating point <-> fixed point conversions
9390 * +----+---+---+-----------+------+---+-------+--------+-------+------+------+
9392 * +----+---+---+-----------+------+---+-------+--------+-------+------+------+
9521 /* Floating point <-> integer conversions
9523 * +----+---+---+-----------+------+---+-------+-----+-------------+----+----+
9525 * +----+---+---+-----------+------+---+-------+-----+-------------+----+----+
9575 case 0b01100110: /* FMOV half <-> 32-bit int */ in disas_fp_int_conv()
9577 case 0b11100110: /* FMOV half <-> 64-bit int */ in disas_fp_int_conv()
9583 case 0b00000110: /* FMOV 32-bit */ in disas_fp_int_conv()
9585 case 0b10100110: /* FMOV 64-bit */ in disas_fp_int_conv()
9587 case 0b11001110: /* FMOV top half of 128-bit */ in disas_fp_int_conv()
9613 /* FP-specific subcases of table C3-6 (SIMD and FP data processing)
9615 * +---+---+---+---------+-----------------------------+
9617 * +---+---+---+---------+-----------------------------+
9633 /* Floating point data-processing (2 source) */ in disas_data_proc_fp()
9651 /* Floating point data-processing (1 source) */ in disas_data_proc_fp()
9658 /* Floating point <-> integer conversions */ in disas_data_proc_fp()
9759 /* UCVTF/SCVTF - Integer to FP conversion */
9794 fracbits = (16 << size) - immhb; in handle_simd_shift_intfp_conv()
9803 /* FCVTZS, FVCVTZU - FP to fixedpoint conversion */
9842 fracbits = (16 << size) - immhb; in handle_simd_shift_fpint_conv()
9907 * +-----+---+-------------+------+------+--------+---+------+------+
9909 * +-----+---+-------------+------+------+--------+---+------+------+
9957 /* Handle 64->64 opcodes which are shared between the scalar and in handle_2misc_64()
9958 * vector 2-reg-misc groups. We cover every integer opcode where size == 3 in handle_2misc_64()
9959 * is valid in either group and also the double-precision fp ops. in handle_2misc_64()
9989 /* 64 bit integer comparison against zero, result is test ? -1 : 0. */ in handle_2misc_64()
10258 /* Handle 2-reg-misc ops which are narrowing (so each 2*size element in handle_2misc_narrow()
10373 * +-----+---+-----------+------+-----------+--------+-----+------+------+
10375 * +-----+---+-----------+------+-----------+--------+-----+------+------+
10481 handle_2misc_narrow(s, true, opcode, u, false, size - 1, rn, rd); in disas_simd_scalar_two_reg_misc()
10561 * +---+---+---+-------------+------+------+--------+---+------+------+
10563 * +---+---+---+-------------+------+------+--------+---+------+------+
10610 /* Handle 2-reg-misc ops which are widening (so each size element in handle_2misc_widening()
10617 /* 32 -> 64 bit fp conversion */ in handle_2misc_widening()
10632 /* 16 -> 32 bit fp conversion */ in handle_2misc_widening()
10656 int grp_size = 3 - opsz; in handle_rev()
10694 int revmask = (1 << grp_size) - 1; in handle_rev()
10724 /* Implement the pairwise operations from 2-misc: in handle_2misc_pairwise()
10727 * double-width result element in the output (possibly accumulating). in handle_2misc_pairwise()
10735 /* 32 + 32 -> 64 op */ in handle_2misc_pairwise()
10817 * +---+---+---+-----------+------+-----------+--------+-----+------+------+
10819 * +---+---+---+-----------+------+-----------+--------+-----+------+------+
10830 int rmode = -1; in disas_simd_two_reg_misc()
11008 /* handle_2misc_narrow does a 2*size -> size operation, but these in disas_simd_two_reg_misc()
11014 handle_2misc_narrow(s, false, opcode, 0, is_q, size - 1, rn, rd); in disas_simd_two_reg_misc()
11024 handle_2misc_narrow(s, false, opcode, 0, is_q, size - 1, rn, rd); in disas_simd_two_reg_misc()
11133 /* All 64-bit element operations can be shared with scalar 2misc */ in disas_simd_two_reg_misc()
11284 * +---+---+---+---+---------+---+-------------+--------+-----+------+------+
11286 * +---+---+---+---+---------+---+-------------+--------+-----+------+------+
11292 * in the vector form and are un-allocated for the scalar decode. Also
11307 int rmode = -1; in disas_simd_two_reg_misc_fp16()
11549 /* C3.6 Data processing - SIMD, inc Crypto
11566 /* Note that this is called with all non-FP cases from in disas_data_proc_simd()
11567 * table C3-6 so it must UNDEF for entries not specifically in disas_data_proc_simd()
11578 /* C3.6 Data processing - SIMD and floating point */
11596 s->is_nonstreaming = true; in trans_FAIL()
11604 * @btype: PSTATE.BTYPE, and is non-zero
11608 * - branch target identifiers,
11609 * - paciasp, pacibsp,
11610 * - BRK insn
11611 * - HLT insn
11657 case 0xd: /* Data processing - register */ in disas_a64_legacy()
11661 case 0xf: /* Data processing - SIMD and floating point */ in disas_a64_legacy()
11676 CPUARMTBFlags tb_flags = arm_tbflags_from_tb(dc->base.tb); in aarch64_tr_init_disas_context()
11679 dc->isar = &arm_cpu->isar; in aarch64_tr_init_disas_context()
11680 dc->condjmp = 0; in aarch64_tr_init_disas_context()
11681 dc->pc_save = dc->base.pc_first; in aarch64_tr_init_disas_context()
11682 dc->aarch64 = true; in aarch64_tr_init_disas_context()
11683 dc->thumb = false; in aarch64_tr_init_disas_context()
11684 dc->sctlr_b = 0; in aarch64_tr_init_disas_context()
11685 dc->be_data = EX_TBFLAG_ANY(tb_flags, BE_DATA) ? MO_BE : MO_LE; in aarch64_tr_init_disas_context()
11686 dc->condexec_mask = 0; in aarch64_tr_init_disas_context()
11687 dc->condexec_cond = 0; in aarch64_tr_init_disas_context()
11689 dc->mmu_idx = core_to_aa64_mmu_idx(core_mmu_idx); in aarch64_tr_init_disas_context()
11690 dc->tbii = EX_TBFLAG_A64(tb_flags, TBII); in aarch64_tr_init_disas_context()
11691 dc->tbid = EX_TBFLAG_A64(tb_flags, TBID); in aarch64_tr_init_disas_context()
11692 dc->tcma = EX_TBFLAG_A64(tb_flags, TCMA); in aarch64_tr_init_disas_context()
11693 dc->current_el = arm_mmu_idx_to_el(dc->mmu_idx); in aarch64_tr_init_disas_context()
11695 dc->user = (dc->current_el == 0); in aarch64_tr_init_disas_context()
11697 dc->fp_excp_el = EX_TBFLAG_ANY(tb_flags, FPEXC_EL); in aarch64_tr_init_disas_context()
11698 dc->align_mem = EX_TBFLAG_ANY(tb_flags, ALIGN_MEM); in aarch64_tr_init_disas_context()
11699 dc->pstate_il = EX_TBFLAG_ANY(tb_flags, PSTATE__IL); in aarch64_tr_init_disas_context()
11700 dc->fgt_active = EX_TBFLAG_ANY(tb_flags, FGT_ACTIVE); in aarch64_tr_init_disas_context()
11701 dc->fgt_svc = EX_TBFLAG_ANY(tb_flags, FGT_SVC); in aarch64_tr_init_disas_context()
11702 dc->trap_eret = EX_TBFLAG_A64(tb_flags, TRAP_ERET); in aarch64_tr_init_disas_context()
11703 dc->sve_excp_el = EX_TBFLAG_A64(tb_flags, SVEEXC_EL); in aarch64_tr_init_disas_context()
11704 dc->sme_excp_el = EX_TBFLAG_A64(tb_flags, SMEEXC_EL); in aarch64_tr_init_disas_context()
11705 dc->vl = (EX_TBFLAG_A64(tb_flags, VL) + 1) * 16; in aarch64_tr_init_disas_context()
11706 dc->svl = (EX_TBFLAG_A64(tb_flags, SVL) + 1) * 16; in aarch64_tr_init_disas_context()
11707 dc->pauth_active = EX_TBFLAG_A64(tb_flags, PAUTH_ACTIVE); in aarch64_tr_init_disas_context()
11708 dc->bt = EX_TBFLAG_A64(tb_flags, BT); in aarch64_tr_init_disas_context()
11709 dc->btype = EX_TBFLAG_A64(tb_flags, BTYPE); in aarch64_tr_init_disas_context()
11710 dc->unpriv = EX_TBFLAG_A64(tb_flags, UNPRIV); in aarch64_tr_init_disas_context()
11711 dc->ata[0] = EX_TBFLAG_A64(tb_flags, ATA); in aarch64_tr_init_disas_context()
11712 dc->ata[1] = EX_TBFLAG_A64(tb_flags, ATA0); in aarch64_tr_init_disas_context()
11713 dc->mte_active[0] = EX_TBFLAG_A64(tb_flags, MTE_ACTIVE); in aarch64_tr_init_disas_context()
11714 dc->mte_active[1] = EX_TBFLAG_A64(tb_flags, MTE0_ACTIVE); in aarch64_tr_init_disas_context()
11715 dc->pstate_sm = EX_TBFLAG_A64(tb_flags, PSTATE_SM); in aarch64_tr_init_disas_context()
11716 dc->pstate_za = EX_TBFLAG_A64(tb_flags, PSTATE_ZA); in aarch64_tr_init_disas_context()
11717 dc->sme_trap_nonstreaming = EX_TBFLAG_A64(tb_flags, SME_TRAP_NONSTREAMING); in aarch64_tr_init_disas_context()
11718 dc->naa = EX_TBFLAG_A64(tb_flags, NAA); in aarch64_tr_init_disas_context()
11719 dc->nv = EX_TBFLAG_A64(tb_flags, NV); in aarch64_tr_init_disas_context()
11720 dc->nv1 = EX_TBFLAG_A64(tb_flags, NV1); in aarch64_tr_init_disas_context()
11721 dc->nv2 = EX_TBFLAG_A64(tb_flags, NV2); in aarch64_tr_init_disas_context()
11722 dc->nv2_mem_e20 = EX_TBFLAG_A64(tb_flags, NV2_MEM_E20); in aarch64_tr_init_disas_context()
11723 dc->nv2_mem_be = EX_TBFLAG_A64(tb_flags, NV2_MEM_BE); in aarch64_tr_init_disas_context()
11724 dc->vec_len = 0; in aarch64_tr_init_disas_context()
11725 dc->vec_stride = 0; in aarch64_tr_init_disas_context()
11726 dc->cp_regs = arm_cpu->cp_regs; in aarch64_tr_init_disas_context()
11727 dc->features = env->features; in aarch64_tr_init_disas_context()
11728 dc->dcz_blocksize = arm_cpu->dcz_blocksize; in aarch64_tr_init_disas_context()
11729 dc->gm_blocksize = arm_cpu->gm_blocksize; in aarch64_tr_init_disas_context()
11733 tcg_debug_assert(dc->tbid & 1); in aarch64_tr_init_disas_context()
11736 dc->lse2 = dc_isar_feature(aa64_lse2, dc); in aarch64_tr_init_disas_context()
11738 /* Single step state. The code-generation logic here is: in aarch64_tr_init_disas_context()
11740 * generate code with no special handling for single-stepping (except in aarch64_tr_init_disas_context()
11744 * SS_ACTIVE == 1, PSTATE.SS == 1: (active-not-pending) in aarch64_tr_init_disas_context()
11749 * SS_ACTIVE == 1, PSTATE.SS == 0: (active-pending) in aarch64_tr_init_disas_context()
11753 dc->ss_active = EX_TBFLAG_ANY(tb_flags, SS_ACTIVE); in aarch64_tr_init_disas_context()
11754 dc->pstate_ss = EX_TBFLAG_ANY(tb_flags, PSTATE__SS); in aarch64_tr_init_disas_context()
11755 dc->is_ldex = false; in aarch64_tr_init_disas_context()
11758 bound = -(dc->base.pc_first | TARGET_PAGE_MASK) / 4; in aarch64_tr_init_disas_context()
11761 if (dc->ss_active) { in aarch64_tr_init_disas_context()
11764 dc->base.max_insns = MIN(dc->base.max_insns, bound); in aarch64_tr_init_disas_context()
11774 target_ulong pc_arg = dc->base.pc_next; in aarch64_tr_insn_start()
11776 if (tb_cflags(dcbase->tb) & CF_PCREL) { in aarch64_tr_insn_start()
11780 dc->insn_start_updated = false; in aarch64_tr_insn_start()
11787 uint64_t pc = s->base.pc_next; in aarch64_tr_translate_insn()
11791 if (s->ss_active && !s->pstate_ss) { in aarch64_tr_translate_insn()
11792 /* Singlestep state is Active-pending. in aarch64_tr_translate_insn()
11802 assert(s->base.num_insns == 1); in aarch64_tr_translate_insn()
11804 s->base.is_jmp = DISAS_NORETURN; in aarch64_tr_translate_insn()
11805 s->base.pc_next = pc + 4; in aarch64_tr_translate_insn()
11816 assert(s->base.num_insns == 1); in aarch64_tr_translate_insn()
11818 s->base.is_jmp = DISAS_NORETURN; in aarch64_tr_translate_insn()
11819 s->base.pc_next = QEMU_ALIGN_UP(pc, 4); in aarch64_tr_translate_insn()
11823 s->pc_curr = pc; in aarch64_tr_translate_insn()
11824 insn = arm_ldl_code(env, &s->base, pc, s->sctlr_b); in aarch64_tr_translate_insn()
11825 s->insn = insn; in aarch64_tr_translate_insn()
11826 s->base.pc_next = pc + 4; in aarch64_tr_translate_insn()
11828 s->fp_access_checked = false; in aarch64_tr_translate_insn()
11829 s->sve_access_checked = false; in aarch64_tr_translate_insn()
11831 if (s->pstate_il) { in aarch64_tr_translate_insn()
11841 if (s->base.num_insns == 1) { in aarch64_tr_translate_insn()
11842 /* First insn can have btype set to non-zero. */ in aarch64_tr_translate_insn()
11843 tcg_debug_assert(s->btype >= 0); in aarch64_tr_translate_insn()
11847 * priority -- below debugging exceptions but above most in aarch64_tr_translate_insn()
11854 if (s->btype != 0 in aarch64_tr_translate_insn()
11855 && !btype_destination_ok(insn, s->bt, s->btype)) { in aarch64_tr_translate_insn()
11860 tcg_debug_assert(s->btype == 0); in aarch64_tr_translate_insn()
11864 s->is_nonstreaming = false; in aarch64_tr_translate_insn()
11865 if (s->sme_trap_nonstreaming) { in aarch64_tr_translate_insn()
11877 * Note that we set btype == -1 when the insn sets btype. in aarch64_tr_translate_insn()
11879 if (s->btype > 0 && s->base.is_jmp != DISAS_NORETURN) { in aarch64_tr_translate_insn()
11888 if (unlikely(dc->ss_active)) { in aarch64_tr_tb_stop()
11892 * (and thus a tb-jump is not possible when singlestepping). in aarch64_tr_tb_stop()
11894 switch (dc->base.is_jmp) { in aarch64_tr_tb_stop()
11906 switch (dc->base.is_jmp) { in aarch64_tr_tb_stop()