Lines Matching +full:build +full:- +full:some +full:- +full:softmmu
22 #include "translate-a64.h"
74 #include "decode-sme-fa64.c.inc"
75 #include "decode-a64.c.inc"
112 ARMMMUIdx useridx = s->mmu_idx; in get_a64_user_mem_index()
114 if (unpriv && s->unpriv) { in get_a64_user_mem_index()
116 * We have pre-computed the condition for AccType_UNPRIV. in get_a64_user_mem_index()
144 /* BTYPE is a 2-bit field, and 0 should be done with reset_btype. */ in set_btype()
147 s->btype = -1; in set_btype()
152 if (s->btype != 0) { in reset_btype()
154 s->btype = 0; in reset_btype()
160 assert(s->pc_save != -1); in gen_pc_plus_diff()
161 if (tb_cflags(s->base.tb) & CF_PCREL) { in gen_pc_plus_diff()
162 tcg_gen_addi_i64(dest, cpu_pc, (s->pc_curr - s->pc_save) + diff); in gen_pc_plus_diff()
164 tcg_gen_movi_i64(dest, s->pc_curr + diff); in gen_pc_plus_diff()
171 s->pc_save = s->pc_curr + diff; in gen_a64_update_pc()
179 * then the address is zero-extended, clearing bits [63:56]
183 * the address is sign-extended from bit 55 into bits [63:56]
193 } else if (!regime_has_2_ranges(s->mmu_idx)) { in gen_top_byte_ignore()
197 /* Sign-extend from bit 55. */ in gen_top_byte_ignore()
224 gen_top_byte_ignore(s, cpu_pc, src, s->tbii); in gen_a64_set_pc()
225 s->pc_save = -1; in gen_a64_set_pc()
232 * for the tag to be present in the FAR_ELx register. But for user-only
237 * of the write-back address.
244 gen_top_byte_ignore(s, clean, addr, s->tbid); in clean_data_tbi()
277 if (tag_checked && s->mte_active[is_unpriv]) { in gen_mte_check1_mmuidx()
282 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid); in gen_mte_check1_mmuidx()
283 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma); in gen_mte_check1_mmuidx()
286 desc = FIELD_DP32(desc, MTEDESC, SIZEM1, memop_size(memop) - 1); in gen_mte_check1_mmuidx()
309 if (tag_checked && s->mte_active[0]) { in gen_mte_checkN()
314 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid); in gen_mte_checkN()
315 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma); in gen_mte_checkN()
318 desc = FIELD_DP32(desc, MTEDESC, SIZEM1, total_size - 1); in gen_mte_checkN()
331 * naturally aligned, but it must not cross a 16-byte boundary.
374 * If size == MO_128, this is a LDXP, and the operation is single-copy in check_atomic_align()
405 } else if (!s->naa) { in check_ordered_align()
423 * Sign-extend the 32-bit value so that the GE/LT comparisons work in a64_test_cc()
426 c64->cond = c32.cond; in a64_test_cc()
427 c64->value = tcg_temp_new_i64(); in a64_test_cc()
428 tcg_gen_ext_i32_i64(c64->value, c32.value); in a64_test_cc()
433 gen_helper_rebuild_hflags_a64(tcg_env, tcg_constant_i32(s->current_el)); in gen_rebuild_hflags()
440 s->base.is_jmp = DISAS_NORETURN; in gen_exception_internal_insn()
447 s->base.is_jmp = DISAS_NORETURN; in gen_exception_bkpt_insn()
452 /* We just completed step of an insn. Move from Active-not-pending in gen_step_complete_exception()
453 * to Active-pending, and then also take the swstep exception. in gen_step_complete_exception()
462 gen_swstep_exception(s, 1, s->is_ldex); in gen_step_complete_exception()
463 s->base.is_jmp = DISAS_NORETURN; in gen_step_complete_exception()
468 if (s->ss_active) { in use_goto_tb()
471 return translator_use_goto_tb(&s->base, dest); in use_goto_tb()
476 if (use_goto_tb(s, s->pc_curr + diff)) { in gen_goto_tb()
478 * For pcrel, the pc must always be up-to-date on entry to in gen_goto_tb()
485 if (tb_cflags(s->base.tb) & CF_PCREL) { in gen_goto_tb()
492 tcg_gen_exit_tb(s->base.tb, n); in gen_goto_tb()
493 s->base.is_jmp = DISAS_NORETURN; in gen_goto_tb()
496 if (s->ss_active) { in gen_goto_tb()
500 s->base.is_jmp = DISAS_NORETURN; in gen_goto_tb()
538 * representing the register contents. This TCGv is an auto-freed
622 * Clear the bits above an N-bit vector, for N = (is_q ? 128 : 64).
652 * - if FPCR.NEP == 0, clear the high elements of reg
653 * - if FPCR.NEP == 1, set the high elements of reg from mergereg
660 if (!s->fpcr_nep) { in write_fp_dreg_merging()
676 * Write a single-prec result, but only clear the higher elements
682 if (!s->fpcr_nep) { in write_fp_sreg_merging()
694 * Write a half-prec result, but only clear the higher elements
701 if (!s->fpcr_nep) { in write_fp_hreg_merging()
712 /* Expand a 2-operand AdvSIMD vector operation using an expander function. */
720 /* Expand a 2-operand + immediate AdvSIMD vector operation using
730 /* Expand a 3-operand AdvSIMD vector operation using an expander function. */
738 /* Expand a 4-operand AdvSIMD vector operation using an expander function. */
747 /* Expand a 2-operand operation using an out-of-line helper. */
756 /* Expand a 3-operand operation using an out-of-line helper. */
766 /* Expand a 3-operand + fpstatus pointer + simd data value operation using
767 * an out-of-line helper.
780 /* Expand a 4-operand operation using an out-of-line helper. */
792 * Expand a 4-operand operation using an out-of-line helper that takes
808 * Expand a 4-operand + fpstatus pointer + simd data value operation using
809 * an out-of-line helper.
904 if (dc->fpcr_ah) { in gen_vfp_maybe_ah_negh()
913 if (dc->fpcr_ah) { in gen_vfp_maybe_ah_negs()
922 if (dc->fpcr_ah) { in gen_vfp_maybe_ah_negd()
1000 /* dest = T0 - T1; compute C, N, V and Z flags */
1206 /* This always zero-extends and writes to a full 128 bit wide vector */ in do_fp_ld()
1366 * unallocated-encoding checks (otherwise the syndrome information
1371 if (s->fp_excp_el) { in fp_access_check_only()
1372 assert(!s->fp_access_checked); in fp_access_check_only()
1373 s->fp_access_checked = -1; in fp_access_check_only()
1377 s->fp_excp_el); in fp_access_check_only()
1380 s->fp_access_checked = 1; in fp_access_check_only()
1386 if (s->sme_trap_nonstreaming && s->is_nonstreaming) { in nonstreaming_check()
1400 * Return <0 for non-supported element sizes, with MO_16 controlled by
1411 return -1; in fp_access_check_scalar_hsd()
1415 return -1; in fp_access_check_scalar_hsd()
1426 return -1; in fp_access_check_vector_hsd()
1433 return -1; in fp_access_check_vector_hsd()
1437 return -1; in fp_access_check_vector_hsd()
1452 if (s->pstate_sm) { in sve_access_check()
1462 s->sve_access_checked = (ret ? 1 : -1); in sve_access_check()
1467 if (s->sve_excp_el) { in sve_access_check()
1469 assert(!s->sve_access_checked); in sve_access_check()
1471 syn_sve_access_trap(), s->sve_excp_el); in sve_access_check()
1472 s->sve_access_checked = -1; in sve_access_check()
1475 s->sve_access_checked = 1; in sve_access_check()
1486 if (s->sme_excp_el) { in sme_access_check()
1489 s->sme_excp_el); in sme_access_check()
1503 if (s->sme_excp_el in sme_enabled_check()
1504 && (!s->fp_excp_el || s->sme_excp_el <= s->fp_excp_el)) { in sme_enabled_check()
1506 s->fp_access_checked = (ret ? 1 : -1); in sme_enabled_check()
1518 if (FIELD_EX64(req, SVCR, SM) && !s->pstate_sm) { in sme_enabled_check_with_svcr()
1523 if (FIELD_EX64(req, SVCR, ZA) && !s->pstate_za) { in sme_enabled_check_with_svcr()
1538 if (!a->q && a->esz == MO_64) { in do_gvec_op2_ool()
1542 gen_gvec_op2_ool(s, a->q, a->rd, a->rn, data, fn); in do_gvec_op2_ool()
1550 if (!a->q && a->esz == MO_64) { in do_gvec_op3_ool()
1554 gen_gvec_op3_ool(s, a->q, a->rd, a->rn, a->rm, data, fn); in do_gvec_op3_ool()
1561 if (!a->q && a->esz == MO_64) { in do_gvec_fn3()
1565 gen_gvec_fn3(s, a->q, a->rd, a->rn, a->rm, fn, a->esz); in do_gvec_fn3()
1572 if (a->esz == MO_64) { in do_gvec_fn3_no64()
1576 gen_gvec_fn3(s, a->q, a->rd, a->rn, a->rm, fn, a->esz); in do_gvec_fn3_no64()
1583 if (a->esz == MO_8) { in do_gvec_fn3_no8_no64()
1591 if (!a->q && a->esz == MO_64) { in do_gvec_fn4()
1595 gen_gvec_fn4(s, a->q, a->rd, a->rn, a->rm, a->ra, fn, a->esz); in do_gvec_fn4()
1618 * or SCTLR bits) there is a check that SP is 16-aligned on every in gen_check_sp_alignment()
1619 * SP-relative load or store (with an exception generated if it is not). in gen_check_sp_alignment()
1639 gen_goto_tb(s, 0, a->imm); in trans_B()
1647 gen_goto_tb(s, 0, a->imm); in trans_BL()
1657 tcg_cmp = read_cpu_reg(s, a->rt, a->sf); in trans_CBZ()
1661 tcg_gen_brcondi_i64(a->nz ? TCG_COND_NE : TCG_COND_EQ, in trans_CBZ()
1665 gen_goto_tb(s, 1, a->imm); in trans_CBZ()
1675 tcg_gen_andi_i64(tcg_cmp, cpu_reg(s, a->rt), 1ULL << a->bitpos); in trans_TBZ()
1680 tcg_gen_brcondi_i64(a->nz ? TCG_COND_NE : TCG_COND_EQ, in trans_TBZ()
1684 gen_goto_tb(s, 1, a->imm); in trans_TBZ()
1691 if (a->c && !dc_isar_feature(aa64_hbc, s)) { in trans_B_cond()
1695 if (a->cond < 0x0e) { in trans_B_cond()
1698 arm_gen_test_cc(a->cond, match.label); in trans_B_cond()
1701 gen_goto_tb(s, 1, a->imm); in trans_B_cond()
1704 gen_goto_tb(s, 0, a->imm); in trans_B_cond()
1712 /* BR to {x16,x17} or !guard -> 1, else 3. */ in set_btype_for_br()
1719 s->btype = -1; in set_btype_for_br()
1734 set_btype_for_br(s, a->rn); in trans_BR()
1735 gen_a64_set_pc(s, cpu_reg(s, a->rn)); in trans_BR()
1736 s->base.is_jmp = DISAS_JUMP; in trans_BR()
1742 TCGv_i64 dst = cpu_reg(s, a->rn); in trans_BLR()
1752 s->base.is_jmp = DISAS_JUMP; in trans_BLR()
1758 gen_a64_set_pc(s, cpu_reg(s, a->rn)); in trans_RET()
1759 s->base.is_jmp = DISAS_JUMP; in trans_RET()
1772 if (!s->pauth_active) { in auth_branch_target()
1793 dst = auth_branch_target(s, cpu_reg(s, a->rn), tcg_constant_i64(0), !a->m); in trans_BRAZ()
1794 set_btype_for_br(s, a->rn); in trans_BRAZ()
1796 s->base.is_jmp = DISAS_JUMP; in trans_BRAZ()
1808 dst = auth_branch_target(s, cpu_reg(s, a->rn), tcg_constant_i64(0), !a->m); in trans_BLRAZ()
1818 s->base.is_jmp = DISAS_JUMP; in trans_BLRAZ()
1830 dst = auth_branch_target(s, cpu_reg(s, 30), cpu_X[31], !a->m); in trans_RETA()
1832 s->base.is_jmp = DISAS_JUMP; in trans_RETA()
1843 dst = auth_branch_target(s, cpu_reg(s,a->rn), cpu_reg_sp(s, a->rm), !a->m); in trans_BRA()
1845 set_btype_for_br(s, a->rn); in trans_BRA()
1846 s->base.is_jmp = DISAS_JUMP; in trans_BRA()
1857 dst = auth_branch_target(s, cpu_reg(s, a->rn), cpu_reg_sp(s, a->rm), !a->m); in trans_BLRA()
1867 s->base.is_jmp = DISAS_JUMP; in trans_BLRA()
1875 if (s->current_el == 0) { in trans_ERET()
1878 if (s->trap_eret) { in trans_ERET()
1884 offsetof(CPUARMState, elr_el[s->current_el])); in trans_ERET()
1886 translator_io_start(&s->base); in trans_ERET()
1889 /* Must exit loop to check un-masked IRQs */ in trans_ERET()
1890 s->base.is_jmp = DISAS_EXIT; in trans_ERET()
1901 if (s->current_el == 0) { in trans_ERETA()
1905 if (s->trap_eret) { in trans_ERETA()
1906 gen_exception_insn_el(s, 0, EXCP_UDEF, syn_erettrap(a->m ? 3 : 2), 2); in trans_ERETA()
1911 offsetof(CPUARMState, elr_el[s->current_el])); in trans_ERETA()
1913 dst = auth_branch_target(s, dst, cpu_X[31], !a->m); in trans_ERETA()
1915 translator_io_start(&s->base); in trans_ERETA()
1918 /* Must exit loop to check un-masked IRQs */ in trans_ERETA()
1919 s->base.is_jmp = DISAS_EXIT; in trans_ERETA()
1936 if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) { in trans_YIELD()
1937 s->base.is_jmp = DISAS_YIELD; in trans_YIELD()
1944 s->base.is_jmp = DISAS_WFI; in trans_WFI()
1956 if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) { in trans_WFE()
1957 s->base.is_jmp = DISAS_WFE; in trans_WFE()
1972 * check ss_active so that single-stepping a WFIT doesn't halt. in trans_WFIT()
1974 if (s->ss_active) { in trans_WFIT()
1980 gen_helper_wfit(tcg_env, cpu_reg(s, a->rd)); in trans_WFIT()
1982 s->base.is_jmp = DISAS_EXIT; in trans_WFIT()
1997 if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) { in trans_WFET()
1998 s->base.is_jmp = DISAS_WFE; in trans_WFET()
2005 if (s->pauth_active) { in trans_XPACLRI()
2013 if (s->pauth_active) { in trans_PACIA1716()
2021 if (s->pauth_active) { in trans_PACIB1716()
2029 if (s->pauth_active) { in trans_AUTIA1716()
2037 if (s->pauth_active) { in trans_AUTIB1716()
2056 if (s->current_el <= 1 && arm_dc_feature(s, ARM_FEATURE_EL2)) { in trans_ESB()
2065 if (s->pauth_active) { in trans_PACIAZ()
2073 if (s->pauth_active) { in trans_PACIASP()
2081 if (s->pauth_active) { in trans_PACIBZ()
2089 if (s->pauth_active) { in trans_PACIBSP()
2097 if (s->pauth_active) { in trans_AUTIAZ()
2105 if (s->pauth_active) { in trans_AUTIASP()
2113 if (s->pauth_active) { in trans_AUTIBZ()
2121 if (s->pauth_active) { in trans_AUTIBSP()
2129 tcg_gen_movi_i64(cpu_exclusive_addr, -1); in trans_CLREX()
2138 switch (a->types) { in trans_DSB_DMB()
2166 * self-modifying code correctly and also to take in trans_ISB()
2213 * ~-(C | Z) in trans_XAFLAG()
2214 * (C | Z) - 1 in trans_XAFLAG()
2223 /* (!C & Z) << 31 -> -(Z & ~C) */ in trans_XAFLAG()
2239 tcg_gen_sari_i32(cpu_VF, cpu_VF, 31); /* V ? -1 : 0 */ in trans_AXFLAG()
2242 /* !(Z | V) -> !(!ZF | V) -> ZF & !V -> ZF & ~VF */ in trans_AXFLAG()
2253 if (!dc_isar_feature(aa64_uao, s) || s->current_el == 0) { in trans_MSR_i_UAO()
2256 if (a->imm & 1) { in trans_MSR_i_UAO()
2262 s->base.is_jmp = DISAS_TOO_MANY; in trans_MSR_i_UAO()
2268 if (!dc_isar_feature(aa64_pan, s) || s->current_el == 0) { in trans_MSR_i_PAN()
2271 if (a->imm & 1) { in trans_MSR_i_PAN()
2277 s->base.is_jmp = DISAS_TOO_MANY; in trans_MSR_i_PAN()
2283 if (s->current_el == 0) { in trans_MSR_i_SPSEL()
2286 gen_helper_msr_i_spsel(tcg_env, tcg_constant_i32(a->imm & PSTATE_SP)); in trans_MSR_i_SPSEL()
2287 s->base.is_jmp = DISAS_TOO_MANY; in trans_MSR_i_SPSEL()
2296 if (a->imm & 1) { in trans_MSR_i_SBSS()
2302 s->base.is_jmp = DISAS_TOO_MANY; in trans_MSR_i_SBSS()
2311 if (a->imm & 1) { in trans_MSR_i_DIT()
2317 s->base.is_jmp = DISAS_TOO_MANY; in trans_MSR_i_DIT()
2324 /* Full MTE is enabled -- set the TCO bit as directed. */ in trans_MSR_i_TCO()
2325 if (a->imm & 1) { in trans_MSR_i_TCO()
2332 s->base.is_jmp = DISAS_UPDATE_NOCHAIN; in trans_MSR_i_TCO()
2335 /* Only "instructions accessible at EL0" -- PSTATE.TCO is WI. */ in trans_MSR_i_TCO()
2345 gen_helper_msr_i_daifset(tcg_env, tcg_constant_i32(a->imm)); in trans_MSR_i_DAIFSET()
2346 s->base.is_jmp = DISAS_TOO_MANY; in trans_MSR_i_DAIFSET()
2352 gen_helper_msr_i_daifclear(tcg_env, tcg_constant_i32(a->imm)); in trans_MSR_i_DAIFCLEAR()
2353 /* Exit the cpu loop to re-evaluate pending IRQs. */ in trans_MSR_i_DAIFCLEAR()
2354 s->base.is_jmp = DISAS_UPDATE_EXIT; in trans_MSR_i_DAIFCLEAR()
2360 if (!dc_isar_feature(aa64_nmi, s) || s->current_el == 0) { in trans_MSR_i_ALLINT()
2364 if (a->imm == 0) { in trans_MSR_i_ALLINT()
2366 } else if (s->current_el > 1) { in trans_MSR_i_ALLINT()
2372 /* Exit the cpu loop to re-evaluate pending IRQs. */ in trans_MSR_i_ALLINT()
2373 s->base.is_jmp = DISAS_UPDATE_EXIT; in trans_MSR_i_ALLINT()
2379 if (!dc_isar_feature(aa64_sme, s) || a->mask == 0) { in trans_MSR_i_SVCR()
2383 int old = s->pstate_sm | (s->pstate_za << 1); in trans_MSR_i_SVCR()
2384 int new = a->imm * 3; in trans_MSR_i_SVCR()
2386 if ((old ^ new) & a->mask) { in trans_MSR_i_SVCR()
2389 tcg_constant_i32(a->mask)); in trans_MSR_i_SVCR()
2390 s->base.is_jmp = DISAS_TOO_MANY; in trans_MSR_i_SVCR()
2401 /* build bit 31, N */ in gen_get_nzcv()
2403 /* build bit 30, Z */ in gen_get_nzcv()
2406 /* build bit 29, C */ in gen_get_nzcv()
2408 /* build bit 28, V */ in gen_get_nzcv()
2458 /* MRS - move from system register
2459 * MSR (register) - move to system register
2471 const ARMCPRegInfo *ri = get_arm_cp_reginfo(s->cp_regs, key); in handle_sys()
2486 switch (s->current_el) { in handle_sys()
2509 if (s->nv2 && ri->nv2_redirect_offset) { in handle_sys()
2511 * Some registers always redirect to memory; some only do so if in handle_sys()
2512 * HCR_EL2.NV1 is 0, and some only if NV1 is 1 (these come in in handle_sys()
2515 if (ri->nv2_redirect_offset & NV2_REDIR_NV1) { in handle_sys()
2516 nv2_mem_redirect = s->nv1; in handle_sys()
2517 } else if (ri->nv2_redirect_offset & NV2_REDIR_NO_NV1) { in handle_sys()
2518 nv2_mem_redirect = !s->nv1; in handle_sys()
2525 if (!cp_access_ok(s->current_el, ri, isread)) { in handle_sys()
2532 if (s->nv2 && (ri->type & ARM_CP_NV2_REDIRECT)) { in handle_sys()
2542 * NV2 redirect-to-memory takes precedence over trap to EL2 or in handle_sys()
2545 } else if (s->nv && arm_cpreg_traps_in_nv(ri)) { in handle_sys()
2552 * over the trap-to-EL2. Most trapped-by-FEAT_NV registers have in handle_sys()
2554 * the trap-to-EL3 controls which would apply to that register in handle_sys()
2555 * at EL2 don't take priority over the FEAT_NV trap-to-EL2.) in handle_sys()
2564 if (ri->accessfn || (ri->fgt && s->fgt_active)) { in handle_sys()
2574 } else if (ri->type & ARM_CP_RAISES_EXC) { in handle_sys()
2583 if ((ri->type & ARM_CP_FPU) && !fp_access_check_only(s)) { in handle_sys()
2585 } else if ((ri->type & ARM_CP_SVE) && !sve_access_check(s)) { in handle_sys()
2587 } else if ((ri->type & ARM_CP_SME) && !sme_access_check(s)) { in handle_sys()
2604 * fine-grained-traps on EL1 also do not apply here. in handle_sys()
2608 ri = get_arm_cp_reginfo(s->cp_regs, key); in handle_sys()
2610 assert(cp_access_ok(s->current_el, ri, isread)); in handle_sys()
2615 assert(!(ri->type & ARM_CP_RAISES_EXC)); in handle_sys()
2624 * The access is 64-bit single copy atomic, guaranteed aligned because in handle_sys()
2633 ARMMMUIdx armmemidx = s->nv2_mem_e20 ? ARMMMUIdx_E20_2 : ARMMMUIdx_E2; in handle_sys()
2637 mop |= (s->nv2_mem_be ? MO_BE : MO_LE); in handle_sys()
2641 (ri->nv2_redirect_offset & ~NV2_REDIR_FLAG_MASK)); in handle_sys()
2655 switch (ri->type & ARM_CP_SPECIAL_MASK) { in handle_sys()
2675 int el = s->nv ? 2 : s->current_el; in handle_sys()
2682 if (s->mte_active[0]) { in handle_sys()
2686 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid); in handle_sys()
2687 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma); in handle_sys()
2709 if (s->ata[0]) { in handle_sys()
2726 if (s->ata[0]) { in handle_sys()
2738 if (ri->type & ARM_CP_IO) { in handle_sys()
2740 need_exit_tb = translator_io_start(&s->base); in handle_sys()
2746 if (ri->type & ARM_CP_CONST) { in handle_sys()
2747 tcg_gen_movi_i64(tcg_rt, ri->resetvalue); in handle_sys()
2748 } else if (ri->readfn) { in handle_sys()
2754 tcg_gen_ld_i64(tcg_rt, tcg_env, ri->fieldoffset); in handle_sys()
2757 if (ri->type & ARM_CP_CONST) { in handle_sys()
2760 } else if (ri->writefn) { in handle_sys()
2766 tcg_gen_st_i64(tcg_rt, tcg_env, ri->fieldoffset); in handle_sys()
2770 if (!isread && !(ri->type & ARM_CP_SUPPRESS_TB_END)) { in handle_sys()
2784 s->base.is_jmp = DISAS_UPDATE_EXIT; in handle_sys()
2790 handle_sys(s, a->l, a->op0, a->op1, a->op2, a->crn, a->crm, a->rt); in trans_SYS()
2797 * For SVC, HVC and SMC we advance the single-step state in trans_SVC()
2799 * mandated, to ensure that single-stepping a system call in trans_SVC()
2802 uint32_t syndrome = syn_aa64_svc(a->imm); in trans_SVC()
2803 if (s->fgt_svc) { in trans_SVC()
2814 int target_el = s->current_el == 3 ? 3 : 2; in trans_HVC()
2816 if (s->current_el == 0) { in trans_HVC()
2828 gen_exception_insn_el(s, 4, EXCP_HVC, syn_aa64_hvc(a->imm), target_el); in trans_HVC()
2834 if (s->current_el == 0) { in trans_SMC()
2839 gen_helper_pre_smc(tcg_env, tcg_constant_i32(syn_aa64_smc(a->imm))); in trans_SMC()
2842 gen_exception_insn_el(s, 4, EXCP_SMC, syn_aa64_smc(a->imm), 3); in trans_SMC()
2848 gen_exception_bkpt_insn(s, syn_aa64_bkpt(a->imm)); in trans_BRK()
2861 if (semihosting_enabled(s->current_el == 0) && a->imm == 0xf000) { in trans_HLT()
2877 * races in multi-threaded linux-user and when MTTCG softmmu is
2887 s->is_ldex = true; in gen_load_exclusive()
2896 if (s->be_data == MO_LE) { in gen_load_exclusive()
2908 if (s->be_data == MO_LE) { in gen_load_exclusive()
2928 /* if (env->exclusive_addr == addr && env->exclusive_val == [addr] in gen_store_exclusive()
2929 * && (!is_pair || env->exclusive_high == [addr + datasize])) { in gen_store_exclusive()
2938 * env->exclusive_addr = -1; in gen_store_exclusive()
2964 * It is possible to trigger an alignment fault pre-LSE2, e.g. with an in gen_store_exclusive()
2965 * unaligned 4-byte write within the range of an aligned 8-byte load. in gen_store_exclusive()
2966 * With LSE2, the store would need to cross a 16-byte boundary when the in gen_store_exclusive()
2986 if (s->be_data == MO_LE) { in gen_store_exclusive()
3000 if (s->be_data == MO_LE) { in gen_store_exclusive()
3015 if (s->be_data == MO_LE) { in gen_store_exclusive()
3038 tcg_gen_movi_i64(cpu_exclusive_addr, -1); in gen_store_exclusive()
3082 if (s->be_data == MO_LE) { in gen_compare_and_swap_pair()
3092 if (s->be_data == MO_LE) { in gen_compare_and_swap_pair()
3101 if (s->be_data == MO_LE) { in gen_compare_and_swap_pair()
3111 if (s->be_data == MO_LE) { in gen_compare_and_swap_pair()
3122 * is accessing a 32-bit or 64-bit register. This logic is derived
3131 * do a zero-extend from 32 to 64 after the load. in ldst_iss_sf()
3143 if (a->rn == 31) { in trans_STXR()
3146 if (a->lasr) { in trans_STXR()
3149 gen_store_exclusive(s, a->rs, a->rt, a->rt2, a->rn, a->sz, false); in trans_STXR()
3155 if (a->rn == 31) { in trans_LDXR()
3158 gen_load_exclusive(s, a->rt, a->rt2, a->rn, a->sz, false); in trans_LDXR()
3159 if (a->lasr) { in trans_LDXR()
3169 bool iss_sf = ldst_iss_sf(a->sz, false, false); in trans_STLR()
3172 * StoreLORelease is the same as Store-Release for QEMU, but in trans_STLR()
3173 * needs the feature-test. in trans_STLR()
3175 if (!a->lasr && !dc_isar_feature(aa64_lor, s)) { in trans_STLR()
3178 /* Generate ISS for non-exclusive accesses including LASR. */ in trans_STLR()
3179 if (a->rn == 31) { in trans_STLR()
3183 memop = check_ordered_align(s, a->rn, 0, true, a->sz); in trans_STLR()
3184 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn), in trans_STLR()
3185 true, a->rn != 31, memop); in trans_STLR()
3186 do_gpr_st(s, cpu_reg(s, a->rt), clean_addr, memop, true, a->rt, in trans_STLR()
3187 iss_sf, a->lasr); in trans_STLR()
3195 bool iss_sf = ldst_iss_sf(a->sz, false, false); in trans_LDAR()
3197 /* LoadLOAcquire is the same as Load-Acquire for QEMU. */ in trans_LDAR()
3198 if (!a->lasr && !dc_isar_feature(aa64_lor, s)) { in trans_LDAR()
3201 /* Generate ISS for non-exclusive accesses including LASR. */ in trans_LDAR()
3202 if (a->rn == 31) { in trans_LDAR()
3205 memop = check_ordered_align(s, a->rn, 0, false, a->sz); in trans_LDAR()
3206 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn), in trans_LDAR()
3207 false, a->rn != 31, memop); in trans_LDAR()
3208 do_gpr_ld(s, cpu_reg(s, a->rt), clean_addr, memop, false, true, in trans_LDAR()
3209 a->rt, iss_sf, a->lasr); in trans_LDAR()
3216 if (a->rn == 31) { in trans_STXP()
3219 if (a->lasr) { in trans_STXP()
3222 gen_store_exclusive(s, a->rs, a->rt, a->rt2, a->rn, a->sz, true); in trans_STXP()
3228 if (a->rn == 31) { in trans_LDXP()
3231 gen_load_exclusive(s, a->rt, a->rt2, a->rn, a->sz, true); in trans_LDXP()
3232 if (a->lasr) { in trans_LDXP()
3243 if (((a->rt | a->rs) & 1) != 0) { in trans_CASP()
3247 gen_compare_and_swap_pair(s, a->rs, a->rt, a->rn, a->sz); in trans_CASP()
3256 gen_compare_and_swap(s, a->rs, a->rt, a->rn, a->sz); in trans_CAS()
3262 bool iss_sf = ldst_iss_sf(a->sz, a->sign, false); in trans_LD_lit()
3263 TCGv_i64 tcg_rt = cpu_reg(s, a->rt); in trans_LD_lit()
3265 MemOp memop = finalize_memop(s, a->sz + a->sign * MO_SIGN); in trans_LD_lit()
3267 gen_pc_plus_diff(s, clean_addr, a->imm); in trans_LD_lit()
3269 false, true, a->rt, iss_sf, false); in trans_LD_lit()
3282 memop = finalize_memop_asimd(s, a->sz); in trans_LD_lit_v()
3284 gen_pc_plus_diff(s, clean_addr, a->imm); in trans_LD_lit_v()
3285 do_fp_ld(s, a->rt, clean_addr, memop); in trans_LD_lit_v()
3293 if (a->rn == 31) { in op_addr_ldstpair_pre()
3297 *dirty_addr = read_cpu_reg_sp(s, a->rn, 1); in op_addr_ldstpair_pre()
3298 if (!a->p) { in op_addr_ldstpair_pre()
3303 (a->w || a->rn != 31), 2 << a->sz, mop); in op_addr_ldstpair_pre()
3309 if (a->w) { in op_addr_ldstpair_post()
3310 if (a->p) { in op_addr_ldstpair_post()
3313 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), dirty_addr); in op_addr_ldstpair_post()
3319 uint64_t offset = a->imm << a->sz; in trans_STP()
3321 MemOp mop = finalize_memop(s, a->sz); in trans_STP()
3324 tcg_rt = cpu_reg(s, a->rt); in trans_STP()
3325 tcg_rt2 = cpu_reg(s, a->rt2); in trans_STP()
3327 * We built mop above for the single logical access -- rebuild it in trans_STP()
3330 * With LSE2, non-sign-extending pairs are treated atomically if in trans_STP()
3332 * within a 16-byte block and that element will be atomic. in trans_STP()
3336 mop = a->sz + 1; in trans_STP()
3337 if (s->align_mem) { in trans_STP()
3338 mop |= (a->sz == 2 ? MO_ALIGN_4 : MO_ALIGN_8); in trans_STP()
3341 if (a->sz == 2) { in trans_STP()
3344 if (s->be_data == MO_LE) { in trans_STP()
3353 if (s->be_data == MO_LE) { in trans_STP()
3366 uint64_t offset = a->imm << a->sz; in trans_LDP()
3368 MemOp mop = finalize_memop(s, a->sz); in trans_LDP()
3371 tcg_rt = cpu_reg(s, a->rt); in trans_LDP()
3372 tcg_rt2 = cpu_reg(s, a->rt2); in trans_LDP()
3375 * We built mop above for the single logical access -- rebuild it in trans_LDP()
3378 * With LSE2, non-sign-extending pairs are treated atomically if in trans_LDP()
3380 * within a 16-byte block and that element will be atomic. in trans_LDP()
3384 * This treats sign-extending loads like zero-extending loads, in trans_LDP()
3387 mop = a->sz + 1; in trans_LDP()
3388 if (s->align_mem) { in trans_LDP()
3389 mop |= (a->sz == 2 ? MO_ALIGN_4 : MO_ALIGN_8); in trans_LDP()
3392 if (a->sz == 2) { in trans_LDP()
3393 int o2 = s->be_data == MO_LE ? 32 : 0; in trans_LDP()
3397 if (a->sign) { in trans_LDP()
3408 if (s->be_data == MO_LE) { in trans_LDP()
3420 uint64_t offset = a->imm << a->sz; in trans_STP_v()
3429 mop = finalize_memop_asimd(s, a->sz); in trans_STP_v()
3431 do_fp_st(s, a->rt, clean_addr, mop); in trans_STP_v()
3432 tcg_gen_addi_i64(clean_addr, clean_addr, 1 << a->sz); in trans_STP_v()
3433 do_fp_st(s, a->rt2, clean_addr, mop); in trans_STP_v()
3440 uint64_t offset = a->imm << a->sz; in trans_LDP_v()
3449 mop = finalize_memop_asimd(s, a->sz); in trans_LDP_v()
3451 do_fp_ld(s, a->rt, clean_addr, mop); in trans_LDP_v()
3452 tcg_gen_addi_i64(clean_addr, clean_addr, 1 << a->sz); in trans_LDP_v()
3453 do_fp_ld(s, a->rt2, clean_addr, mop); in trans_LDP_v()
3461 uint64_t offset = a->imm << LOG2_TAG_GRANULE; in trans_STGP()
3466 tcg_debug_assert(a->sz == MO_64); in trans_STGP()
3472 if (a->rn == 31) { in trans_STGP()
3476 dirty_addr = read_cpu_reg_sp(s, a->rn, 1); in trans_STGP()
3477 if (!a->p) { in trans_STGP()
3482 tcg_rt = cpu_reg(s, a->rt); in trans_STGP()
3483 tcg_rt2 = cpu_reg(s, a->rt2); in trans_STGP()
3486 * STGP is defined as two 8-byte memory operations, aligned to TAG_GRANULE, in trans_STGP()
3487 * and one tag operation. We implement it as one single aligned 16-byte in trans_STGP()
3489 * MO_ATOM_IFALIGN_PAIR produces 8-byte atomicity for the memory store. in trans_STGP()
3494 if (s->be_data == MO_LE) { in trans_STGP()
3502 if (s->ata[0]) { in trans_STGP()
3503 if (tb_cflags(s->base.tb) & CF_PARALLEL) { in trans_STGP()
3520 if (a->rn == 31) { in op_addr_ldst_imm_pre()
3524 *dirty_addr = read_cpu_reg_sp(s, a->rn, 1); in op_addr_ldst_imm_pre()
3525 if (!a->p) { in op_addr_ldst_imm_pre()
3528 memidx = get_a64_user_mem_index(s, a->unpriv); in op_addr_ldst_imm_pre()
3530 a->w || a->rn != 31, in op_addr_ldst_imm_pre()
3531 mop, a->unpriv, memidx); in op_addr_ldst_imm_pre()
3537 if (a->w) { in op_addr_ldst_imm_post()
3538 if (a->p) { in op_addr_ldst_imm_post()
3541 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), dirty_addr); in op_addr_ldst_imm_post()
3547 bool iss_sf, iss_valid = !a->w; in trans_STR_i()
3549 int memidx = get_a64_user_mem_index(s, a->unpriv); in trans_STR_i()
3550 MemOp mop = finalize_memop(s, a->sz + a->sign * MO_SIGN); in trans_STR_i()
3552 op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, true, mop); in trans_STR_i()
3554 tcg_rt = cpu_reg(s, a->rt); in trans_STR_i()
3555 iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext); in trans_STR_i()
3558 iss_valid, a->rt, iss_sf, false); in trans_STR_i()
3559 op_addr_ldst_imm_post(s, a, dirty_addr, a->imm); in trans_STR_i()
3565 bool iss_sf, iss_valid = !a->w; in trans_LDR_i()
3567 int memidx = get_a64_user_mem_index(s, a->unpriv); in trans_LDR_i()
3568 MemOp mop = finalize_memop(s, a->sz + a->sign * MO_SIGN); in trans_LDR_i()
3570 op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, false, mop); in trans_LDR_i()
3572 tcg_rt = cpu_reg(s, a->rt); in trans_LDR_i()
3573 iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext); in trans_LDR_i()
3576 a->ext, memidx, iss_valid, a->rt, iss_sf, false); in trans_LDR_i()
3577 op_addr_ldst_imm_post(s, a, dirty_addr, a->imm); in trans_LDR_i()
3589 mop = finalize_memop_asimd(s, a->sz); in trans_STR_v_i()
3590 op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, true, mop); in trans_STR_v_i()
3591 do_fp_st(s, a->rt, clean_addr, mop); in trans_STR_v_i()
3592 op_addr_ldst_imm_post(s, a, dirty_addr, a->imm); in trans_STR_v_i()
3604 mop = finalize_memop_asimd(s, a->sz); in trans_LDR_v_i()
3605 op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, false, mop); in trans_LDR_v_i()
3606 do_fp_ld(s, a->rt, clean_addr, mop); in trans_LDR_v_i()
3607 op_addr_ldst_imm_post(s, a, dirty_addr, a->imm); in trans_LDR_v_i()
3617 if (a->rn == 31) { in op_addr_ldst_pre()
3620 *dirty_addr = read_cpu_reg_sp(s, a->rn, 1); in op_addr_ldst_pre()
3622 tcg_rm = read_cpu_reg(s, a->rm, 1); in op_addr_ldst_pre()
3623 ext_and_shift_reg(tcg_rm, tcg_rm, a->opt, a->s ? a->sz : 0); in op_addr_ldst_pre()
3632 bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext); in trans_LDR()
3635 if (extract32(a->opt, 1, 1) == 0) { in trans_LDR()
3639 memop = finalize_memop(s, a->sz + a->sign * MO_SIGN); in trans_LDR()
3641 tcg_rt = cpu_reg(s, a->rt); in trans_LDR()
3643 a->ext, true, a->rt, iss_sf, false); in trans_LDR()
3650 bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext); in trans_STR()
3653 if (extract32(a->opt, 1, 1) == 0) { in trans_STR()
3657 memop = finalize_memop(s, a->sz); in trans_STR()
3659 tcg_rt = cpu_reg(s, a->rt); in trans_STR()
3660 do_gpr_st(s, tcg_rt, clean_addr, memop, true, a->rt, iss_sf, false); in trans_STR()
3669 if (extract32(a->opt, 1, 1) == 0) { in trans_LDR_v()
3677 memop = finalize_memop_asimd(s, a->sz); in trans_LDR_v()
3679 do_fp_ld(s, a->rt, clean_addr, memop); in trans_LDR_v()
3688 if (extract32(a->opt, 1, 1) == 0) { in trans_STR_v()
3696 memop = finalize_memop_asimd(s, a->sz); in trans_STR_v()
3698 do_fp_st(s, a->rt, clean_addr, memop); in trans_STR_v()
3706 MemOp mop = a->sz | sign; in do_atomic_ld()
3709 if (a->rn == 31) { in do_atomic_ld()
3712 mop = check_atomic_align(s, a->rn, mop); in do_atomic_ld()
3713 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn), false, in do_atomic_ld()
3714 a->rn != 31, mop); in do_atomic_ld()
3715 tcg_rs = read_cpu_reg(s, a->rs, true); in do_atomic_ld()
3716 tcg_rt = cpu_reg(s, a->rt); in do_atomic_ld()
3727 switch (a->sz) { in do_atomic_ld()
3758 bool iss_sf = ldst_iss_sf(a->sz, false, false); in TRANS_FEAT()
3766 if (a->rn == 31) { in TRANS_FEAT()
3769 mop = check_ordered_align(s, a->rn, 0, false, a->sz); in TRANS_FEAT()
3770 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn), false, in TRANS_FEAT()
3771 a->rn != 31, mop); in TRANS_FEAT()
3774 * fetch-and-do-something op. in TRANS_FEAT()
3776 * full load-acquire (we only need "load-acquire processor consistent"), in TRANS_FEAT()
3779 do_gpr_ld(s, cpu_reg(s, a->rt), clean_addr, mop, false, in TRANS_FEAT()
3780 true, a->rt, iss_sf, true); in TRANS_FEAT()
3795 if (a->rn == 31) { in trans_LDRA()
3798 dirty_addr = read_cpu_reg_sp(s, a->rn, 1); in trans_LDRA()
3800 if (s->pauth_active) { in trans_LDRA()
3801 if (!a->m) { in trans_LDRA()
3810 tcg_gen_addi_i64(dirty_addr, dirty_addr, a->imm); in trans_LDRA()
3816 a->w || a->rn != 31, memop); in trans_LDRA()
3818 tcg_rt = cpu_reg(s, a->rt); in trans_LDRA()
3820 /* extend */ false, /* iss_valid */ !a->w, in trans_LDRA()
3821 /* iss_srt */ a->rt, /* iss_sf */ true, /* iss_ar */ false); in trans_LDRA()
3823 if (a->w) { in trans_LDRA()
3824 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), dirty_addr); in trans_LDRA()
3832 MemOp mop = a->sz | (a->sign ? MO_SIGN : 0); in trans_LDAPR_i()
3833 bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext); in trans_LDAPR_i()
3839 if (a->rn == 31) { in trans_LDAPR_i()
3843 mop = check_ordered_align(s, a->rn, a->imm, false, mop); in trans_LDAPR_i()
3844 dirty_addr = read_cpu_reg_sp(s, a->rn, 1); in trans_LDAPR_i()
3845 tcg_gen_addi_i64(dirty_addr, dirty_addr, a->imm); in trans_LDAPR_i()
3849 * Load-AcquirePC semantics; we implement as the slightly more in trans_LDAPR_i()
3850 * restrictive Load-Acquire. in trans_LDAPR_i()
3852 do_gpr_ld(s, cpu_reg(s, a->rt), clean_addr, mop, a->ext, true, in trans_LDAPR_i()
3853 a->rt, iss_sf, true); in trans_LDAPR_i()
3861 MemOp mop = a->sz; in trans_STLR_i()
3862 bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext); in trans_STLR_i()
3868 /* TODO: ARMv8.4-LSE SCTLR.nAA */ in trans_STLR_i()
3870 if (a->rn == 31) { in trans_STLR_i()
3874 mop = check_ordered_align(s, a->rn, a->imm, true, mop); in trans_STLR_i()
3875 dirty_addr = read_cpu_reg_sp(s, a->rn, 1); in trans_STLR_i()
3876 tcg_gen_addi_i64(dirty_addr, dirty_addr, a->imm); in trans_STLR_i()
3879 /* Store-Release semantics */ in trans_STLR_i()
3881 do_gpr_st(s, cpu_reg(s, a->rt), clean_addr, mop, true, a->rt, iss_sf, true); in trans_STLR_i()
3893 int size = a->sz; in trans_LD_mult()
3895 if (!a->p && a->rm != 0) { in trans_LD_mult()
3896 /* For non-postindexed accesses the Rm field must be 0 */ in trans_LD_mult()
3899 if (size == 3 && !a->q && a->selem != 1) { in trans_LD_mult()
3906 if (a->rn == 31) { in trans_LD_mult()
3910 /* For our purposes, bytes are always little-endian. */ in trans_LD_mult()
3911 endian = s->be_data; in trans_LD_mult()
3916 total = a->rpt * a->selem * (a->q ? 16 : 8); in trans_LD_mult()
3917 tcg_rn = cpu_reg_sp(s, a->rn); in trans_LD_mult()
3921 * promote consecutive little-endian elements below. in trans_LD_mult()
3923 clean_addr = gen_mte_checkN(s, tcg_rn, false, a->p || a->rn != 31, total, in trans_LD_mult()
3927 * Consecutive little-endian elements from a single register in trans_LD_mult()
3928 * can be promoted to a larger little-endian operation. in trans_LD_mult()
3931 if (a->selem == 1 && endian == MO_LE) { in trans_LD_mult()
3935 if (!s->align_mem) { in trans_LD_mult()
3940 elements = (a->q ? 16 : 8) >> size; in trans_LD_mult()
3942 for (r = 0; r < a->rpt; r++) { in trans_LD_mult()
3946 for (xs = 0; xs < a->selem; xs++) { in trans_LD_mult()
3947 int tt = (a->rt + r + xs) % 32; in trans_LD_mult()
3955 * For non-quad operations, setting a slice of the low 64 bits of in trans_LD_mult()
3961 for (r = 0; r < a->rpt * a->selem; r++) { in trans_LD_mult()
3962 int tt = (a->rt + r) % 32; in trans_LD_mult()
3963 clear_vec_high(s, a->q, tt); in trans_LD_mult()
3966 if (a->p) { in trans_LD_mult()
3967 if (a->rm == 31) { in trans_LD_mult()
3970 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm)); in trans_LD_mult()
3984 int size = a->sz; in trans_ST_mult()
3986 if (!a->p && a->rm != 0) { in trans_ST_mult()
3987 /* For non-postindexed accesses the Rm field must be 0 */ in trans_ST_mult()
3990 if (size == 3 && !a->q && a->selem != 1) { in trans_ST_mult()
3997 if (a->rn == 31) { in trans_ST_mult()
4001 /* For our purposes, bytes are always little-endian. */ in trans_ST_mult()
4002 endian = s->be_data; in trans_ST_mult()
4007 total = a->rpt * a->selem * (a->q ? 16 : 8); in trans_ST_mult()
4008 tcg_rn = cpu_reg_sp(s, a->rn); in trans_ST_mult()
4012 * promote consecutive little-endian elements below. in trans_ST_mult()
4014 clean_addr = gen_mte_checkN(s, tcg_rn, true, a->p || a->rn != 31, total, in trans_ST_mult()
4018 * Consecutive little-endian elements from a single register in trans_ST_mult()
4019 * can be promoted to a larger little-endian operation. in trans_ST_mult()
4022 if (a->selem == 1 && endian == MO_LE) { in trans_ST_mult()
4026 if (!s->align_mem) { in trans_ST_mult()
4031 elements = (a->q ? 16 : 8) >> size; in trans_ST_mult()
4033 for (r = 0; r < a->rpt; r++) { in trans_ST_mult()
4037 for (xs = 0; xs < a->selem; xs++) { in trans_ST_mult()
4038 int tt = (a->rt + r + xs) % 32; in trans_ST_mult()
4045 if (a->p) { in trans_ST_mult()
4046 if (a->rm == 31) { in trans_ST_mult()
4049 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm)); in trans_ST_mult()
4061 if (!a->p && a->rm != 0) { in trans_ST_single()
4068 if (a->rn == 31) { in trans_ST_single()
4072 total = a->selem << a->scale; in trans_ST_single()
4073 tcg_rn = cpu_reg_sp(s, a->rn); in trans_ST_single()
4075 mop = finalize_memop_asimd(s, a->scale); in trans_ST_single()
4076 clean_addr = gen_mte_checkN(s, tcg_rn, true, a->p || a->rn != 31, in trans_ST_single()
4079 tcg_ebytes = tcg_constant_i64(1 << a->scale); in trans_ST_single()
4080 for (xs = 0, rt = a->rt; xs < a->selem; xs++, rt = (rt + 1) % 32) { in trans_ST_single()
4081 do_vec_st(s, rt, a->index, clean_addr, mop); in trans_ST_single()
4085 if (a->p) { in trans_ST_single()
4086 if (a->rm == 31) { in trans_ST_single()
4089 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm)); in trans_ST_single()
4101 if (!a->p && a->rm != 0) { in trans_LD_single()
4108 if (a->rn == 31) { in trans_LD_single()
4112 total = a->selem << a->scale; in trans_LD_single()
4113 tcg_rn = cpu_reg_sp(s, a->rn); in trans_LD_single()
4115 mop = finalize_memop_asimd(s, a->scale); in trans_LD_single()
4116 clean_addr = gen_mte_checkN(s, tcg_rn, false, a->p || a->rn != 31, in trans_LD_single()
4119 tcg_ebytes = tcg_constant_i64(1 << a->scale); in trans_LD_single()
4120 for (xs = 0, rt = a->rt; xs < a->selem; xs++, rt = (rt + 1) % 32) { in trans_LD_single()
4121 do_vec_ld(s, rt, a->index, clean_addr, mop); in trans_LD_single()
4125 if (a->p) { in trans_LD_single()
4126 if (a->rm == 31) { in trans_LD_single()
4129 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm)); in trans_LD_single()
4141 if (!a->p && a->rm != 0) { in trans_LD_single_repl()
4148 if (a->rn == 31) { in trans_LD_single_repl()
4152 total = a->selem << a->scale; in trans_LD_single_repl()
4153 tcg_rn = cpu_reg_sp(s, a->rn); in trans_LD_single_repl()
4155 mop = finalize_memop_asimd(s, a->scale); in trans_LD_single_repl()
4156 clean_addr = gen_mte_checkN(s, tcg_rn, false, a->p || a->rn != 31, in trans_LD_single_repl()
4159 tcg_ebytes = tcg_constant_i64(1 << a->scale); in trans_LD_single_repl()
4160 for (xs = 0, rt = a->rt; xs < a->selem; xs++, rt = (rt + 1) % 32) { in trans_LD_single_repl()
4165 tcg_gen_gvec_dup_i64(a->scale, vec_full_reg_offset(s, rt), in trans_LD_single_repl()
4166 (a->q + 1) * 8, vec_full_reg_size(s), tcg_tmp); in trans_LD_single_repl()
4170 if (a->p) { in trans_LD_single_repl()
4171 if (a->rm == 31) { in trans_LD_single_repl()
4174 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm)); in trans_LD_single_repl()
4183 int size = 4 << s->dcz_blocksize; in trans_STZGM()
4188 if (s->current_el == 0) { in trans_STZGM()
4192 if (a->rn == 31) { in trans_STZGM()
4196 addr = read_cpu_reg_sp(s, a->rn, true); in trans_STZGM()
4197 tcg_gen_addi_i64(addr, addr, a->imm); in trans_STZGM()
4198 tcg_rt = cpu_reg(s, a->rt); in trans_STZGM()
4200 if (s->ata[0]) { in trans_STZGM()
4204 * The non-tags portion of STZGM is mostly like DC_ZVA, in trans_STZGM()
4208 tcg_gen_andi_i64(clean_addr, clean_addr, -size); in trans_STZGM()
4220 if (s->current_el == 0) { in trans_STGM()
4224 if (a->rn == 31) { in trans_STGM()
4228 addr = read_cpu_reg_sp(s, a->rn, true); in trans_STGM()
4229 tcg_gen_addi_i64(addr, addr, a->imm); in trans_STGM()
4230 tcg_rt = cpu_reg(s, a->rt); in trans_STGM()
4232 if (s->ata[0]) { in trans_STGM()
4236 int size = 4 << s->gm_blocksize; in trans_STGM()
4239 tcg_gen_andi_i64(clean_addr, clean_addr, -size); in trans_STGM()
4252 if (s->current_el == 0) { in trans_LDGM()
4256 if (a->rn == 31) { in trans_LDGM()
4260 addr = read_cpu_reg_sp(s, a->rn, true); in trans_LDGM()
4261 tcg_gen_addi_i64(addr, addr, a->imm); in trans_LDGM()
4262 tcg_rt = cpu_reg(s, a->rt); in trans_LDGM()
4264 if (s->ata[0]) { in trans_LDGM()
4268 int size = 4 << s->gm_blocksize; in trans_LDGM()
4271 tcg_gen_andi_i64(clean_addr, clean_addr, -size); in trans_LDGM()
4287 if (a->rn == 31) { in trans_LDG()
4291 addr = read_cpu_reg_sp(s, a->rn, true); in trans_LDG()
4292 if (!a->p) { in trans_LDG()
4293 /* pre-index or signed offset */ in trans_LDG()
4294 tcg_gen_addi_i64(addr, addr, a->imm); in trans_LDG()
4297 tcg_gen_andi_i64(addr, addr, -TAG_GRANULE); in trans_LDG()
4298 tcg_rt = cpu_reg(s, a->rt); in trans_LDG()
4299 if (s->ata[0]) { in trans_LDG()
4311 if (a->w) { in trans_LDG()
4312 /* pre-index or post-index */ in trans_LDG()
4313 if (a->p) { in trans_LDG()
4314 /* post-index */ in trans_LDG()
4315 tcg_gen_addi_i64(addr, addr, a->imm); in trans_LDG()
4317 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), addr); in trans_LDG()
4326 if (a->rn == 31) { in do_STG()
4330 addr = read_cpu_reg_sp(s, a->rn, true); in do_STG()
4331 if (!a->p) { in do_STG()
4332 /* pre-index or signed offset */ in do_STG()
4333 tcg_gen_addi_i64(addr, addr, a->imm); in do_STG()
4335 tcg_rt = cpu_reg_sp(s, a->rt); in do_STG()
4336 if (!s->ata[0]) { in do_STG()
4340 * at least for system mode; user-only won't enforce alignment. in do_STG()
4347 } else if (tb_cflags(s->base.tb) & CF_PARALLEL) { in do_STG()
4370 /* This is 1 or 2 atomic 16-byte operations. */ in do_STG()
4378 if (a->w) { in do_STG()
4379 /* pre-index or post-index */ in do_STG()
4380 if (a->p) { in do_STG()
4381 /* post-index */ in do_STG()
4382 tcg_gen_addi_i64(addr, addr, a->imm); in do_STG()
4384 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), addr); in do_STG()
4411 if (a->rs == a->rn || a->rs == a->rd || a->rn == a->rd || in do_SET()
4412 a->rd == 31 || a->rn == 31) { in do_SET()
4416 memidx = get_a64_user_mem_index(s, a->unpriv); in do_SET()
4422 syndrome = syn_mop(true, is_setg, (a->nontemp << 1) | a->unpriv, in do_SET()
4423 is_epilogue, false, true, a->rd, a->rs, a->rn); in do_SET()
4425 if (is_setg ? s->ata[a->unpriv] : s->mte_active[a->unpriv]) { in do_SET()
4427 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid); in do_SET()
4428 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma); in do_SET()
4457 bool wunpriv = extract32(a->options, 0, 1); in do_CPY()
4458 bool runpriv = extract32(a->options, 1, 1); in do_CPY()
4465 if (a->rs == a->rn || a->rs == a->rd || a->rn == a->rd || in do_CPY()
4466 a->rd == 31 || a->rs == 31 || a->rn == 31) { in do_CPY()
4477 syndrome = syn_mop(false, false, a->options, is_epilogue, in do_CPY()
4478 false, true, a->rd, a->rs, a->rn); in do_CPY()
4481 if (s->mte_active[runpriv]) { in do_CPY()
4482 rdesc = FIELD_DP32(rdesc, MTEDESC, TBI, s->tbid); in do_CPY()
4483 rdesc = FIELD_DP32(rdesc, MTEDESC, TCMA, s->tcma); in do_CPY()
4485 if (s->mte_active[wunpriv]) { in do_CPY()
4486 wdesc = FIELD_DP32(wdesc, MTEDESC, TBI, s->tbid); in do_CPY()
4487 wdesc = FIELD_DP32(wdesc, MTEDESC, TCMA, s->tcma); in do_CPY()
4516 TCGv_i64 tcg_rn = rn_sp ? cpu_reg_sp(s, a->rn) : cpu_reg(s, a->rn); in gen_rri()
4517 TCGv_i64 tcg_rd = rd_sp ? cpu_reg_sp(s, a->rd) : cpu_reg(s, a->rd); in gen_rri()
4518 TCGv_i64 tcg_imm = tcg_constant_i64(a->imm); in gen_rri()
4521 if (!a->sf) { in gen_rri()
4528 * PC-rel. addressing
4533 gen_pc_plus_diff(s, cpu_reg(s, a->rd), a->imm); in trans_ADR()
4539 int64_t offset = (int64_t)a->imm << 12; in trans_ADRP()
4542 offset -= s->pc_curr & 0xfff; in trans_ADRP()
4543 gen_pc_plus_diff(s, cpu_reg(s, a->rd), offset); in trans_ADRP()
4552 TRANS(ADDS_i, gen_rri, a, 0, 1, a->sf ? gen_add64_CC : gen_add32_CC)
4553 TRANS(SUBS_i, gen_rri, a, 0, 1, a->sf ? gen_sub64_CC : gen_sub32_CC)
4565 imm = a->uimm6 << LOG2_TAG_GRANULE; in gen_add_sub_imm_with_tags()
4567 imm = -imm; in gen_add_sub_imm_with_tags()
4570 tcg_rn = cpu_reg_sp(s, a->rn); in gen_add_sub_imm_with_tags()
4571 tcg_rd = cpu_reg_sp(s, a->rd); in gen_add_sub_imm_with_tags()
4573 if (s->ata[0]) { in gen_add_sub_imm_with_tags()
4576 tcg_constant_i32(a->uimm4)); in gen_add_sub_imm_with_tags()
4623 * of between 1 and e-1 non-zero bits, rotated within the in logic_imm_decode_wmask()
4624 * element by between 0 and e-1 bits. in logic_imm_decode_wmask()
4628 * 64 bit elements: immn = 1, imms = <length of run - 1> in logic_imm_decode_wmask()
4629 * 32 bit elements: immn = 0, imms = 0 : <length of run - 1> in logic_imm_decode_wmask()
4630 * 16 bit elements: immn = 0, imms = 10 : <length of run - 1> in logic_imm_decode_wmask()
4631 * 8 bit elements: immn = 0, imms = 110 : <length of run - 1> in logic_imm_decode_wmask()
4632 * 4 bit elements: immn = 0, imms = 1110 : <length of run - 1> in logic_imm_decode_wmask()
4633 * 2 bit elements: immn = 0, imms = 11110 : <length of run - 1> in logic_imm_decode_wmask()
4636 * Further, <length of run - 1> all-ones is a reserved pattern. in logic_imm_decode_wmask()
4642 len = 31 - clz32((immn << 6) | (~imms & 0x3f)); in logic_imm_decode_wmask()
4649 levels = e - 1; in logic_imm_decode_wmask()
4654 /* <length of run - 1> mustn't be all-ones. */ in logic_imm_decode_wmask()
4663 mask = (mask >> r) | (mask << (e - r)); in logic_imm_decode_wmask()
4678 /* Some immediate field values are reserved. */ in gen_rri_log()
4679 if (!logic_imm_decode_wmask(&imm, extract32(a->dbm, 12, 1), in gen_rri_log()
4680 extract32(a->dbm, 0, 6), in gen_rri_log()
4681 extract32(a->dbm, 6, 6))) { in gen_rri_log()
4684 if (!a->sf) { in gen_rri_log()
4688 tcg_rd = set_cc ? cpu_reg(s, a->rd) : cpu_reg_sp(s, a->rd); in gen_rri_log()
4689 tcg_rn = cpu_reg(s, a->rn); in gen_rri_log()
4693 gen_logic_CC(a->sf, tcg_rd); in gen_rri_log()
4695 if (!a->sf) { in gen_rri_log()
4712 int pos = a->hw << 4; in TRANS()
4713 tcg_gen_movi_i64(cpu_reg(s, a->rd), (uint64_t)a->imm << pos); in TRANS()
4719 int pos = a->hw << 4; in trans_MOVN()
4720 uint64_t imm = a->imm; in trans_MOVN()
4723 if (!a->sf) { in trans_MOVN()
4726 tcg_gen_movi_i64(cpu_reg(s, a->rd), imm); in trans_MOVN()
4732 int pos = a->hw << 4; in trans_MOVK()
4735 tcg_rd = cpu_reg(s, a->rd); in trans_MOVK()
4736 tcg_im = tcg_constant_i64(a->imm); in trans_MOVK()
4738 if (!a->sf) { in trans_MOVK()
4750 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); in trans_SBFM()
4751 TCGv_i64 tcg_tmp = read_cpu_reg(s, a->rn, 1); in trans_SBFM()
4752 unsigned int bitsize = a->sf ? 64 : 32; in trans_SBFM()
4753 unsigned int ri = a->immr; in trans_SBFM()
4754 unsigned int si = a->imms; in trans_SBFM()
4758 /* Wd<s-r:0> = Wn<s:r> */ in trans_SBFM()
4759 len = (si - ri) + 1; in trans_SBFM()
4761 if (!a->sf) { in trans_SBFM()
4765 /* Wd<32+s-r,32-r> = Wn<s:0> */ in trans_SBFM()
4767 pos = (bitsize - ri) & (bitsize - 1); in trans_SBFM()
4781 * bitsize, therefore no final zero-extension is unneeded for !sf. in trans_SBFM()
4790 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); in trans_UBFM()
4791 TCGv_i64 tcg_tmp = read_cpu_reg(s, a->rn, 1); in trans_UBFM()
4792 unsigned int bitsize = a->sf ? 64 : 32; in trans_UBFM()
4793 unsigned int ri = a->immr; in trans_UBFM()
4794 unsigned int si = a->imms; in trans_UBFM()
4797 tcg_rd = cpu_reg(s, a->rd); in trans_UBFM()
4798 tcg_tmp = read_cpu_reg(s, a->rn, 1); in trans_UBFM()
4801 /* Wd<s-r:0> = Wn<s:r> */ in trans_UBFM()
4802 len = (si - ri) + 1; in trans_UBFM()
4805 /* Wd<32+s-r,32-r> = Wn<s:0> */ in trans_UBFM()
4807 pos = (bitsize - ri) & (bitsize - 1); in trans_UBFM()
4815 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); in trans_BFM()
4816 TCGv_i64 tcg_tmp = read_cpu_reg(s, a->rn, 1); in trans_BFM()
4817 unsigned int bitsize = a->sf ? 64 : 32; in trans_BFM()
4818 unsigned int ri = a->immr; in trans_BFM()
4819 unsigned int si = a->imms; in trans_BFM()
4822 tcg_rd = cpu_reg(s, a->rd); in trans_BFM()
4823 tcg_tmp = read_cpu_reg(s, a->rn, 1); in trans_BFM()
4826 /* Wd<s-r:0> = Wn<s:r> */ in trans_BFM()
4828 len = (si - ri) + 1; in trans_BFM()
4831 /* Wd<32+s-r,32-r> = Wn<s:0> */ in trans_BFM()
4833 pos = (bitsize - ri) & (bitsize - 1); in trans_BFM()
4837 if (!a->sf) { in trans_BFM()
4847 tcg_rd = cpu_reg(s, a->rd); in trans_EXTR()
4849 if (unlikely(a->imm == 0)) { in trans_EXTR()
4854 if (a->sf) { in trans_EXTR()
4855 tcg_gen_mov_i64(tcg_rd, cpu_reg(s, a->rm)); in trans_EXTR()
4857 tcg_gen_ext32u_i64(tcg_rd, cpu_reg(s, a->rm)); in trans_EXTR()
4860 tcg_rm = cpu_reg(s, a->rm); in trans_EXTR()
4861 tcg_rn = cpu_reg(s, a->rn); in trans_EXTR()
4863 if (a->sf) { in trans_EXTR()
4865 tcg_gen_extract2_i64(tcg_rd, tcg_rm, tcg_rn, a->imm); in trans_EXTR()
4870 if (a->rm == a->rn) { in trans_EXTR()
4871 tcg_gen_rotri_i32(t0, t0, a->imm); in trans_EXTR()
4875 tcg_gen_extract2_i32(t0, t0, t1, a->imm); in trans_EXTR()
4886 int len = (a->len + 1) * 16; in trans_TBL_TBX()
4888 tcg_gen_gvec_2_ptr(vec_full_reg_offset(s, a->rd), in trans_TBL_TBX()
4889 vec_full_reg_offset(s, a->rm), tcg_env, in trans_TBL_TBX()
4890 a->q ? 16 : 8, vec_full_reg_size(s), in trans_TBL_TBX()
4891 (len << 6) | (a->tbx << 5) | a->rn, in trans_TBL_TBX()
4902 MemOp esz = a->esz; in do_simd_permute()
4903 int datasize = a->q ? 16 : 8; in do_simd_permute()
4907 if (esz == MO_64 && !a->q) { in do_simd_permute()
4915 tcg_res[1] = a->q ? tcg_temp_new_i64() : NULL; in do_simd_permute()
4922 read_vec_element(s, tcg_ele, (idx & elements ? a->rm : a->rn), in do_simd_permute()
4923 idx & (elements - 1), esz); in do_simd_permute()
4934 for (int i = a->q; i >= 0; --i) { in do_simd_permute()
4935 write_vec_element(s, tcg_res[i], a->rd, i, MO_64); in do_simd_permute()
4937 clear_vec_high(s, a->q, a->rd); in do_simd_permute()
5012 read_vec_element_i32(s, tcg_op1, a->rn, 3, MO_32); in TRANS_FEAT()
5013 read_vec_element_i32(s, tcg_op2, a->rm, 3, MO_32); in TRANS_FEAT()
5014 read_vec_element_i32(s, tcg_op3, a->ra, 3, MO_32); in TRANS_FEAT()
5022 clear_vec(s, a->rd); in TRANS_FEAT()
5023 write_vec_element_i32(s, tcg_res, a->rd, 3, MO_32); in TRANS_FEAT()
5031 gen_gvec_op3_ool(s, true, a->rd, a->rn, a->rm, a->imm, fn); in do_crypto3i()
5046 gen_gvec_xar(MO_64, vec_full_reg_offset(s, a->rd), in TRANS_FEAT()
5047 vec_full_reg_offset(s, a->rn), in TRANS_FEAT()
5048 vec_full_reg_offset(s, a->rm), a->imm, 16, in TRANS_FEAT()
5074 if (!decode_esz_idx(a->imm, &esz, &idx)) { in trans_DUP_element_s()
5080 * zero-extends it into the bottom of the destination register. in trans_DUP_element_s()
5083 read_vec_element(s, tmp, a->rn, idx, esz); in trans_DUP_element_s()
5084 write_fp_dreg(s, a->rd, tmp); in trans_DUP_element_s()
5094 if (!decode_esz_idx(a->imm, &esz, &idx)) { in trans_DUP_element_v()
5097 if (esz == MO_64 && !a->q) { in trans_DUP_element_v()
5101 tcg_gen_gvec_dup_mem(esz, vec_full_reg_offset(s, a->rd), in trans_DUP_element_v()
5102 vec_reg_offset(s, a->rn, idx, esz), in trans_DUP_element_v()
5103 a->q ? 16 : 8, vec_full_reg_size(s)); in trans_DUP_element_v()
5113 if (!decode_esz_idx(a->imm, &esz, &idx)) { in trans_DUP_general()
5116 if (esz == MO_64 && !a->q) { in trans_DUP_general()
5120 tcg_gen_gvec_dup_i64(esz, vec_full_reg_offset(s, a->rd), in trans_DUP_general()
5121 a->q ? 16 : 8, vec_full_reg_size(s), in trans_DUP_general()
5122 cpu_reg(s, a->rn)); in trans_DUP_general()
5132 if (!decode_esz_idx(a->imm, &esz, &idx)) { in do_smov_umov()
5136 if (esz == MO_64 || (esz == MO_32 && !a->q)) { in do_smov_umov()
5140 if (esz == MO_64 ? !a->q : a->q) { in do_smov_umov()
5145 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); in do_smov_umov()
5146 read_vec_element(s, tcg_rd, a->rn, idx, esz | is_signed); in do_smov_umov()
5147 if (is_signed && !a->q) { in do_smov_umov()
5162 if (!decode_esz_idx(a->imm, &esz, &idx)) { in TRANS()
5166 write_vec_element(s, cpu_reg(s, a->rn), a->rd, idx, esz); in TRANS()
5167 clear_vec_high(s, true, a->rd); in TRANS()
5177 if (!decode_esz_idx(a->di, &esz, &didx)) { in trans_INS_element()
5180 sidx = a->si >> esz; in trans_INS_element()
5184 read_vec_element(s, tmp, a->rn, sidx, esz); in trans_INS_element()
5185 write_vec_element(s, tmp, a->rd, didx, esz); in trans_INS_element()
5187 /* INS is considered a 128-bit write for SVE. */ in trans_INS_element()
5188 clear_vec_high(s, true, a->rd); in trans_INS_element()
5207 switch (a->esz) { in do_fp3_scalar_with_fpsttype()
5210 TCGv_i64 t0 = read_fp_dreg(s, a->rn); in do_fp3_scalar_with_fpsttype()
5211 TCGv_i64 t1 = read_fp_dreg(s, a->rm); in do_fp3_scalar_with_fpsttype()
5212 f->gen_d(t0, t0, t1, fpstatus_ptr(fpsttype)); in do_fp3_scalar_with_fpsttype()
5213 write_fp_dreg_merging(s, a->rd, mergereg, t0); in do_fp3_scalar_with_fpsttype()
5218 TCGv_i32 t0 = read_fp_sreg(s, a->rn); in do_fp3_scalar_with_fpsttype()
5219 TCGv_i32 t1 = read_fp_sreg(s, a->rm); in do_fp3_scalar_with_fpsttype()
5220 f->gen_s(t0, t0, t1, fpstatus_ptr(fpsttype)); in do_fp3_scalar_with_fpsttype()
5221 write_fp_sreg_merging(s, a->rd, mergereg, t0); in do_fp3_scalar_with_fpsttype()
5229 TCGv_i32 t0 = read_fp_hreg(s, a->rn); in do_fp3_scalar_with_fpsttype()
5230 TCGv_i32 t1 = read_fp_hreg(s, a->rm); in do_fp3_scalar_with_fpsttype()
5231 f->gen_h(t0, t0, t1, fpstatus_ptr(fpsttype)); in do_fp3_scalar_with_fpsttype()
5232 write_fp_hreg_merging(s, a->rd, mergereg, t0); in do_fp3_scalar_with_fpsttype()
5245 a->esz == MO_16 ? in do_fp3_scalar()
5253 return do_fp3_scalar_with_fpsttype(s, a, s->fpcr_ah ? fah : fnormal, in do_fp3_scalar_ah_2fn()
5254 mergereg, select_ah_fpst(s, a->esz)); in do_fp3_scalar_ah_2fn()
5257 /* Some insns need to call different helpers when FPCR.AH == 1 */
5263 return do_fp3_scalar(s, a, s->fpcr_ah ? fah : fnormal, mergereg); in do_fp3_scalar_2fn()
5271 TRANS(FADD_s, do_fp3_scalar, a, &f_scalar_fadd, a->rn)
5278 TRANS(FSUB_s, do_fp3_scalar, a, &f_scalar_fsub, a->rn)
5285 TRANS(FDIV_s, do_fp3_scalar, a, &f_scalar_fdiv, a->rn)
5292 TRANS(FMUL_s, do_fp3_scalar, a, &f_scalar_fmul, a->rn)
5304 TRANS(FMAX_s, do_fp3_scalar_2fn, a, &f_scalar_fmax, &f_scalar_fmax_ah, a->rn)
5316 TRANS(FMIN_s, do_fp3_scalar_2fn, a, &f_scalar_fmin, &f_scalar_fmin_ah, a->rn)
5323 TRANS(FMAXNM_s, do_fp3_scalar, a, &f_scalar_fmaxnm, a->rn)
5330 TRANS(FMINNM_s, do_fp3_scalar, a, &f_scalar_fminnm, a->rn)
5337 TRANS(FMULX_s, do_fp3_scalar, a, &f_scalar_fmulx, a->rn)
5385 TRANS(FNMUL_s, do_fp3_scalar_2fn, a, &f_scalar_fnmul, &f_scalar_ah_fnmul, a->rn)
5392 TRANS(FCMEQ_s, do_fp3_scalar, a, &f_scalar_fcmeq, a->rm)
5399 TRANS(FCMGE_s, do_fp3_scalar, a, &f_scalar_fcmge, a->rm)
5406 TRANS(FCMGT_s, do_fp3_scalar, a, &f_scalar_fcmgt, a->rm)
5413 TRANS(FACGE_s, do_fp3_scalar, a, &f_scalar_facge, a->rm)
5420 TRANS(FACGT_s, do_fp3_scalar, a, &f_scalar_facgt, a->rm)
5468 TRANS(FABD_s, do_fp3_scalar_2fn, a, &f_scalar_fabd, &f_scalar_ah_fabd, a->rn)
5481 &f_scalar_frecps, &f_scalar_ah_frecps, a->rn)
5494 &f_scalar_frsqrts, &f_scalar_ah_frsqrts, a->rn)
5499 switch (a->esz) { in do_fcmp0_s()
5502 TCGv_i64 t0 = read_fp_dreg(s, a->rn); in do_fcmp0_s()
5505 f->gen_d(t0, t1, t0, fpstatus_ptr(FPST_A64)); in do_fcmp0_s()
5507 f->gen_d(t0, t0, t1, fpstatus_ptr(FPST_A64)); in do_fcmp0_s()
5509 write_fp_dreg(s, a->rd, t0); in do_fcmp0_s()
5514 TCGv_i32 t0 = read_fp_sreg(s, a->rn); in do_fcmp0_s()
5517 f->gen_s(t0, t1, t0, fpstatus_ptr(FPST_A64)); in do_fcmp0_s()
5519 f->gen_s(t0, t0, t1, fpstatus_ptr(FPST_A64)); in do_fcmp0_s()
5521 write_fp_sreg(s, a->rd, t0); in do_fcmp0_s()
5529 TCGv_i32 t0 = read_fp_hreg(s, a->rn); in do_fcmp0_s()
5532 f->gen_h(t0, t1, t0, fpstatus_ptr(FPST_A64_F16)); in do_fcmp0_s()
5534 f->gen_h(t0, t0, t1, fpstatus_ptr(FPST_A64_F16)); in do_fcmp0_s()
5536 write_fp_sreg(s, a->rd, t0); in do_fcmp0_s()
5557 MemOp esz = a->esz; in do_satacc_s()
5567 read_vec_element(s, t1, a->rn, 0, esz | sgn_n); in do_satacc_s()
5568 read_vec_element(s, t2, a->rm, 0, esz | sgn_m); in do_satacc_s()
5578 write_fp_dreg(s, a->rd, t0); in do_satacc_s()
5597 read_vec_element(s, t0, a->rn, 0, MO_64); in TRANS()
5598 read_vec_element(s, t1, a->rm, 0, MO_64); in TRANS()
5600 write_fp_dreg(s, a->rd, t0); in TRANS()
5622 if (a->esz == MO_64) { in do_env_scalar2()
5623 TCGv_i64 t0 = read_fp_dreg(s, a->rn); in do_env_scalar2()
5624 TCGv_i64 t1 = read_fp_dreg(s, a->rm); in do_env_scalar2()
5625 f->gen_d(t0, tcg_env, t0, t1); in do_env_scalar2()
5626 write_fp_dreg(s, a->rd, t0); in do_env_scalar2()
5631 read_vec_element_i32(s, t0, a->rn, 0, a->esz); in do_env_scalar2()
5632 read_vec_element_i32(s, t1, a->rm, 0, a->esz); in do_env_scalar2()
5633 f->gen_bhs[a->esz](t0, tcg_env, t0, t1); in do_env_scalar2()
5634 write_fp_sreg(s, a->rd, t0); in do_env_scalar2()
5674 if (a->esz == MO_16 || a->esz == MO_32) { in do_env_scalar2_hs()
5699 if (a->esz != MO_16 && a->esz != MO_32) { in do_env_scalar3_hs()
5709 read_vec_element_i32(s, t0, a->rn, 0, a->esz); in do_env_scalar3_hs()
5710 read_vec_element_i32(s, t1, a->rm, 0, a->esz); in do_env_scalar3_hs()
5711 read_vec_element_i32(s, t2, a->rd, 0, a->esz); in do_env_scalar3_hs()
5712 f->gen_hs[a->esz - 1](t0, tcg_env, t0, t1, t2); in do_env_scalar3_hs()
5713 write_fp_sreg(s, a->rd, t0); in do_env_scalar3_hs()
5730 TCGv_i64 t0 = read_fp_dreg(s, a->rn); in do_cmop_d()
5731 TCGv_i64 t1 = read_fp_dreg(s, a->rm); in do_cmop_d()
5733 write_fp_dreg(s, a->rd, t0); in do_cmop_d()
5750 MemOp esz = a->esz; in TRANS()
5751 int check = fp_access_check_vector_hsd(s, a->q, esz); in TRANS()
5757 gen_gvec_op3_fpst(s, a->q, a->rd, a->rn, a->rm, fpsttype, in TRANS()
5758 data, fns[esz - 1]); in TRANS()
5766 a->esz == MO_16 ? in do_fp3_vector()
5774 return do_fp3_vector(s, a, data, s->fpcr_ah ? fah : fnormal); in do_fp3_vector_2fn()
5781 return do_fp3_vector_with_fpsttype(s, a, data, s->fpcr_ah ? fah : fnormal, in do_fp3_vector_ah_2fn()
5782 select_ah_fpst(s, a->esz)); in do_fp3_vector_ah_2fn()
5997 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, a->rd), in do_fmlal()
5998 vec_full_reg_offset(s, a->rn), in do_fmlal()
5999 vec_full_reg_offset(s, a->rm), tcg_env, in do_fmlal()
6000 a->q ? 16 : 8, vec_full_reg_size(s), in do_fmlal()
6031 TRANS(BSL_v, do_bitsel, a->q, a->rd, a->rd, a->rn, a->rm)
6032 TRANS(BIT_v, do_bitsel, a->q, a->rd, a->rm, a->rn, a->rd)
6033 TRANS(BIF_v, do_bitsel, a->q, a->rd, a->rm, a->rd, a->rn)
6074 if (a->esz == MO_64 && !a->q) { in TRANS()
6078 tcg_gen_gvec_cmp(cond, a->esz, in TRANS()
6079 vec_full_reg_offset(s, a->rd), in TRANS()
6080 vec_full_reg_offset(s, a->rn), in TRANS()
6081 vec_full_reg_offset(s, a->rm), in TRANS()
6082 a->q ? 16 : 8, vec_full_reg_size(s)); in TRANS()
6103 gen_gvec_op4_ool(s, a->q, a->rd, a->rn, a->rm, a->rd, 0, fn); in TRANS()
6112 gen_gvec_op4_env(s, a->q, a->rd, a->rn, a->rm, a->rd, 0, fn); in do_dot_vector_env()
6133 gen_gvec_op4_fpst(s, true, a->rd, a->rn, a->rm, a->rd, in TRANS_FEAT()
6134 s->fpcr_ah ? FPST_AH : FPST_A64, a->q, in TRANS_FEAT()
6149 TRANS_FEAT(FCADD_90, aa64_fcma, do_fp3_vector, a, 0 | (s->fpcr_ah << 1),
6151 TRANS_FEAT(FCADD_270, aa64_fcma, do_fp3_vector, a, 1 | (s->fpcr_ah << 1),
6167 check = fp_access_check_vector_hsd(s, a->q, a->esz); in trans_FCMLA_v()
6172 gen_gvec_op4_fpst(s, a->q, a->rd, a->rn, a->rm, a->rd, in trans_FCMLA_v()
6173 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64, in trans_FCMLA_v()
6174 a->rot | (s->fpcr_ah << 2), fn[a->esz]); in trans_FCMLA_v()
6181 * These read from the top or bottom half of a 128-bit vector.
6182 * After widening, optionally accumulate with a 128-bit vector.
6203 /* There are no 64x64->128 bit operations. */ in do_3op_widening()
6221 top_swap = top ? 0 : half - 1; in do_3op_widening()
6256 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6259 a->esz, a->q, a->rd, a->rn, a->rm, -1,
6262 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6265 a->esz, a->q, a->rd, a->rn, a->rm, -1,
6268 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6271 a->esz, a->q, a->rd, a->rn, a->rm, -1,
6275 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
6278 a->esz, a->q, a->rd, a->rn, a->rm, a->idx,
6281 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
6284 a->esz, a->q, a->rd, a->rn, a->rm, a->idx,
6287 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
6290 a->esz, a->q, a->rd, a->rn, a->rm, a->idx,
6328 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6331 a->esz, a->q, a->rd, a->rn, a->rm, -1,
6334 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6337 a->esz, a->q, a->rd, a->rn, a->rm, -1,
6340 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6343 a->esz, a->q, a->rd, a->rn, a->rm, -1,
6346 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6349 a->esz, a->q, a->rd, a->rn, a->rm, -1,
6403 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6404 a->esz == MO_16 ? gen_sqdmull_h : gen_sqdmull_s, false)
6406 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6407 a->esz == MO_16 ? gen_sqdmlal_h : gen_sqdmlal_s, true)
6409 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6410 a->esz == MO_16 ? gen_sqdmlsl_h : gen_sqdmlsl_s, true)
6413 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
6414 a->esz == MO_16 ? gen_sqdmull_h : gen_sqdmull_s, false)
6416 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
6417 a->esz == MO_16 ? gen_sqdmlal_h : gen_sqdmlal_s, true)
6419 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
6420 a->esz == MO_16 ? gen_sqdmlsl_h : gen_sqdmlsl_s, true)
6426 MemOp esz = a->esz; in do_addsub_wide()
6428 bool top = a->q; in do_addsub_wide()
6429 int top_swap = top ? 0 : half - 1; in do_addsub_wide()
6432 /* There are no 64x64->128 bit operations. */ in do_addsub_wide()
6445 read_vec_element(s, tcg_op1, a->rm, elt + top_half, esz | sign); in do_addsub_wide()
6446 read_vec_element(s, tcg_op0, a->rn, elt, esz + 1); in do_addsub_wide()
6452 write_vec_element(s, tcg_op0, a->rd, elt, esz + 1); in do_addsub_wide()
6454 clear_vec_high(s, 1, a->rd); in do_addsub_wide()
6467 MemOp esz = a->esz; in TRANS()
6469 bool top = a->q; in TRANS()
6471 uint64_t rbit = 1ull << (ebits - 1); in TRANS()
6474 /* There are no 128x128->64 bit operations. */ in TRANS()
6489 top_swap = top ? half - 1 : 0; in TRANS()
6495 read_vec_element(s, tcg_op1, a->rm, elt, esz + 1); in TRANS()
6496 read_vec_element(s, tcg_op0, a->rn, elt, esz + 1); in TRANS()
6506 write_vec_element(s, tcg_op0, a->rd, elt + top_half, esz); in TRANS()
6508 clear_vec_high(s, top, a->rd); in TRANS()
6521 gen_gvec_op3_ool(s, true, a->rd, a->rn, a->rm, a->q, fn); in TRANS()
6535 switch (a->esz) { in TRANS()
6538 TCGv_i64 t0 = read_fp_dreg(s, a->rn); in TRANS()
6541 read_vec_element(s, t1, a->rm, a->idx, MO_64); in TRANS()
6542 f->gen_d(t0, t0, t1, fpstatus_ptr(FPST_A64)); in TRANS()
6543 write_fp_dreg_merging(s, a->rd, a->rn, t0); in TRANS()
6548 TCGv_i32 t0 = read_fp_sreg(s, a->rn); in TRANS()
6551 read_vec_element_i32(s, t1, a->rm, a->idx, MO_32); in TRANS()
6552 f->gen_s(t0, t0, t1, fpstatus_ptr(FPST_A64)); in TRANS()
6553 write_fp_sreg_merging(s, a->rd, a->rn, t0); in TRANS()
6561 TCGv_i32 t0 = read_fp_hreg(s, a->rn); in TRANS()
6564 read_vec_element_i32(s, t1, a->rm, a->idx, MO_16); in TRANS()
6565 f->gen_h(t0, t0, t1, fpstatus_ptr(FPST_A64_F16)); in TRANS()
6566 write_fp_hreg_merging(s, a->rd, a->rn, t0); in TRANS()
6580 switch (a->esz) { in do_fmla_scalar_idx()
6583 TCGv_i64 t0 = read_fp_dreg(s, a->rd); in do_fmla_scalar_idx()
6584 TCGv_i64 t1 = read_fp_dreg(s, a->rn); in do_fmla_scalar_idx()
6587 read_vec_element(s, t2, a->rm, a->idx, MO_64); in do_fmla_scalar_idx()
6592 write_fp_dreg_merging(s, a->rd, a->rd, t0); in do_fmla_scalar_idx()
6597 TCGv_i32 t0 = read_fp_sreg(s, a->rd); in do_fmla_scalar_idx()
6598 TCGv_i32 t1 = read_fp_sreg(s, a->rn); in do_fmla_scalar_idx()
6601 read_vec_element_i32(s, t2, a->rm, a->idx, MO_32); in do_fmla_scalar_idx()
6606 write_fp_sreg_merging(s, a->rd, a->rd, t0); in do_fmla_scalar_idx()
6614 TCGv_i32 t0 = read_fp_hreg(s, a->rd); in do_fmla_scalar_idx()
6615 TCGv_i32 t1 = read_fp_hreg(s, a->rn); in do_fmla_scalar_idx()
6618 read_vec_element_i32(s, t2, a->rm, a->idx, MO_16); in do_fmla_scalar_idx()
6624 write_fp_hreg_merging(s, a->rd, a->rd, t0); in do_fmla_scalar_idx()
6639 if (a->esz < MO_16 || a->esz > MO_32) { in TRANS()
6646 read_vec_element_i32(s, t0, a->rn, 0, a->esz); in TRANS()
6647 read_vec_element_i32(s, t1, a->rm, a->idx, a->esz); in TRANS()
6648 f->gen_bhs[a->esz](t0, tcg_env, t0, t1); in TRANS()
6649 write_fp_sreg(s, a->rd, t0); in TRANS()
6660 if (a->esz < MO_16 || a->esz > MO_32) { in do_env_scalar3_idx_hs()
6668 read_vec_element_i32(s, t0, a->rn, 0, a->esz); in do_env_scalar3_idx_hs()
6669 read_vec_element_i32(s, t1, a->rm, a->idx, a->esz); in do_env_scalar3_idx_hs()
6670 read_vec_element_i32(s, t2, a->rd, 0, a->esz); in do_env_scalar3_idx_hs()
6671 f->gen_hs[a->esz - 1](t0, tcg_env, t0, t1, t2); in do_env_scalar3_idx_hs()
6672 write_fp_sreg(s, a->rd, t0); in do_env_scalar3_idx_hs()
6689 read_vec_element(s, t0, a->rd, 0, a->esz + 1); in do_scalar_muladd_widening_idx()
6691 read_vec_element(s, t1, a->rn, 0, a->esz | MO_SIGN); in do_scalar_muladd_widening_idx()
6692 read_vec_element(s, t2, a->rm, a->idx, a->esz | MO_SIGN); in do_scalar_muladd_widening_idx()
6696 clear_vec(s, a->rd); in do_scalar_muladd_widening_idx()
6697 write_vec_element(s, t0, a->rd, 0, a->esz + 1); in do_scalar_muladd_widening_idx()
6703 a->esz == MO_16 ? gen_sqdmull_h : gen_sqdmull_s, false)
6705 a->esz == MO_16 ? gen_sqdmlal_h : gen_sqdmlal_s, true)
6707 a->esz == MO_16 ? gen_sqdmlsl_h : gen_sqdmlsl_s, true)
6712 MemOp esz = a->esz; in do_fp3_vector_idx()
6713 int check = fp_access_check_vector_hsd(s, a->q, esz); in do_fp3_vector_idx()
6719 gen_gvec_op3_fpst(s, a->q, a->rd, a->rn, a->rm, in do_fp3_vector_idx()
6721 a->idx, fns[esz - 1]); in do_fp3_vector_idx()
6752 MemOp esz = a->esz; in TRANS()
6753 int check = fp_access_check_vector_hsd(s, a->q, esz); in TRANS()
6759 gen_gvec_op4_fpst(s, a->q, a->rd, a->rn, a->rm, a->rd, in TRANS()
6761 a->idx, fns[neg ? 1 + s->fpcr_ah : 0][esz - 1]); in TRANS()
6771 int data = (a->idx << 2) | (is_2 << 1) | is_s; in TRANS()
6772 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, a->rd), in TRANS()
6773 vec_full_reg_offset(s, a->rn), in TRANS()
6774 vec_full_reg_offset(s, a->rm), tcg_env, in TRANS()
6775 a->q ? 16 : 8, vec_full_reg_size(s), in TRANS()
6789 assert(a->esz == MO_16 || a->esz == MO_32); in TRANS_FEAT()
6791 gen_gvec_op3_ool(s, a->q, a->rd, a->rn, a->rm, a->idx, fns[a->esz - 1]); in TRANS_FEAT()
6809 assert(a->esz == MO_16 || a->esz == MO_32); in TRANS()
6811 gen_gvec_op4_ool(s, a->q, a->rd, a->rn, a->rm, a->rd, in TRANS()
6812 a->idx, fns[a->esz - 1][sub]); in TRANS()
6823 assert(a->esz == MO_16 || a->esz == MO_32); in TRANS()
6825 tcg_gen_gvec_4_ool(vec_full_reg_offset(s, a->rd), in TRANS()
6826 vec_full_reg_offset(s, a->rn), in TRANS()
6827 vec_full_reg_offset(s, a->rm), in TRANS()
6829 a->q ? 16 : 8, vec_full_reg_size(s), in TRANS()
6830 a->idx, fns[a->esz - 1]); in TRANS()
6865 gen_gvec_op4_ool(s, a->q, a->rd, a->rn, a->rm, a->rd, a->idx, fn); in TRANS_FEAT()
6874 gen_gvec_op4_env(s, a->q, a->rd, a->rn, a->rm, a->rd, a->idx, fn); in do_dot_vector_idx_env()
6895 gen_gvec_op4_fpst(s, true, a->rd, a->rn, a->rm, a->rd, in TRANS_FEAT()
6896 s->fpcr_ah ? FPST_AH : FPST_A64, in TRANS_FEAT()
6897 (a->idx << 1) | a->q, in TRANS_FEAT()
6910 switch (a->esz) { in trans_FCMLA_vi()
6924 gen_gvec_op4_fpst(s, a->q, a->rd, a->rn, a->rm, a->rd, in trans_FCMLA_vi()
6925 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64, in trans_FCMLA_vi()
6926 (s->fpcr_ah << 4) | (a->idx << 2) | a->rot, fn); in trans_FCMLA_vi()
6937 switch (a->esz) { in do_fp3_scalar_pair()
6943 read_vec_element(s, t0, a->rn, 0, MO_64); in do_fp3_scalar_pair()
6944 read_vec_element(s, t1, a->rn, 1, MO_64); in do_fp3_scalar_pair()
6945 f->gen_d(t0, t0, t1, fpstatus_ptr(FPST_A64)); in do_fp3_scalar_pair()
6946 write_fp_dreg(s, a->rd, t0); in do_fp3_scalar_pair()
6954 read_vec_element_i32(s, t0, a->rn, 0, MO_32); in do_fp3_scalar_pair()
6955 read_vec_element_i32(s, t1, a->rn, 1, MO_32); in do_fp3_scalar_pair()
6956 f->gen_s(t0, t0, t1, fpstatus_ptr(FPST_A64)); in do_fp3_scalar_pair()
6957 write_fp_sreg(s, a->rd, t0); in do_fp3_scalar_pair()
6968 read_vec_element_i32(s, t0, a->rn, 0, MO_16); in do_fp3_scalar_pair()
6969 read_vec_element_i32(s, t1, a->rn, 1, MO_16); in do_fp3_scalar_pair()
6970 f->gen_h(t0, t0, t1, fpstatus_ptr(FPST_A64_F16)); in do_fp3_scalar_pair()
6971 write_fp_sreg(s, a->rd, t0); in do_fp3_scalar_pair()
6984 return do_fp3_scalar_pair(s, a, s->fpcr_ah ? fah : fnormal); in do_fp3_scalar_pair_2fn()
6999 read_vec_element(s, t0, a->rn, 0, MO_64); in trans_ADDP_s()
7000 read_vec_element(s, t1, a->rn, 1, MO_64); in trans_ADDP_s()
7002 write_fp_dreg(s, a->rd, t0); in trans_ADDP_s()
7008 * Floating-point conditional select
7015 int check = fp_access_check_scalar_hsd(s, a->esz); in trans_FCSEL()
7024 read_vec_element(s, t_true, a->rn, 0, a->esz); in trans_FCSEL()
7025 read_vec_element(s, t_false, a->rm, 0, a->esz); in trans_FCSEL()
7027 a64_test_cc(&c, a->cond); in trans_FCSEL()
7033 * and we've already done the zero-extension. in trans_FCSEL()
7035 write_fp_dreg(s, a->rd, t_true); in trans_FCSEL()
7046 TCGv_i64 lo = read_fp_dreg(s, a->rn); in trans_EXT_d()
7047 if (a->imm != 0) { in trans_EXT_d()
7048 TCGv_i64 hi = read_fp_dreg(s, a->rm); in trans_EXT_d()
7049 tcg_gen_extract2_i64(lo, lo, hi, a->imm * 8); in trans_EXT_d()
7051 write_fp_dreg(s, a->rd, lo); in trans_EXT_d()
7059 int pos = (a->imm & 7) * 8; in trans_EXT_q()
7060 int elt = a->imm >> 3; in trans_EXT_q()
7069 read_vec_element(s, lo, a->rn, elt, MO_64); in trans_EXT_q()
7071 read_vec_element(s, hi, elt & 2 ? a->rm : a->rn, elt & 1, MO_64); in trans_EXT_q()
7077 read_vec_element(s, hh, a->rm, elt & 1, MO_64); in trans_EXT_q()
7081 write_vec_element(s, lo, a->rd, 0, MO_64); in trans_EXT_q()
7082 write_vec_element(s, hi, a->rd, 1, MO_64); in trans_EXT_q()
7083 clear_vec_high(s, true, a->rd); in trans_EXT_q()
7088 * Floating-point data-processing (3 source)
7096 * These are fused multiply-add. Note that doing the negations here in do_fmadd()
7098 * its sign bit flipped if it is a negated-input. in do_fmadd()
7100 switch (a->esz) { in do_fmadd()
7103 TCGv_i64 tn = read_fp_dreg(s, a->rn); in do_fmadd()
7104 TCGv_i64 tm = read_fp_dreg(s, a->rm); in do_fmadd()
7105 TCGv_i64 ta = read_fp_dreg(s, a->ra); in do_fmadd()
7115 write_fp_dreg_merging(s, a->rd, a->ra, ta); in do_fmadd()
7121 TCGv_i32 tn = read_fp_sreg(s, a->rn); in do_fmadd()
7122 TCGv_i32 tm = read_fp_sreg(s, a->rm); in do_fmadd()
7123 TCGv_i32 ta = read_fp_sreg(s, a->ra); in do_fmadd()
7133 write_fp_sreg_merging(s, a->rd, a->ra, ta); in do_fmadd()
7142 TCGv_i32 tn = read_fp_hreg(s, a->rn); in do_fmadd()
7143 TCGv_i32 tm = read_fp_hreg(s, a->rm); in do_fmadd()
7144 TCGv_i32 ta = read_fp_hreg(s, a->ra); in do_fmadd()
7154 write_fp_hreg_merging(s, a->rd, a->ra, ta); in do_fmadd()
7177 MemOp src_mop = a->esz | src_sign; in TRANS()
7178 int elements = (a->q ? 16 : 8) >> a->esz; in TRANS()
7191 read_vec_element(s, tcg_res, a->rn, 0, src_mop); in TRANS()
7193 read_vec_element(s, tcg_elt, a->rn, i, src_mop); in TRANS()
7197 tcg_gen_ext_i64(tcg_res, tcg_res, a->esz + widen); in TRANS()
7198 write_fp_dreg(s, a->rd, tcg_res); in TRANS()
7245 MemOp esz = a->esz; in do_fp_reduction()
7246 int elts = (a->q ? 16 : 8) >> esz; in do_fp_reduction()
7248 TCGv_i32 res = do_reduction_op(s, a->rn, esz, 0, elts, fpst, in do_fp_reduction()
7249 s->fpcr_ah ? fah : fnormal); in do_fp_reduction()
7250 write_fp_sreg(s, a->rd, res); in do_fp_reduction()
7272 * Floating-point Immediate in TRANS_FEAT()
7277 int check = fp_access_check_scalar_hsd(s, a->esz); in TRANS_FEAT()
7284 imm = vfp_expand_imm(a->esz, a->imm); in TRANS_FEAT()
7285 write_fp_dreg(s, a->rd, tcg_constant_i64(imm)); in TRANS_FEAT()
7351 int check = fp_access_check_scalar_hsd(s, a->esz); in trans_FCMP()
7357 handle_fp_compare(s, a->esz, a->rn, a->rm, a->z, a->e); in trans_FCMP()
7365 int check = fp_access_check_scalar_hsd(s, a->esz); in trans_FCCMP()
7371 if (a->cond < 0x0e) { /* not always */ in trans_FCCMP()
7374 arm_gen_test_cc(a->cond, label_match); in trans_FCCMP()
7376 gen_set_nzcv(tcg_constant_i64(a->nzcv << 28)); in trans_FCCMP()
7381 handle_fp_compare(s, a->esz, a->rn, a->rm, false, a->e); in trans_FCCMP()
7399 tcg_gen_gvec_dup_imm(MO_16, vec_full_reg_offset(s, a->rd), in trans_FMOVI_v_h()
7400 a->q ? 16 : 8, vec_full_reg_size(s), in trans_FMOVI_v_h()
7401 vfp_expand_imm(MO_16, a->abcdefgh)); in trans_FMOVI_v_h()
7417 if ((a->cmode & 1) && a->cmode < 12) { in trans_Vimm()
7419 fn = a->op ? tcg_gen_gvec_andi : tcg_gen_gvec_ori; in trans_Vimm()
7422 if (a->cmode == 15 && a->op == 1 && a->q == 0) { in trans_Vimm()
7429 uint64_t imm = asimd_imm_const(a->abcdefgh, a->cmode, a->op); in trans_Vimm()
7430 gen_gvec_fn2i(s, a->q, a->rd, a->rd, imm, fn, MO_64); in trans_Vimm()
7442 gen_gvec_fn2i(s, a->q, a->rd, a->rn, a->imm, fn, a->esz); in do_vec_shift_imm()
7465 int esz = a->esz; in TRANS()
7479 read_vec_element(s, tcg_rn, a->rn, a->q, MO_64); in TRANS()
7488 tcg_gen_shli_i64(tcg_rd, tcg_rd, a->imm); in TRANS()
7489 write_vec_element(s, tcg_rd, a->rd, i, esz + 1); in TRANS()
7491 clear_vec_high(s, true, a->rd); in TRANS()
7530 TCGv_i64 rnd = tcg_constant_i64(1ull << (shift - 1)); in gen_srshr_bhs()
7542 TCGv_i64 rnd = tcg_constant_i64(1ull << (shift - 1)); in gen_urshr_bhs()
7556 /* Extension of sign bit (0,-1) plus sign bit (0,1) is zero. */ in gen_srshr_d()
7560 tcg_gen_extract_i64(rnd, src, shift - 1, 1); in gen_srshr_d()
7576 tcg_gen_extract_i64(rnd, src, shift - 1, 1); in gen_urshr_d()
7599 tcg_gen_deposit_i64(dst, dst, src, 0, 64 - shift); in gen_sri_d()
7605 tcg_gen_deposit_i64(dst, dst, src, shift, 64 - shift); in gen_sli_d()
7612 int esz = a->esz; in do_vec_shift_imm_narrow()
7629 read_vec_element(s, tcg_rn, a->rn, i, (esz + 1) | sign); in do_vec_shift_imm_narrow()
7630 fn(tcg_rn, tcg_rn, a->imm); in do_vec_shift_imm_narrow()
7634 write_vec_element(s, tcg_rd, a->rd, a->q, MO_64); in do_vec_shift_imm_narrow()
7635 clear_vec_high(s, a->q, a->rd); in do_vec_shift_imm_narrow()
7823 read_vec_element(s, rn, a->rn, 0, a->esz | sign); in TRANS()
7825 read_vec_element(s, rd, a->rd, 0, a->esz | sign); in TRANS()
7827 fn(rd, rn, a->imm); in TRANS()
7828 write_fp_dreg(s, a->rd, rd); in TRANS()
7927 /* Note that the helpers sign-extend their inputs, so don't do it here. */
7928 TRANS(SQSHL_si, do_scalar_shift_imm, a, f_scalar_sqshli[a->esz], false, 0)
7929 TRANS(UQSHL_si, do_scalar_shift_imm, a, f_scalar_uqshli[a->esz], false, 0)
7930 TRANS(SQSHLU_si, do_scalar_shift_imm, a, f_scalar_sqshlui[a->esz], false, 0)
7936 MemOp esz = a->esz; in do_scalar_shift_imm_narrow()
7944 read_vec_element(s, rn, a->rn, 0, (esz + 1) | sign); in do_scalar_shift_imm_narrow()
7945 fns[esz](rd, rn, a->imm); in do_scalar_shift_imm_narrow()
7949 write_fp_dreg(s, a->rd, rd); in do_scalar_shift_imm_narrow()
7964 tcg_rd = cpu_reg(s, a->rd); in TRANS()
7966 if (!a->sf && is_signed) { in TRANS()
7969 tcg_gen_ext32s_i64(tcg_n, cpu_reg(s, a->rn)); in TRANS()
7970 tcg_gen_ext32s_i64(tcg_m, cpu_reg(s, a->rm)); in TRANS()
7972 tcg_n = read_cpu_reg(s, a->rn, a->sf); in TRANS()
7973 tcg_m = read_cpu_reg(s, a->rm, a->sf); in TRANS()
7982 if (!a->sf) { /* zero extend final result */ in TRANS()
8055 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); in do_shift_reg()
8056 TCGv_i64 tcg_rn = read_cpu_reg(s, a->rn, a->sf); in do_shift_reg()
8058 tcg_gen_andi_i64(tcg_shift, cpu_reg(s, a->rm), a->sf ? 63 : 31); in do_shift_reg()
8059 shift_reg(tcg_rd, tcg_rn, a->sf, shift_type, tcg_shift); in do_shift_reg()
8073 switch (a->esz) { in TRANS()
8078 tcg_gen_extract_i64(tcg_val, cpu_reg(s, a->rm), 0, 8 << a->esz); in TRANS()
8081 tcg_val = cpu_reg(s, a->rm); in TRANS()
8086 tcg_acc = cpu_reg(s, a->rn); in TRANS()
8087 tcg_bytes = tcg_constant_i32(1 << a->esz); in TRANS()
8088 tcg_rd = cpu_reg(s, a->rd); in TRANS()
8103 TCGv_i64 tcg_n = read_cpu_reg_sp(s, a->rn, true); in TRANS_FEAT()
8104 TCGv_i64 tcg_m = read_cpu_reg_sp(s, a->rm, true); in TRANS_FEAT()
8105 TCGv_i64 tcg_d = cpu_reg(s, a->rd); in TRANS_FEAT()
8124 TCGv_i64 tcg_rd = cpu_reg_sp(s, a->rd); in TRANS_FEAT()
8125 TCGv_i64 tcg_rn = cpu_reg_sp(s, a->rn); in TRANS_FEAT()
8127 if (s->ata[0]) { in TRANS_FEAT()
8128 gen_helper_irg(tcg_rd, tcg_env, tcg_rn, cpu_reg(s, a->rm)); in TRANS_FEAT()
8142 tcg_gen_extract_i64(t, cpu_reg_sp(s, a->rn), 56, 4); in trans_GMI()
8144 tcg_gen_or_i64(cpu_reg(s, a->rd), cpu_reg(s, a->rm), t); in trans_GMI()
8153 gen_helper_pacga(cpu_reg(s, a->rd), tcg_env, in trans_PACGA()
8154 cpu_reg(s, a->rn), cpu_reg_sp(s, a->rm)); in trans_PACGA()
8209 TRANS(RBIT, gen_rr, a->rd, a->rn, a->sf ? gen_helper_rbit64 : gen_rbit32)
8210 TRANS(REV16, gen_rr, a->rd, a->rn, a->sf ? gen_rev16_64 : gen_rev16_32)
8211 TRANS(REV32, gen_rr, a->rd, a->rn, a->sf ? gen_rev32 : gen_rev_32)
8212 TRANS(REV64, gen_rr, a->rd, a->rn, tcg_gen_bswap64_i64)
8237 TRANS(CLZ, gen_rr, a->rd, a->rn, a->sf ? gen_clz64 : gen_clz32)
8238 TRANS(CLS, gen_rr, a->rd, a->rn, a->sf ? tcg_gen_clrsb_i64 : gen_cls32)
8244 if (a->z) { in gen_pacaut()
8245 if (a->rn != 31) { in gen_pacaut()
8250 tcg_rn = cpu_reg_sp(s, a->rn); in gen_pacaut()
8252 if (s->pauth_active) { in gen_pacaut()
8253 tcg_rd = cpu_reg(s, a->rd); in gen_pacaut()
8271 if (s->pauth_active) { in TRANS_FEAT()
8278 TRANS_FEAT(XPACI, aa64_pauth, do_xpac, a->rd, gen_helper_xpaci)
8279 TRANS_FEAT(XPACD, aa64_pauth, do_xpac, a->rd, gen_helper_xpacd)
8286 if (!a->sf && (a->sa & (1 << 5))) { in do_logic_reg()
8290 tcg_rd = cpu_reg(s, a->rd); in do_logic_reg()
8291 tcg_rn = cpu_reg(s, a->rn); in do_logic_reg()
8293 tcg_rm = read_cpu_reg(s, a->rm, a->sf); in do_logic_reg()
8294 if (a->sa) { in do_logic_reg()
8295 shift_reg_imm(tcg_rm, tcg_rm, a->sf, a->st, a->sa); in do_logic_reg()
8298 (a->n ? inv_fn : fn)(tcg_rd, tcg_rn, tcg_rm); in do_logic_reg()
8299 if (!a->sf) { in do_logic_reg()
8303 gen_logic_CC(a->sf, tcg_rd); in do_logic_reg()
8312 * register-register MOV and MVN, so it is worth special casing. in trans_ORR_r()
8314 if (a->sa == 0 && a->st == 0 && a->rn == 31) { in trans_ORR_r()
8315 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); in trans_ORR_r()
8316 TCGv_i64 tcg_rm = cpu_reg(s, a->rm); in trans_ORR_r()
8318 if (a->n) { in trans_ORR_r()
8320 if (!a->sf) { in trans_ORR_r()
8324 if (a->sf) { in trans_ORR_r()
8345 if (a->sa > 4) { in TRANS()
8349 /* non-flag setting ops may use SP */ in TRANS()
8351 tcg_rd = cpu_reg_sp(s, a->rd); in TRANS()
8353 tcg_rd = cpu_reg(s, a->rd); in TRANS()
8355 tcg_rn = read_cpu_reg_sp(s, a->rn, a->sf); in TRANS()
8357 tcg_rm = read_cpu_reg(s, a->rm, a->sf); in TRANS()
8358 ext_and_shift_reg(tcg_rm, tcg_rm, a->st, a->sa); in TRANS()
8369 gen_sub_CC(a->sf, tcg_result, tcg_rn, tcg_rm); in TRANS()
8371 gen_add_CC(a->sf, tcg_result, tcg_rn, tcg_rm); in TRANS()
8375 if (a->sf) { in TRANS()
8393 if (a->st == 3 || (!a->sf && (a->sa & 32))) { in TRANS()
8397 tcg_rd = cpu_reg(s, a->rd); in TRANS()
8398 tcg_rn = read_cpu_reg(s, a->rn, a->sf); in TRANS()
8399 tcg_rm = read_cpu_reg(s, a->rm, a->sf); in TRANS()
8401 shift_reg_imm(tcg_rm, tcg_rm, a->sf, a->st, a->sa); in TRANS()
8412 gen_sub_CC(a->sf, tcg_result, tcg_rn, tcg_rm); in TRANS()
8414 gen_add_CC(a->sf, tcg_result, tcg_rn, tcg_rm); in TRANS()
8418 if (a->sf) { in TRANS()
8435 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); in TRANS()
8436 TCGv_i64 tcg_rn = cpu_reg(s, a->rn); in TRANS()
8437 TCGv_i64 tcg_rm = cpu_reg(s, a->rm); in TRANS()
8449 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); in TRANS()
8453 tcg_op1 = cpu_reg(s, a->rn); in TRANS()
8454 tcg_op2 = cpu_reg(s, a->rm); in TRANS()
8458 tcg_gen_ext_i64(tcg_op1, cpu_reg(s, a->rn), mop); in TRANS()
8459 tcg_gen_ext_i64(tcg_op2, cpu_reg(s, a->rm), mop); in TRANS()
8462 if (a->ra == 31 && !is_sub) { in TRANS()
8463 /* Special-case MADD with rA == XZR; it is the standard MUL alias */ in TRANS()
8467 TCGv_i64 tcg_ra = cpu_reg(s, a->ra); in TRANS()
8498 tcg_rd = cpu_reg(s, a->rd); in TRANS()
8499 tcg_rn = cpu_reg(s, a->rn); in TRANS()
8503 tcg_gen_not_i64(tcg_y, cpu_reg(s, a->rm)); in TRANS()
8505 tcg_y = cpu_reg(s, a->rm); in TRANS()
8509 gen_adc_CC(a->sf, tcg_rd, tcg_rn, tcg_y); in TRANS()
8511 gen_adc(a->sf, tcg_rd, tcg_rn, tcg_y); in TRANS()
8523 int mask = a->mask; in TRANS()
8531 tcg_rn = read_cpu_reg(s, a->rn, 1); in TRANS()
8532 tcg_gen_rotri_i64(tcg_rn, tcg_rn, a->imm); in TRANS()
8538 tcg_gen_shli_i32(cpu_NF, nzcv, 31 - 3); in TRANS()
8548 tcg_gen_shli_i32(cpu_VF, nzcv, 31 - 0); in TRANS()
8559 tcg_gen_shli_i32(cpu_VF, tmp, shift - 1); in do_setf()
8565 TRANS_FEAT(SETF8, aa64_condm_4, do_setf, a->rn, 24)
8566 TRANS_FEAT(SETF16, aa64_condm_4, do_setf, a->rn, 16)
8581 arm_test_cc(&c, a->cond); in trans_CCMP()
8585 if (a->imm) { in trans_CCMP()
8586 tcg_y = tcg_constant_i64(a->y); in trans_CCMP()
8588 tcg_y = cpu_reg(s, a->y); in trans_CCMP()
8590 tcg_rn = cpu_reg(s, a->rn); in trans_CCMP()
8593 if (a->op) { in trans_CCMP()
8594 gen_sub_CC(a->sf, tcg_tmp, tcg_rn, tcg_y); in trans_CCMP()
8596 gen_add_CC(a->sf, tcg_tmp, tcg_rn, tcg_y); in trans_CCMP()
8601 * to help with this: T1 = (COND ? 0 : -1), T2 = (COND ? -1 : 0). in trans_CCMP()
8608 nzcv = a->nzcv; in trans_CCMP()
8651 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); in trans_CSEL()
8655 a64_test_cc(&c, a->cond); in trans_CSEL()
8657 if (a->rn == 31 && a->rm == 31 && (a->else_inc ^ a->else_inv)) { in trans_CSEL()
8659 if (a->else_inv) { in trans_CSEL()
8667 TCGv_i64 t_true = cpu_reg(s, a->rn); in trans_CSEL()
8668 TCGv_i64 t_false = read_cpu_reg(s, a->rm, 1); in trans_CSEL()
8670 if (a->else_inv && a->else_inc) { in trans_CSEL()
8672 } else if (a->else_inv) { in trans_CSEL()
8674 } else if (a->else_inc) { in trans_CSEL()
8680 if (!a->sf) { in trans_CSEL()
8696 switch (a->esz) { in do_fp1_scalar_int()
8699 TCGv_i64 t = read_fp_dreg(s, a->rn); in do_fp1_scalar_int()
8700 f->gen_d(t, t); in do_fp1_scalar_int()
8702 write_fp_dreg_merging(s, a->rd, a->rd, t); in do_fp1_scalar_int()
8704 write_fp_dreg(s, a->rd, t); in do_fp1_scalar_int()
8710 TCGv_i32 t = read_fp_sreg(s, a->rn); in do_fp1_scalar_int()
8711 f->gen_s(t, t); in do_fp1_scalar_int()
8713 write_fp_sreg_merging(s, a->rd, a->rd, t); in do_fp1_scalar_int()
8715 write_fp_sreg(s, a->rd, t); in do_fp1_scalar_int()
8724 TCGv_i32 t = read_fp_hreg(s, a->rn); in do_fp1_scalar_int()
8725 f->gen_h(t, t); in do_fp1_scalar_int()
8727 write_fp_hreg_merging(s, a->rd, a->rd, t); in do_fp1_scalar_int()
8729 write_fp_sreg(s, a->rd, t); in do_fp1_scalar_int()
8743 return do_fp1_scalar_int(s, a, s->fpcr_ah ? fah : fnormal, true); in do_fp1_scalar_int_2fn()
8791 int check = fp_access_check_scalar_hsd(s, a->esz); in do_fp1_scalar_with_fpsttype()
8802 switch (a->esz) { in do_fp1_scalar_with_fpsttype()
8804 t64 = read_fp_dreg(s, a->rn); in do_fp1_scalar_with_fpsttype()
8805 f->gen_d(t64, t64, fpst); in do_fp1_scalar_with_fpsttype()
8806 write_fp_dreg_merging(s, a->rd, a->rd, t64); in do_fp1_scalar_with_fpsttype()
8809 t32 = read_fp_sreg(s, a->rn); in do_fp1_scalar_with_fpsttype()
8810 f->gen_s(t32, t32, fpst); in do_fp1_scalar_with_fpsttype()
8811 write_fp_sreg_merging(s, a->rd, a->rd, t32); in do_fp1_scalar_with_fpsttype()
8814 t32 = read_fp_hreg(s, a->rn); in do_fp1_scalar_with_fpsttype()
8815 f->gen_h(t32, t32, fpst); in do_fp1_scalar_with_fpsttype()
8816 write_fp_hreg_merging(s, a->rd, a->rd, t32); in do_fp1_scalar_with_fpsttype()
8832 a->esz == MO_16 ? in do_fp1_scalar()
8839 return do_fp1_scalar_with_fpsttype(s, a, f, rmode, select_ah_fpst(s, a->esz)); in do_fp1_scalar_ah()
8847 TRANS(FSQRT_s, do_fp1_scalar, a, &f_scalar_fsqrt, -1)
8859 TRANS(FRINTI_s, do_fp1_scalar, a, &f_scalar_frint, -1)
8866 TRANS(FRINTX_s, do_fp1_scalar, a, &f_scalar_frintx, -1)
8870 ARMFPStatusFlavour fpsttype = s->fpcr_ah ? FPST_AH : FPST_A64; in trans_BFCVT_s()
8878 check = fp_access_check_scalar_hsd(s, a->esz); in trans_BFCVT_s()
8884 t32 = read_fp_sreg(s, a->rn); in trans_BFCVT_s()
8886 write_fp_hreg_merging(s, a->rd, a->rd, t32); in trans_BFCVT_s()
8897 TRANS_FEAT(FRINT32X_s, aa64_frint, do_fp1_scalar, a, &f_scalar_frint32, -1)
8906 TRANS_FEAT(FRINT64X_s, aa64_frint, do_fp1_scalar, a, &f_scalar_frint64, -1)
8919 s->fpcr_ah && dc_isar_feature(aa64_rpres, s) ?
8920 &f_scalar_frecpe_rpres : &f_scalar_frecpe, -1)
8927 TRANS(FRECPX_s, do_fp1_scalar_ah, a, &f_scalar_frecpx, -1)
8940 s->fpcr_ah && dc_isar_feature(aa64_rpres, s) ?
8941 &f_scalar_frsqrte_rpres : &f_scalar_frsqrte, -1)
8946 TCGv_i32 tcg_rn = read_fp_sreg(s, a->rn); in trans_FCVT_s_ds()
8951 write_fp_dreg_merging(s, a->rd, a->rd, tcg_rd); in trans_FCVT_s_ds()
8959 TCGv_i32 tmp = read_fp_sreg(s, a->rn); in trans_FCVT_s_hs()
8965 write_fp_hreg_merging(s, a->rd, a->rd, tmp); in trans_FCVT_s_hs()
8973 TCGv_i64 tcg_rn = read_fp_dreg(s, a->rn); in trans_FCVT_s_sd()
8978 write_fp_sreg_merging(s, a->rd, a->rd, tcg_rd); in trans_FCVT_s_sd()
8986 TCGv_i64 tcg_rn = read_fp_dreg(s, a->rn); in trans_FCVT_s_hd()
8993 write_fp_hreg_merging(s, a->rd, a->rd, tcg_rd); in trans_FCVT_s_hd()
9001 TCGv_i32 tcg_rn = read_fp_hreg(s, a->rn); in trans_FCVT_s_sh()
9007 write_fp_sreg_merging(s, a->rd, a->rd, tcg_rd); in trans_FCVT_s_sh()
9015 TCGv_i32 tcg_rn = read_fp_hreg(s, a->rn); in trans_FCVT_s_dh()
9021 write_fp_dreg_merging(s, a->rd, a->rd, tcg_rd); in trans_FCVT_s_dh()
9076 int check = fp_access_check_scalar_hsd(s, a->esz); in do_cvtf_g()
9082 if (a->sf) { in do_cvtf_g()
9083 tcg_int = cpu_reg(s, a->rn); in do_cvtf_g()
9085 tcg_int = read_cpu_reg(s, a->rn, true); in do_cvtf_g()
9092 return do_cvtf_scalar(s, a->esz, a->rd, a->shift, tcg_int, is_signed); in do_cvtf_g()
9107 int check = fp_access_check_scalar_hsd(s, a->esz); in TRANS()
9114 read_vec_element(s, tcg_int, a->rn, 0, a->esz | (is_signed ? MO_SIGN : 0)); in TRANS()
9115 return do_cvtf_scalar(s, a->esz, a->rd, a->shift, tcg_int, is_signed); in TRANS()
9222 int check = fp_access_check_scalar_hsd(s, a->esz); in do_fcvt_g()
9228 tcg_int = cpu_reg(s, a->rd); in do_fcvt_g()
9229 do_fcvt_scalar(s, (a->sf ? MO_64 : MO_32) | (is_signed ? MO_SIGN : 0), in do_fcvt_g()
9230 a->esz, tcg_int, a->shift, a->rn, rmode); in do_fcvt_g()
9232 if (!a->sf) { in do_fcvt_g()
9259 int check = fp_access_check_scalar_hsd(s, a->esz); in TRANS()
9266 do_fcvt_scalar(s, a->esz | (is_signed ? MO_SIGN : 0), in TRANS()
9267 a->esz, tcg_int, a->shift, a->rn, rmode); in TRANS()
9269 if (!s->fpcr_nep) { in TRANS()
9270 clear_vec(s, a->rd); in TRANS()
9272 write_vec_element(s, tcg_int, a->rd, 0, a->esz); in TRANS()
9293 TCGv_i64 t = read_fp_dreg(s, a->rn); in TRANS()
9298 tcg_gen_ext32u_i64(cpu_reg(s, a->rd), t); in TRANS()
9313 TCGv_i64 tcg_rn = cpu_reg(s, a->rn); in trans_FMOV_hx()
9316 write_fp_dreg(s, a->rd, tmp); in trans_FMOV_hx()
9324 TCGv_i64 tcg_rn = cpu_reg(s, a->rn); in trans_FMOV_sw()
9327 write_fp_dreg(s, a->rd, tmp); in trans_FMOV_sw()
9335 TCGv_i64 tcg_rn = cpu_reg(s, a->rn); in trans_FMOV_dx()
9336 write_fp_dreg(s, a->rd, tcg_rn); in trans_FMOV_dx()
9344 TCGv_i64 tcg_rn = cpu_reg(s, a->rn); in trans_FMOV_ux()
9345 tcg_gen_st_i64(tcg_rn, tcg_env, fp_reg_hi_offset(s, a->rd)); in trans_FMOV_ux()
9346 clear_vec_high(s, true, a->rd); in trans_FMOV_ux()
9357 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); in trans_FMOV_xh()
9358 tcg_gen_ld16u_i64(tcg_rd, tcg_env, fp_reg_offset(s, a->rn, MO_16)); in trans_FMOV_xh()
9366 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); in trans_FMOV_ws()
9367 tcg_gen_ld32u_i64(tcg_rd, tcg_env, fp_reg_offset(s, a->rn, MO_32)); in trans_FMOV_ws()
9375 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); in trans_FMOV_xd()
9376 tcg_gen_ld_i64(tcg_rd, tcg_env, fp_reg_offset(s, a->rn, MO_64)); in trans_FMOV_xd()
9384 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); in trans_FMOV_xu()
9385 tcg_gen_ld_i64(tcg_rd, tcg_env, fp_reg_hi_offset(s, a->rn)); in trans_FMOV_xu()
9400 if (a->esz == MO_64) { in do_env_scalar1()
9401 TCGv_i64 t = read_fp_dreg(s, a->rn); in do_env_scalar1()
9402 f->gen_d(t, tcg_env, t); in do_env_scalar1()
9403 write_fp_dreg(s, a->rd, t); in do_env_scalar1()
9407 read_vec_element_i32(s, t, a->rn, 0, a->esz); in do_env_scalar1()
9408 f->gen_bhs[a->esz](t, tcg_env, t); in do_env_scalar1()
9409 write_fp_sreg(s, a->rd, t); in do_env_scalar1()
9416 if (a->esz == MO_64 && !a->q) { in do_env_vector1()
9422 if (a->esz == MO_64) { in do_env_vector1()
9426 read_vec_element(s, t, a->rn, i, MO_64); in do_env_vector1()
9427 f->gen_d(t, tcg_env, t); in do_env_vector1()
9428 write_vec_element(s, t, a->rd, i, MO_64); in do_env_vector1()
9432 int n = (a->q ? 16 : 8) >> a->esz; in do_env_vector1()
9435 read_vec_element_i32(s, t, a->rn, i, a->esz); in do_env_vector1()
9436 f->gen_bhs[a->esz](t, tcg_env, t); in do_env_vector1()
9437 write_vec_element_i32(s, t, a->rd, i, a->esz); in do_env_vector1()
9440 clear_vec_high(s, a->q, a->rd); in do_env_vector1()
9465 TCGv_i64 t = read_fp_dreg(s, a->rn); in do_scalar1_d()
9467 write_fp_dreg(s, a->rd, t); in do_scalar1_d()
9478 TCGv_i64 t = read_fp_dreg(s, a->rn); in TRANS()
9480 write_fp_dreg(s, a->rd, t); in TRANS()
9494 if (a->esz == MO_64) { in TRANS()
9500 read_vec_element(s, t, a->rn, 0, a->esz + 1); in TRANS()
9501 fn[a->esz](t, t); in TRANS()
9502 clear_vec(s, a->rd); in TRANS()
9503 write_vec_element(s, t, a->rd, 0, a->esz); in TRANS()
9552 TCGv_i64 src = read_fp_dreg(s, a->rn); in TRANS()
9555 write_fp_sreg_merging(s, a->rd, a->rd, dst); in TRANS()
9564 if (!a->q && a->esz == MO_64) { in do_gvec_fn2()
9568 gen_gvec_fn2(s, a->q, a->rd, a->rn, fn, a->esz); in do_gvec_fn2()
9590 if (a->esz == MO_64) { in TRANS()
9594 gen_gvec_fn2(s, a->q, a->rd, a->rn, fn, a->esz); in TRANS()
9610 if (a->esz == MO_64) { in TRANS()
9617 read_vec_element(s, t0, a->rn, 0, MO_64); in TRANS()
9618 read_vec_element(s, t1, a->rn, 1, MO_64); in TRANS()
9619 fn[a->esz](t0, t0); in TRANS()
9620 fn[a->esz](t1, t1); in TRANS()
9621 write_vec_element(s, t0, a->rd, a->q ? 2 : 0, MO_32); in TRANS()
9622 write_vec_element(s, t1, a->rd, a->q ? 3 : 1, MO_32); in TRANS()
9623 clear_vec_high(s, a->q, a->rd); in TRANS()
9713 f_vector_bfcvtn[s->fpcr_ah]) in TRANS_FEAT()
9727 if (a->esz == MO_64) { in TRANS_FEAT()
9735 widenfn = widenfns[a->esz]; in TRANS_FEAT()
9736 part = a->q ? 2 : 0; in TRANS_FEAT()
9739 read_vec_element_i32(s, tcg_op, a->rn, part + pass, MO_32); in TRANS_FEAT()
9742 tcg_gen_shli_i64(tcg_res[pass], tcg_res[pass], 8 << a->esz); in TRANS_FEAT()
9746 write_vec_element(s, tcg_res[pass], a->rd, pass, MO_64); in TRANS_FEAT()
9753 int check = fp_access_check_vector_hsd(s, a->q, a->esz); in do_fabs_fneg_v()
9759 gen_gvec_fn2(s, a->q, a->rd, a->rn, fn, a->esz); in do_fabs_fneg_v()
9771 int check = fp_access_check_vector_hsd(s, a->q, a->esz); in TRANS()
9777 fpst = fpstatus_ptr(a->esz == MO_16 ? FPST_A64_F16 : FPST_A64); in TRANS()
9782 if (a->esz == MO_64) { in TRANS()
9786 read_vec_element(s, t64, a->rn, pass, MO_64); in TRANS()
9787 f->gen_d(t64, t64, fpst); in TRANS()
9788 write_vec_element(s, t64, a->rd, pass, MO_64); in TRANS()
9793 = (a->esz == MO_16 ? f->gen_h : f->gen_s); in TRANS()
9795 for (int pass = 0, n = (a->q ? 16 : 8) >> a->esz; pass < n; ++pass) { in TRANS()
9796 read_vec_element_i32(s, t32, a->rn, pass, a->esz); in TRANS()
9798 write_vec_element_i32(s, t32, a->rd, pass, a->esz); in TRANS()
9801 clear_vec_high(s, a->q, a->rd); in TRANS()
9809 TRANS(FSQRT_v, do_fp1_vector, a, &f_scalar_fsqrt, -1)
9816 TRANS(FRINTI_v, do_fp1_vector, a, &f_scalar_frint, -1)
9817 TRANS(FRINTX_v, do_fp1_vector, a, &f_scalar_frintx, -1)
9821 TRANS_FEAT(FRINT32X_v, aa64_frint, do_fp1_vector, a, &f_scalar_frint32, -1)
9824 TRANS_FEAT(FRINT64X_v, aa64_frint, do_fp1_vector, a, &f_scalar_frint64, -1)
9842 data, fns[esz - 1]); in do_gvec_op2_fpst_with_fpsttype()
9869 a->esz, a->q, a->rd, a->rn, 0, f_scvtf_v)
9871 a->esz, a->q, a->rd, a->rn, a->shift, f_scvtf_v)
9879 a->esz, a->q, a->rd, a->rn, 0, f_ucvtf_v)
9881 a->esz, a->q, a->rd, a->rn, a->shift, f_ucvtf_v)
9889 a->esz, a->q, a->rd, a->rn, a->shift, f_fcvtzs_vf)
9897 a->esz, a->q, a->rd, a->rn, a->shift, f_fcvtzu_vf)
9912 a->esz, a->q, a->rd, a->rn, float_round_nearest_even, f_fcvt_s_vi)
9914 a->esz, a->q, a->rd, a->rn, float_round_nearest_even, f_fcvt_u_vi)
9916 a->esz, a->q, a->rd, a->rn, float_round_up, f_fcvt_s_vi)
9918 a->esz, a->q, a->rd, a->rn, float_round_up, f_fcvt_u_vi)
9920 a->esz, a->q, a->rd, a->rn, float_round_down, f_fcvt_s_vi)
9922 a->esz, a->q, a->rd, a->rn, float_round_down, f_fcvt_u_vi)
9924 a->esz, a->q, a->rd, a->rn, float_round_to_zero, f_fcvt_s_vi)
9926 a->esz, a->q, a->rd, a->rn, float_round_to_zero, f_fcvt_u_vi)
9928 a->esz, a->q, a->rd, a->rn, float_round_ties_away, f_fcvt_s_vi)
9930 a->esz, a->q, a->rd, a->rn, float_round_ties_away, f_fcvt_u_vi)
9937 TRANS(FCMEQ0_v, do_gvec_op2_fpst, a->esz, a->q, a->rd, a->rn, 0, f_fceq0)
9944 TRANS(FCMGT0_v, do_gvec_op2_fpst, a->esz, a->q, a->rd, a->rn, 0, f_fcgt0)
9951 TRANS(FCMGE0_v, do_gvec_op2_fpst, a->esz, a->q, a->rd, a->rn, 0, f_fcge0)
9958 TRANS(FCMLT0_v, do_gvec_op2_fpst, a->esz, a->q, a->rd, a->rn, 0, f_fclt0)
9965 TRANS(FCMLE0_v, do_gvec_op2_fpst, a->esz, a->q, a->rd, a->rn, 0, f_fcle0)
9977 TRANS(FRECPE_v, do_gvec_op2_ah_fpst, a->esz, a->q, a->rd, a->rn, 0,
9978 s->fpcr_ah && dc_isar_feature(aa64_rpres, s) ? f_frecpe_rpres : f_frecpe)
9990 TRANS(FRSQRTE_v, do_gvec_op2_ah_fpst, a->esz, a->q, a->rd, a->rn, 0,
9991 s->fpcr_ah && dc_isar_feature(aa64_rpres, s) ? f_frsqrte_rpres : f_frsqrte)
9995 /* Handle 2-reg-misc ops which are widening (so each size element in trans_FCVTL_v()
10006 if (a->esz == MO_64) { in trans_FCVTL_v()
10007 /* 32 -> 64 bit fp conversion */ in trans_FCVTL_v()
10010 int srcelt = a->q ? 2 : 0; in trans_FCVTL_v()
10016 read_vec_element_i32(s, tcg_op, a->rn, srcelt + pass, MO_32); in trans_FCVTL_v()
10020 write_vec_element(s, tcg_res[pass], a->rd, pass, MO_64); in trans_FCVTL_v()
10023 /* 16 -> 32 bit fp conversion */ in trans_FCVTL_v()
10024 int srcelt = a->q ? 4 : 0; in trans_FCVTL_v()
10032 read_vec_element_i32(s, tcg_res[pass], a->rn, srcelt + pass, MO_16); in trans_FCVTL_v()
10037 write_vec_element_i32(s, tcg_res[pass], a->rd, pass, MO_32); in trans_FCVTL_v()
10040 clear_vec_high(s, true, a->rd); in trans_FCVTL_v()
10051 s->is_nonstreaming = true; in trans_FAIL()
10059 * @btype: PSTATE.BTYPE, and is non-zero
10063 * - branch target identifiers,
10064 * - paciasp, pacibsp,
10065 * - BRK insn
10066 * - HLT insn
10113 CPUARMTBFlags tb_flags = arm_tbflags_from_tb(dc->base.tb); in aarch64_tr_init_disas_context()
10116 dc->isar = &arm_cpu->isar; in aarch64_tr_init_disas_context()
10117 dc->condjmp = 0; in aarch64_tr_init_disas_context()
10118 dc->pc_save = dc->base.pc_first; in aarch64_tr_init_disas_context()
10119 dc->aarch64 = true; in aarch64_tr_init_disas_context()
10120 dc->thumb = false; in aarch64_tr_init_disas_context()
10121 dc->sctlr_b = 0; in aarch64_tr_init_disas_context()
10122 dc->be_data = EX_TBFLAG_ANY(tb_flags, BE_DATA) ? MO_BE : MO_LE; in aarch64_tr_init_disas_context()
10123 dc->condexec_mask = 0; in aarch64_tr_init_disas_context()
10124 dc->condexec_cond = 0; in aarch64_tr_init_disas_context()
10126 dc->mmu_idx = core_to_aa64_mmu_idx(core_mmu_idx); in aarch64_tr_init_disas_context()
10127 dc->tbii = EX_TBFLAG_A64(tb_flags, TBII); in aarch64_tr_init_disas_context()
10128 dc->tbid = EX_TBFLAG_A64(tb_flags, TBID); in aarch64_tr_init_disas_context()
10129 dc->tcma = EX_TBFLAG_A64(tb_flags, TCMA); in aarch64_tr_init_disas_context()
10130 dc->current_el = arm_mmu_idx_to_el(dc->mmu_idx); in aarch64_tr_init_disas_context()
10132 dc->user = (dc->current_el == 0); in aarch64_tr_init_disas_context()
10134 dc->fp_excp_el = EX_TBFLAG_ANY(tb_flags, FPEXC_EL); in aarch64_tr_init_disas_context()
10135 dc->align_mem = EX_TBFLAG_ANY(tb_flags, ALIGN_MEM); in aarch64_tr_init_disas_context()
10136 dc->pstate_il = EX_TBFLAG_ANY(tb_flags, PSTATE__IL); in aarch64_tr_init_disas_context()
10137 dc->fgt_active = EX_TBFLAG_ANY(tb_flags, FGT_ACTIVE); in aarch64_tr_init_disas_context()
10138 dc->fgt_svc = EX_TBFLAG_ANY(tb_flags, FGT_SVC); in aarch64_tr_init_disas_context()
10139 dc->trap_eret = EX_TBFLAG_A64(tb_flags, TRAP_ERET); in aarch64_tr_init_disas_context()
10140 dc->sve_excp_el = EX_TBFLAG_A64(tb_flags, SVEEXC_EL); in aarch64_tr_init_disas_context()
10141 dc->sme_excp_el = EX_TBFLAG_A64(tb_flags, SMEEXC_EL); in aarch64_tr_init_disas_context()
10142 dc->zt0_excp_el = EX_TBFLAG_A64(tb_flags, ZT0EXC_EL); in aarch64_tr_init_disas_context()
10143 dc->vl = (EX_TBFLAG_A64(tb_flags, VL) + 1) * 16; in aarch64_tr_init_disas_context()
10144 dc->svl = (EX_TBFLAG_A64(tb_flags, SVL) + 1) * 16; in aarch64_tr_init_disas_context()
10145 dc->max_svl = arm_cpu->sme_max_vq * 16; in aarch64_tr_init_disas_context()
10146 dc->pauth_active = EX_TBFLAG_A64(tb_flags, PAUTH_ACTIVE); in aarch64_tr_init_disas_context()
10147 dc->bt = EX_TBFLAG_A64(tb_flags, BT); in aarch64_tr_init_disas_context()
10148 dc->btype = EX_TBFLAG_A64(tb_flags, BTYPE); in aarch64_tr_init_disas_context()
10149 dc->unpriv = EX_TBFLAG_A64(tb_flags, UNPRIV); in aarch64_tr_init_disas_context()
10150 dc->ata[0] = EX_TBFLAG_A64(tb_flags, ATA); in aarch64_tr_init_disas_context()
10151 dc->ata[1] = EX_TBFLAG_A64(tb_flags, ATA0); in aarch64_tr_init_disas_context()
10152 dc->mte_active[0] = EX_TBFLAG_A64(tb_flags, MTE_ACTIVE); in aarch64_tr_init_disas_context()
10153 dc->mte_active[1] = EX_TBFLAG_A64(tb_flags, MTE0_ACTIVE); in aarch64_tr_init_disas_context()
10154 dc->pstate_sm = EX_TBFLAG_A64(tb_flags, PSTATE_SM); in aarch64_tr_init_disas_context()
10155 dc->pstate_za = EX_TBFLAG_A64(tb_flags, PSTATE_ZA); in aarch64_tr_init_disas_context()
10156 dc->sme_trap_nonstreaming = EX_TBFLAG_A64(tb_flags, SME_TRAP_NONSTREAMING); in aarch64_tr_init_disas_context()
10157 dc->naa = EX_TBFLAG_A64(tb_flags, NAA); in aarch64_tr_init_disas_context()
10158 dc->nv = EX_TBFLAG_A64(tb_flags, NV); in aarch64_tr_init_disas_context()
10159 dc->nv1 = EX_TBFLAG_A64(tb_flags, NV1); in aarch64_tr_init_disas_context()
10160 dc->nv2 = EX_TBFLAG_A64(tb_flags, NV2); in aarch64_tr_init_disas_context()
10161 dc->nv2_mem_e20 = EX_TBFLAG_A64(tb_flags, NV2_MEM_E20); in aarch64_tr_init_disas_context()
10162 dc->nv2_mem_be = EX_TBFLAG_A64(tb_flags, NV2_MEM_BE); in aarch64_tr_init_disas_context()
10163 dc->fpcr_ah = EX_TBFLAG_A64(tb_flags, AH); in aarch64_tr_init_disas_context()
10164 dc->fpcr_nep = EX_TBFLAG_A64(tb_flags, NEP); in aarch64_tr_init_disas_context()
10165 dc->vec_len = 0; in aarch64_tr_init_disas_context()
10166 dc->vec_stride = 0; in aarch64_tr_init_disas_context()
10167 dc->cp_regs = arm_cpu->cp_regs; in aarch64_tr_init_disas_context()
10168 dc->features = env->features; in aarch64_tr_init_disas_context()
10169 dc->dcz_blocksize = arm_cpu->dcz_blocksize; in aarch64_tr_init_disas_context()
10170 dc->gm_blocksize = arm_cpu->gm_blocksize; in aarch64_tr_init_disas_context()
10174 tcg_debug_assert(dc->tbid & 1); in aarch64_tr_init_disas_context()
10177 dc->lse2 = dc_isar_feature(aa64_lse2, dc); in aarch64_tr_init_disas_context()
10179 /* Single step state. The code-generation logic here is: in aarch64_tr_init_disas_context()
10181 * generate code with no special handling for single-stepping (except in aarch64_tr_init_disas_context()
10185 * SS_ACTIVE == 1, PSTATE.SS == 1: (active-not-pending) in aarch64_tr_init_disas_context()
10190 * SS_ACTIVE == 1, PSTATE.SS == 0: (active-pending) in aarch64_tr_init_disas_context()
10194 dc->ss_active = EX_TBFLAG_ANY(tb_flags, SS_ACTIVE); in aarch64_tr_init_disas_context()
10195 dc->pstate_ss = EX_TBFLAG_ANY(tb_flags, PSTATE__SS); in aarch64_tr_init_disas_context()
10196 dc->is_ldex = false; in aarch64_tr_init_disas_context()
10199 bound = -(dc->base.pc_first | TARGET_PAGE_MASK) / 4; in aarch64_tr_init_disas_context()
10202 if (dc->ss_active) { in aarch64_tr_init_disas_context()
10205 dc->base.max_insns = MIN(dc->base.max_insns, bound); in aarch64_tr_init_disas_context()
10215 target_ulong pc_arg = dc->base.pc_next; in aarch64_tr_insn_start()
10217 if (tb_cflags(dcbase->tb) & CF_PCREL) { in aarch64_tr_insn_start()
10221 dc->insn_start_updated = false; in aarch64_tr_insn_start()
10228 uint64_t pc = s->base.pc_next; in aarch64_tr_translate_insn()
10232 if (s->ss_active && !s->pstate_ss) { in aarch64_tr_translate_insn()
10233 /* Singlestep state is Active-pending. in aarch64_tr_translate_insn()
10243 assert(s->base.num_insns == 1); in aarch64_tr_translate_insn()
10245 s->base.is_jmp = DISAS_NORETURN; in aarch64_tr_translate_insn()
10246 s->base.pc_next = pc + 4; in aarch64_tr_translate_insn()
10257 assert(s->base.num_insns == 1); in aarch64_tr_translate_insn()
10259 s->base.is_jmp = DISAS_NORETURN; in aarch64_tr_translate_insn()
10260 s->base.pc_next = QEMU_ALIGN_UP(pc, 4); in aarch64_tr_translate_insn()
10264 s->pc_curr = pc; in aarch64_tr_translate_insn()
10265 insn = arm_ldl_code(env, &s->base, pc, s->sctlr_b); in aarch64_tr_translate_insn()
10266 s->insn = insn; in aarch64_tr_translate_insn()
10267 s->base.pc_next = pc + 4; in aarch64_tr_translate_insn()
10269 s->fp_access_checked = 0; in aarch64_tr_translate_insn()
10270 s->sve_access_checked = 0; in aarch64_tr_translate_insn()
10272 if (s->pstate_il) { in aarch64_tr_translate_insn()
10282 if (s->base.num_insns == 1) { in aarch64_tr_translate_insn()
10283 /* First insn can have btype set to non-zero. */ in aarch64_tr_translate_insn()
10284 tcg_debug_assert(s->btype >= 0); in aarch64_tr_translate_insn()
10288 * priority -- below debugging exceptions but above most in aarch64_tr_translate_insn()
10295 if (s->btype != 0 in aarch64_tr_translate_insn()
10296 && !btype_destination_ok(insn, s->bt, s->btype)) { in aarch64_tr_translate_insn()
10301 tcg_debug_assert(s->btype == 0); in aarch64_tr_translate_insn()
10305 s->is_nonstreaming = false; in aarch64_tr_translate_insn()
10306 if (s->sme_trap_nonstreaming) { in aarch64_tr_translate_insn()
10318 * Note that we set btype == -1 when the insn sets btype. in aarch64_tr_translate_insn()
10320 if (s->btype > 0 && s->base.is_jmp != DISAS_NORETURN) { in aarch64_tr_translate_insn()
10329 if (unlikely(dc->ss_active)) { in aarch64_tr_tb_stop()
10333 * (and thus a tb-jump is not possible when singlestepping). in aarch64_tr_tb_stop()
10335 switch (dc->base.is_jmp) { in aarch64_tr_tb_stop()
10347 switch (dc->base.is_jmp) { in aarch64_tr_tb_stop()