Lines Matching +full:aarch64_be +full:- +full:linux +full:- +full:user
22 #include "translate-a64.h"
43 * Returns -1 for unallocated encoding; diagnose later.
48 return 31 - clz32(x); in tszimm_esz()
54 * We won't use the tszimm_shr() value if tszimm_esz() returns -1 (the in tszimm_shr()
62 return (16 << esz) - x; in tszimm_shr()
73 return x - (8 << esz); in tszimm_shl()
87 /* Convert a 2-bit memory size (msz) to a 4-bit data type (dtype)
100 #include "decode-sve.c.inc"
106 /* Invoke an out-of-line helper on 2 Zregs. */
143 return gen_gvec_fpst_zz(s, fn, a->rd, a->rn, data, in gen_gvec_fpst_ah_arg_zz()
144 select_ah_fpst(s, a->esz)); in gen_gvec_fpst_ah_arg_zz()
147 /* Invoke an out-of-line helper on 3 Zregs. */
167 return gen_gvec_ool_zzz(s, fn, a->rd, a->rn, a->rm, data); in gen_gvec_ool_arg_zzz()
170 /* Invoke an out-of-line helper on 3 Zregs, plus float_status. */
194 if (a->esz == MO_8 && !dc_isar_feature(aa64_sve_b16b16, s)) { in gen_gvec_fpst_arg_zzz()
197 return gen_gvec_fpst_zzz(s, fn, a->rd, a->rn, a->rm, data, in gen_gvec_fpst_arg_zzz()
198 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64); in gen_gvec_fpst_arg_zzz()
204 return gen_gvec_fpst_zzz(s, fn, a->rd, a->rn, a->rm, data, in gen_gvec_fpst_ah_arg_zzz()
205 select_ah_fpst(s, a->esz)); in gen_gvec_fpst_ah_arg_zzz()
208 /* Invoke an out-of-line helper on 4 Zregs. */
229 return gen_gvec_ool_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, data); in gen_gvec_ool_arg_zzzz()
235 return gen_gvec_ool_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, a->index); in gen_gvec_ool_arg_zzxz()
238 /* Invoke an out-of-line helper on 4 Zregs, plus a pointer. */
276 return gen_gvec_env_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, data); in gen_gvec_env_arg_zzzz()
282 return gen_gvec_env_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, a->index); in gen_gvec_env_arg_zzxz()
285 /* Invoke an out-of-line helper on 4 Zregs, 1 Preg, plus fpst. */
307 /* Invoke an out-of-line helper on 2 Zregs and a predicate. */
327 return gen_gvec_ool_zzp(s, fn, a->rd, a->rn, a->pg, data); in gen_gvec_ool_arg_zpz()
333 return gen_gvec_ool_zzp(s, fn, a->rd, a->rn, a->pg, a->imm); in gen_gvec_ool_arg_zpzi()
359 return gen_gvec_fpst_zzp(s, fn, a->rd, a->rn, a->pg, data, flavour); in gen_gvec_fpst_arg_zpz()
362 /* Invoke an out-of-line helper on 3 Zregs and a predicate. */
383 return gen_gvec_ool_zzzp(s, fn, a->rd, a->rn, a->rm, a->pg, data); in gen_gvec_ool_arg_zpzz()
386 /* Invoke an out-of-line helper on 3 Zregs and a predicate. */
411 if (a->esz == MO_8 && !dc_isar_feature(aa64_sve_b16b16, s)) { in gen_gvec_fpst_arg_zpzz()
414 return gen_gvec_fpst_zzzp(s, fn, a->rd, a->rn, a->rm, a->pg, 0, in gen_gvec_fpst_arg_zpzz()
415 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64); in gen_gvec_fpst_arg_zpzz()
436 if (a->esz < 0) { in gen_gvec_fn_arg_zzi()
437 /* Invalid tsz encoding -- see tszimm_esz. */ in gen_gvec_fn_arg_zzi()
440 return gen_gvec_fn_zzi(s, gvec_fn, a->esz, a->rd, a->rn, a->imm); in gen_gvec_fn_arg_zzi()
462 return gen_gvec_fn_zzz(s, fn, a->esz, a->rd, a->rn, a->rm); in gen_gvec_fn_arg_zzz()
474 gvec_fn(a->esz, vec_full_reg_offset(s, a->rd), in gen_gvec_fn_arg_zzzz()
475 vec_full_reg_offset(s, a->rn), in gen_gvec_fn_arg_zzzz()
476 vec_full_reg_offset(s, a->rm), in gen_gvec_fn_arg_zzzz()
477 vec_full_reg_offset(s, a->ra), vsz, vsz); in gen_gvec_fn_arg_zzzz()
493 /* Initialize a Zreg with replications of a 64-bit immediate. */
570 *** SVE Logical - Unpredicated Group
580 if (a->esz < 0 || !dc_isar_feature(aa64_sve2, s)) { in TRANS_FEAT()
585 gen_gvec_xar(a->esz, vec_full_reg_offset(s, a->rd), in TRANS_FEAT()
586 vec_full_reg_offset(s, a->rn), in TRANS_FEAT()
587 vec_full_reg_offset(s, a->rm), a->imm, vsz, vsz); in TRANS_FEAT()
701 *** SVE Integer Arithmetic - Unpredicated Group in TRANS_FEAT()
712 *** SVE Integer Arithmetic - Binary Predicated Group in TRANS_FEAT()
733 name##_zpzz_fns[a->esz], a, 0)
761 TRANS_FEAT(SDIV_zpzz, aa64_sve, gen_gvec_ool_arg_zpzz, sdiv_fns[a->esz], a, 0)
766 TRANS_FEAT(UDIV_zpzz, aa64_sve, gen_gvec_ool_arg_zpzz, udiv_fns[a->esz], a, 0)
768 TRANS_FEAT(SEL_zpzz, aa64_sve, do_sel_z, a->rd, a->rn, a->rm, a->pg, a->esz)
771 *** SVE Integer Arithmetic - Unary Predicated Group
779 TRANS_FEAT(NAME, FEAT, gen_gvec_ool_arg_zpz, name##_fns[a->esz], a, 0)
802 s->fpcr_ah ? fabs_ah_fns[a->esz] : fabs_fns[a->esz], a, 0)
813 s->fpcr_ah ? fneg_ah_fns[a->esz] : fneg_fns[a->esz], a, 0)
819 TRANS_FEAT(SXTB, aa64_sve, gen_gvec_ool_arg_zpz, sxtb_fns[a->esz], a, 0)
825 TRANS_FEAT(UXTB, aa64_sve, gen_gvec_ool_arg_zpz, uxtb_fns[a->esz], a, 0)
830 TRANS_FEAT(SXTH, aa64_sve, gen_gvec_ool_arg_zpz, sxth_fns[a->esz], a, 0)
835 TRANS_FEAT(UXTH, aa64_sve, gen_gvec_ool_arg_zpz, uxth_fns[a->esz], a, 0)
838 a->esz == 3 ? gen_helper_sve_sxtw_d : NULL, a, 0)
840 a->esz == 3 ? gen_helper_sve_uxtw_d : NULL, a, 0)
847 gen_gvec_ool_arg_zpz, addqv_fns[a->esz], a, 0)
854 gen_gvec_ool_arg_zpz, smaxqv_fns[a->esz], a, 0)
861 gen_gvec_ool_arg_zpz, sminqv_fns[a->esz], a, 0)
868 gen_gvec_ool_arg_zpz, umaxqv_fns[a->esz], a, 0)
875 gen_gvec_ool_arg_zpz, uminqv_fns[a->esz], a, 0)
902 tcg_gen_addi_ptr(t_zn, tcg_env, vec_full_reg_offset(s, a->rn)); in do_vpz_ool()
903 tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, a->pg)); in do_vpz_ool()
906 write_fp_dreg(s, a->rd, temp); in do_vpz_ool()
915 TRANS_FEAT(NAME, aa64_sve, do_vpz_ool, a, name##_fns[a->esz])
931 TRANS_FEAT(SADDV, aa64_sve, do_vpz_ool, a, saddv_fns[a->esz]) in TRANS_FEAT()
936 *** SVE Shift by Immediate - Predicated Group in TRANS_FEAT()
958 if (a->esz < 0) { in do_shift_zpzi()
959 /* Invalid tsz encoding -- see tszimm_esz. */ in do_shift_zpzi()
965 * For arithmetic right-shift, it's the same as by one less. in do_shift_zpzi()
968 max = 8 << a->esz; in do_shift_zpzi()
969 if (a->imm >= max) { in do_shift_zpzi()
971 a->imm = max - 1; in do_shift_zpzi()
973 return do_movz_zpz(s, a->rd, a->rd, a->pg, a->esz, true); in do_shift_zpzi()
976 return gen_gvec_ool_arg_zpzi(s, fns[a->esz], a); in do_shift_zpzi()
1008 a->esz < 0 ? NULL : sqshl_zpzi_fns[a->esz], a)
1015 a->esz < 0 ? NULL : uqshl_zpzi_fns[a->esz], a)
1022 a->esz < 0 ? NULL : srshr_fns[a->esz], a)
1029 a->esz < 0 ? NULL : urshr_fns[a->esz], a)
1036 a->esz < 0 ? NULL : sqshlu_fns[a->esz], a)
1039 *** SVE Bitwise Shift - Predicated Group
1048 a->esz < 0 ? NULL : name##_zpzw_fns[a->esz], a, 0)
1057 *** SVE Bitwise Shift - Unpredicated Group in DO_ZPZW()
1064 if (a->esz < 0) { in DO_ZPZW()
1065 /* Invalid tsz encoding -- see tszimm_esz. */ in DO_ZPZW()
1071 arithmetic right-shift, it's the same as by one less. in DO_ZPZW()
1073 if (a->imm >= 8 << a->esz) { in DO_ZPZW()
1075 a->imm = (8 << a->esz) - 1; in DO_ZPZW()
1077 do_dupi_z(s, a->rd, 0); in DO_ZPZW()
1081 gvec_fn(a->esz, vec_full_reg_offset(s, a->rd), in DO_ZPZW()
1082 vec_full_reg_offset(s, a->rn), a->imm, vsz, vsz); in DO_ZPZW()
1097 name##_zzw_fns[a->esz], a, 0)
1106 *** SVE Integer Multiply-Add Group
1114 tcg_gen_gvec_5_ool(vec_full_reg_offset(s, a->rd),
1115 vec_full_reg_offset(s, a->ra),
1116 vec_full_reg_offset(s, a->rn),
1117 vec_full_reg_offset(s, a->rm),
1118 pred_full_reg_offset(s, a->pg),
1128 TRANS_FEAT(MLA, aa64_sve, do_zpzzz_ool, a, mla_fns[a->esz])
1134 TRANS_FEAT(MLS, aa64_sve, do_zpzzz_ool, a, mls_fns[a->esz]) in TRANS_FEAT()
1175 TRANS_FEAT(INDEX_ii, aa64_sve, do_index, a->esz, a->rd,
1176 tcg_constant_i64(a->imm1), tcg_constant_i64(a->imm2))
1177 TRANS_FEAT(INDEX_ir, aa64_sve, do_index, a->esz, a->rd,
1178 tcg_constant_i64(a->imm), cpu_reg(s, a->rm))
1179 TRANS_FEAT(INDEX_ri, aa64_sve, do_index, a->esz, a->rd,
1180 cpu_reg(s, a->rn), tcg_constant_i64(a->imm))
1181 TRANS_FEAT(INDEX_rr, aa64_sve, do_index, a->esz, a->rd,
1182 cpu_reg(s, a->rn), cpu_reg(s, a->rm))
1194 TCGv_i64 rd = cpu_reg_sp(s, a->rd); in trans_ADDVL()
1195 TCGv_i64 rn = cpu_reg_sp(s, a->rn); in trans_ADDVL()
1196 tcg_gen_addi_i64(rd, rn, a->imm * vec_full_reg_size(s)); in trans_ADDVL()
1207 TCGv_i64 rd = cpu_reg_sp(s, a->rd); in trans_ADDSVL()
1208 TCGv_i64 rn = cpu_reg_sp(s, a->rn); in trans_ADDSVL()
1209 tcg_gen_addi_i64(rd, rn, a->imm * streaming_vec_reg_size(s)); in trans_ADDSVL()
1220 TCGv_i64 rd = cpu_reg_sp(s, a->rd); in trans_ADDPL()
1221 TCGv_i64 rn = cpu_reg_sp(s, a->rn); in trans_ADDPL()
1222 tcg_gen_addi_i64(rd, rn, a->imm * pred_full_reg_size(s)); in trans_ADDPL()
1233 TCGv_i64 rd = cpu_reg_sp(s, a->rd); in trans_ADDSPL()
1234 TCGv_i64 rn = cpu_reg_sp(s, a->rn); in trans_ADDSPL()
1235 tcg_gen_addi_i64(rd, rn, a->imm * streaming_pred_reg_size(s)); in trans_ADDSPL()
1246 TCGv_i64 reg = cpu_reg(s, a->rd); in trans_RDVL()
1247 tcg_gen_movi_i64(reg, a->imm * vec_full_reg_size(s)); in trans_RDVL()
1258 TCGv_i64 reg = cpu_reg(s, a->rd); in trans_RDSVL()
1259 tcg_gen_movi_i64(reg, a->imm * streaming_vec_reg_size(s)); in trans_RDSVL()
1270 return gen_gvec_ool_zzz(s, fn, a->rd, a->rn, a->rm, a->imm); in do_adr()
1279 *** SVE Integer Misc - Unpredicated Group
1287 fexpa_fns[a->esz], a->rd, a->rn, s->fpcr_ah)
1294 ftssel_fns[a->esz], a, s->fpcr_ah)
1308 int dofs = pred_full_reg_offset(s, a->rd); in do_pppp_flags()
1309 int nofs = pred_full_reg_offset(s, a->rn); in do_pppp_flags()
1310 int mofs = pred_full_reg_offset(s, a->rm); in do_pppp_flags()
1311 int gofs = pred_full_reg_offset(s, a->pg); in do_pppp_flags()
1313 if (!a->s) { in do_pppp_flags()
1329 gvec_op->fni8(pd, pn, pm, pg); in do_pppp_flags()
1340 if (a->rd == a->pg) { in do_pppp_flags()
1376 if (!a->s) { in trans_AND_pppp()
1377 if (a->rn == a->rm) { in trans_AND_pppp()
1378 if (a->pg == a->rn) { in trans_AND_pppp()
1379 return do_mov_p(s, a->rd, a->rn); in trans_AND_pppp()
1381 return gen_gvec_fn_ppp(s, tcg_gen_gvec_and, a->rd, a->rn, a->pg); in trans_AND_pppp()
1382 } else if (a->pg == a->rn || a->pg == a->rm) { in trans_AND_pppp()
1383 return gen_gvec_fn_ppp(s, tcg_gen_gvec_and, a->rd, a->rn, a->rm); in trans_AND_pppp()
1414 if (!a->s && a->pg == a->rn) { in trans_BIC_pppp()
1415 return gen_gvec_fn_ppp(s, tcg_gen_gvec_andc, a->rd, a->rn, a->rm); in trans_BIC_pppp()
1446 if (!a->s && a->pg == a->rm) { in trans_EOR_pppp()
1447 return gen_gvec_fn_ppp(s, tcg_gen_gvec_andc, a->rd, a->pg, a->rn); in trans_EOR_pppp()
1454 if (a->s || !dc_isar_feature(aa64_sve, s)) { in trans_SEL_pppp()
1459 tcg_gen_gvec_bitsel(MO_8, pred_full_reg_offset(s, a->rd), in trans_SEL_pppp()
1460 pred_full_reg_offset(s, a->pg), in trans_SEL_pppp()
1461 pred_full_reg_offset(s, a->rn), in trans_SEL_pppp()
1462 pred_full_reg_offset(s, a->rm), psz, psz); in trans_SEL_pppp()
1492 if (!a->s && a->pg == a->rn && a->rn == a->rm) { in trans_ORR_pppp()
1493 return do_mov_p(s, a->rd, a->rn); in trans_ORR_pppp()
1592 int nofs = pred_full_reg_offset(s, a->rn); in trans_PTEST()
1593 int gofs = pred_full_reg_offset(s, a->pg); in trans_PTEST()
1634 bound = 16 << (pattern - 9); in decode_pred_count()
1637 return elements - elements % 4; in decode_pred_count()
1639 return elements - elements % 3; in decode_pred_count()
1718 tcg_gen_movi_i32(cpu_NF, -(word != 0)); in do_predset()
1726 TRANS_FEAT(PTRUE, aa64_sve, do_predset, a->esz, a->rd, a->pat, a->s)
1735 int val = (1 << 15) | (1 << a->esz); in trans_PTRUE_cnt()
1738 tcg_gen_gvec_dup_imm(MO_64, pred_full_reg_offset(s, a->rd), in trans_PTRUE_cnt()
1749 TRANS_FEAT(PFALSE, aa64_sve, do_predset, 0, a->rd, 32, false)
1757 .rd = a->rd, .pg = a->pg, .s = a->s, in trans_RDFFR_p()
1761 s->is_nonstreaming = true; in trans_RDFFR_p()
1765 TRANS_FEAT_NONSTREAMING(RDFFR, aa64_sve, do_mov_p, a->rd, FFR_PRED_NUM)
1766 TRANS_FEAT_NONSTREAMING(WRFFR, aa64_sve, do_mov_p, FFR_PRED_NUM, a->rn)
1782 desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz); in do_pfirst_pnext()
1784 tcg_gen_addi_ptr(t_pd, tcg_env, pred_full_reg_offset(s, a->rd)); in do_pfirst_pnext()
1785 tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, a->rn)); in do_pfirst_pnext()
1801 /* Perform an inline saturating addition of a 32-bit value within in TRANS_FEAT()
1802 * a 64-bit register. The second operand is known to be positive, in TRANS_FEAT()
1809 /* Use normal 64-bit arithmetic to detect 32-bit overflow. */ in TRANS_FEAT()
1826 /* Similarly with 64-bit values. */
1839 t2 = tcg_constant_i64(-1); in do_sat_addsub_64()
1954 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz); in trans_CNT_r()
1955 tcg_gen_movi_i64(cpu_reg(s, a->rd), numelem * a->imm); in trans_CNT_r()
1967 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz); in trans_INCDEC_r()
1968 int inc = numelem * a->imm * (a->d ? -1 : 1); in trans_INCDEC_r()
1969 TCGv_i64 reg = cpu_reg(s, a->rd); in trans_INCDEC_r()
1986 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz); in trans_SINCDEC_r_32()
1987 int inc = numelem * a->imm; in trans_SINCDEC_r_32()
1988 TCGv_i64 reg = cpu_reg(s, a->rd); in trans_SINCDEC_r_32()
1990 /* Use normal 64-bit arithmetic to detect 32-bit overflow. */ in trans_SINCDEC_r_32()
1992 if (a->u) { in trans_SINCDEC_r_32()
1998 do_sat_addsub_32(reg, tcg_constant_i64(inc), a->u, a->d); in trans_SINCDEC_r_32()
2013 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz); in trans_SINCDEC_r_64()
2014 int inc = numelem * a->imm; in trans_SINCDEC_r_64()
2015 TCGv_i64 reg = cpu_reg(s, a->rd); in trans_SINCDEC_r_64()
2018 do_sat_addsub_64(reg, tcg_constant_i64(inc), a->u, a->d); in trans_SINCDEC_r_64()
2025 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) { in trans_INCDEC_v()
2030 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz); in trans_INCDEC_v()
2031 int inc = numelem * a->imm; in trans_INCDEC_v()
2035 tcg_gen_gvec_adds(a->esz, vec_full_reg_offset(s, a->rd), in trans_INCDEC_v()
2036 vec_full_reg_offset(s, a->rn), in trans_INCDEC_v()
2037 tcg_constant_i64(a->d ? -inc : inc), in trans_INCDEC_v()
2041 do_mov_z(s, a->rd, a->rn); in trans_INCDEC_v()
2048 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) { in trans_SINCDEC_v()
2053 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz); in trans_SINCDEC_v()
2054 int inc = numelem * a->imm; in trans_SINCDEC_v()
2058 do_sat_addsub_vec(s, a->esz, a->rd, a->rn, in trans_SINCDEC_v()
2059 tcg_constant_i64(inc), a->u, a->d); in trans_SINCDEC_v()
2062 do_mov_z(s, a->rd, a->rn); in trans_SINCDEC_v()
2074 if (!logic_imm_decode_wmask(&imm, extract32(a->dbm, 12, 1), in do_zz_dbm()
2075 extract32(a->dbm, 0, 6), in do_zz_dbm()
2076 extract32(a->dbm, 6, 6))) { in do_zz_dbm()
2079 return gen_gvec_fn_zzi(s, gvec_fn, MO_64, a->rd, a->rn, imm); in do_zz_dbm()
2093 if (!logic_imm_decode_wmask(&imm, extract32(a->dbm, 12, 1), in TRANS_FEAT()
2094 extract32(a->dbm, 0, 6), in TRANS_FEAT()
2095 extract32(a->dbm, 6, 6))) { in TRANS_FEAT()
2099 do_dupi_z(s, a->rd, imm); in TRANS_FEAT()
2105 *** SVE Integer Wide Immediate - Predicated Group
2134 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) { in trans_FCPY()
2139 uint64_t imm = vfp_expand_imm(a->esz, a->imm); in trans_FCPY()
2140 do_cpy_m(s, a->esz, a->rd, a->rn, a->pg, tcg_constant_i64(imm)); in trans_FCPY()
2151 do_cpy_m(s, a->esz, a->rd, a->rn, a->pg, tcg_constant_i64(a->imm)); in trans_CPY_m_i()
2168 tcg_gen_gvec_2i_ool(vec_full_reg_offset(s, a->rd), in trans_CPY_z_i()
2169 pred_full_reg_offset(s, a->pg), in trans_CPY_z_i()
2170 tcg_constant_i64(a->imm), in trans_CPY_z_i()
2171 vsz, vsz, 0, fns[a->esz]); in trans_CPY_z_i()
2188 unsigned n_siz = vsz - n_ofs; in do_EXT()
2210 TRANS_FEAT(EXT, aa64_sve, do_EXT, a->rd, a->rn, a->rm, a->imm)
2211 TRANS_FEAT(EXT_sve2, aa64_sve2, do_EXT, a->rd, a->rn, (a->rn + 1) % 32, a->imm)
2224 imm = a->imm; in trans_EXTQ()
2231 dofs = vec_full_reg_offset(s, a->rd); in trans_EXTQ()
2232 sofs2 = vec_full_reg_offset(s, a->rn); in trans_EXTQ()
2263 *** SVE Permute - Unpredicated Group
2273 tcg_gen_gvec_dup_i64(a->esz, vec_full_reg_offset(s, a->rd), in trans_DUP_s()
2274 vsz, vsz, cpu_reg_sp(s, a->rn)); in trans_DUP_s()
2284 if ((a->imm & 0x1f) == 0) { in trans_DUP_x()
2289 unsigned dofs = vec_full_reg_offset(s, a->rd); in trans_DUP_x()
2292 esz = ctz32(a->imm); in trans_DUP_x()
2293 index = a->imm >> (esz + 1); in trans_DUP_x()
2296 unsigned nofs = vec_reg_offset(s, a->rn, index, esz); in trans_DUP_x()
2300 * While dup_mem handles 128-bit elements, dup_imm does not. in trans_DUP_x()
2321 dofs = vec_full_reg_offset(s, a->rd); in trans_DUPQ()
2322 nofs = vec_reg_offset(s, a->rn, a->imm, a->esz); in trans_DUPQ()
2325 tcg_gen_gvec_dup_mem(a->esz, dofs + i, nofs + i, 16, 16); in trans_DUPQ()
2342 tcg_gen_addi_ptr(t_zd, tcg_env, vec_full_reg_offset(s, a->rd)); in do_insr_i64()
2343 tcg_gen_addi_ptr(t_zn, tcg_env, vec_full_reg_offset(s, a->rn)); in do_insr_i64()
2345 fns[a->esz](t_zd, t_zn, val, desc); in do_insr_i64()
2355 tcg_gen_ld_i64(t, tcg_env, vec_reg_offset(s, a->rm, 0, MO_64)); in trans_INSR_f()
2367 do_insr_i64(s, a, cpu_reg(s, a->rm)); in trans_INSR_r()
2376 TRANS_FEAT(REV_v, aa64_sve, gen_gvec_ool_zz, rev_fns[a->esz], a->rd, a->rn, 0)
2382 TRANS_FEAT(TBL, aa64_sve, gen_gvec_ool_arg_zzz, sve_tbl_fns[a->esz], a, 0)
2388 TRANS_FEAT(TBL_sve2, aa64_sve2, gen_gvec_ool_zzzz, sve2_tbl_fns[a->esz],
2389 a->rd, a->rn, (a->rn + 1) % 32, a->rm, 0)
2396 tblq_fns[a->esz], a, 0)
2402 TRANS_FEAT(TBX, aa64_sve2, gen_gvec_ool_arg_zzz, tbx_fns[a->esz], a, 0)
2409 tbxq_fns[a->esz], a, 0)
2428 if (a->esz != MO_8) { in trans_PMOV_pv()
2429 tcg_gen_gvec_2_ool(pred_full_reg_offset(s, a->rd), in trans_PMOV_pv()
2430 vec_full_reg_offset(s, a->rn), in trans_PMOV_pv()
2431 vl, vl, a->imm, fns[a->esz]); in trans_PMOV_pv()
2436 * Copy the low PL bytes from vector Zn, zero-extending to a in trans_PMOV_pv()
2441 pofs = pred_full_reg_offset(s, a->rd); in trans_PMOV_pv()
2442 vofs = vec_full_reg_offset(s, a->rn); in trans_PMOV_pv()
2492 if (a->esz == MO_8) { in trans_PMOV_vp()
2498 tcg_gen_gvec_mov(MO_64, vec_full_reg_offset(s, a->rd), in trans_PMOV_vp()
2499 pred_full_reg_offset(s, a->rn), in trans_PMOV_vp()
2502 tcg_gen_gvec_2_ool(vec_full_reg_offset(s, a->rd), in trans_PMOV_vp()
2503 pred_full_reg_offset(s, a->rn), in trans_PMOV_vp()
2504 vl, vl, a->imm, fns[a->esz]); in trans_PMOV_vp()
2518 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) { in trans_UNPK()
2523 tcg_gen_gvec_2_ool(vec_full_reg_offset(s, a->rd), in trans_UNPK()
2524 vec_full_reg_offset(s, a->rn) in trans_UNPK()
2525 + (a->h ? vsz / 2 : 0), in trans_UNPK()
2526 vsz, vsz, 0, fns[a->esz][a->u]); in trans_UNPK()
2532 *** SVE Permute - Predicates Group
2550 desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz); in do_perm_pred3()
2553 tcg_gen_addi_ptr(t_d, tcg_env, pred_full_reg_offset(s, a->rd)); in do_perm_pred3()
2554 tcg_gen_addi_ptr(t_n, tcg_env, pred_full_reg_offset(s, a->rn)); in do_perm_pred3()
2555 tcg_gen_addi_ptr(t_m, tcg_env, pred_full_reg_offset(s, a->rm)); in do_perm_pred3()
2573 tcg_gen_addi_ptr(t_d, tcg_env, pred_full_reg_offset(s, a->rd)); in do_perm_pred2()
2574 tcg_gen_addi_ptr(t_n, tcg_env, pred_full_reg_offset(s, a->rn)); in do_perm_pred2()
2577 desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz); in do_perm_pred2()
2596 *** SVE Permute - Interleaving Group
2607 tcg_gen_gvec_3_ool(vec_full_reg_offset(s, a->rd), in do_interleave_q()
2608 vec_full_reg_offset(s, a->rn), in do_interleave_q()
2609 vec_full_reg_offset(s, a->rm), in do_interleave_q()
2621 zip_fns[a->esz], a, 0)
2623 zip_fns[a->esz], a, vec_full_reg_size(s) / 2)
2636 zipq_fns[a->esz], a, 0)
2638 zipq_fns[a->esz], a, 16 / 2)
2645 uzp_fns[a->esz], a, 0)
2647 uzp_fns[a->esz], a, 1 << a->esz)
2659 uzpq_fns[a->esz], a, 0)
2661 uzpq_fns[a->esz], a, 1 << a->esz)
2669 trn_fns[a->esz], a, 0)
2671 trn_fns[a->esz], a, 1 << a->esz)
2679 *** SVE Permute Vector - Predicated Group
2686 compact_fns[a->esz], a, 0)
2690 * indication; e.g. not found for esz=3 is -8.
2717 tcg_gen_andi_i32(last, last, vsz - 1); in incr_last_active()
2731 tcg_gen_andi_i32(last, last, vsz - 1); in wrap_last_active()
2733 TCGv_i32 max = tcg_constant_i32(vsz - (1 << esz)); in wrap_last_active()
2776 tcg_gen_xori_i32(last, last, 8 - (1 << esz)); in load_last_active()
2791 unsigned vsz, esz = a->esz; in do_clast_vector()
2800 find_last_active(s, last, esz, a->pg); in do_clast_vector()
2802 /* There is of course no movcond for a 2048-bit vector, in do_clast_vector()
2811 ele = load_last_active(s, last, a->rm, esz); in do_clast_vector()
2814 tcg_gen_gvec_dup_i64(esz, vec_full_reg_offset(s, a->rd), vsz, vsz, ele); in do_clast_vector()
2817 if (a->rd != a->rn) { in do_clast_vector()
2822 do_mov_z(s, a->rd, a->rn); in do_clast_vector()
2852 * adjusting for tcg_env->vfp.zregs[rm], it is still a valid address in TRANS_FEAT()
2866 int esz = a->esz; in do_clast_fp()
2867 int ofs = vec_reg_offset(s, a->rd, 0, esz); in do_clast_fp()
2870 do_clast_scalar(s, esz, a->pg, a->rn, before, reg); in do_clast_fp()
2871 write_fp_dreg(s, a->rd, reg); in do_clast_fp()
2888 reg = cpu_reg(s, a->rd); in TRANS_FEAT()
2889 switch (a->esz) { in TRANS_FEAT()
2905 do_clast_scalar(s, a->esz, a->pg, a->rn, before, reg); in TRANS_FEAT()
2932 TCGv_i64 val = do_last_scalar(s, a->esz, a->pg, a->rn, before); in do_last_fp()
2933 write_fp_dreg(s, a->rd, val); in do_last_fp()
2945 TCGv_i64 val = do_last_scalar(s, a->esz, a->pg, a->rn, before); in TRANS_FEAT()
2946 tcg_gen_mov_i64(cpu_reg(s, a->rd), val); in TRANS_FEAT()
2960 do_cpy_m(s, a->esz, a->rd, a->rd, a->pg, cpu_reg_sp(s, a->rn)); in TRANS_FEAT()
2971 int ofs = vec_reg_offset(s, a->rn, 0, a->esz); in trans_CPY_m_v()
2972 TCGv_i64 t = load_esz(tcg_env, ofs, a->esz); in trans_CPY_m_v()
2973 do_cpy_m(s, a->esz, a->rd, a->rd, a->pg, t); in trans_CPY_m_v()
2982 TRANS_FEAT(REVB, aa64_sve, gen_gvec_ool_arg_zpz, revb_fns[a->esz], a, 0)
2987 TRANS_FEAT(REVH, aa64_sve, gen_gvec_ool_arg_zpz, revh_fns[a->esz], a, 0)
2990 a->esz == 3 ? gen_helper_sve_revw_d : NULL, a, 0)
2995 gen_helper_sve_splice, a, a->esz)
2998 a->rd, a->rn, (a->rn + 1) % 32, a->pg, a->esz)
3001 *** SVE Integer Compare - Vectors Group
3025 tcg_gen_addi_ptr(pd, tcg_env, pred_full_reg_offset(s, a->rd)); in do_ppzz_flags()
3026 tcg_gen_addi_ptr(zn, tcg_env, vec_full_reg_offset(s, a->rn)); in do_ppzz_flags()
3027 tcg_gen_addi_ptr(zm, tcg_env, vec_full_reg_offset(s, a->rm)); in do_ppzz_flags()
3028 tcg_gen_addi_ptr(pg, tcg_env, pred_full_reg_offset(s, a->pg)); in do_ppzz_flags()
3042 a, name##_ppzz_fns[a->esz])
3059 a, name##_ppzw_fns[a->esz])
3075 *** SVE Integer Compare - Immediate Groups
3098 tcg_gen_addi_ptr(pd, tcg_env, pred_full_reg_offset(s, a->rd));
3099 tcg_gen_addi_ptr(zn, tcg_env, vec_full_reg_offset(s, a->rn));
3100 tcg_gen_addi_ptr(pg, tcg_env, pred_full_reg_offset(s, a->pg));
3102 gen_fn(t, pd, zn, pg, tcg_constant_i32(simd_desc(vsz, vsz, a->imm)));
3114 name##_ppzi_fns[a->esz])
3149 tcg_gen_addi_ptr(d, tcg_env, pred_full_reg_offset(s, a->rd)); in DO_PPZI()
3150 tcg_gen_addi_ptr(n, tcg_env, pred_full_reg_offset(s, a->rn)); in DO_PPZI()
3151 tcg_gen_addi_ptr(m, tcg_env, pred_full_reg_offset(s, a->rm)); in DO_PPZI()
3152 tcg_gen_addi_ptr(g, tcg_env, pred_full_reg_offset(s, a->pg)); in DO_PPZI()
3154 if (a->s) { in DO_PPZI()
3179 tcg_gen_addi_ptr(d, tcg_env, pred_full_reg_offset(s, a->rd)); in do_brk2()
3180 tcg_gen_addi_ptr(n, tcg_env, pred_full_reg_offset(s, a->rn)); in do_brk2()
3181 tcg_gen_addi_ptr(g, tcg_env, pred_full_reg_offset(s, a->pg)); in do_brk2()
3183 if (a->s) { in do_brk2()
3257 do_cntp(s, cpu_reg(s, a->rd), a->esz, a->rn, a->pg); in trans_CNTP()
3281 pred_full_reg_offset(s, a->rn) ^ in trans_CNTP_c()
3285 desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz); in trans_CNTP_c()
3286 desc = FIELD_DP32(desc, PREDDESC, DATA, a->vl); in trans_CNTP_c()
3288 gen_helper_sve2p1_cntp_c(cpu_reg(s, a->rd), t_png, tcg_constant_i32(desc)); in trans_CNTP_c()
3298 TCGv_i64 reg = cpu_reg(s, a->rd); in trans_INCDECP_r()
3301 do_cntp(s, val, a->esz, a->pg, a->pg); in trans_INCDECP_r()
3302 if (a->d) { in trans_INCDECP_r()
3313 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) { in trans_INCDECP_z()
3319 GVecGen2sFn *gvec_fn = a->d ? tcg_gen_gvec_subs : tcg_gen_gvec_adds; in trans_INCDECP_z()
3321 do_cntp(s, val, a->esz, a->pg, a->pg); in trans_INCDECP_z()
3322 gvec_fn(a->esz, vec_full_reg_offset(s, a->rd), in trans_INCDECP_z()
3323 vec_full_reg_offset(s, a->rn), val, vsz, vsz); in trans_INCDECP_z()
3334 TCGv_i64 reg = cpu_reg(s, a->rd); in trans_SINCDECP_r_32()
3337 do_cntp(s, val, a->esz, a->pg, a->pg); in trans_SINCDECP_r_32()
3338 do_sat_addsub_32(reg, val, a->u, a->d); in trans_SINCDECP_r_32()
3349 TCGv_i64 reg = cpu_reg(s, a->rd); in trans_SINCDECP_r_64()
3352 do_cntp(s, val, a->esz, a->pg, a->pg); in trans_SINCDECP_r_64()
3353 do_sat_addsub_64(reg, val, a->u, a->d); in trans_SINCDECP_r_64()
3360 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) { in trans_SINCDECP_z()
3365 do_cntp(s, val, a->esz, a->pg, a->pg); in trans_SINCDECP_z()
3366 do_sat_addsub_vec(s, a->esz, a->rd, a->rn, val, a->u, a->d); in trans_SINCDECP_z()
3384 TCGCond cond = (a->ne ? TCG_COND_NE : TCG_COND_EQ); in trans_CTERM()
3385 TCGv_i64 rn = read_cpu_reg(s, a->rn, a->sf); in trans_CTERM()
3386 TCGv_i64 rm = read_cpu_reg(s, a->rm, a->sf); in trans_CTERM()
3413 /* Note that GE/HS has a->eq == 0 and GT/HI has a->eq == 1. */ in do_WHILE()
3414 bool eq = a->eq == lt; in do_WHILE()
3420 op0 = read_cpu_reg(s, a->rn, 1); in do_WHILE()
3421 op1 = read_cpu_reg(s, a->rm, 1); in do_WHILE()
3423 if (!a->sf) { in do_WHILE()
3424 if (a->u) { in do_WHILE()
3441 if (a->u) { in do_WHILE()
3442 maxval = a->sf ? UINT64_MAX : UINT32_MAX; in do_WHILE()
3445 maxval = a->sf ? INT64_MAX : INT32_MAX; in do_WHILE()
3450 if (a->u) { in do_WHILE()
3454 maxval = a->sf ? INT64_MIN : INT32_MIN; in do_WHILE()
3459 tmax = tcg_constant_i64((vsz << scale) >> a->esz); in do_WHILE()
3465 * For the less-than while, if op1 is maxval (and the only time in do_WHILE()
3466 * the addition above could overflow), then we produce an all-true in do_WHILE()
3470 * Similarly, the greater-than while has the same issue with the in do_WHILE()
3484 /* Since we're bounded, pass as a 32-bit type. */ in do_WHILE()
3489 desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz); in do_WHILE()
3493 tcg_gen_addi_ptr(ptr, tcg_env, pred_full_reg_offset(s, a->rd)); in do_WHILE()
3535 op0 = read_cpu_reg(s, a->rn, 1); in trans_WHILE_ptr()
3536 op1 = read_cpu_reg(s, a->rm, 1); in trans_WHILE_ptr()
3538 tmax = tcg_constant_i64(vsz >> a->esz); in trans_WHILE_ptr()
3541 if (a->rw) { in trans_WHILE_ptr()
3543 /* diff = abs(op1 - op0), noting that op0/1 are unsigned. */ in trans_WHILE_ptr()
3549 tcg_gen_shri_i64(diff, diff, a->esz); in trans_WHILE_ptr()
3556 tcg_gen_shri_i64(diff, diff, a->esz); in trans_WHILE_ptr()
3564 /* Since we're bounded, pass as a 32-bit type. */ in trans_WHILE_ptr()
3569 desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz); in trans_WHILE_ptr()
3572 tcg_gen_addi_ptr(ptr, tcg_env, pred_full_reg_offset(s, a->rd)); in trans_WHILE_ptr()
3591 pred_full_reg_offset(s, a->rn) ^ in do_pext()
3598 int rd = (a->rd + i) % 16; in do_pext()
3599 int part = a->imm * n + i; in do_pext()
3603 desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz); in do_pext()
3616 *** SVE Integer Wide Immediate - Unpredicated Group
3621 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) { in trans_FDUP()
3626 int dofs = vec_full_reg_offset(s, a->rd); in trans_FDUP()
3630 imm = vfp_expand_imm(a->esz, a->imm); in trans_FDUP()
3631 tcg_gen_gvec_dup_imm(a->esz, dofs, vsz, vsz, imm); in trans_FDUP()
3643 int dofs = vec_full_reg_offset(s, a->rd); in trans_DUP_i()
3644 tcg_gen_gvec_dup_imm(a->esz, dofs, vsz, vsz, a->imm); in trans_DUP_i()
3653 a->imm = -a->imm; in TRANS_FEAT()
3693 tcg_gen_gvec_2s(vec_full_reg_offset(s, a->rd), in trans_SUBR_zzi()
3694 vec_full_reg_offset(s, a->rn), in trans_SUBR_zzi()
3695 vsz, vsz, tcg_constant_i64(a->imm), &op[a->esz]); in trans_SUBR_zzi()
3705 do_sat_addsub_vec(s, a->esz, a->rd, a->rn, in TRANS_FEAT()
3706 tcg_constant_i64(a->imm), u, d); in TRANS_FEAT()
3720 tcg_gen_gvec_2i_ool(vec_full_reg_offset(s, a->rd), in TRANS_FEAT()
3721 vec_full_reg_offset(s, a->rn), in TRANS_FEAT()
3722 tcg_constant_i64(a->imm), vsz, vsz, 0, fn); in TRANS_FEAT()
3732 TRANS_FEAT(NAME##_zzi, aa64_sve, do_zzi_ool, a, name##i_fns[a->esz])
3746 dot_fns[a->u][a->sz], a->rd, a->rn, a->rm, a->ra, 0)
3749 * SVE Multiply - Indexed
3773 a->rd, a->rn, a->rm, a->index) in TRANS_FEAT()
3791 a->rd, a->rn, a->rm, (a->index << 1) | TOP) in TRANS_FEAT()
3833 a->rd, a->rn, a->rm, a->ra, (a->index << 1) | TOP) in TRANS_FEAT()
3869 a->rd, a->rn, a->rm, a->ra, (a->index << 2) | a->rot) in TRANS_FEAT()
3883 *** SVE Floating Point Multiply-Add Indexed Group in TRANS_FEAT()
3890 if (a->esz == MO_8 && !dc_isar_feature(aa64_sve_b16b16, s)) { in TRANS_FEAT()
3893 return gen_gvec_fpst_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, a->index, in TRANS_FEAT()
3894 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64); in TRANS_FEAT()
3901 TRANS_FEAT(FMLA_zzxz, aa64_sve, do_fmla_zzxz, a, fmla_idx_fns[a->esz])
3910 fmls_idx_fns[a->esz][s->fpcr_ah])
3921 fmul_idx_fns[a->esz], a->rd, a->rn, a->rm, a->index,
3922 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
3953 tcg_gen_addi_ptr(t_zn, tcg_env, vec_full_reg_offset(s, a->rn)); in do_reduce()
3954 tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, a->pg)); in do_reduce()
3955 status = fpstatus_ptr(a->esz == MO_16 ? FPST_A64_F16 : FPST_A64); in do_reduce()
3959 write_fp_dreg(s, a->rd, temp); in do_reduce()
3968 TRANS_FEAT(NAME, aa64_sve, do_reduce, a, name##_fns[a->esz])
3980 s->fpcr_ah ? name##_ah_fns[a->esz] : name##_fns[a->esz])
3995 faddqv_fns[a->esz], a, 0,
3996 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
4003 fmaxnmqv_fns[a->esz], a, 0,
4004 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
4011 fminnmqv_fns[a->esz], a, 0,
4012 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
4023 (s->fpcr_ah ? fmaxqv_ah_fns : fmaxqv_fns)[a->esz], a, 0,
4024 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
4035 (s->fpcr_ah ? fminqv_ah_fns : fminqv_fns)[a->esz], a, 0,
4036 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
4039 *** SVE Floating Point Unary Operations - Unpredicated Group
4051 s->fpcr_ah && dc_isar_feature(aa64_rpres, s) ?
4052 frecpe_rpres_fns[a->esz] : frecpe_fns[a->esz], a, 0)
4063 s->fpcr_ah && dc_isar_feature(aa64_rpres, s) ?
4064 frsqrte_rpres_fns[a->esz] : frsqrte_fns[a->esz], a, 0)
4079 fpstatus_ptr(a->esz == MO_16 ? FPST_A64_F16 : FPST_A64); in do_ppz_fp()
4081 tcg_gen_gvec_3_ptr(pred_full_reg_offset(s, a->rd), in do_ppz_fp()
4082 vec_full_reg_offset(s, a->rn), in do_ppz_fp()
4083 pred_full_reg_offset(s, a->pg), in do_ppz_fp()
4094 TRANS_FEAT(NAME, aa64_sve, do_ppz_fp, a, name##_fns[a->esz])
4106 *** SVE floating-point trig multiply-add coefficient
4114 ftmad_fns[a->esz], a->rd, a->rn, a->rm,
4115 a->imm | (s->fpcr_ah << 3),
4116 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
4136 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) { in trans_FADDA()
4139 s->is_nonstreaming = true; in trans_FADDA()
4144 t_val = load_esz(tcg_env, vec_reg_offset(s, a->rn, 0, a->esz), a->esz); in trans_FADDA()
4147 tcg_gen_addi_ptr(t_rm, tcg_env, vec_full_reg_offset(s, a->rm)); in trans_FADDA()
4148 tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, a->pg)); in trans_FADDA()
4149 t_fpst = fpstatus_ptr(a->esz == MO_16 ? FPST_A64_F16 : FPST_A64); in trans_FADDA()
4152 fns[a->esz - 1](t_val, t_val, t_rm, t_pg, t_fpst, t_desc); in trans_FADDA()
4154 write_fp_dreg(s, a->rd, t_val); in trans_FADDA()
4159 *** SVE Floating Point Arithmetic - Unpredicated Group
4167 TRANS_FEAT(NAME, aa64_sve, gen_gvec_fpst_arg_zzz, name##_fns[a->esz], a, 0)
4179 s->fpcr_ah ? name##_ah_fns[a->esz] : name##_fns[a->esz], a, 0)
4194 ftsmul_fns[a->esz], a, 0)
4197 *** SVE Floating Point Arithmetic - Predicated Group
4205 TRANS_FEAT(NAME, FEAT, gen_gvec_fpst_arg_zpzz, name##_zpzz_fns[a->esz], a)
4217 s->fpcr_ah ? name##_ah_zpzz_fns[a->esz] : \
4218 name##_zpzz_fns[a->esz], a)
4226 TRANS_FEAT(NAME, FEAT, gen_gvec_fpst_arg_zpzz, name##_zpzz_fns[a->esz], a)
4238 s->fpcr_ah ? name##_ah_zpzz_fns[a->esz] : \
4239 name##_zpzz_fns[a->esz], a)
4282 do_fp_scalar(s, a->rd, a->rn, a->pg, a->esz == MO_16, in do_fp_imm()
4295 { -1, -1 }, \
4301 name##_const[a->esz][a->imm], name##_fns[a->esz])
4315 { -1, -1 }, \
4321 name##_const[a->esz][a->imm], \
4322 s->fpcr_ah ? name##_ah_fns[a->esz] : name##_fns[a->esz])
4343 TCGv_ptr status = fpstatus_ptr(a->esz == MO_16 ? FPST_A64_F16 : FPST_A64); in DO_FP_IMM()
4344 tcg_gen_gvec_4_ptr(pred_full_reg_offset(s, a->rd), in DO_FP_IMM()
4345 vec_full_reg_offset(s, a->rn), in DO_FP_IMM()
4346 vec_full_reg_offset(s, a->rm), in DO_FP_IMM()
4347 pred_full_reg_offset(s, a->pg), in DO_FP_IMM()
4358 TRANS_FEAT(NAME##_ppzz, aa64_sve, do_fp_cmp, a, name##_fns[a->esz])
4374 TRANS_FEAT(FCADD, aa64_sve, gen_gvec_fpst_zzzp, fcadd_fns[a->esz],
4375 a->rd, a->rn, a->rm, a->pg, a->rot | (s->fpcr_ah << 1),
4376 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
4382 if (a->esz == MO_8 && !dc_isar_feature(aa64_sve_b16b16, s)) { in do_fmla_zpzzz()
4385 return gen_gvec_fpst_zzzzp(s, fn, a->rd, a->rn, a->rm, a->ra, a->pg, 0, in do_fmla_zpzzz()
4386 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64); in do_fmla_zpzzz()
4399 s->fpcr_ah ? name##_ah_fns[a->esz] : name##_fns[a->esz])
4413 TRANS_FEAT(FCMLA_zpzzz, aa64_sve, gen_gvec_fpst_zzzzp, fcmla_fns[a->esz],
4414 a->rd, a->rn, a->rm, a->ra, a->pg, a->rot | (s->fpcr_ah << 2),
4415 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
4420 TRANS_FEAT(FCMLA_zzxz, aa64_sve, gen_gvec_fpst_zzzz, fcmla_idx_fns[a->esz],
4421 a->rd, a->rn, a->rm, a->ra, a->index * 4 + a->rot,
4422 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
4435 s->fpcr_ah ? FPST_AH : FPST_A64)
4483 TRANS_FEAT(FRINTI, aa64_sve, gen_gvec_fpst_arg_zpz, frint_fns[a->esz],
4484 a, 0, a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
4492 TRANS_FEAT(FRINTX, aa64_sve, gen_gvec_fpst_arg_zpz, frintx_fns[a->esz],
4493 a, 0, a->esz == MO_16 ? FPST_A64_F16 : FPST_A64);
4510 status = fpstatus_ptr(a->esz == MO_16 ? FPST_A64_F16 : FPST_A64); in do_frint_mode()
4513 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, a->rd), in do_frint_mode()
4514 vec_full_reg_offset(s, a->rn), in do_frint_mode()
4515 pred_full_reg_offset(s, a->pg), in do_frint_mode()
4523 FPROUNDING_TIEEVEN, frint_fns[a->esz])
4525 FPROUNDING_POSINF, frint_fns[a->esz])
4527 FPROUNDING_NEGINF, frint_fns[a->esz])
4529 FPROUNDING_ZERO, frint_fns[a->esz])
4531 FPROUNDING_TIEAWAY, frint_fns[a->esz])
4537 TRANS_FEAT(FRECPX, aa64_sve, gen_gvec_fpst_arg_zpz, frecpx_fns[a->esz],
4538 a, 0, select_ah_fpst(s, a->esz))
4544 TRANS_FEAT(FSQRT, aa64_sve, gen_gvec_fpst_arg_zpz, fsqrt_fns[a->esz],
4545 a, 0, a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
4582 *** SVE Memory - 32-bit Gather and Unsized Contiguous Group
4605 * are defined as a stream of bytes, which equates to little-endian in gen_sve_ldr()
4657 * Note that we still store the entire 64-bit unit into tcg_env. in gen_sve_ldr()
4665 len_remain -= 8; in gen_sve_ldr()
4714 * are defined as a stream of bytes, which equates to little-endian in gen_sve_str()
4716 * a little-endian store for aarch64_be-linux-user out of line. in gen_sve_str()
4771 len_remain -= 8; in gen_sve_str()
4811 int off = vec_full_reg_offset(s, a->rd); in trans_LDR_zri()
4812 gen_sve_ldr(s, tcg_env, off, size, a->rn, a->imm * size, in trans_LDR_zri()
4813 s->align_mem ? MO_ALIGN_16 : MO_UNALN); in trans_LDR_zri()
4825 int off = pred_full_reg_offset(s, a->rd); in trans_LDR_pri()
4826 gen_sve_ldr(s, tcg_env, off, size, a->rn, a->imm * size, in trans_LDR_pri()
4827 s->align_mem ? MO_ALIGN_2 : MO_UNALN); in trans_LDR_pri()
4839 int off = vec_full_reg_offset(s, a->rd); in trans_STR_zri()
4840 gen_sve_str(s, tcg_env, off, size, a->rn, a->imm * size, in trans_STR_zri()
4841 s->align_mem ? MO_ALIGN_16 : MO_UNALN); in trans_STR_zri()
4853 int off = pred_full_reg_offset(s, a->rd); in trans_STR_pri()
4854 gen_sve_str(s, tcg_env, off, size, a->rn, a->imm * size, in trans_STR_pri()
4855 s->align_mem ? MO_ALIGN_2 : MO_UNALN); in trans_STR_pri()
4861 *** SVE Memory - Contiguous Load Group
4894 sizem1 = (nregs << msz) - 1; in make_svemte_desc()
4897 if (s->mte_active[0]) { in make_svemte_desc()
4899 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid); in make_svemte_desc()
4900 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma); in make_svemte_desc()
4915 if (!s->mte_active[0]) { in do_mem_zpa()
4934 { /* mte inactive, little-endian */
4965 /* mte inactive, big-endian */
4997 { /* mte active, little-endian */
5038 /* mte active, big-endian */
5085 = ldr_fns[s->mte_active[0]][s->be_data == MO_BE][dtype][nreg]; in do_ld_zpa()
5097 if (a->rm == 31) { in trans_LD_zprr()
5101 /* dtypes 16-18 are artificial, representing 128-bit element */ in trans_LD_zprr()
5102 switch (a->dtype) { in trans_LD_zprr()
5112 s->is_nonstreaming = true; in trans_LD_zprr()
5125 tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), dtype_msz(a->dtype)); in trans_LD_zprr()
5126 tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn)); in trans_LD_zprr()
5127 do_ld_zpa(s, a->rd, a->pg, addr, a->dtype, a->nreg); in trans_LD_zprr()
5134 /* dtypes 16-18 are artificial, representing 128-bit element */ in trans_LD_zpri()
5135 switch (a->dtype) { in trans_LD_zpri()
5145 s->is_nonstreaming = true; in trans_LD_zpri()
5158 int elements = vsz >> dtype_esz[a->dtype]; in trans_LD_zpri()
5161 tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn), in trans_LD_zpri()
5162 (a->imm * elements * (a->nreg + 1)) in trans_LD_zpri()
5163 << dtype_msz(a->dtype)); in trans_LD_zpri()
5164 do_ld_zpa(s, a->rd, a->pg, addr, a->dtype, a->nreg); in trans_LD_zpri()
5172 { /* mte inactive, little-endian */ in trans_LDFF1_zprr()
5193 /* mte inactive, big-endian */ in trans_LDFF1_zprr()
5214 { /* mte active, little-endian */ in trans_LDFF1_zprr()
5235 /* mte active, big-endian */ in trans_LDFF1_zprr()
5260 s->is_nonstreaming = true; in trans_LDFF1_zprr()
5263 tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), dtype_msz(a->dtype)); in trans_LDFF1_zprr()
5264 tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn)); in trans_LDFF1_zprr()
5265 do_mem_zpa(s, a->rd, a->pg, addr, a->dtype, 1, false, in trans_LDFF1_zprr()
5266 fns[s->mte_active[0]][s->be_data == MO_BE][a->dtype]); in trans_LDFF1_zprr()
5274 { /* mte inactive, little-endian */ in trans_LDNF1_zpri()
5295 /* mte inactive, big-endian */ in trans_LDNF1_zpri()
5316 { /* mte inactive, little-endian */ in trans_LDNF1_zpri()
5337 /* mte inactive, big-endian */ in trans_LDNF1_zpri()
5362 s->is_nonstreaming = true; in trans_LDNF1_zpri()
5365 int elements = vsz >> dtype_esz[a->dtype]; in trans_LDNF1_zpri()
5366 int off = (a->imm * elements) << dtype_msz(a->dtype); in trans_LDNF1_zpri()
5369 tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn), off); in trans_LDNF1_zpri()
5370 do_mem_zpa(s, a->rd, a->pg, addr, a->dtype, 1, false, in trans_LDNF1_zpri()
5371 fns[s->mte_active[0]][s->be_data == MO_BE][a->dtype]); in trans_LDNF1_zpri()
5384 if (!s->mte_active[0]) { in do_ldrq()
5391 * Zero-extend the first 16 bits of the predicate into a temporary. in do_ldrq()
5410 = ldr_fns[s->mte_active[0]][s->be_data == MO_BE][dtype][0]; in do_ldrq()
5417 tcg_gen_gvec_dup_mem(4, doff + 16, doff, vsz - 16, vsz - 16); in do_ldrq()
5423 if (a->rm == 31 || !dc_isar_feature(aa64_sve, s)) { in trans_LD1RQ_zprr()
5427 int msz = dtype_msz(a->dtype); in trans_LD1RQ_zprr()
5429 tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), msz); in trans_LD1RQ_zprr()
5430 tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn)); in trans_LD1RQ_zprr()
5431 do_ldrq(s, a->rd, a->pg, addr, a->dtype); in trans_LD1RQ_zprr()
5443 tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn), a->imm * 16); in trans_LD1RQ_zpri()
5444 do_ldrq(s, a->rd, a->pg, addr, a->dtype); in trans_LD1RQ_zpri()
5468 if (!s->mte_active[0]) { in do_ldro()
5475 * Zero-extend the first 32 bits of the predicate into a temporary. in do_ldro()
5494 = ldr_fns[s->mte_active[0]][s->be_data == MO_BE][dtype][0]; in do_ldro()
5506 tcg_gen_gvec_dup_mem(5, doff + 32, doff, vsz_r32 - 32, vsz_r32 - 32); in do_ldro()
5508 vsz -= vsz_r32; in do_ldro()
5519 if (a->rm == 31) { in trans_LD1RO_zprr()
5522 s->is_nonstreaming = true; in trans_LD1RO_zprr()
5525 tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), dtype_msz(a->dtype)); in trans_LD1RO_zprr()
5526 tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn)); in trans_LD1RO_zprr()
5527 do_ldro(s, a->rd, a->pg, addr, a->dtype); in trans_LD1RO_zprr()
5537 s->is_nonstreaming = true; in trans_LD1RO_zpri()
5540 tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn), a->imm * 32); in trans_LD1RO_zpri()
5541 do_ldro(s, a->rd, a->pg, addr, a->dtype); in trans_LD1RO_zpri()
5551 unsigned esz = dtype_esz[a->dtype]; in trans_LD1R_zpri()
5552 unsigned msz = dtype_msz(a->dtype); in trans_LD1R_zpri()
5573 tcg_gen_ld_i64(temp, tcg_env, pred_full_reg_offset(s, a->pg)); in trans_LD1R_zpri()
5578 find_last_active(s, t32, esz, a->pg); in trans_LD1R_zpri()
5584 tcg_gen_addi_i64(temp, cpu_reg_sp(s, a->rn), a->imm << msz); in trans_LD1R_zpri()
5586 memop = finalize_memop(s, dtype_mop[a->dtype]); in trans_LD1R_zpri()
5591 tcg_gen_gvec_dup_i64(esz, vec_full_reg_offset(s, a->rd), in trans_LD1R_zpri()
5596 return do_movz_zpz(s, a->rd, a->rd, a->pg, esz, false); in trans_LD1R_zpri()
5728 int be = s->be_data == MO_BE; in do_st_zpa()
5732 fn = fn_single[s->mte_active[0]][be][msz][esz]; in do_st_zpa()
5734 /* ST2, ST3, ST4 -- msz == esz, enforced by encoding */ in do_st_zpa()
5736 fn = fn_multiple[s->mte_active[0]][be][nreg - 1][msz]; in do_st_zpa()
5744 if (a->rm == 31 || a->msz > a->esz) { in trans_ST_zprr()
5747 switch (a->esz) { in trans_ST_zprr()
5754 if (a->nreg == 0) { in trans_ST_zprr()
5755 assert(a->msz < a->esz); in trans_ST_zprr()
5759 s->is_nonstreaming = true; in trans_ST_zprr()
5772 tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), a->msz); in trans_ST_zprr()
5773 tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn)); in trans_ST_zprr()
5774 do_st_zpa(s, a->rd, a->pg, addr, a->msz, a->esz, a->nreg); in trans_ST_zprr()
5781 if (a->msz > a->esz) { in trans_ST_zpri()
5784 switch (a->esz) { in trans_ST_zpri()
5791 if (a->nreg == 0) { in trans_ST_zpri()
5792 assert(a->msz < a->esz); in trans_ST_zpri()
5796 s->is_nonstreaming = true; in trans_ST_zpri()
5809 int elements = vsz >> a->esz; in trans_ST_zpri()
5812 tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn), in trans_ST_zpri()
5813 (a->imm * elements * (a->nreg + 1)) << a->msz); in trans_ST_zpri()
5814 do_st_zpa(s, a->rd, a->pg, addr, a->msz, a->esz, a->nreg); in trans_ST_zpri()
5844 { /* Little-endian */
5858 /* First-fault */
5872 { /* Big-endian */
5886 /* First-fault */
5900 { /* Little-endian */
5914 /* First-fault */
5928 { /* Big-endian */
5942 /* First-fault */
5957 /* Note that we overload xs=2 to indicate 64-bit offset. */
5961 { /* Little-endian */
5987 /* First-fault */
6012 { /* Big-endian */
6038 /* First-fault */
6064 { /* Little-endian */
6090 /* First-fault */
6115 { /* Big-endian */
6141 /* First-fault */
6179 bool be = s->be_data == MO_BE; in trans_LD1_zprz()
6180 bool mte = s->mte_active[0]; in trans_LD1_zprz()
6185 s->is_nonstreaming = true; in trans_LD1_zprz()
6190 switch (a->esz) { in trans_LD1_zprz()
6192 fn = gather_load_fn32[mte][be][a->ff][a->xs][a->u][a->msz]; in trans_LD1_zprz()
6195 fn = gather_load_fn64[mte][be][a->ff][a->xs][a->u][a->msz]; in trans_LD1_zprz()
6202 do_mem_zpz(s, a->rd, a->pg, a->rm, a->scale * a->msz, in trans_LD1_zprz()
6203 cpu_reg_sp(s, a->rn), a->msz, false, fn); in trans_LD1_zprz()
6210 bool be = s->be_data == MO_BE; in trans_LD1Q()
6211 bool mte = s->mte_active[0]; in trans_LD1Q()
6216 s->is_nonstreaming = true; in trans_LD1Q()
6225 * Unlike LD1_zprz, a->rm is the scalar register and it can be XZR, not XSP. in trans_LD1Q()
6226 * a->rn is the vector register. in trans_LD1Q()
6228 do_mem_zpz(s, a->rd, a->pg, a->rn, 0, in trans_LD1Q()
6229 cpu_reg(s, a->rm), MO_128, false, fn); in trans_LD1Q()
6236 bool be = s->be_data == MO_BE; in trans_LD1_zpiz()
6237 bool mte = s->mte_active[0]; in trans_LD1_zpiz()
6239 if (a->esz < a->msz || (a->esz == a->msz && !a->u)) { in trans_LD1_zpiz()
6245 s->is_nonstreaming = true; in trans_LD1_zpiz()
6250 switch (a->esz) { in trans_LD1_zpiz()
6252 fn = gather_load_fn32[mte][be][a->ff][0][a->u][a->msz]; in trans_LD1_zpiz()
6255 fn = gather_load_fn64[mte][be][a->ff][2][a->u][a->msz]; in trans_LD1_zpiz()
6263 do_mem_zpz(s, a->rd, a->pg, a->rn, 0, in trans_LD1_zpiz()
6264 tcg_constant_i64(a->imm << a->msz), a->msz, false, fn); in trans_LD1_zpiz()
6271 bool be = s->be_data == MO_BE; in trans_LDNT1_zprz()
6272 bool mte = s->mte_active[0]; in trans_LDNT1_zprz()
6274 if (a->esz < a->msz + !a->u) { in trans_LDNT1_zprz()
6280 s->is_nonstreaming = true; in trans_LDNT1_zprz()
6285 switch (a->esz) { in trans_LDNT1_zprz()
6287 fn = gather_load_fn32[mte][be][0][0][a->u][a->msz]; in trans_LDNT1_zprz()
6290 fn = gather_load_fn64[mte][be][0][2][a->u][a->msz]; in trans_LDNT1_zprz()
6295 do_mem_zpz(s, a->rd, a->pg, a->rn, 0, in trans_LDNT1_zprz()
6296 cpu_reg(s, a->rm), a->msz, false, fn); in trans_LDNT1_zprz()
6303 { /* Little-endian */
6310 { /* Big-endian */
6318 { /* Little-endian */
6325 { /* Big-endian */
6334 /* Note that we overload xs=2 to indicate 64-bit offset. */
6337 { /* Little-endian */
6350 { /* Big-endian */
6364 { /* Little-endian */
6377 { /* Big-endian */
6403 bool be = s->be_data == MO_BE; in trans_ST1_zprz()
6404 bool mte = s->mte_active[0]; in trans_ST1_zprz()
6406 if (a->esz < a->msz || (a->msz == 0 && a->scale)) { in trans_ST1_zprz()
6412 s->is_nonstreaming = true; in trans_ST1_zprz()
6416 switch (a->esz) { in trans_ST1_zprz()
6418 fn = scatter_store_fn32[mte][be][a->xs][a->msz]; in trans_ST1_zprz()
6421 fn = scatter_store_fn64[mte][be][a->xs][a->msz]; in trans_ST1_zprz()
6426 do_mem_zpz(s, a->rd, a->pg, a->rm, a->scale * a->msz, in trans_ST1_zprz()
6427 cpu_reg_sp(s, a->rn), a->msz, true, fn); in trans_ST1_zprz()
6434 bool be = s->be_data == MO_BE; in trans_ST1Q()
6435 bool mte = s->mte_active[0]; in trans_ST1Q()
6440 s->is_nonstreaming = true; in trans_ST1Q()
6446 * Unlike ST1_zprz, a->rm is the scalar register, and it in trans_ST1Q()
6447 * can be XZR, not XSP. a->rn is the vector register. in trans_ST1Q()
6449 do_mem_zpz(s, a->rd, a->pg, a->rn, 0, in trans_ST1Q()
6450 cpu_reg(s, a->rm), MO_128, true, fn); in trans_ST1Q()
6457 bool be = s->be_data == MO_BE; in trans_ST1_zpiz()
6458 bool mte = s->mte_active[0]; in trans_ST1_zpiz()
6460 if (a->esz < a->msz) { in trans_ST1_zpiz()
6466 s->is_nonstreaming = true; in trans_ST1_zpiz()
6471 switch (a->esz) { in trans_ST1_zpiz()
6473 fn = scatter_store_fn32[mte][be][0][a->msz]; in trans_ST1_zpiz()
6476 fn = scatter_store_fn64[mte][be][2][a->msz]; in trans_ST1_zpiz()
6484 do_mem_zpz(s, a->rd, a->pg, a->rn, 0, in trans_ST1_zpiz()
6485 tcg_constant_i64(a->imm << a->msz), a->msz, true, fn); in trans_ST1_zpiz()
6492 bool be = s->be_data == MO_BE; in trans_STNT1_zprz()
6493 bool mte = s->mte_active[0]; in trans_STNT1_zprz()
6495 if (a->esz < a->msz) { in trans_STNT1_zprz()
6501 s->is_nonstreaming = true; in trans_STNT1_zprz()
6506 switch (a->esz) { in trans_STNT1_zprz()
6508 fn = scatter_store_fn32[mte][be][0][a->msz]; in trans_STNT1_zprz()
6511 fn = scatter_store_fn64[mte][be][2][a->msz]; in trans_STNT1_zprz()
6517 do_mem_zpz(s, a->rd, a->pg, a->rn, 0, in trans_STNT1_zprz()
6518 cpu_reg(s, a->rm), a->msz, true, fn); in trans_STNT1_zprz()
6538 if (a->rm == 31 || !dc_isar_feature(aa64_sve, s)) { in trans_PRF_rr()
6552 s->is_nonstreaming = true; in trans_PRF_ns()
6571 TRANS_FEAT(MOVPRFX, aa64_sve, do_mov_z, a->rd, a->rn)
6572 TRANS_FEAT(MOVPRFX_m, aa64_sve, do_sel_z, a->rd, a->rn, a->rd, a->pg, a->esz)
6573 TRANS_FEAT(MOVPRFX_z, aa64_sve, do_movz_zpz, a->rd, a->rn, a->pg, a->esz, false)
6576 * SVE2 Integer Multiply - Unpredicated
6587 smulh_zzz_fns[a->esz], a, 0)
6594 umulh_zzz_fns[a->esz], a, 0)
6604 sqrdmulh_zzz_fns[a->esz], a, 0)
6607 * SVE2 Integer - Predicated
6615 sadlp_fns[a->esz], a, 0)
6622 uadlp_fns[a->esz], a, 0)
6629 a->esz == 2 ? gen_helper_sve2_urecpe_s : NULL, a, 0)
6632 a->esz == 2 ? gen_helper_sve2_ursqrte_s : NULL, a, 0)
6638 TRANS_FEAT(SQABS, aa64_sve2, gen_gvec_ool_arg_zpz, sqabs_fns[a->esz], a, 0)
6644 TRANS_FEAT(SQNEG, aa64_sve2, gen_gvec_ool_arg_zpz, sqneg_fns[a->esz], a, 0)
6684 saddl_fns[a->esz], a, 0)
6686 saddl_fns[a->esz], a, 3)
6688 saddl_fns[a->esz], a, 2)
6695 ssubl_fns[a->esz], a, 0)
6697 ssubl_fns[a->esz], a, 3)
6699 ssubl_fns[a->esz], a, 2)
6701 ssubl_fns[a->esz], a, 1)
6708 sabdl_fns[a->esz], a, 0)
6710 sabdl_fns[a->esz], a, 3)
6717 uaddl_fns[a->esz], a, 0)
6719 uaddl_fns[a->esz], a, 3)
6726 usubl_fns[a->esz], a, 0)
6728 usubl_fns[a->esz], a, 3)
6735 uabdl_fns[a->esz], a, 0)
6737 uabdl_fns[a->esz], a, 3)
6744 sqdmull_fns[a->esz], a, 0)
6746 sqdmull_fns[a->esz], a, 3)
6753 smull_fns[a->esz], a, 0)
6755 smull_fns[a->esz], a, 3)
6762 umull_fns[a->esz], a, 0)
6764 umull_fns[a->esz], a, 3)
6770 TRANS_FEAT(EORBT, aa64_sve2, gen_gvec_ool_arg_zzz, eoril_fns[a->esz], a, 2)
6771 TRANS_FEAT(EORTB, aa64_sve2, gen_gvec_ool_arg_zzz, eoril_fns[a->esz], a, 1)
6780 if (a->esz == 0) { in do_trans_pmull()
6784 s->is_nonstreaming = true; in do_trans_pmull()
6788 return gen_gvec_ool_arg_zzz(s, fns[a->esz], a, sel); in do_trans_pmull()
6798 TRANS_FEAT(SADDWB, aa64_sve2, gen_gvec_ool_arg_zzz, saddw_fns[a->esz], a, 0)
6799 TRANS_FEAT(SADDWT, aa64_sve2, gen_gvec_ool_arg_zzz, saddw_fns[a->esz], a, 1)
6805 TRANS_FEAT(SSUBWB, aa64_sve2, gen_gvec_ool_arg_zzz, ssubw_fns[a->esz], a, 0)
6806 TRANS_FEAT(SSUBWT, aa64_sve2, gen_gvec_ool_arg_zzz, ssubw_fns[a->esz], a, 1)
6812 TRANS_FEAT(UADDWB, aa64_sve2, gen_gvec_ool_arg_zzz, uaddw_fns[a->esz], a, 0)
6813 TRANS_FEAT(UADDWT, aa64_sve2, gen_gvec_ool_arg_zzz, uaddw_fns[a->esz], a, 1)
6819 TRANS_FEAT(USUBWB, aa64_sve2, gen_gvec_ool_arg_zzz, usubw_fns[a->esz], a, 0)
6820 TRANS_FEAT(USUBWT, aa64_sve2, gen_gvec_ool_arg_zzz, usubw_fns[a->esz], a, 1)
6839 tcg_gen_sari_vec(vece, d, d, halfbits - shl); in gen_sshll_vec()
6855 shift = shl - top * halfbits; in gen_ushll_i64()
6857 tcg_gen_shri_i64(d, n, -shift); in gen_ushll_i64()
6901 tcg_gen_shri_vec(vece, d, d, halfbits - shl); in gen_ushll_vec()
6910 if (a->esz < 0 || a->esz > 2) { in do_shll_tb()
6915 tcg_gen_gvec_2i(vec_full_reg_offset(s, a->rd), in do_shll_tb()
6916 vec_full_reg_offset(s, a->rn), in do_shll_tb()
6917 vsz, vsz, (a->imm << 1) | sel, in do_shll_tb()
6918 &ops[a->esz]); in do_shll_tb()
6971 bext_fns[a->esz], a, 0)
6978 bdep_fns[a->esz], a, 0)
6985 bgrp_fns[a->esz], a, 0)
6992 cadd_fns[a->esz], a, 0)
6994 cadd_fns[a->esz], a, 1)
7001 sqcadd_fns[a->esz], a, 0)
7003 sqcadd_fns[a->esz], a, 1)
7009 TRANS_FEAT(SABALB, aa64_sve2, gen_gvec_ool_arg_zzzz, sabal_fns[a->esz], a, 0)
7010 TRANS_FEAT(SABALT, aa64_sve2, gen_gvec_ool_arg_zzzz, sabal_fns[a->esz], a, 1)
7016 TRANS_FEAT(UABALB, aa64_sve2, gen_gvec_ool_arg_zzzz, uabal_fns[a->esz], a, 0)
7017 TRANS_FEAT(UABALT, aa64_sve2, gen_gvec_ool_arg_zzzz, uabal_fns[a->esz], a, 1)
7029 return gen_gvec_ool_arg_zzzz(s, fns[a->esz & 1], a, (a->esz & 2) | sel); in do_adcl()
7048 if (a->esz < 0 || a->esz > MO_32 || a->imm != 0) { in TRANS_FEAT()
7053 tcg_gen_gvec_2(vec_full_reg_offset(s, a->rd), in TRANS_FEAT()
7054 vec_full_reg_offset(s, a->rn), in TRANS_FEAT()
7055 vsz, vsz, &ops[a->esz]); in TRANS_FEAT()
7067 int64_t mask = (1ull << halfbits) - 1; in gen_sqxtnb_vec()
7068 int64_t min = -1ull << (halfbits - 1); in gen_sqxtnb_vec()
7069 int64_t max = -min - 1; in gen_sqxtnb_vec()
7095 int64_t mask = (1ull << halfbits) - 1; in TRANS_FEAT()
7096 int64_t min = -1ull << (halfbits - 1); in TRANS_FEAT()
7097 int64_t max = -min - 1; in TRANS_FEAT()
7131 int64_t max = (1ull << halfbits) - 1; in gen_uqxtnb_vec()
7155 int64_t max = (1ull << halfbits) - 1; in TRANS_FEAT()
7189 int64_t max = (1ull << halfbits) - 1; in gen_sqxtunb_vec()
7214 int64_t max = (1ull << halfbits) - 1; in TRANS_FEAT()
7245 if (a->esz < 0 || a->esz > MO_32) { in TRANS_FEAT()
7248 assert(a->imm > 0 && a->imm <= (8 << a->esz)); in TRANS_FEAT()
7251 tcg_gen_gvec_2i(vec_full_reg_offset(s, a->rd), in TRANS_FEAT()
7252 vec_full_reg_offset(s, a->rn), in TRANS_FEAT()
7253 vsz, vsz, a->imm, &ops[a->esz]); in TRANS_FEAT()
7316 tcg_gen_shli_i64(n, n, halfbits - shr); in TRANS_FEAT()
7343 tcg_gen_shli_vec(vece, n, n, halfbits - shr); in gen_shrnt_vec()
7469 int64_t max = MAKE_64BIT_MASK(0, halfbits - 1); in TRANS_FEAT()
7470 int64_t min = -max - 1; in TRANS_FEAT()
7502 int64_t max = MAKE_64BIT_MASK(0, halfbits - 1); in TRANS_FEAT()
7503 int64_t min = -max - 1; in TRANS_FEAT()
7634 name##_fns[a->esz], a, 0)
7649 TRANS_FEAT_NONSTREAMING(MATCH, aa64_sve2, do_ppzz_flags, a, match_fns[a->esz])
7654 TRANS_FEAT_NONSTREAMING(NMATCH, aa64_sve2, do_ppzz_flags, a, nmatch_fns[a->esz])
7660 histcnt_fns[a->esz], a, 0)
7663 a->esz == 0 ? gen_helper_sve2_histseg : NULL, a, 0)
7675 if (vec_full_reg_size(s) < 4 * memop_size(a->esz)) { in DO_ZPZZ_FP()
7678 gen_gvec_fpst_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, 0, FPST_A64); in DO_ZPZZ_FP()
7688 * SVE Integer Multiply-Add (unpredicated)
7696 sqdmlal_zzzw_fns[a->esz], a, 0)
7698 sqdmlal_zzzw_fns[a->esz], a, 3)
7700 sqdmlal_zzzw_fns[a->esz], a, 2)
7707 sqdmlsl_zzzw_fns[a->esz], a, 0)
7709 sqdmlsl_zzzw_fns[a->esz], a, 3)
7711 sqdmlsl_zzzw_fns[a->esz], a, 2)
7718 sqrdmlah_fns[a->esz], a, 0)
7725 sqrdmlsh_fns[a->esz], a, 0)
7732 smlal_zzzw_fns[a->esz], a, 0)
7734 smlal_zzzw_fns[a->esz], a, 1)
7741 umlal_zzzw_fns[a->esz], a, 0)
7743 umlal_zzzw_fns[a->esz], a, 1)
7750 smlsl_zzzw_fns[a->esz], a, 0)
7752 smlsl_zzzw_fns[a->esz], a, 1)
7759 umlsl_zzzw_fns[a->esz], a, 0)
7761 umlsl_zzzw_fns[a->esz], a, 1)
7768 cmla_fns[a->esz], a->rd, a->rn, a->rm, a->ra, a->rot)
7774 cdot_fns[a->esz], a->rd, a->rn, a->rm, a->ra, a->rot)
7781 sqrdcmlah_fns[a->esz], a->rd, a->rn, a->rm, a->ra, a->rot)
7792 gen_helper_crypto_aesmc, a->rd, a->rd, 0)
7794 gen_helper_crypto_aesimc, a->rd, a->rd, 0)
7816 s->fpcr_ah ? FPST_AH : FPST_A64)
7832 TRANS_FEAT(FLOGB, aa64_sve2, gen_gvec_fpst_arg_zpz, flogb_fns[a->esz],
7833 a, 0, a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
7838 a->rd, a->rn, a->rm, a->ra, in do_FMLAL_zzzw()
7850 a->rd, a->rn, a->rm, a->ra, in TRANS_FEAT()
7851 (a->index << 3) | (sel << 1) | sub, tcg_env); in TRANS_FEAT()
7882 a->rd, a->rn, a->rm, a->ra, sel, in TRANS_FEAT()
7883 s->fpcr_ah ? FPST_AH : FPST_A64); in TRANS_FEAT()
7892 a->rd, a->rn, a->rm, a->ra, in TRANS_FEAT()
7893 (a->index << 1) | sel, in TRANS_FEAT()
7894 s->fpcr_ah ? FPST_AH : FPST_A64); in TRANS_FEAT()
7902 if (s->fpcr_ah) { in TRANS_FEAT()
7904 a->rd, a->rn, a->rm, a->ra, sel, FPST_AH); in TRANS_FEAT()
7907 a->rd, a->rn, a->rm, a->ra, sel, FPST_A64); in TRANS_FEAT()
7916 if (s->fpcr_ah) { in TRANS_FEAT()
7918 a->rd, a->rn, a->rm, a->ra, in TRANS_FEAT()
7919 (a->index << 1) | sel, FPST_AH); in TRANS_FEAT()
7922 a->rd, a->rn, a->rm, a->ra, in TRANS_FEAT()
7923 (a->index << 1) | sel, FPST_A64); in TRANS_FEAT()
7934 int elements = vl >> a->esz; in TRANS_FEAT()
7951 tcg_gen_addi_i64(tmp, cpu_reg(s, a->rv), a->imm); in TRANS_FEAT()
7953 tcg_gen_andi_i64(tmp, tmp, elements - 1); in TRANS_FEAT()
7959 tcg_gen_shli_i64(tmp, tmp, a->esz); in TRANS_FEAT()
7969 tcg_gen_ld8u_i64(tmp, ptr, pred_full_reg_offset(s, a->pm)); in TRANS_FEAT()
7978 tcg_gen_gvec_ands(MO_64, pred_full_reg_offset(s, a->pd), in TRANS_FEAT()
7979 pred_full_reg_offset(s, a->pn), tmp, pl, pl); in TRANS_FEAT()
8095 if (a->esz == MO_8 in TRANS_FEAT()
8102 assert(a->rd == a->ra); in TRANS_FEAT()
8103 return gen_gvec_fpst_zzz(s, fn[a->esz], a->rd, a->rn, a->rm, 1, in TRANS_FEAT()
8104 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64); in TRANS_FEAT()
8108 gen_helper_sme2_sqcvtn_sh, a->rd, a->rn, 0)
8110 gen_helper_sme2_uqcvtn_sh, a->rd, a->rn, 0)
8112 gen_helper_sme2_sqcvtun_sh, a->rd, a->rn, 0)
8143 bool be = s->be_data == MO_BE; in gen_ldst_c()
8155 /* Ignore non-temporal bit */ in gen_ldst_c()
8165 if (!s->mte_active[0]) { in gen_ldst_c()
8191 tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), a->esz); in gen_ldst_zcrr_c()
8192 tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn)); in gen_ldst_zcrr_c()
8193 return gen_ldst_c(s, addr, a->rd, a->png, a->esz, is_write, in gen_ldst_zcrr_c()
8194 a->nreg, strided); in gen_ldst_zcrr_c()
8202 tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn), in gen_ldst_zcri_c()
8203 a->imm * a->nreg * vec_full_reg_size(s)); in gen_ldst_zcri_c()
8204 return gen_ldst_c(s, addr, a->rd, a->png, a->esz, is_write, in gen_ldst_zcri_c()
8205 a->nreg, strided); in gen_ldst_zcri_c()