Lines Matching refs:a
141 arg_rr_esz *a, int data) in gen_gvec_fpst_arg_zz() argument
143 return gen_gvec_fpst_zz(s, fn, a->rd, a->rn, data, in gen_gvec_fpst_arg_zz()
144 a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR); in gen_gvec_fpst_arg_zz()
165 arg_rrr_esz *a, int data) in gen_gvec_ool_arg_zzz() argument
167 return gen_gvec_ool_zzz(s, fn, a->rd, a->rn, a->rm, data); in gen_gvec_ool_arg_zzz()
191 arg_rrr_esz *a, int data) in gen_gvec_fpst_arg_zzz() argument
193 return gen_gvec_fpst_zzz(s, fn, a->rd, a->rn, a->rm, data, in gen_gvec_fpst_arg_zzz()
194 a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR); in gen_gvec_fpst_arg_zzz()
216 arg_rrrr_esz *a, int data) in gen_gvec_ool_arg_zzzz() argument
218 return gen_gvec_ool_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, data); in gen_gvec_ool_arg_zzzz()
222 arg_rrxr_esz *a) in gen_gvec_ool_arg_zzxz() argument
224 return gen_gvec_ool_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, a->index); in gen_gvec_ool_arg_zzxz()
295 arg_rpr_esz *a, int data) in gen_gvec_ool_arg_zpz() argument
297 return gen_gvec_ool_zzp(s, fn, a->rd, a->rn, a->pg, data); in gen_gvec_ool_arg_zpz()
301 arg_rpri_esz *a) in gen_gvec_ool_arg_zpzi() argument
303 return gen_gvec_ool_zzp(s, fn, a->rd, a->rn, a->pg, a->imm); in gen_gvec_ool_arg_zpzi()
326 arg_rpr_esz *a, int data, in gen_gvec_fpst_arg_zpz() argument
329 return gen_gvec_fpst_zzp(s, fn, a->rd, a->rn, a->pg, data, flavour); in gen_gvec_fpst_arg_zpz()
351 arg_rprr_esz *a, int data) in gen_gvec_ool_arg_zpzz() argument
353 return gen_gvec_ool_zzzp(s, fn, a->rd, a->rn, a->rm, a->pg, data); in gen_gvec_ool_arg_zpzz()
378 arg_rprr_esz *a) in gen_gvec_fpst_arg_zpzz() argument
380 return gen_gvec_fpst_zzzp(s, fn, a->rd, a->rn, a->rm, a->pg, 0, in gen_gvec_fpst_arg_zpzz()
381 a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR); in gen_gvec_fpst_arg_zpzz()
400 arg_rri_esz *a) in gen_gvec_fn_arg_zzi() argument
402 if (a->esz < 0) { in gen_gvec_fn_arg_zzi()
406 return gen_gvec_fn_zzi(s, gvec_fn, a->esz, a->rd, a->rn, a->imm); in gen_gvec_fn_arg_zzi()
426 arg_rrr_esz *a) in gen_gvec_fn_arg_zzz() argument
428 return gen_gvec_fn_zzz(s, fn, a->esz, a->rd, a->rn, a->rm); in gen_gvec_fn_arg_zzz()
433 arg_rrrr_esz *a) in gen_gvec_fn_arg_zzzz() argument
440 gvec_fn(a->esz, vec_full_reg_offset(s, a->rd), in gen_gvec_fn_arg_zzzz()
441 vec_full_reg_offset(s, a->rn), in gen_gvec_fn_arg_zzzz()
442 vec_full_reg_offset(s, a->rm), in gen_gvec_fn_arg_zzzz()
443 vec_full_reg_offset(s, a->ra), vsz, vsz); in gen_gvec_fn_arg_zzzz()
529 static bool trans_INVALID(DisasContext *s, arg_INVALID *a) in trans_INVALID() argument
539 TRANS_FEAT(AND_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_and, a) in TRANS_FEAT() argument
540 TRANS_FEAT(ORR_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_or, a) in TRANS_FEAT()
541 TRANS_FEAT(EOR_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_xor, a) in TRANS_FEAT()
542 TRANS_FEAT(BIC_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_andc, a) in TRANS_FEAT()
544 static bool trans_XAR(DisasContext *s, arg_rrri_esz *a) in TRANS_FEAT()
546 if (a->esz < 0 || !dc_isar_feature(aa64_sve2, s)) { in TRANS_FEAT()
551 gen_gvec_xar(a->esz, vec_full_reg_offset(s, a->rd), in TRANS_FEAT()
552 vec_full_reg_offset(s, a->rn), in TRANS_FEAT()
553 vec_full_reg_offset(s, a->rm), a->imm, vsz, vsz); in TRANS_FEAT()
558 TRANS_FEAT(EOR3, aa64_sve2, gen_gvec_fn_arg_zzzz, gen_gvec_eor3, a) in TRANS_FEAT() argument
559 TRANS_FEAT(BCAX, aa64_sve2, gen_gvec_fn_arg_zzzz, gen_gvec_bcax, a) in TRANS_FEAT()
562 uint32_t a, uint32_t oprsz, uint32_t maxsz) in TRANS_FEAT()
565 tcg_gen_gvec_bitsel(vece, d, a, n, m, oprsz, maxsz); in TRANS_FEAT()
568 TRANS_FEAT(BSL, aa64_sve2, gen_gvec_fn_arg_zzzz, gen_bsl, a) in TRANS_FEAT() argument
591 uint32_t a, uint32_t oprsz, uint32_t maxsz) in gen_bsl1n() argument
600 tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op); in gen_bsl1n()
603 TRANS_FEAT(BSL1N, aa64_sve2, gen_gvec_fn_arg_zzzz, gen_bsl1n, a) in TRANS_FEAT() argument
635 uint32_t a, uint32_t oprsz, uint32_t maxsz) in gen_bsl2n() argument
644 tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op); in gen_bsl2n()
647 TRANS_FEAT(BSL2N, aa64_sve2, gen_gvec_fn_arg_zzzz, gen_bsl2n, a) in TRANS_FEAT() argument
664 uint32_t a, uint32_t oprsz, uint32_t maxsz) in gen_nbsl() argument
673 tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op); in gen_nbsl()
676 TRANS_FEAT(NBSL, aa64_sve2, gen_gvec_fn_arg_zzzz, gen_nbsl, a) in TRANS_FEAT() argument
682 TRANS_FEAT(ADD_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_add, a) in TRANS_FEAT()
683 TRANS_FEAT(SUB_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_sub, a) in TRANS_FEAT()
684 TRANS_FEAT(SQADD_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_ssadd, a) in TRANS_FEAT()
685 TRANS_FEAT(SQSUB_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_sssub, a) in TRANS_FEAT()
686 TRANS_FEAT(UQADD_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_usadd, a) in TRANS_FEAT()
687 TRANS_FEAT(UQSUB_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_ussub, a) in TRANS_FEAT()
711 name##_zpzz_fns[a->esz], a, 0)
739 TRANS_FEAT(SDIV_zpzz, aa64_sve, gen_gvec_ool_arg_zpzz, sdiv_fns[a->esz], a, 0)
744 TRANS_FEAT(UDIV_zpzz, aa64_sve, gen_gvec_ool_arg_zpzz, udiv_fns[a->esz], a, 0)
746 TRANS_FEAT(SEL_zpzz, aa64_sve, do_sel_z, a->rd, a->rn, a->rm, a->pg, a->esz)
757 TRANS_FEAT(NAME, FEAT, gen_gvec_ool_arg_zpz, name##_fns[a->esz], a, 0)
772 TRANS_FEAT(FABS, aa64_sve, gen_gvec_ool_arg_zpz, fabs_fns[a->esz], a, 0)
778 TRANS_FEAT(FNEG, aa64_sve, gen_gvec_ool_arg_zpz, fneg_fns[a->esz], a, 0)
784 TRANS_FEAT(SXTB, aa64_sve, gen_gvec_ool_arg_zpz, sxtb_fns[a->esz], a, 0)
790 TRANS_FEAT(UXTB, aa64_sve, gen_gvec_ool_arg_zpz, uxtb_fns[a->esz], a, 0)
795 TRANS_FEAT(SXTH, aa64_sve, gen_gvec_ool_arg_zpz, sxth_fns[a->esz], a, 0)
800 TRANS_FEAT(UXTH, aa64_sve, gen_gvec_ool_arg_zpz, uxth_fns[a->esz], a, 0)
803 a->esz == 3 ? gen_helper_sve_sxtw_d : NULL, a, 0)
805 a->esz == 3 ? gen_helper_sve_uxtw_d : NULL, a, 0)
812 static bool do_vpz_ool(DisasContext *s, arg_rpr_esz *a, in do_vpz_ool() argument
832 tcg_gen_addi_ptr(t_zn, tcg_env, vec_full_reg_offset(s, a->rn)); in do_vpz_ool()
833 tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, a->pg)); in do_vpz_ool()
836 write_fp_dreg(s, a->rd, temp); in do_vpz_ool()
845 TRANS_FEAT(NAME, aa64_sve, do_vpz_ool, a, name##_fns[a->esz])
861 TRANS_FEAT(SADDV, aa64_sve, do_vpz_ool, a, saddv_fns[a->esz]) in TRANS_FEAT() argument
883 static bool do_shift_zpzi(DisasContext *s, arg_rpri_esz *a, bool asr, in do_shift_zpzi() argument
888 if (a->esz < 0) { in do_shift_zpzi()
898 max = 8 << a->esz; in do_shift_zpzi()
899 if (a->imm >= max) { in do_shift_zpzi()
901 a->imm = max - 1; in do_shift_zpzi()
903 return do_movz_zpz(s, a->rd, a->rd, a->pg, a->esz, true); in do_shift_zpzi()
906 return gen_gvec_ool_arg_zpzi(s, fns[a->esz], a); in do_shift_zpzi()
913 TRANS_FEAT(ASR_zpzi, aa64_sve, do_shift_zpzi, a, true, asr_zpzi_fns)
919 TRANS_FEAT(LSR_zpzi, aa64_sve, do_shift_zpzi, a, false, lsr_zpzi_fns)
925 TRANS_FEAT(LSL_zpzi, aa64_sve, do_shift_zpzi, a, false, lsl_zpzi_fns)
931 TRANS_FEAT(ASRD, aa64_sve, do_shift_zpzi, a, false, asrd_fns)
938 a->esz < 0 ? NULL : sqshl_zpzi_fns[a->esz], a)
945 a->esz < 0 ? NULL : uqshl_zpzi_fns[a->esz], a)
952 a->esz < 0 ? NULL : srshr_fns[a->esz], a)
959 a->esz < 0 ? NULL : urshr_fns[a->esz], a)
966 a->esz < 0 ? NULL : sqshlu_fns[a->esz], a)
978 a->esz < 0 ? NULL : name##_zpzw_fns[a->esz], a, 0)
990 static bool do_shift_imm(DisasContext *s, arg_rri_esz *a, bool asr, in DO_ZPZW()
994 if (a->esz < 0) { in DO_ZPZW()
1003 if (a->imm >= 8 << a->esz) { in DO_ZPZW()
1005 a->imm = (8 << a->esz) - 1; in DO_ZPZW()
1007 do_dupi_z(s, a->rd, 0); in DO_ZPZW()
1011 gvec_fn(a->esz, vec_full_reg_offset(s, a->rd), in DO_ZPZW()
1012 vec_full_reg_offset(s, a->rn), a->imm, vsz, vsz); in DO_ZPZW()
1017 TRANS_FEAT(ASR_zzi, aa64_sve, do_shift_imm, a, true, tcg_gen_gvec_sari) in TRANS_FEAT() argument
1018 TRANS_FEAT(LSR_zzi, aa64_sve, do_shift_imm, a, false, tcg_gen_gvec_shri) in TRANS_FEAT()
1019 TRANS_FEAT(LSL_zzi, aa64_sve, do_shift_imm, a, false, tcg_gen_gvec_shli) in TRANS_FEAT()
1027 name##_zzw_fns[a->esz], a, 0)
1039 static bool do_zpzzz_ool(DisasContext *s, arg_rprrr_esz *a,
1044 tcg_gen_gvec_5_ool(vec_full_reg_offset(s, a->rd),
1045 vec_full_reg_offset(s, a->ra),
1046 vec_full_reg_offset(s, a->rn),
1047 vec_full_reg_offset(s, a->rm),
1048 pred_full_reg_offset(s, a->pg),
1058 TRANS_FEAT(MLA, aa64_sve, do_zpzzz_ool, a, mla_fns[a->esz])
1064 TRANS_FEAT(MLS, aa64_sve, do_zpzzz_ool, a, mls_fns[a->esz]) in TRANS_FEAT() argument
1105 TRANS_FEAT(INDEX_ii, aa64_sve, do_index, a->esz, a->rd,
1106 tcg_constant_i64(a->imm1), tcg_constant_i64(a->imm2))
1107 TRANS_FEAT(INDEX_ir, aa64_sve, do_index, a->esz, a->rd,
1108 tcg_constant_i64(a->imm), cpu_reg(s, a->rm))
1109 TRANS_FEAT(INDEX_ri, aa64_sve, do_index, a->esz, a->rd,
1110 cpu_reg(s, a->rn), tcg_constant_i64(a->imm))
1111 TRANS_FEAT(INDEX_rr, aa64_sve, do_index, a->esz, a->rd,
1112 cpu_reg(s, a->rn), cpu_reg(s, a->rm))
1118 static bool trans_ADDVL(DisasContext *s, arg_ADDVL *a) in trans_ADDVL() argument
1124 TCGv_i64 rd = cpu_reg_sp(s, a->rd); in trans_ADDVL()
1125 TCGv_i64 rn = cpu_reg_sp(s, a->rn); in trans_ADDVL()
1126 tcg_gen_addi_i64(rd, rn, a->imm * vec_full_reg_size(s)); in trans_ADDVL()
1131 static bool trans_ADDSVL(DisasContext *s, arg_ADDSVL *a) in trans_ADDSVL() argument
1137 TCGv_i64 rd = cpu_reg_sp(s, a->rd); in trans_ADDSVL()
1138 TCGv_i64 rn = cpu_reg_sp(s, a->rn); in trans_ADDSVL()
1139 tcg_gen_addi_i64(rd, rn, a->imm * streaming_vec_reg_size(s)); in trans_ADDSVL()
1144 static bool trans_ADDPL(DisasContext *s, arg_ADDPL *a) in trans_ADDPL() argument
1150 TCGv_i64 rd = cpu_reg_sp(s, a->rd); in trans_ADDPL()
1151 TCGv_i64 rn = cpu_reg_sp(s, a->rn); in trans_ADDPL()
1152 tcg_gen_addi_i64(rd, rn, a->imm * pred_full_reg_size(s)); in trans_ADDPL()
1157 static bool trans_ADDSPL(DisasContext *s, arg_ADDSPL *a) in trans_ADDSPL() argument
1163 TCGv_i64 rd = cpu_reg_sp(s, a->rd); in trans_ADDSPL()
1164 TCGv_i64 rn = cpu_reg_sp(s, a->rn); in trans_ADDSPL()
1165 tcg_gen_addi_i64(rd, rn, a->imm * streaming_pred_reg_size(s)); in trans_ADDSPL()
1170 static bool trans_RDVL(DisasContext *s, arg_RDVL *a) in trans_RDVL() argument
1176 TCGv_i64 reg = cpu_reg(s, a->rd); in trans_RDVL()
1177 tcg_gen_movi_i64(reg, a->imm * vec_full_reg_size(s)); in trans_RDVL()
1182 static bool trans_RDSVL(DisasContext *s, arg_RDSVL *a) in trans_RDSVL() argument
1188 TCGv_i64 reg = cpu_reg(s, a->rd); in trans_RDSVL()
1189 tcg_gen_movi_i64(reg, a->imm * streaming_vec_reg_size(s)); in trans_RDSVL()
1198 static bool do_adr(DisasContext *s, arg_rrri *a, gen_helper_gvec_3 *fn) in do_adr() argument
1200 return gen_gvec_ool_zzz(s, fn, a->rd, a->rn, a->rm, a->imm); in do_adr()
1203 TRANS_FEAT_NONSTREAMING(ADR_p32, aa64_sve, do_adr, a, gen_helper_sve_adr_p32)
1204 TRANS_FEAT_NONSTREAMING(ADR_p64, aa64_sve, do_adr, a, gen_helper_sve_adr_p64)
1205 TRANS_FEAT_NONSTREAMING(ADR_s32, aa64_sve, do_adr, a, gen_helper_sve_adr_s32)
1206 TRANS_FEAT_NONSTREAMING(ADR_u32, aa64_sve, do_adr, a, gen_helper_sve_adr_u32)
1217 fexpa_fns[a->esz], a->rd, a->rn, 0)
1224 ftssel_fns[a->esz], a, 0)
1230 static bool do_pppp_flags(DisasContext *s, arg_rprr_s *a, in do_pppp_flags() argument
1238 int dofs = pred_full_reg_offset(s, a->rd); in do_pppp_flags()
1239 int nofs = pred_full_reg_offset(s, a->rn); in do_pppp_flags()
1240 int mofs = pred_full_reg_offset(s, a->rm); in do_pppp_flags()
1241 int gofs = pred_full_reg_offset(s, a->pg); in do_pppp_flags()
1243 if (!a->s) { in do_pppp_flags()
1270 if (a->rd == a->pg) { in do_pppp_flags()
1294 static bool trans_AND_pppp(DisasContext *s, arg_rprr_s *a) in trans_AND_pppp() argument
1306 if (!a->s) { in trans_AND_pppp()
1307 if (a->rn == a->rm) { in trans_AND_pppp()
1308 if (a->pg == a->rn) { in trans_AND_pppp()
1309 return do_mov_p(s, a->rd, a->rn); in trans_AND_pppp()
1311 return gen_gvec_fn_ppp(s, tcg_gen_gvec_and, a->rd, a->rn, a->pg); in trans_AND_pppp()
1312 } else if (a->pg == a->rn || a->pg == a->rm) { in trans_AND_pppp()
1313 return gen_gvec_fn_ppp(s, tcg_gen_gvec_and, a->rd, a->rn, a->rm); in trans_AND_pppp()
1316 return do_pppp_flags(s, a, &op); in trans_AND_pppp()
1332 static bool trans_BIC_pppp(DisasContext *s, arg_rprr_s *a) in trans_BIC_pppp() argument
1344 if (!a->s && a->pg == a->rn) { in trans_BIC_pppp()
1345 return gen_gvec_fn_ppp(s, tcg_gen_gvec_andc, a->rd, a->rn, a->rm); in trans_BIC_pppp()
1347 return do_pppp_flags(s, a, &op); in trans_BIC_pppp()
1363 static bool trans_EOR_pppp(DisasContext *s, arg_rprr_s *a) in trans_EOR_pppp() argument
1376 if (!a->s && a->pg == a->rm) { in trans_EOR_pppp()
1377 return gen_gvec_fn_ppp(s, tcg_gen_gvec_andc, a->rd, a->pg, a->rn); in trans_EOR_pppp()
1379 return do_pppp_flags(s, a, &op); in trans_EOR_pppp()
1382 static bool trans_SEL_pppp(DisasContext *s, arg_rprr_s *a) in trans_SEL_pppp() argument
1384 if (a->s || !dc_isar_feature(aa64_sve, s)) { in trans_SEL_pppp()
1389 tcg_gen_gvec_bitsel(MO_8, pred_full_reg_offset(s, a->rd), in trans_SEL_pppp()
1390 pred_full_reg_offset(s, a->pg), in trans_SEL_pppp()
1391 pred_full_reg_offset(s, a->rn), in trans_SEL_pppp()
1392 pred_full_reg_offset(s, a->rm), psz, psz); in trans_SEL_pppp()
1410 static bool trans_ORR_pppp(DisasContext *s, arg_rprr_s *a) in trans_ORR_pppp() argument
1422 if (!a->s && a->pg == a->rn && a->rn == a->rm) { in trans_ORR_pppp()
1423 return do_mov_p(s, a->rd, a->rn); in trans_ORR_pppp()
1425 return do_pppp_flags(s, a, &op); in trans_ORR_pppp()
1441 static bool trans_ORN_pppp(DisasContext *s, arg_rprr_s *a) in trans_ORN_pppp() argument
1453 return do_pppp_flags(s, a, &op); in trans_ORN_pppp()
1469 static bool trans_NOR_pppp(DisasContext *s, arg_rprr_s *a) in trans_NOR_pppp() argument
1481 return do_pppp_flags(s, a, &op); in trans_NOR_pppp()
1497 static bool trans_NAND_pppp(DisasContext *s, arg_rprr_s *a) in trans_NAND_pppp() argument
1509 return do_pppp_flags(s, a, &op); in trans_NAND_pppp()
1516 static bool trans_PTEST(DisasContext *s, arg_PTEST *a) in trans_PTEST() argument
1522 int nofs = pred_full_reg_offset(s, a->rn); in trans_PTEST()
1523 int gofs = pred_full_reg_offset(s, a->pg); in trans_PTEST()
1656 TRANS_FEAT(PTRUE, aa64_sve, do_predset, a->esz, a->rd, a->pat, a->s)
1663 TRANS_FEAT(PFALSE, aa64_sve, do_predset, 0, a->rd, 32, false)
1665 static bool trans_RDFFR_p(DisasContext *s, arg_RDFFR_p *a) in trans_RDFFR_p() argument
1671 .rd = a->rd, .pg = a->pg, .s = a->s, in trans_RDFFR_p()
1679 TRANS_FEAT_NONSTREAMING(RDFFR, aa64_sve, do_mov_p, a->rd, FFR_PRED_NUM)
1680 TRANS_FEAT_NONSTREAMING(WRFFR, aa64_sve, do_mov_p, FFR_PRED_NUM, a->rn)
1682 static bool do_pfirst_pnext(DisasContext *s, arg_rr_esz *a, in do_pfirst_pnext() argument
1696 desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz); in do_pfirst_pnext()
1698 tcg_gen_addi_ptr(t_pd, tcg_env, pred_full_reg_offset(s, a->rd)); in do_pfirst_pnext()
1699 tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, a->rn)); in do_pfirst_pnext()
1708 TRANS_FEAT(PFIRST, aa64_sve, do_pfirst_pnext, a, gen_helper_sve_pfirst) in TRANS_FEAT() argument
1709 TRANS_FEAT(PNEXT, aa64_sve, do_pfirst_pnext, a, gen_helper_sve_pnext) in TRANS_FEAT()
1861 static bool trans_CNT_r(DisasContext *s, arg_CNT_r *a) in trans_CNT_r() argument
1868 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz); in trans_CNT_r()
1869 tcg_gen_movi_i64(cpu_reg(s, a->rd), numelem * a->imm); in trans_CNT_r()
1874 static bool trans_INCDEC_r(DisasContext *s, arg_incdec_cnt *a) in trans_INCDEC_r() argument
1881 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz); in trans_INCDEC_r()
1882 int inc = numelem * a->imm * (a->d ? -1 : 1); in trans_INCDEC_r()
1883 TCGv_i64 reg = cpu_reg(s, a->rd); in trans_INCDEC_r()
1890 static bool trans_SINCDEC_r_32(DisasContext *s, arg_incdec_cnt *a) in trans_SINCDEC_r_32() argument
1900 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz); in trans_SINCDEC_r_32()
1901 int inc = numelem * a->imm; in trans_SINCDEC_r_32()
1902 TCGv_i64 reg = cpu_reg(s, a->rd); in trans_SINCDEC_r_32()
1906 if (a->u) { in trans_SINCDEC_r_32()
1912 do_sat_addsub_32(reg, tcg_constant_i64(inc), a->u, a->d); in trans_SINCDEC_r_32()
1917 static bool trans_SINCDEC_r_64(DisasContext *s, arg_incdec_cnt *a) in trans_SINCDEC_r_64() argument
1927 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz); in trans_SINCDEC_r_64()
1928 int inc = numelem * a->imm; in trans_SINCDEC_r_64()
1929 TCGv_i64 reg = cpu_reg(s, a->rd); in trans_SINCDEC_r_64()
1932 do_sat_addsub_64(reg, tcg_constant_i64(inc), a->u, a->d); in trans_SINCDEC_r_64()
1937 static bool trans_INCDEC_v(DisasContext *s, arg_incdec2_cnt *a) in trans_INCDEC_v() argument
1939 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) { in trans_INCDEC_v()
1944 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz); in trans_INCDEC_v()
1945 int inc = numelem * a->imm; in trans_INCDEC_v()
1949 tcg_gen_gvec_adds(a->esz, vec_full_reg_offset(s, a->rd), in trans_INCDEC_v()
1950 vec_full_reg_offset(s, a->rn), in trans_INCDEC_v()
1951 tcg_constant_i64(a->d ? -inc : inc), in trans_INCDEC_v()
1955 do_mov_z(s, a->rd, a->rn); in trans_INCDEC_v()
1960 static bool trans_SINCDEC_v(DisasContext *s, arg_incdec2_cnt *a) in trans_SINCDEC_v() argument
1962 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) { in trans_SINCDEC_v()
1967 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz); in trans_SINCDEC_v()
1968 int inc = numelem * a->imm; in trans_SINCDEC_v()
1972 do_sat_addsub_vec(s, a->esz, a->rd, a->rn, in trans_SINCDEC_v()
1973 tcg_constant_i64(inc), a->u, a->d); in trans_SINCDEC_v()
1976 do_mov_z(s, a->rd, a->rn); in trans_SINCDEC_v()
1985 static bool do_zz_dbm(DisasContext *s, arg_rr_dbm *a, GVecGen2iFn *gvec_fn) in do_zz_dbm() argument
1988 if (!logic_imm_decode_wmask(&imm, extract32(a->dbm, 12, 1), in do_zz_dbm()
1989 extract32(a->dbm, 0, 6), in do_zz_dbm()
1990 extract32(a->dbm, 6, 6))) { in do_zz_dbm()
1993 return gen_gvec_fn_zzi(s, gvec_fn, MO_64, a->rd, a->rn, imm); in do_zz_dbm()
1996 TRANS_FEAT(AND_zzi, aa64_sve, do_zz_dbm, a, tcg_gen_gvec_andi) in TRANS_FEAT() argument
1997 TRANS_FEAT(ORR_zzi, aa64_sve, do_zz_dbm, a, tcg_gen_gvec_ori) in TRANS_FEAT()
1998 TRANS_FEAT(EOR_zzi, aa64_sve, do_zz_dbm, a, tcg_gen_gvec_xori) in TRANS_FEAT()
2000 static bool trans_DUPM(DisasContext *s, arg_DUPM *a) in TRANS_FEAT()
2007 if (!logic_imm_decode_wmask(&imm, extract32(a->dbm, 12, 1), in TRANS_FEAT()
2008 extract32(a->dbm, 0, 6), in TRANS_FEAT()
2009 extract32(a->dbm, 6, 6))) { in TRANS_FEAT()
2013 do_dupi_z(s, a->rd, imm); in TRANS_FEAT()
2046 static bool trans_FCPY(DisasContext *s, arg_FCPY *a) in trans_FCPY() argument
2048 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) { in trans_FCPY()
2053 uint64_t imm = vfp_expand_imm(a->esz, a->imm); in trans_FCPY()
2054 do_cpy_m(s, a->esz, a->rd, a->rn, a->pg, tcg_constant_i64(imm)); in trans_FCPY()
2059 static bool trans_CPY_m_i(DisasContext *s, arg_rpri_esz *a) in trans_CPY_m_i() argument
2065 do_cpy_m(s, a->esz, a->rd, a->rn, a->pg, tcg_constant_i64(a->imm)); in trans_CPY_m_i()
2070 static bool trans_CPY_z_i(DisasContext *s, arg_CPY_z_i *a) in trans_CPY_z_i() argument
2082 tcg_gen_gvec_2i_ool(vec_full_reg_offset(s, a->rd), in trans_CPY_z_i()
2083 pred_full_reg_offset(s, a->pg), in trans_CPY_z_i()
2084 tcg_constant_i64(a->imm), in trans_CPY_z_i()
2085 vsz, vsz, 0, fns[a->esz]); in trans_CPY_z_i()
2124 TRANS_FEAT(EXT, aa64_sve, do_EXT, a->rd, a->rn, a->rm, a->imm)
2125 TRANS_FEAT(EXT_sve2, aa64_sve2, do_EXT, a->rd, a->rn, (a->rn + 1) % 32, a->imm)
2131 static bool trans_DUP_s(DisasContext *s, arg_DUP_s *a) in trans_DUP_s() argument
2138 tcg_gen_gvec_dup_i64(a->esz, vec_full_reg_offset(s, a->rd), in trans_DUP_s()
2139 vsz, vsz, cpu_reg_sp(s, a->rn)); in trans_DUP_s()
2144 static bool trans_DUP_x(DisasContext *s, arg_DUP_x *a) in trans_DUP_x() argument
2149 if ((a->imm & 0x1f) == 0) { in trans_DUP_x()
2154 unsigned dofs = vec_full_reg_offset(s, a->rd); in trans_DUP_x()
2157 esz = ctz32(a->imm); in trans_DUP_x()
2158 index = a->imm >> (esz + 1); in trans_DUP_x()
2161 unsigned nofs = vec_reg_offset(s, a->rn, index, esz); in trans_DUP_x()
2174 static void do_insr_i64(DisasContext *s, arg_rrr_esz *a, TCGv_i64 val) in do_insr_i64() argument
2186 tcg_gen_addi_ptr(t_zd, tcg_env, vec_full_reg_offset(s, a->rd)); in do_insr_i64()
2187 tcg_gen_addi_ptr(t_zn, tcg_env, vec_full_reg_offset(s, a->rn)); in do_insr_i64()
2189 fns[a->esz](t_zd, t_zn, val, desc); in do_insr_i64()
2192 static bool trans_INSR_f(DisasContext *s, arg_rrr_esz *a) in trans_INSR_f() argument
2199 tcg_gen_ld_i64(t, tcg_env, vec_reg_offset(s, a->rm, 0, MO_64)); in trans_INSR_f()
2200 do_insr_i64(s, a, t); in trans_INSR_f()
2205 static bool trans_INSR_r(DisasContext *s, arg_rrr_esz *a) in trans_INSR_r() argument
2211 do_insr_i64(s, a, cpu_reg(s, a->rm)); in trans_INSR_r()
2220 TRANS_FEAT(REV_v, aa64_sve, gen_gvec_ool_zz, rev_fns[a->esz], a->rd, a->rn, 0)
2226 TRANS_FEAT(TBL, aa64_sve, gen_gvec_ool_arg_zzz, sve_tbl_fns[a->esz], a, 0)
2232 TRANS_FEAT(TBL_sve2, aa64_sve2, gen_gvec_ool_zzzz, sve2_tbl_fns[a->esz],
2233 a->rd, a->rn, (a->rn + 1) % 32, a->rm, 0)
2239 TRANS_FEAT(TBX, aa64_sve2, gen_gvec_ool_arg_zzz, tbx_fns[a->esz], a, 0)
2241 static bool trans_UNPK(DisasContext *s, arg_UNPK *a) in trans_UNPK() argument
2250 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) { in trans_UNPK()
2255 tcg_gen_gvec_2_ool(vec_full_reg_offset(s, a->rd), in trans_UNPK()
2256 vec_full_reg_offset(s, a->rn) in trans_UNPK()
2257 + (a->h ? vsz / 2 : 0), in trans_UNPK()
2258 vsz, vsz, 0, fns[a->esz][a->u]); in trans_UNPK()
2267 static bool do_perm_pred3(DisasContext *s, arg_rrr_esz *a, bool high_odd, in do_perm_pred3() argument
2282 desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz); in do_perm_pred3()
2285 tcg_gen_addi_ptr(t_d, tcg_env, pred_full_reg_offset(s, a->rd)); in do_perm_pred3()
2286 tcg_gen_addi_ptr(t_n, tcg_env, pred_full_reg_offset(s, a->rn)); in do_perm_pred3()
2287 tcg_gen_addi_ptr(t_m, tcg_env, pred_full_reg_offset(s, a->rm)); in do_perm_pred3()
2293 static bool do_perm_pred2(DisasContext *s, arg_rr_esz *a, bool high_odd, in do_perm_pred2() argument
2305 tcg_gen_addi_ptr(t_d, tcg_env, pred_full_reg_offset(s, a->rd)); in do_perm_pred2()
2306 tcg_gen_addi_ptr(t_n, tcg_env, pred_full_reg_offset(s, a->rn)); in do_perm_pred2()
2309 desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz); in do_perm_pred2()
2316 TRANS_FEAT(ZIP1_p, aa64_sve, do_perm_pred3, a, 0, gen_helper_sve_zip_p)
2317 TRANS_FEAT(ZIP2_p, aa64_sve, do_perm_pred3, a, 1, gen_helper_sve_zip_p)
2318 TRANS_FEAT(UZP1_p, aa64_sve, do_perm_pred3, a, 0, gen_helper_sve_uzp_p)
2319 TRANS_FEAT(UZP2_p, aa64_sve, do_perm_pred3, a, 1, gen_helper_sve_uzp_p)
2320 TRANS_FEAT(TRN1_p, aa64_sve, do_perm_pred3, a, 0, gen_helper_sve_trn_p)
2321 TRANS_FEAT(TRN2_p, aa64_sve, do_perm_pred3, a, 1, gen_helper_sve_trn_p)
2323 TRANS_FEAT(REV_p, aa64_sve, do_perm_pred2, a, 0, gen_helper_sve_rev_p)
2324 TRANS_FEAT(PUNPKLO, aa64_sve, do_perm_pred2, a, 0, gen_helper_sve_punpk_p)
2325 TRANS_FEAT(PUNPKHI, aa64_sve, do_perm_pred2, a, 1, gen_helper_sve_punpk_p)
2336 zip_fns[a->esz], a, 0)
2338 zip_fns[a->esz], a, vec_full_reg_size(s) / 2)
2341 gen_helper_sve2_zip_q, a, 0)
2343 gen_helper_sve2_zip_q, a,
2352 uzp_fns[a->esz], a, 0)
2354 uzp_fns[a->esz], a, 1 << a->esz)
2357 gen_helper_sve2_uzp_q, a, 0)
2359 gen_helper_sve2_uzp_q, a, 16)
2367 trn_fns[a->esz], a, 0)
2369 trn_fns[a->esz], a, 1 << a->esz)
2372 gen_helper_sve2_trn_q, a, 0)
2374 gen_helper_sve2_trn_q, a, 16)
2384 compact_fns[a->esz], a, 0)
2484 static bool do_clast_vector(DisasContext *s, arg_rprr_esz *a, bool before) in do_clast_vector() argument
2489 unsigned vsz, esz = a->esz; in do_clast_vector()
2498 find_last_active(s, last, esz, a->pg); in do_clast_vector()
2509 ele = load_last_active(s, last, a->rm, esz); in do_clast_vector()
2512 tcg_gen_gvec_dup_i64(esz, vec_full_reg_offset(s, a->rd), vsz, vsz, ele); in do_clast_vector()
2515 if (a->rd != a->rn) { in do_clast_vector()
2520 do_mov_z(s, a->rd, a->rn); in do_clast_vector()
2529 TRANS_FEAT(CLASTA_z, aa64_sve, do_clast_vector, a, false) in TRANS_FEAT() argument
2530 TRANS_FEAT(CLASTB_z, aa64_sve, do_clast_vector, a, true) in TRANS_FEAT()
2561 static bool do_clast_fp(DisasContext *s, arg_rpr_esz *a, bool before) in do_clast_fp() argument
2564 int esz = a->esz; in do_clast_fp()
2565 int ofs = vec_reg_offset(s, a->rd, 0, esz); in do_clast_fp()
2568 do_clast_scalar(s, esz, a->pg, a->rn, before, reg); in do_clast_fp()
2569 write_fp_dreg(s, a->rd, reg); in do_clast_fp()
2574 TRANS_FEAT(CLASTA_v, aa64_sve, do_clast_fp, a, false) in TRANS_FEAT() argument
2575 TRANS_FEAT(CLASTB_v, aa64_sve, do_clast_fp, a, true) in TRANS_FEAT()
2578 static bool do_clast_general(DisasContext *s, arg_rpr_esz *a, bool before) in TRANS_FEAT()
2586 reg = cpu_reg(s, a->rd); in TRANS_FEAT()
2587 switch (a->esz) { in TRANS_FEAT()
2603 do_clast_scalar(s, a->esz, a->pg, a->rn, before, reg); in TRANS_FEAT()
2607 TRANS_FEAT(CLASTA_r, aa64_sve, do_clast_general, a, false) in TRANS_FEAT() argument
2608 TRANS_FEAT(CLASTB_r, aa64_sve, do_clast_general, a, true) in TRANS_FEAT()
2627 static bool do_last_fp(DisasContext *s, arg_rpr_esz *a, bool before) in do_last_fp() argument
2630 TCGv_i64 val = do_last_scalar(s, a->esz, a->pg, a->rn, before); in do_last_fp()
2631 write_fp_dreg(s, a->rd, val); in do_last_fp()
2636 TRANS_FEAT(LASTA_v, aa64_sve, do_last_fp, a, false) in TRANS_FEAT() argument
2637 TRANS_FEAT(LASTB_v, aa64_sve, do_last_fp, a, true) in TRANS_FEAT()
2640 static bool do_last_general(DisasContext *s, arg_rpr_esz *a, bool before) in TRANS_FEAT()
2643 TCGv_i64 val = do_last_scalar(s, a->esz, a->pg, a->rn, before); in TRANS_FEAT()
2644 tcg_gen_mov_i64(cpu_reg(s, a->rd), val); in TRANS_FEAT()
2649 TRANS_FEAT(LASTA_r, aa64_sve, do_last_general, a, false) in TRANS_FEAT() argument
2650 TRANS_FEAT(LASTB_r, aa64_sve, do_last_general, a, true) in TRANS_FEAT()
2652 static bool trans_CPY_m_r(DisasContext *s, arg_rpr_esz *a) in TRANS_FEAT()
2658 do_cpy_m(s, a->esz, a->rd, a->rd, a->pg, cpu_reg_sp(s, a->rn)); in TRANS_FEAT()
2663 static bool trans_CPY_m_v(DisasContext *s, arg_rpr_esz *a) in trans_CPY_m_v() argument
2669 int ofs = vec_reg_offset(s, a->rn, 0, a->esz); in trans_CPY_m_v()
2670 TCGv_i64 t = load_esz(tcg_env, ofs, a->esz); in trans_CPY_m_v()
2671 do_cpy_m(s, a->esz, a->rd, a->rd, a->pg, t); in trans_CPY_m_v()
2680 TRANS_FEAT(REVB, aa64_sve, gen_gvec_ool_arg_zpz, revb_fns[a->esz], a, 0)
2685 TRANS_FEAT(REVH, aa64_sve, gen_gvec_ool_arg_zpz, revh_fns[a->esz], a, 0)
2688 a->esz == 3 ? gen_helper_sve_revw_d : NULL, a, 0)
2690 TRANS_FEAT(REVD, aa64_sme, gen_gvec_ool_arg_zpz, gen_helper_sme_revd_q, a, 0)
2693 gen_helper_sve_splice, a, a->esz)
2696 a->rd, a->rn, (a->rn + 1) % 32, a->pg, a->esz)
2702 static bool do_ppzz_flags(DisasContext *s, arg_rprr_esz *a, in do_ppzz_flags() argument
2723 tcg_gen_addi_ptr(pd, tcg_env, pred_full_reg_offset(s, a->rd)); in do_ppzz_flags()
2724 tcg_gen_addi_ptr(zn, tcg_env, vec_full_reg_offset(s, a->rn)); in do_ppzz_flags()
2725 tcg_gen_addi_ptr(zm, tcg_env, vec_full_reg_offset(s, a->rm)); in do_ppzz_flags()
2726 tcg_gen_addi_ptr(pg, tcg_env, pred_full_reg_offset(s, a->pg)); in do_ppzz_flags()
2740 a, name##_ppzz_fns[a->esz])
2757 a, name##_ppzw_fns[a->esz])
2776 static bool do_ppzi_flags(DisasContext *s, arg_rpri_esz *a,
2796 tcg_gen_addi_ptr(pd, tcg_env, pred_full_reg_offset(s, a->rd));
2797 tcg_gen_addi_ptr(zn, tcg_env, vec_full_reg_offset(s, a->rn));
2798 tcg_gen_addi_ptr(pg, tcg_env, pred_full_reg_offset(s, a->pg));
2800 gen_fn(t, pd, zn, pg, tcg_constant_i32(simd_desc(vsz, vsz, a->imm)));
2811 TRANS_FEAT(NAME##_ppzi, aa64_sve, do_ppzi_flags, a, \
2812 name##_ppzi_fns[a->esz])
2831 static bool do_brk3(DisasContext *s, arg_rprr_s *a, in DO_PPZI()
2847 tcg_gen_addi_ptr(d, tcg_env, pred_full_reg_offset(s, a->rd)); in DO_PPZI()
2848 tcg_gen_addi_ptr(n, tcg_env, pred_full_reg_offset(s, a->rn)); in DO_PPZI()
2849 tcg_gen_addi_ptr(m, tcg_env, pred_full_reg_offset(s, a->rm)); in DO_PPZI()
2850 tcg_gen_addi_ptr(g, tcg_env, pred_full_reg_offset(s, a->pg)); in DO_PPZI()
2852 if (a->s) { in DO_PPZI()
2862 static bool do_brk2(DisasContext *s, arg_rpr_s *a, in do_brk2() argument
2877 tcg_gen_addi_ptr(d, tcg_env, pred_full_reg_offset(s, a->rd)); in do_brk2()
2878 tcg_gen_addi_ptr(n, tcg_env, pred_full_reg_offset(s, a->rn)); in do_brk2()
2879 tcg_gen_addi_ptr(g, tcg_env, pred_full_reg_offset(s, a->pg)); in do_brk2()
2881 if (a->s) { in do_brk2()
2891 TRANS_FEAT(BRKPA, aa64_sve, do_brk3, a, in TRANS_FEAT() argument
2893 TRANS_FEAT(BRKPB, aa64_sve, do_brk3, a, in TRANS_FEAT()
2896 TRANS_FEAT(BRKA_m, aa64_sve, do_brk2, a, in TRANS_FEAT()
2898 TRANS_FEAT(BRKB_m, aa64_sve, do_brk2, a, in TRANS_FEAT()
2901 TRANS_FEAT(BRKA_z, aa64_sve, do_brk2, a, in TRANS_FEAT()
2903 TRANS_FEAT(BRKB_z, aa64_sve, do_brk2, a, in TRANS_FEAT()
2906 TRANS_FEAT(BRKN, aa64_sve, do_brk2, a, in TRANS_FEAT()
2949 static bool trans_CNTP(DisasContext *s, arg_CNTP *a) in trans_CNTP() argument
2955 do_cntp(s, cpu_reg(s, a->rd), a->esz, a->rn, a->pg); in trans_CNTP()
2960 static bool trans_INCDECP_r(DisasContext *s, arg_incdec_pred *a) in trans_INCDECP_r() argument
2966 TCGv_i64 reg = cpu_reg(s, a->rd); in trans_INCDECP_r()
2969 do_cntp(s, val, a->esz, a->pg, a->pg); in trans_INCDECP_r()
2970 if (a->d) { in trans_INCDECP_r()
2979 static bool trans_INCDECP_z(DisasContext *s, arg_incdec2_pred *a) in trans_INCDECP_z() argument
2981 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) { in trans_INCDECP_z()
2987 GVecGen2sFn *gvec_fn = a->d ? tcg_gen_gvec_subs : tcg_gen_gvec_adds; in trans_INCDECP_z()
2989 do_cntp(s, val, a->esz, a->pg, a->pg); in trans_INCDECP_z()
2990 gvec_fn(a->esz, vec_full_reg_offset(s, a->rd), in trans_INCDECP_z()
2991 vec_full_reg_offset(s, a->rn), val, vsz, vsz); in trans_INCDECP_z()
2996 static bool trans_SINCDECP_r_32(DisasContext *s, arg_incdec_pred *a) in trans_SINCDECP_r_32() argument
3002 TCGv_i64 reg = cpu_reg(s, a->rd); in trans_SINCDECP_r_32()
3005 do_cntp(s, val, a->esz, a->pg, a->pg); in trans_SINCDECP_r_32()
3006 do_sat_addsub_32(reg, val, a->u, a->d); in trans_SINCDECP_r_32()
3011 static bool trans_SINCDECP_r_64(DisasContext *s, arg_incdec_pred *a) in trans_SINCDECP_r_64() argument
3017 TCGv_i64 reg = cpu_reg(s, a->rd); in trans_SINCDECP_r_64()
3020 do_cntp(s, val, a->esz, a->pg, a->pg); in trans_SINCDECP_r_64()
3021 do_sat_addsub_64(reg, val, a->u, a->d); in trans_SINCDECP_r_64()
3026 static bool trans_SINCDECP_z(DisasContext *s, arg_incdec2_pred *a) in trans_SINCDECP_z() argument
3028 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) { in trans_SINCDECP_z()
3033 do_cntp(s, val, a->esz, a->pg, a->pg); in trans_SINCDECP_z()
3034 do_sat_addsub_vec(s, a->esz, a->rd, a->rn, val, a->u, a->d); in trans_SINCDECP_z()
3043 static bool trans_CTERM(DisasContext *s, arg_CTERM *a) in trans_CTERM() argument
3052 TCGCond cond = (a->ne ? TCG_COND_NE : TCG_COND_EQ); in trans_CTERM()
3053 TCGv_i64 rn = read_cpu_reg(s, a->rn, a->sf); in trans_CTERM()
3054 TCGv_i64 rm = read_cpu_reg(s, a->rm, a->sf); in trans_CTERM()
3070 static bool trans_WHILE(DisasContext *s, arg_WHILE *a) in trans_WHILE() argument
3080 bool eq = a->eq == a->lt; in trans_WHILE()
3083 if (a->lt in trans_WHILE()
3092 op0 = read_cpu_reg(s, a->rn, 1); in trans_WHILE()
3093 op1 = read_cpu_reg(s, a->rm, 1); in trans_WHILE()
3095 if (!a->sf) { in trans_WHILE()
3096 if (a->u) { in trans_WHILE()
3111 if (a->lt) { in trans_WHILE()
3113 if (a->u) { in trans_WHILE()
3114 maxval = a->sf ? UINT64_MAX : UINT32_MAX; in trans_WHILE()
3117 maxval = a->sf ? INT64_MAX : INT32_MAX; in trans_WHILE()
3122 if (a->u) { in trans_WHILE()
3126 maxval = a->sf ? INT64_MIN : INT32_MIN; in trans_WHILE()
3131 tmax = tcg_constant_i64(vsz >> a->esz); in trans_WHILE()
3161 tcg_gen_shli_i32(t2, t2, a->esz); in trans_WHILE()
3164 desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz); in trans_WHILE()
3167 tcg_gen_addi_ptr(ptr, tcg_env, pred_full_reg_offset(s, a->rd)); in trans_WHILE()
3169 if (a->lt) { in trans_WHILE()
3178 static bool trans_WHILE_ptr(DisasContext *s, arg_WHILE_ptr *a) in trans_WHILE_ptr() argument
3193 op0 = read_cpu_reg(s, a->rn, 1); in trans_WHILE_ptr()
3194 op1 = read_cpu_reg(s, a->rm, 1); in trans_WHILE_ptr()
3199 if (a->rw) { in trans_WHILE_ptr()
3207 tcg_gen_andi_i64(diff, diff, -1 << a->esz); in trans_WHILE_ptr()
3214 tcg_gen_andi_i64(diff, diff, -1 << a->esz); in trans_WHILE_ptr()
3227 desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz); in trans_WHILE_ptr()
3230 tcg_gen_addi_ptr(ptr, tcg_env, pred_full_reg_offset(s, a->rd)); in trans_WHILE_ptr()
3241 static bool trans_FDUP(DisasContext *s, arg_FDUP *a) in trans_FDUP() argument
3243 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) { in trans_FDUP()
3248 int dofs = vec_full_reg_offset(s, a->rd); in trans_FDUP()
3252 imm = vfp_expand_imm(a->esz, a->imm); in trans_FDUP()
3253 tcg_gen_gvec_dup_imm(a->esz, dofs, vsz, vsz, imm); in trans_FDUP()
3258 static bool trans_DUP_i(DisasContext *s, arg_DUP_i *a) in trans_DUP_i() argument
3265 int dofs = vec_full_reg_offset(s, a->rd); in trans_DUP_i()
3266 tcg_gen_gvec_dup_imm(a->esz, dofs, vsz, vsz, a->imm); in trans_DUP_i()
3271 TRANS_FEAT(ADD_zzi, aa64_sve, gen_gvec_fn_arg_zzi, tcg_gen_gvec_addi, a) in TRANS_FEAT() argument
3273 static bool trans_SUB_zzi(DisasContext *s, arg_rri_esz *a) in TRANS_FEAT()
3275 a->imm = -a->imm; in TRANS_FEAT()
3276 return trans_ADD_zzi(s, a); in TRANS_FEAT()
3279 static bool trans_SUBR_zzi(DisasContext *s, arg_rri_esz *a) in trans_SUBR_zzi() argument
3315 tcg_gen_gvec_2s(vec_full_reg_offset(s, a->rd), in trans_SUBR_zzi()
3316 vec_full_reg_offset(s, a->rn), in trans_SUBR_zzi()
3317 vsz, vsz, tcg_constant_i64(a->imm), &op[a->esz]); in trans_SUBR_zzi()
3322 TRANS_FEAT(MUL_zzi, aa64_sve, gen_gvec_fn_arg_zzi, tcg_gen_gvec_muli, a) in TRANS_FEAT() argument
3324 static bool do_zzi_sat(DisasContext *s, arg_rri_esz *a, bool u, bool d) in TRANS_FEAT()
3327 do_sat_addsub_vec(s, a->esz, a->rd, a->rn, in TRANS_FEAT()
3328 tcg_constant_i64(a->imm), u, d); in TRANS_FEAT()
3333 TRANS_FEAT(SQADD_zzi, aa64_sve, do_zzi_sat, a, false, false) in TRANS_FEAT() argument
3334 TRANS_FEAT(UQADD_zzi, aa64_sve, do_zzi_sat, a, true, false) in TRANS_FEAT()
3335 TRANS_FEAT(SQSUB_zzi, aa64_sve, do_zzi_sat, a, false, true) in TRANS_FEAT()
3336 TRANS_FEAT(UQSUB_zzi, aa64_sve, do_zzi_sat, a, true, true) in TRANS_FEAT()
3338 static bool do_zzi_ool(DisasContext *s, arg_rri_esz *a, gen_helper_gvec_2i *fn) in TRANS_FEAT()
3342 tcg_gen_gvec_2i_ool(vec_full_reg_offset(s, a->rd), in TRANS_FEAT()
3343 vec_full_reg_offset(s, a->rn), in TRANS_FEAT()
3344 tcg_constant_i64(a->imm), vsz, vsz, 0, fn); in TRANS_FEAT()
3354 TRANS_FEAT(NAME##_zzi, aa64_sve, do_zzi_ool, a, name##i_fns[a->esz])
3368 dot_fns[a->u][a->sz], a->rd, a->rn, a->rm, a->ra, 0)
3375 gen_helper_gvec_sdot_idx_b, a) in TRANS_FEAT()
3377 gen_helper_gvec_sdot_idx_h, a) in TRANS_FEAT()
3379 gen_helper_gvec_udot_idx_b, a) in TRANS_FEAT()
3381 gen_helper_gvec_udot_idx_h, a) in TRANS_FEAT()
3384 gen_helper_gvec_sudot_idx_b, a) in TRANS_FEAT()
3386 gen_helper_gvec_usdot_idx_b, a) in TRANS_FEAT()
3390 a->rd, a->rn, a->rm, a->index) in TRANS_FEAT()
3408 a->rd, a->rn, a->rm, (a->index << 1) | TOP) in TRANS_FEAT()
3428 TRANS_FEAT(NAME, aa64_sve2, gen_gvec_ool_arg_zzxz, FUNC, a) in TRANS_FEAT()
3450 a->rd, a->rn, a->rm, a->ra, (a->index << 1) | TOP) in TRANS_FEAT()
3486 a->rd, a->rn, a->rm, a->ra, (a->index << 2) | a->rot) in TRANS_FEAT()
3503 static bool do_FMLA_zzxz(DisasContext *s, arg_rrxr_esz *a, bool sub) in TRANS_FEAT()
3511 return gen_gvec_fpst_zzzz(s, fns[a->esz], a->rd, a->rn, a->rm, a->ra, in TRANS_FEAT()
3512 (a->index << 1) | sub, in TRANS_FEAT()
3513 a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR); in TRANS_FEAT()
3516 TRANS_FEAT(FMLA_zzxz, aa64_sve, do_FMLA_zzxz, a, false)
3517 TRANS_FEAT(FMLS_zzxz, aa64_sve, do_FMLA_zzxz, a, true)
3528 fmul_idx_fns[a->esz], a->rd, a->rn, a->rm, a->index,
3529 a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR)
3538 static bool do_reduce(DisasContext *s, arg_rpr_esz *a, in do_reduce() argument
3560 tcg_gen_addi_ptr(t_zn, tcg_env, vec_full_reg_offset(s, a->rn)); in do_reduce()
3561 tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, a->pg)); in do_reduce()
3562 status = fpstatus_ptr(a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR); in do_reduce()
3566 write_fp_dreg(s, a->rd, temp); in do_reduce()
3575 TRANS_FEAT(NAME, aa64_sve, do_reduce, a, name##_fns[a->esz])
3593 TRANS_FEAT(FRECPE, aa64_sve, gen_gvec_fpst_arg_zz, frecpe_fns[a->esz], a, 0)
3599 TRANS_FEAT(FRSQRTE, aa64_sve, gen_gvec_fpst_arg_zz, frsqrte_fns[a->esz], a, 0)
3605 static bool do_ppz_fp(DisasContext *s, arg_rpr_esz *a, in do_ppz_fp() argument
3614 fpstatus_ptr(a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR); in do_ppz_fp()
3616 tcg_gen_gvec_3_ptr(pred_full_reg_offset(s, a->rd), in do_ppz_fp()
3617 vec_full_reg_offset(s, a->rn), in do_ppz_fp()
3618 pred_full_reg_offset(s, a->pg), in do_ppz_fp()
3629 TRANS_FEAT(NAME, aa64_sve, do_ppz_fp, a, name##_fns[a->esz])
3649 ftmad_fns[a->esz], a->rd, a->rn, a->rm, a->imm,
3650 a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR)
3656 static bool trans_FADDA(DisasContext *s, arg_rprr_esz *a) in trans_FADDA() argument
3670 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) { in trans_FADDA()
3678 t_val = load_esz(tcg_env, vec_reg_offset(s, a->rn, 0, a->esz), a->esz); in trans_FADDA()
3681 tcg_gen_addi_ptr(t_rm, tcg_env, vec_full_reg_offset(s, a->rm)); in trans_FADDA()
3682 tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, a->pg)); in trans_FADDA()
3683 t_fpst = fpstatus_ptr(a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR); in trans_FADDA()
3686 fns[a->esz - 1](t_val, t_val, t_rm, t_pg, t_fpst, t_desc); in trans_FADDA()
3688 write_fp_dreg(s, a->rd, t_val); in trans_FADDA()
3701 TRANS_FEAT(NAME, aa64_sve, gen_gvec_fpst_arg_zzz, name##_fns[a->esz], a, 0)
3716 ftsmul_fns[a->esz], a, 0)
3727 TRANS_FEAT(NAME, FEAT, gen_gvec_fpst_arg_zpzz, name##_zpzz_fns[a->esz], a)
3763 static bool do_fp_imm(DisasContext *s, arg_rpri_esz *a, uint64_t imm, in do_fp_imm() argument
3770 do_fp_scalar(s, a->rd, a->rn, a->pg, a->esz == MO_16, in do_fp_imm()
3788 TRANS_FEAT(NAME##_zpzi, aa64_sve, do_fp_imm, a, \
3789 name##_const[a->esz][a->imm], name##_fns[a->esz])
3802 static bool do_fp_cmp(DisasContext *s, arg_rprr_esz *a, in DO_FP_IMM()
3810 TCGv_ptr status = fpstatus_ptr(a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR); in DO_FP_IMM()
3811 tcg_gen_gvec_4_ptr(pred_full_reg_offset(s, a->rd), in DO_FP_IMM()
3812 vec_full_reg_offset(s, a->rn), in DO_FP_IMM()
3813 vec_full_reg_offset(s, a->rm), in DO_FP_IMM()
3814 pred_full_reg_offset(s, a->pg), in DO_FP_IMM()
3825 TRANS_FEAT(NAME##_ppzz, aa64_sve, do_fp_cmp, a, name##_fns[a->esz])
3841 TRANS_FEAT(FCADD, aa64_sve, gen_gvec_fpst_zzzp, fcadd_fns[a->esz],
3842 a->rd, a->rn, a->rm, a->pg, a->rot,
3843 a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR)
3850 TRANS_FEAT(NAME, aa64_sve, gen_gvec_fpst_zzzzp, name##_fns[a->esz], \
3851 a->rd, a->rn, a->rm, a->ra, a->pg, 0, \
3852 a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR)
3865 TRANS_FEAT(FCMLA_zpzzz, aa64_sve, gen_gvec_fpst_zzzzp, fcmla_fns[a->esz],
3866 a->rd, a->rn, a->rm, a->ra, a->pg, a->rot,
3867 a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR)
3872 TRANS_FEAT(FCMLA_zzxz, aa64_sve, gen_gvec_fpst_zzzz, fcmla_idx_fns[a->esz],
3873 a->rd, a->rn, a->rm, a->ra, a->index * 4 + a->rot,
3874 a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR)
3881 gen_helper_sve_fcvt_sh, a, 0, FPST_FPCR)
3883 gen_helper_sve_fcvt_hs, a, 0, FPST_FPCR)
3886 gen_helper_sve_bfcvt, a, 0, FPST_FPCR)
3889 gen_helper_sve_fcvt_dh, a, 0, FPST_FPCR)
3891 gen_helper_sve_fcvt_hd, a, 0, FPST_FPCR)
3893 gen_helper_sve_fcvt_ds, a, 0, FPST_FPCR)
3895 gen_helper_sve_fcvt_sd, a, 0, FPST_FPCR)
3898 gen_helper_sve_fcvtzs_hh, a, 0, FPST_FPCR_F16)
3900 gen_helper_sve_fcvtzu_hh, a, 0, FPST_FPCR_F16)
3902 gen_helper_sve_fcvtzs_hs, a, 0, FPST_FPCR_F16)
3904 gen_helper_sve_fcvtzu_hs, a, 0, FPST_FPCR_F16)
3906 gen_helper_sve_fcvtzs_hd, a, 0, FPST_FPCR_F16)
3908 gen_helper_sve_fcvtzu_hd, a, 0, FPST_FPCR_F16)
3911 gen_helper_sve_fcvtzs_ss, a, 0, FPST_FPCR)
3913 gen_helper_sve_fcvtzu_ss, a, 0, FPST_FPCR)
3915 gen_helper_sve_fcvtzs_sd, a, 0, FPST_FPCR)
3917 gen_helper_sve_fcvtzu_sd, a, 0, FPST_FPCR)
3919 gen_helper_sve_fcvtzs_ds, a, 0, FPST_FPCR)
3921 gen_helper_sve_fcvtzu_ds, a, 0, FPST_FPCR)
3924 gen_helper_sve_fcvtzs_dd, a, 0, FPST_FPCR)
3926 gen_helper_sve_fcvtzu_dd, a, 0, FPST_FPCR)
3934 TRANS_FEAT(FRINTI, aa64_sve, gen_gvec_fpst_arg_zpz, frint_fns[a->esz],
3935 a, 0, a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR)
3943 TRANS_FEAT(FRINTX, aa64_sve, gen_gvec_fpst_arg_zpz, frintx_fns[a->esz],
3944 a, 0, a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
3946 static bool do_frint_mode(DisasContext *s, arg_rpr_esz *a, in do_frint_mode() argument
3961 status = fpstatus_ptr(a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR); in do_frint_mode()
3964 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, a->rd), in do_frint_mode()
3965 vec_full_reg_offset(s, a->rn), in do_frint_mode()
3966 pred_full_reg_offset(s, a->pg), in do_frint_mode()
3973 TRANS_FEAT(FRINTN, aa64_sve, do_frint_mode, a,
3974 FPROUNDING_TIEEVEN, frint_fns[a->esz])
3975 TRANS_FEAT(FRINTP, aa64_sve, do_frint_mode, a,
3976 FPROUNDING_POSINF, frint_fns[a->esz])
3977 TRANS_FEAT(FRINTM, aa64_sve, do_frint_mode, a,
3978 FPROUNDING_NEGINF, frint_fns[a->esz])
3979 TRANS_FEAT(FRINTZ, aa64_sve, do_frint_mode, a,
3980 FPROUNDING_ZERO, frint_fns[a->esz])
3981 TRANS_FEAT(FRINTA, aa64_sve, do_frint_mode, a,
3982 FPROUNDING_TIEAWAY, frint_fns[a->esz])
3988 TRANS_FEAT(FRECPX, aa64_sve, gen_gvec_fpst_arg_zpz, frecpx_fns[a->esz],
3989 a, 0, a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR)
3995 TRANS_FEAT(FSQRT, aa64_sve, gen_gvec_fpst_arg_zpz, fsqrt_fns[a->esz],
3996 a, 0, a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR)
3999 gen_helper_sve_scvt_hh, a, 0, FPST_FPCR_F16)
4001 gen_helper_sve_scvt_sh, a, 0, FPST_FPCR_F16)
4003 gen_helper_sve_scvt_dh, a, 0, FPST_FPCR_F16)
4006 gen_helper_sve_scvt_ss, a, 0, FPST_FPCR)
4008 gen_helper_sve_scvt_ds, a, 0, FPST_FPCR)
4011 gen_helper_sve_scvt_sd, a, 0, FPST_FPCR)
4013 gen_helper_sve_scvt_dd, a, 0, FPST_FPCR)
4016 gen_helper_sve_ucvt_hh, a, 0, FPST_FPCR_F16)
4018 gen_helper_sve_ucvt_sh, a, 0, FPST_FPCR_F16)
4020 gen_helper_sve_ucvt_dh, a, 0, FPST_FPCR_F16)
4023 gen_helper_sve_ucvt_ss, a, 0, FPST_FPCR)
4025 gen_helper_sve_ucvt_ds, a, 0, FPST_FPCR)
4027 gen_helper_sve_ucvt_sd, a, 0, FPST_FPCR)
4030 gen_helper_sve_ucvt_dd, a, 0, FPST_FPCR)
4239 static bool trans_LDR_zri(DisasContext *s, arg_rri *a) in trans_LDR_zri() argument
4246 int off = vec_full_reg_offset(s, a->rd); in trans_LDR_zri()
4247 gen_sve_ldr(s, tcg_env, off, size, a->rn, a->imm * size); in trans_LDR_zri()
4252 static bool trans_LDR_pri(DisasContext *s, arg_rri *a) in trans_LDR_pri() argument
4259 int off = pred_full_reg_offset(s, a->rd); in trans_LDR_pri()
4260 gen_sve_ldr(s, tcg_env, off, size, a->rn, a->imm * size); in trans_LDR_pri()
4265 static bool trans_STR_zri(DisasContext *s, arg_rri *a) in trans_STR_zri() argument
4272 int off = vec_full_reg_offset(s, a->rd); in trans_STR_zri()
4273 gen_sve_str(s, tcg_env, off, size, a->rn, a->imm * size); in trans_STR_zri()
4278 static bool trans_STR_pri(DisasContext *s, arg_rri *a) in trans_STR_pri() argument
4285 int off = pred_full_reg_offset(s, a->rd); in trans_STR_pri()
4286 gen_sve_str(s, tcg_env, off, size, a->rn, a->imm * size); in trans_STR_pri()
4493 static bool trans_LD_zprr(DisasContext *s, arg_rprr_load *a) in trans_LD_zprr() argument
4495 if (a->rm == 31 || !dc_isar_feature(aa64_sve, s)) { in trans_LD_zprr()
4500 tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), dtype_msz(a->dtype)); in trans_LD_zprr()
4501 tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn)); in trans_LD_zprr()
4502 do_ld_zpa(s, a->rd, a->pg, addr, a->dtype, a->nreg); in trans_LD_zprr()
4507 static bool trans_LD_zpri(DisasContext *s, arg_rpri_load *a) in trans_LD_zpri() argument
4514 int elements = vsz >> dtype_esz[a->dtype]; in trans_LD_zpri()
4517 tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn), in trans_LD_zpri()
4518 (a->imm * elements * (a->nreg + 1)) in trans_LD_zpri()
4519 << dtype_msz(a->dtype)); in trans_LD_zpri()
4520 do_ld_zpa(s, a->rd, a->pg, addr, a->dtype, a->nreg); in trans_LD_zpri()
4525 static bool trans_LDFF1_zprr(DisasContext *s, arg_rprr_load *a) in trans_LDFF1_zprr() argument
4619 tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), dtype_msz(a->dtype)); in trans_LDFF1_zprr()
4620 tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn)); in trans_LDFF1_zprr()
4621 do_mem_zpa(s, a->rd, a->pg, addr, a->dtype, 1, false, in trans_LDFF1_zprr()
4622 fns[s->mte_active[0]][s->be_data == MO_BE][a->dtype]); in trans_LDFF1_zprr()
4627 static bool trans_LDNF1_zpri(DisasContext *s, arg_rpri_load *a) in trans_LDNF1_zpri() argument
4721 int elements = vsz >> dtype_esz[a->dtype]; in trans_LDNF1_zpri()
4722 int off = (a->imm * elements) << dtype_msz(a->dtype); in trans_LDNF1_zpri()
4725 tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn), off); in trans_LDNF1_zpri()
4726 do_mem_zpa(s, a->rd, a->pg, addr, a->dtype, 1, false, in trans_LDNF1_zpri()
4727 fns[s->mte_active[0]][s->be_data == MO_BE][a->dtype]); in trans_LDNF1_zpri()
4777 static bool trans_LD1RQ_zprr(DisasContext *s, arg_rprr_load *a) in trans_LD1RQ_zprr() argument
4779 if (a->rm == 31 || !dc_isar_feature(aa64_sve, s)) { in trans_LD1RQ_zprr()
4783 int msz = dtype_msz(a->dtype); in trans_LD1RQ_zprr()
4785 tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), msz); in trans_LD1RQ_zprr()
4786 tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn)); in trans_LD1RQ_zprr()
4787 do_ldrq(s, a->rd, a->pg, addr, a->dtype); in trans_LD1RQ_zprr()
4792 static bool trans_LD1RQ_zpri(DisasContext *s, arg_rpri_load *a) in trans_LD1RQ_zpri() argument
4799 tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn), a->imm * 16); in trans_LD1RQ_zpri()
4800 do_ldrq(s, a->rd, a->pg, addr, a->dtype); in trans_LD1RQ_zpri()
4870 static bool trans_LD1RO_zprr(DisasContext *s, arg_rprr_load *a) in trans_LD1RO_zprr() argument
4875 if (a->rm == 31) { in trans_LD1RO_zprr()
4881 tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), dtype_msz(a->dtype)); in trans_LD1RO_zprr()
4882 tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn)); in trans_LD1RO_zprr()
4883 do_ldro(s, a->rd, a->pg, addr, a->dtype); in trans_LD1RO_zprr()
4888 static bool trans_LD1RO_zpri(DisasContext *s, arg_rpri_load *a) in trans_LD1RO_zpri() argument
4896 tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn), a->imm * 32); in trans_LD1RO_zpri()
4897 do_ldro(s, a->rd, a->pg, addr, a->dtype); in trans_LD1RO_zpri()
4903 static bool trans_LD1R_zpri(DisasContext *s, arg_rpri_load *a) in trans_LD1R_zpri() argument
4907 unsigned esz = dtype_esz[a->dtype]; in trans_LD1R_zpri()
4908 unsigned msz = dtype_msz(a->dtype); in trans_LD1R_zpri()
4929 tcg_gen_ld_i64(temp, tcg_env, pred_full_reg_offset(s, a->pg)); in trans_LD1R_zpri()
4934 find_last_active(s, t32, esz, a->pg); in trans_LD1R_zpri()
4940 tcg_gen_addi_i64(temp, cpu_reg_sp(s, a->rn), a->imm << msz); in trans_LD1R_zpri()
4942 memop = finalize_memop(s, dtype_mop[a->dtype]); in trans_LD1R_zpri()
4947 tcg_gen_gvec_dup_i64(esz, vec_full_reg_offset(s, a->rd), in trans_LD1R_zpri()
4952 return do_movz_zpz(s, a->rd, a->rd, a->pg, esz, false); in trans_LD1R_zpri()
5078 static bool trans_ST_zprr(DisasContext *s, arg_rprr_store *a) in trans_ST_zprr() argument
5083 if (a->rm == 31 || a->msz > a->esz) { in trans_ST_zprr()
5088 tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), a->msz); in trans_ST_zprr()
5089 tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn)); in trans_ST_zprr()
5090 do_st_zpa(s, a->rd, a->pg, addr, a->msz, a->esz, a->nreg); in trans_ST_zprr()
5095 static bool trans_ST_zpri(DisasContext *s, arg_rpri_store *a) in trans_ST_zpri() argument
5100 if (a->msz > a->esz) { in trans_ST_zpri()
5105 int elements = vsz >> a->esz; in trans_ST_zpri()
5108 tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn), in trans_ST_zpri()
5109 (a->imm * elements * (a->nreg + 1)) << a->msz); in trans_ST_zpri()
5110 do_st_zpa(s, a->rd, a->pg, addr, a->msz, a->esz, a->nreg); in trans_ST_zpri()
5464 static bool trans_LD1_zprz(DisasContext *s, arg_LD1_zprz *a) in trans_LD1_zprz() argument
5478 switch (a->esz) { in trans_LD1_zprz()
5480 fn = gather_load_fn32[mte][be][a->ff][a->xs][a->u][a->msz]; in trans_LD1_zprz()
5483 fn = gather_load_fn64[mte][be][a->ff][a->xs][a->u][a->msz]; in trans_LD1_zprz()
5488 do_mem_zpz(s, a->rd, a->pg, a->rm, a->scale * a->msz, in trans_LD1_zprz()
5489 cpu_reg_sp(s, a->rn), a->msz, false, fn); in trans_LD1_zprz()
5493 static bool trans_LD1_zpiz(DisasContext *s, arg_LD1_zpiz *a) in trans_LD1_zpiz() argument
5499 if (a->esz < a->msz || (a->esz == a->msz && !a->u)) { in trans_LD1_zpiz()
5510 switch (a->esz) { in trans_LD1_zpiz()
5512 fn = gather_load_fn32[mte][be][a->ff][0][a->u][a->msz]; in trans_LD1_zpiz()
5515 fn = gather_load_fn64[mte][be][a->ff][2][a->u][a->msz]; in trans_LD1_zpiz()
5523 do_mem_zpz(s, a->rd, a->pg, a->rn, 0, in trans_LD1_zpiz()
5524 tcg_constant_i64(a->imm << a->msz), a->msz, false, fn); in trans_LD1_zpiz()
5528 static bool trans_LDNT1_zprz(DisasContext *s, arg_LD1_zprz *a) in trans_LDNT1_zprz() argument
5534 if (a->esz < a->msz + !a->u) { in trans_LDNT1_zprz()
5545 switch (a->esz) { in trans_LDNT1_zprz()
5547 fn = gather_load_fn32[mte][be][0][0][a->u][a->msz]; in trans_LDNT1_zprz()
5550 fn = gather_load_fn64[mte][be][0][2][a->u][a->msz]; in trans_LDNT1_zprz()
5555 do_mem_zpz(s, a->rd, a->pg, a->rn, 0, in trans_LDNT1_zprz()
5556 cpu_reg(s, a->rm), a->msz, false, fn); in trans_LDNT1_zprz()
5652 static bool trans_ST1_zprz(DisasContext *s, arg_ST1_zprz *a) in trans_ST1_zprz() argument
5658 if (a->esz < a->msz || (a->msz == 0 && a->scale)) { in trans_ST1_zprz()
5668 switch (a->esz) { in trans_ST1_zprz()
5670 fn = scatter_store_fn32[mte][be][a->xs][a->msz]; in trans_ST1_zprz()
5673 fn = scatter_store_fn64[mte][be][a->xs][a->msz]; in trans_ST1_zprz()
5678 do_mem_zpz(s, a->rd, a->pg, a->rm, a->scale * a->msz, in trans_ST1_zprz()
5679 cpu_reg_sp(s, a->rn), a->msz, true, fn); in trans_ST1_zprz()
5683 static bool trans_ST1_zpiz(DisasContext *s, arg_ST1_zpiz *a) in trans_ST1_zpiz() argument
5689 if (a->esz < a->msz) { in trans_ST1_zpiz()
5700 switch (a->esz) { in trans_ST1_zpiz()
5702 fn = scatter_store_fn32[mte][be][0][a->msz]; in trans_ST1_zpiz()
5705 fn = scatter_store_fn64[mte][be][2][a->msz]; in trans_ST1_zpiz()
5713 do_mem_zpz(s, a->rd, a->pg, a->rn, 0, in trans_ST1_zpiz()
5714 tcg_constant_i64(a->imm << a->msz), a->msz, true, fn); in trans_ST1_zpiz()
5718 static bool trans_STNT1_zprz(DisasContext *s, arg_ST1_zprz *a) in trans_STNT1_zprz() argument
5724 if (a->esz < a->msz) { in trans_STNT1_zprz()
5735 switch (a->esz) { in trans_STNT1_zprz()
5737 fn = scatter_store_fn32[mte][be][0][a->msz]; in trans_STNT1_zprz()
5740 fn = scatter_store_fn64[mte][be][2][a->msz]; in trans_STNT1_zprz()
5746 do_mem_zpz(s, a->rd, a->pg, a->rn, 0, in trans_STNT1_zprz()
5747 cpu_reg(s, a->rm), a->msz, true, fn); in trans_STNT1_zprz()
5755 static bool trans_PRF(DisasContext *s, arg_PRF *a) in trans_PRF() argument
5765 static bool trans_PRF_rr(DisasContext *s, arg_PRF_rr *a) in trans_PRF_rr() argument
5767 if (a->rm == 31 || !dc_isar_feature(aa64_sve, s)) { in trans_PRF_rr()
5775 static bool trans_PRF_ns(DisasContext *s, arg_PRF_ns *a) in trans_PRF_ns() argument
5800 TRANS_FEAT(MOVPRFX, aa64_sve, do_mov_z, a->rd, a->rn)
5801 TRANS_FEAT(MOVPRFX_m, aa64_sve, do_sel_z, a->rd, a->rn, a->rd, a->pg, a->esz)
5802 TRANS_FEAT(MOVPRFX_z, aa64_sve, do_movz_zpz, a->rd, a->rn, a->pg, a->esz, false)
5808 TRANS_FEAT(MUL_zzz, aa64_sve2, gen_gvec_fn_arg_zzz, tcg_gen_gvec_mul, a)
5815 smulh_zzz_fns[a->esz], a, 0)
5822 umulh_zzz_fns[a->esz], a, 0)
5825 gen_helper_gvec_pmul_b, a, 0)
5832 sqdmulh_zzz_fns[a->esz], a, 0)
5839 sqrdmulh_zzz_fns[a->esz], a, 0)
5850 sadlp_fns[a->esz], a, 0)
5857 uadlp_fns[a->esz], a, 0)
5864 a->esz == 2 ? gen_helper_sve2_urecpe_s : NULL, a, 0)
5867 a->esz == 2 ? gen_helper_sve2_ursqrte_s : NULL, a, 0)
5873 TRANS_FEAT(SQABS, aa64_sve2, gen_gvec_ool_arg_zpz, sqabs_fns[a->esz], a, 0)
5879 TRANS_FEAT(SQNEG, aa64_sve2, gen_gvec_ool_arg_zpz, sqneg_fns[a->esz], a, 0)
5919 saddl_fns[a->esz], a, 0)
5921 saddl_fns[a->esz], a, 3)
5923 saddl_fns[a->esz], a, 2)
5930 ssubl_fns[a->esz], a, 0)
5932 ssubl_fns[a->esz], a, 3)
5934 ssubl_fns[a->esz], a, 2)
5936 ssubl_fns[a->esz], a, 1)
5943 sabdl_fns[a->esz], a, 0)
5945 sabdl_fns[a->esz], a, 3)
5952 uaddl_fns[a->esz], a, 0)
5954 uaddl_fns[a->esz], a, 3)
5961 usubl_fns[a->esz], a, 0)
5963 usubl_fns[a->esz], a, 3)
5970 uabdl_fns[a->esz], a, 0)
5972 uabdl_fns[a->esz], a, 3)
5979 sqdmull_fns[a->esz], a, 0)
5981 sqdmull_fns[a->esz], a, 3)
5988 smull_fns[a->esz], a, 0)
5990 smull_fns[a->esz], a, 3)
5997 umull_fns[a->esz], a, 0)
5999 umull_fns[a->esz], a, 3)
6005 TRANS_FEAT(EORBT, aa64_sve2, gen_gvec_ool_arg_zzz, eoril_fns[a->esz], a, 2)
6006 TRANS_FEAT(EORTB, aa64_sve2, gen_gvec_ool_arg_zzz, eoril_fns[a->esz], a, 1)
6008 static bool do_trans_pmull(DisasContext *s, arg_rrr_esz *a, bool sel) in do_trans_pmull() argument
6015 if (a->esz == 0) { in do_trans_pmull()
6023 return gen_gvec_ool_arg_zzz(s, fns[a->esz], a, sel); in do_trans_pmull()
6026 TRANS_FEAT(PMULLB, aa64_sve2, do_trans_pmull, a, false)
6027 TRANS_FEAT(PMULLT, aa64_sve2, do_trans_pmull, a, true)
6033 TRANS_FEAT(SADDWB, aa64_sve2, gen_gvec_ool_arg_zzz, saddw_fns[a->esz], a, 0)
6034 TRANS_FEAT(SADDWT, aa64_sve2, gen_gvec_ool_arg_zzz, saddw_fns[a->esz], a, 1)
6040 TRANS_FEAT(SSUBWB, aa64_sve2, gen_gvec_ool_arg_zzz, ssubw_fns[a->esz], a, 0)
6041 TRANS_FEAT(SSUBWT, aa64_sve2, gen_gvec_ool_arg_zzz, ssubw_fns[a->esz], a, 1)
6047 TRANS_FEAT(UADDWB, aa64_sve2, gen_gvec_ool_arg_zzz, uaddw_fns[a->esz], a, 0)
6048 TRANS_FEAT(UADDWT, aa64_sve2, gen_gvec_ool_arg_zzz, uaddw_fns[a->esz], a, 1)
6054 TRANS_FEAT(USUBWB, aa64_sve2, gen_gvec_ool_arg_zzz, usubw_fns[a->esz], a, 0)
6055 TRANS_FEAT(USUBWT, aa64_sve2, gen_gvec_ool_arg_zzz, usubw_fns[a->esz], a, 1)
6141 static bool do_shll_tb(DisasContext *s, arg_rri_esz *a, in do_shll_tb() argument
6145 if (a->esz < 0 || a->esz > 2) { in do_shll_tb()
6150 tcg_gen_gvec_2i(vec_full_reg_offset(s, a->rd), in do_shll_tb()
6151 vec_full_reg_offset(s, a->rn), in do_shll_tb()
6152 vsz, vsz, (a->imm << 1) | sel, in do_shll_tb()
6153 &ops[a->esz]); in do_shll_tb()
6175 TRANS_FEAT(SSHLLB, aa64_sve2, do_shll_tb, a, sshll_ops, false)
6176 TRANS_FEAT(SSHLLT, aa64_sve2, do_shll_tb, a, sshll_ops, true)
6198 TRANS_FEAT(USHLLB, aa64_sve2, do_shll_tb, a, ushll_ops, false)
6199 TRANS_FEAT(USHLLT, aa64_sve2, do_shll_tb, a, ushll_ops, true)
6206 bext_fns[a->esz], a, 0)
6213 bdep_fns[a->esz], a, 0)
6220 bgrp_fns[a->esz], a, 0)
6227 cadd_fns[a->esz], a, 0)
6229 cadd_fns[a->esz], a, 1)
6236 sqcadd_fns[a->esz], a, 0)
6238 sqcadd_fns[a->esz], a, 1)
6244 TRANS_FEAT(SABALB, aa64_sve2, gen_gvec_ool_arg_zzzz, sabal_fns[a->esz], a, 0)
6245 TRANS_FEAT(SABALT, aa64_sve2, gen_gvec_ool_arg_zzzz, sabal_fns[a->esz], a, 1)
6251 TRANS_FEAT(UABALB, aa64_sve2, gen_gvec_ool_arg_zzzz, uabal_fns[a->esz], a, 0)
6252 TRANS_FEAT(UABALT, aa64_sve2, gen_gvec_ool_arg_zzzz, uabal_fns[a->esz], a, 1)
6254 static bool do_adcl(DisasContext *s, arg_rrrr_esz *a, bool sel) in do_adcl() argument
6264 return gen_gvec_ool_arg_zzzz(s, fns[a->esz & 1], a, (a->esz & 2) | sel); in do_adcl()
6267 TRANS_FEAT(ADCLB, aa64_sve2, do_adcl, a, false) in TRANS_FEAT() argument
6268 TRANS_FEAT(ADCLT, aa64_sve2, do_adcl, a, true) in TRANS_FEAT()
6270 TRANS_FEAT(SSRA, aa64_sve2, gen_gvec_fn_arg_zzi, gen_gvec_ssra, a) in TRANS_FEAT()
6271 TRANS_FEAT(USRA, aa64_sve2, gen_gvec_fn_arg_zzi, gen_gvec_usra, a) in TRANS_FEAT()
6272 TRANS_FEAT(SRSRA, aa64_sve2, gen_gvec_fn_arg_zzi, gen_gvec_srsra, a) in TRANS_FEAT()
6273 TRANS_FEAT(URSRA, aa64_sve2, gen_gvec_fn_arg_zzi, gen_gvec_ursra, a) in TRANS_FEAT()
6274 TRANS_FEAT(SRI, aa64_sve2, gen_gvec_fn_arg_zzi, gen_gvec_sri, a) in TRANS_FEAT()
6275 TRANS_FEAT(SLI, aa64_sve2, gen_gvec_fn_arg_zzi, gen_gvec_sli, a) in TRANS_FEAT()
6277 TRANS_FEAT(SABA, aa64_sve2, gen_gvec_fn_arg_zzz, gen_gvec_saba, a) in TRANS_FEAT()
6278 TRANS_FEAT(UABA, aa64_sve2, gen_gvec_fn_arg_zzz, gen_gvec_uaba, a) in TRANS_FEAT()
6280 static bool do_narrow_extract(DisasContext *s, arg_rri_esz *a, in TRANS_FEAT()
6283 if (a->esz < 0 || a->esz > MO_32 || a->imm != 0) { in TRANS_FEAT()
6288 tcg_gen_gvec_2(vec_full_reg_offset(s, a->rd), in TRANS_FEAT()
6289 vec_full_reg_offset(s, a->rn), in TRANS_FEAT()
6290 vsz, vsz, &ops[a->esz]); in TRANS_FEAT()
6329 TRANS_FEAT(SQXTNB, aa64_sve2, do_narrow_extract, a, sqxtnb_ops) in TRANS_FEAT() argument
6365 TRANS_FEAT(SQXTNT, aa64_sve2, do_narrow_extract, a, sqxtnt_ops)
6395 TRANS_FEAT(UQXTNB, aa64_sve2, do_narrow_extract, a, uqxtnb_ops) in TRANS_FEAT() argument
6426 TRANS_FEAT(UQXTNT, aa64_sve2, do_narrow_extract, a, uqxtnt_ops)
6458 TRANS_FEAT(SQXTUNB, aa64_sve2, do_narrow_extract, a, sqxtunb_ops) in TRANS_FEAT() argument
6491 TRANS_FEAT(SQXTUNT, aa64_sve2, do_narrow_extract, a, sqxtunt_ops) in TRANS_FEAT() argument
6493 static bool do_shr_narrow(DisasContext *s, arg_rri_esz *a, in TRANS_FEAT()
6496 if (a->esz < 0 || a->esz > MO_32) { in TRANS_FEAT()
6499 assert(a->imm > 0 && a->imm <= (8 << a->esz)); in TRANS_FEAT()
6502 tcg_gen_gvec_2i(vec_full_reg_offset(s, a->rd), in TRANS_FEAT()
6503 vec_full_reg_offset(s, a->rn), in TRANS_FEAT()
6504 vsz, vsz, a->imm, &ops[a->esz]); in TRANS_FEAT()
6562 TRANS_FEAT(SHRNB, aa64_sve2, do_shr_narrow, a, shrnb_ops) in TRANS_FEAT() argument
6623 TRANS_FEAT(SHRNT, aa64_sve2, do_shr_narrow, a, shrnt_ops)
6630 TRANS_FEAT(RSHRNB, aa64_sve2, do_shr_narrow, a, rshrnb_ops)
6637 TRANS_FEAT(RSHRNT, aa64_sve2, do_shr_narrow, a, rshrnt_ops) in TRANS_FEAT() argument
6669 TRANS_FEAT(SQSHRUNB, aa64_sve2, do_shr_narrow, a, sqshrunb_ops) in TRANS_FEAT() argument
6707 TRANS_FEAT(SQSHRUNT, aa64_sve2, do_shr_narrow, a, sqshrunt_ops)
6714 TRANS_FEAT(SQRSHRUNB, aa64_sve2, do_shr_narrow, a, sqrshrunb_ops)
6721 TRANS_FEAT(SQRSHRUNT, aa64_sve2, do_shr_narrow, a, sqrshrunt_ops) in TRANS_FEAT() argument
6757 TRANS_FEAT(SQSHRNB, aa64_sve2, do_shr_narrow, a, sqshrnb_ops) in TRANS_FEAT() argument
6798 TRANS_FEAT(SQSHRNT, aa64_sve2, do_shr_narrow, a, sqshrnt_ops)
6805 TRANS_FEAT(SQRSHRNB, aa64_sve2, do_shr_narrow, a, sqrshrnb_ops)
6812 TRANS_FEAT(SQRSHRNT, aa64_sve2, do_shr_narrow, a, sqrshrnt_ops) in TRANS_FEAT() argument
6842 TRANS_FEAT(UQSHRNB, aa64_sve2, do_shr_narrow, a, uqshrnb_ops) in TRANS_FEAT() argument
6877 TRANS_FEAT(UQSHRNT, aa64_sve2, do_shr_narrow, a, uqshrnt_ops)
6884 TRANS_FEAT(UQRSHRNB, aa64_sve2, do_shr_narrow, a, uqrshrnb_ops)
6891 TRANS_FEAT(UQRSHRNT, aa64_sve2, do_shr_narrow, a, uqrshrnt_ops)
6899 name##_fns[a->esz], a, 0)
6914 TRANS_FEAT_NONSTREAMING(MATCH, aa64_sve2, do_ppzz_flags, a, match_fns[a->esz])
6919 TRANS_FEAT_NONSTREAMING(NMATCH, aa64_sve2, do_ppzz_flags, a, nmatch_fns[a->esz])
6925 histcnt_fns[a->esz], a, 0)
6928 a->esz == 0 ? gen_helper_sve2_histseg : NULL, a, 0)
6941 gen_helper_fmmla_s, a->rd, a->rn, a->rm, a->ra,
6944 gen_helper_fmmla_d, a->rd, a->rn, a->rm, a->ra,
6952 sqdmlal_zzzw_fns[a->esz], a, 0)
6954 sqdmlal_zzzw_fns[a->esz], a, 3)
6956 sqdmlal_zzzw_fns[a->esz], a, 2)
6963 sqdmlsl_zzzw_fns[a->esz], a, 0)
6965 sqdmlsl_zzzw_fns[a->esz], a, 3)
6967 sqdmlsl_zzzw_fns[a->esz], a, 2)
6974 sqrdmlah_fns[a->esz], a, 0)
6981 sqrdmlsh_fns[a->esz], a, 0)
6988 smlal_zzzw_fns[a->esz], a, 0)
6990 smlal_zzzw_fns[a->esz], a, 1)
6997 umlal_zzzw_fns[a->esz], a, 0)
6999 umlal_zzzw_fns[a->esz], a, 1)
7006 smlsl_zzzw_fns[a->esz], a, 0)
7008 smlsl_zzzw_fns[a->esz], a, 1)
7015 umlsl_zzzw_fns[a->esz], a, 0)
7017 umlsl_zzzw_fns[a->esz], a, 1)
7024 cmla_fns[a->esz], a->rd, a->rn, a->rm, a->ra, a->rot)
7030 cdot_fns[a->esz], a->rd, a->rn, a->rm, a->ra, a->rot)
7037 sqrdcmlah_fns[a->esz], a->rd, a->rn, a->rm, a->ra, a->rot)
7040 a->esz == 2 ? gen_helper_gvec_usdot_b : NULL, a, 0)
7043 gen_helper_crypto_aesmc, a->rd, a->rd, 0)
7045 gen_helper_crypto_aesimc, a->rd, a->rd, 0)
7048 gen_helper_crypto_aese, a, 0)
7050 gen_helper_crypto_aesd, a, 0)
7053 gen_helper_crypto_sm4e, a, 0)
7055 gen_helper_crypto_sm4ekey, a, 0)
7058 gen_gvec_rax1, a)
7061 gen_helper_sve2_fcvtnt_sh, a, 0, FPST_FPCR)
7063 gen_helper_sve2_fcvtnt_ds, a, 0, FPST_FPCR)
7066 gen_helper_sve_bfcvtnt, a, 0, FPST_FPCR)
7069 gen_helper_sve2_fcvtlt_hs, a, 0, FPST_FPCR)
7071 gen_helper_sve2_fcvtlt_sd, a, 0, FPST_FPCR)
7073 TRANS_FEAT(FCVTX_ds, aa64_sve2, do_frint_mode, a,
7075 TRANS_FEAT(FCVTXNT_ds, aa64_sve2, do_frint_mode, a,
7082 TRANS_FEAT(FLOGB, aa64_sve2, gen_gvec_fpst_arg_zpz, flogb_fns[a->esz],
7083 a, 0, a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR)
7085 static bool do_FMLAL_zzzw(DisasContext *s, arg_rrrr_esz *a, bool sub, bool sel) in do_FMLAL_zzzw() argument
7088 a->rd, a->rn, a->rm, a->ra, in do_FMLAL_zzzw()
7092 TRANS_FEAT(FMLALB_zzzw, aa64_sve2, do_FMLAL_zzzw, a, false, false) in TRANS_FEAT() argument
7093 TRANS_FEAT(FMLALT_zzzw, aa64_sve2, do_FMLAL_zzzw, a, false, true) in TRANS_FEAT()
7094 TRANS_FEAT(FMLSLB_zzzw, aa64_sve2, do_FMLAL_zzzw, a, true, false) in TRANS_FEAT()
7095 TRANS_FEAT(FMLSLT_zzzw, aa64_sve2, do_FMLAL_zzzw, a, true, true) in TRANS_FEAT()
7097 static bool do_FMLAL_zzxw(DisasContext *s, arg_rrxr_esz *a, bool sub, bool sel) in TRANS_FEAT()
7100 a->rd, a->rn, a->rm, a->ra, in TRANS_FEAT()
7101 (a->index << 2) | (sel << 1) | sub, tcg_env); in TRANS_FEAT()
7104 TRANS_FEAT(FMLALB_zzxw, aa64_sve2, do_FMLAL_zzxw, a, false, false) in TRANS_FEAT() argument
7105 TRANS_FEAT(FMLALT_zzxw, aa64_sve2, do_FMLAL_zzxw, a, false, true) in TRANS_FEAT()
7106 TRANS_FEAT(FMLSLB_zzxw, aa64_sve2, do_FMLAL_zzxw, a, true, false) in TRANS_FEAT()
7107 TRANS_FEAT(FMLSLT_zzxw, aa64_sve2, do_FMLAL_zzxw, a, true, true) in TRANS_FEAT()
7110 gen_helper_gvec_smmla_b, a, 0) in TRANS_FEAT()
7112 gen_helper_gvec_usmmla_b, a, 0) in TRANS_FEAT()
7114 gen_helper_gvec_ummla_b, a, 0) in TRANS_FEAT()
7117 gen_helper_gvec_bfdot, a, 0) in TRANS_FEAT()
7119 gen_helper_gvec_bfdot_idx, a) in TRANS_FEAT()
7122 gen_helper_gvec_bfmmla, a, 0) in TRANS_FEAT()
7124 static bool do_BFMLAL_zzzw(DisasContext *s, arg_rrrr_esz *a, bool sel) in TRANS_FEAT()
7127 a->rd, a->rn, a->rm, a->ra, sel, FPST_FPCR); in TRANS_FEAT()
7130 TRANS_FEAT(BFMLALB_zzzw, aa64_sve_bf16, do_BFMLAL_zzzw, a, false) in TRANS_FEAT() argument
7131 TRANS_FEAT(BFMLALT_zzzw, aa64_sve_bf16, do_BFMLAL_zzzw, a, true) in TRANS_FEAT()
7133 static bool do_BFMLAL_zzxw(DisasContext *s, arg_rrxr_esz *a, bool sel) in TRANS_FEAT()
7136 a->rd, a->rn, a->rm, a->ra, in TRANS_FEAT()
7137 (a->index << 1) | sel, FPST_FPCR); in TRANS_FEAT()
7140 TRANS_FEAT(BFMLALB_zzxw, aa64_sve_bf16, do_BFMLAL_zzxw, a, false) in TRANS_FEAT() argument
7141 TRANS_FEAT(BFMLALT_zzxw, aa64_sve_bf16, do_BFMLAL_zzxw, a, true) in TRANS_FEAT()
7143 static bool trans_PSEL(DisasContext *s, arg_psel *a) in TRANS_FEAT()
7147 int elements = vl >> a->esz; in TRANS_FEAT()
7164 tcg_gen_addi_i64(tmp, cpu_reg(s, a->rv), a->imm); in TRANS_FEAT()
7172 tcg_gen_shli_i64(tmp, tmp, a->esz); in TRANS_FEAT()
7182 tcg_gen_ld8u_i64(tmp, ptr, pred_full_reg_offset(s, a->pm)); in TRANS_FEAT()
7190 tcg_gen_gvec_ands(MO_64, pred_full_reg_offset(s, a->pd), in TRANS_FEAT()
7191 pred_full_reg_offset(s, a->pn), tmp, pl, pl); in TRANS_FEAT()
7195 static void gen_sclamp_i32(TCGv_i32 d, TCGv_i32 n, TCGv_i32 m, TCGv_i32 a) in gen_sclamp_i32() argument
7197 tcg_gen_smax_i32(d, a, n); in gen_sclamp_i32()
7201 static void gen_sclamp_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 a) in gen_sclamp_i64() argument
7203 tcg_gen_smax_i64(d, a, n); in gen_sclamp_i64()
7208 TCGv_vec m, TCGv_vec a) in gen_sclamp_vec() argument
7210 tcg_gen_smax_vec(vece, d, a, n); in gen_sclamp_vec()
7215 uint32_t a, uint32_t oprsz, uint32_t maxsz) in gen_sclamp() argument
7241 tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &ops[vece]); in gen_sclamp()
7244 TRANS_FEAT(SCLAMP, aa64_sme, gen_gvec_fn_arg_zzzz, gen_sclamp, a) in TRANS_FEAT() argument
7246 static void gen_uclamp_i32(TCGv_i32 d, TCGv_i32 n, TCGv_i32 m, TCGv_i32 a) in TRANS_FEAT()
7248 tcg_gen_umax_i32(d, a, n); in TRANS_FEAT()
7252 static void gen_uclamp_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 a) in gen_uclamp_i64() argument
7254 tcg_gen_umax_i64(d, a, n); in gen_uclamp_i64()
7259 TCGv_vec m, TCGv_vec a) in gen_uclamp_vec() argument
7261 tcg_gen_umax_vec(vece, d, a, n); in gen_uclamp_vec()
7266 uint32_t a, uint32_t oprsz, uint32_t maxsz) in gen_uclamp() argument
7292 tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &ops[vece]); in gen_uclamp()
7295 TRANS_FEAT(UCLAMP, aa64_sme, gen_gvec_fn_arg_zzzz, gen_uclamp, a)