1 /* 2 * AArch64 SME translation 3 * 4 * Copyright (c) 2022 Linaro, Ltd 5 * 6 * This library is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Lesser General Public 8 * License as published by the Free Software Foundation; either 9 * version 2.1 of the License, or (at your option) any later version. 10 * 11 * This library is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * Lesser General Public License for more details. 15 * 16 * You should have received a copy of the GNU Lesser General Public 17 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 18 */ 19 20 #include "qemu/osdep.h" 21 #include "translate.h" 22 #include "translate-a64.h" 23 24 /* 25 * Include the generated decoder. 26 */ 27 28 #include "decode-sme.c.inc" 29 30 static bool sme2_zt0_enabled_check(DisasContext *s) 31 { 32 if (!sme_za_enabled_check(s)) { 33 return false; 34 } 35 if (s->zt0_excp_el) { 36 gen_exception_insn_el(s, 0, EXCP_UDEF, 37 syn_smetrap(SME_ET_InaccessibleZT0, false), 38 s->zt0_excp_el); 39 return false; 40 } 41 return true; 42 } 43 44 /* Resolve tile.size[rs+imm] to a host pointer. */ 45 static TCGv_ptr get_tile_rowcol(DisasContext *s, int esz, int rs, 46 int tile, int imm, int div_len, 47 int vec_mod, bool vertical) 48 { 49 int pos, len, offset; 50 TCGv_i32 tmp; 51 TCGv_ptr addr; 52 53 /* Compute the final index, which is Rs+imm. */ 54 tmp = tcg_temp_new_i32(); 55 tcg_gen_trunc_tl_i32(tmp, cpu_reg(s, rs)); 56 /* 57 * Round the vector index down to a multiple of vec_mod if necessary. 58 * We do this before adding the offset, to handle cases like 59 * MOVA (tile to vector, 2 registers) where we want to call this 60 * several times in a loop with an increasing offset. We rely on 61 * the instruction encodings always forcing the initial offset in 62 * [rs + offset] to be a multiple of vec_mod. The pseudocode usually 63 * does the round-down after adding the offset rather than before, 64 * but MOVA is an exception. 65 */ 66 if (vec_mod > 1) { 67 tcg_gen_andc_i32(tmp, tmp, tcg_constant_i32(vec_mod - 1)); 68 } 69 tcg_gen_addi_i32(tmp, tmp, imm); 70 71 /* Prepare a power-of-two modulo via extraction of @len bits. */ 72 len = ctz32(streaming_vec_reg_size(s) / div_len) - esz; 73 74 if (!len) { 75 /* 76 * SVL is 128 and the element size is 128. There is exactly 77 * one 128x128 tile in the ZA storage, and so we calculate 78 * (Rs + imm) MOD 1, which is always 0. We need to special case 79 * this because TCG doesn't allow deposit ops with len 0. 80 */ 81 tcg_gen_movi_i32(tmp, 0); 82 } else if (vertical) { 83 /* 84 * Compute the byte offset of the index within the tile: 85 * (index % (svl / size)) * size 86 * = (index % (svl >> esz)) << esz 87 * Perform the power-of-two modulo via extraction of the low @len bits. 88 * Perform the multiply by shifting left by @pos bits. 89 * Perform these operations simultaneously via deposit into zero. 90 */ 91 pos = esz; 92 tcg_gen_deposit_z_i32(tmp, tmp, pos, len); 93 94 /* 95 * For big-endian, adjust the indexed column byte offset within 96 * the uint64_t host words that make up env->zarray[]. 97 */ 98 if (HOST_BIG_ENDIAN && esz < MO_64) { 99 tcg_gen_xori_i32(tmp, tmp, 8 - (1 << esz)); 100 } 101 } else { 102 /* 103 * Compute the byte offset of the index within the tile: 104 * (index % (svl / size)) * (size * sizeof(row)) 105 * = (index % (svl >> esz)) << (esz + log2(sizeof(row))) 106 */ 107 pos = esz + ctz32(sizeof(ARMVectorReg)); 108 tcg_gen_deposit_z_i32(tmp, tmp, pos, len); 109 110 /* Row slices are always aligned and need no endian adjustment. */ 111 } 112 113 /* The tile byte offset within env->zarray is the row. */ 114 offset = tile * sizeof(ARMVectorReg); 115 116 /* Include the byte offset of zarray to make this relative to env. */ 117 offset += offsetof(CPUARMState, za_state.za); 118 tcg_gen_addi_i32(tmp, tmp, offset); 119 120 /* Add the byte offset to env to produce the final pointer. */ 121 addr = tcg_temp_new_ptr(); 122 tcg_gen_ext_i32_ptr(addr, tmp); 123 tcg_gen_add_ptr(addr, addr, tcg_env); 124 125 return addr; 126 } 127 128 /* Resolve ZArray[rs+imm] to a host pointer. */ 129 static TCGv_ptr get_zarray(DisasContext *s, int rs, int imm, 130 int div_len, int vec_mod) 131 { 132 /* ZA[n] equates to ZA0H.B[n]. */ 133 return get_tile_rowcol(s, MO_8, rs, 0, imm, div_len, vec_mod, false); 134 } 135 136 /* 137 * Resolve tile.size[0] to a host pointer. 138 * Used by e.g. outer product insns where we require the entire tile. 139 */ 140 static TCGv_ptr get_tile(DisasContext *s, int esz, int tile) 141 { 142 TCGv_ptr addr = tcg_temp_new_ptr(); 143 int offset; 144 145 offset = tile * sizeof(ARMVectorReg) + offsetof(CPUARMState, za_state.za); 146 147 tcg_gen_addi_ptr(addr, tcg_env, offset); 148 return addr; 149 } 150 151 static bool trans_ZERO(DisasContext *s, arg_ZERO *a) 152 { 153 if (!dc_isar_feature(aa64_sme, s)) { 154 return false; 155 } 156 if (sme_za_enabled_check(s)) { 157 gen_helper_sme_zero(tcg_env, tcg_constant_i32(a->imm), 158 tcg_constant_i32(streaming_vec_reg_size(s))); 159 } 160 return true; 161 } 162 163 static bool trans_ZERO_zt0(DisasContext *s, arg_ZERO_zt0 *a) 164 { 165 if (!dc_isar_feature(aa64_sme2, s)) { 166 return false; 167 } 168 if (sme_enabled_check(s) && sme2_zt0_enabled_check(s)) { 169 tcg_gen_gvec_dup_imm(MO_64, offsetof(CPUARMState, za_state.zt0), 170 sizeof_field(CPUARMState, za_state.zt0), 171 sizeof_field(CPUARMState, za_state.zt0), 0); 172 } 173 return true; 174 } 175 176 static bool trans_ZERO_za(DisasContext *s, arg_ZERO_za *a) 177 { 178 if (!dc_isar_feature(aa64_sme2p1, s)) { 179 return false; 180 } 181 if (sme_smza_enabled_check(s)) { 182 int svl = streaming_vec_reg_size(s); 183 int vstride = svl / a->ngrp; 184 TCGv_ptr t_za = get_zarray(s, a->rv, a->off, a->ngrp, a->nvec); 185 186 for (int r = 0; r < a->ngrp; ++r) { 187 for (int i = 0; i < a->nvec; ++i) { 188 int o_za = (r * vstride + i) * sizeof(ARMVectorReg); 189 tcg_gen_gvec_dup_imm_var(MO_64, t_za, o_za, svl, svl, 0); 190 } 191 } 192 } 193 return true; 194 } 195 196 static bool do_mova_tile(DisasContext *s, arg_mova_p *a, bool to_vec) 197 { 198 static gen_helper_gvec_4 * const h_fns[5] = { 199 gen_helper_sve_sel_zpzz_b, gen_helper_sve_sel_zpzz_h, 200 gen_helper_sve_sel_zpzz_s, gen_helper_sve_sel_zpzz_d, 201 gen_helper_sve_sel_zpzz_q 202 }; 203 static gen_helper_gvec_3 * const cz_fns[5] = { 204 gen_helper_sme_mova_cz_b, gen_helper_sme_mova_cz_h, 205 gen_helper_sme_mova_cz_s, gen_helper_sme_mova_cz_d, 206 gen_helper_sme_mova_cz_q, 207 }; 208 static gen_helper_gvec_3 * const zc_fns[5] = { 209 gen_helper_sme_mova_zc_b, gen_helper_sme_mova_zc_h, 210 gen_helper_sme_mova_zc_s, gen_helper_sme_mova_zc_d, 211 gen_helper_sme_mova_zc_q, 212 }; 213 214 TCGv_ptr t_za, t_zr, t_pg; 215 TCGv_i32 t_desc; 216 int svl; 217 218 if (!sme_smza_enabled_check(s)) { 219 return true; 220 } 221 222 t_za = get_tile_rowcol(s, a->esz, a->rs, a->za, a->off, 1, 0, a->v); 223 t_zr = vec_full_reg_ptr(s, a->zr); 224 t_pg = pred_full_reg_ptr(s, a->pg); 225 226 svl = streaming_vec_reg_size(s); 227 t_desc = tcg_constant_i32(simd_desc(svl, svl, 0)); 228 229 if (a->v) { 230 /* Vertical slice -- use sme mova helpers. */ 231 if (to_vec) { 232 zc_fns[a->esz](t_zr, t_za, t_pg, t_desc); 233 } else { 234 cz_fns[a->esz](t_za, t_zr, t_pg, t_desc); 235 } 236 } else { 237 /* Horizontal slice -- reuse sve sel helpers. */ 238 if (to_vec) { 239 h_fns[a->esz](t_zr, t_za, t_zr, t_pg, t_desc); 240 } else { 241 h_fns[a->esz](t_za, t_zr, t_za, t_pg, t_desc); 242 } 243 } 244 return true; 245 } 246 247 TRANS_FEAT(MOVA_tz, aa64_sme, do_mova_tile, a, false) 248 TRANS_FEAT(MOVA_zt, aa64_sme, do_mova_tile, a, true) 249 250 static bool do_mova_tile_n(DisasContext *s, arg_mova_t *a, int n, 251 bool to_vec, bool zero) 252 { 253 static gen_helper_gvec_2 * const cz_fns[] = { 254 gen_helper_sme2_mova_cz_b, gen_helper_sme2_mova_cz_h, 255 gen_helper_sme2_mova_cz_s, gen_helper_sme2_mova_cz_d, 256 }; 257 static gen_helper_gvec_2 * const zc_fns[] = { 258 gen_helper_sme2_mova_zc_b, gen_helper_sme2_mova_zc_h, 259 gen_helper_sme2_mova_zc_s, gen_helper_sme2_mova_zc_d, 260 }; 261 static gen_helper_gvec_2 * const zc_z_fns[] = { 262 gen_helper_sme2p1_movaz_zc_b, gen_helper_sme2p1_movaz_zc_h, 263 gen_helper_sme2p1_movaz_zc_s, gen_helper_sme2p1_movaz_zc_d, 264 gen_helper_sme2p1_movaz_zc_q, 265 }; 266 TCGv_ptr t_za; 267 int svl, bytes_per_op = n << a->esz; 268 269 /* 270 * The MaxImplementedSVL check happens in the decode pseudocode, 271 * before the SM+ZA enabled check in the operation pseudocode. 272 * This will (currently) only fail for NREG=4, ESZ=MO_64. 273 */ 274 if (s->max_svl < bytes_per_op) { 275 unallocated_encoding(s); 276 return true; 277 } 278 279 assert(a->esz <= MO_64 + zero); 280 281 if (!sme_smza_enabled_check(s)) { 282 return true; 283 } 284 285 svl = streaming_vec_reg_size(s); 286 287 /* 288 * The CurrentVL check happens in the operation pseudocode, 289 * after the SM+ZA enabled check. 290 */ 291 if (svl < bytes_per_op) { 292 unallocated_encoding(s); 293 return true; 294 } 295 296 if (a->v) { 297 TCGv_i32 t_desc = tcg_constant_i32(simd_desc(svl, svl, 0)); 298 299 for (int i = 0; i < n; ++i) { 300 TCGv_ptr t_zr = vec_full_reg_ptr(s, a->zr * n + i); 301 t_za = get_tile_rowcol(s, a->esz, a->rs, a->za, 302 a->off * n + i, 1, n, a->v); 303 if (zero) { 304 zc_z_fns[a->esz](t_zr, t_za, t_desc); 305 } else if (to_vec) { 306 zc_fns[a->esz](t_zr, t_za, t_desc); 307 } else { 308 cz_fns[a->esz](t_za, t_zr, t_desc); 309 } 310 } 311 } else { 312 for (int i = 0; i < n; ++i) { 313 int o_zr = vec_full_reg_offset(s, a->zr * n + i); 314 t_za = get_tile_rowcol(s, a->esz, a->rs, a->za, 315 a->off * n + i, 1, n, a->v); 316 if (to_vec) { 317 tcg_gen_gvec_mov_var(MO_8, tcg_env, o_zr, t_za, 0, svl, svl); 318 if (zero) { 319 tcg_gen_gvec_dup_imm_var(MO_8, t_za, 0, svl, svl, 0); 320 } 321 } else { 322 tcg_gen_gvec_mov_var(MO_8, t_za, 0, tcg_env, o_zr, svl, svl); 323 } 324 } 325 } 326 return true; 327 } 328 329 TRANS_FEAT(MOVA_tz2, aa64_sme2, do_mova_tile_n, a, 2, false, false) 330 TRANS_FEAT(MOVA_tz4, aa64_sme2, do_mova_tile_n, a, 4, false, false) 331 TRANS_FEAT(MOVA_zt2, aa64_sme2, do_mova_tile_n, a, 2, true, false) 332 TRANS_FEAT(MOVA_zt4, aa64_sme2, do_mova_tile_n, a, 4, true, false) 333 334 TRANS_FEAT(MOVAZ_zt, aa64_sme2p1, do_mova_tile_n, a, 1, true, true) 335 TRANS_FEAT(MOVAZ_zt2, aa64_sme2p1, do_mova_tile_n, a, 2, true, true) 336 TRANS_FEAT(MOVAZ_zt4, aa64_sme2p1, do_mova_tile_n, a, 4, true, true) 337 338 static bool do_mova_array_n(DisasContext *s, arg_mova_a *a, int n, 339 bool to_vec, bool zero) 340 { 341 TCGv_ptr t_za; 342 int svl; 343 344 if (!sme_smza_enabled_check(s)) { 345 return true; 346 } 347 348 svl = streaming_vec_reg_size(s); 349 t_za = get_zarray(s, a->rv, a->off, n, 0); 350 351 for (int i = 0; i < n; ++i) { 352 int o_za = (svl / n * sizeof(ARMVectorReg)) * i; 353 int o_zr = vec_full_reg_offset(s, a->zr * n + i); 354 355 if (to_vec) { 356 tcg_gen_gvec_mov_var(MO_8, tcg_env, o_zr, t_za, o_za, svl, svl); 357 if (zero) { 358 tcg_gen_gvec_dup_imm_var(MO_8, t_za, o_za, svl, svl, 0); 359 } 360 } else { 361 tcg_gen_gvec_mov_var(MO_8, t_za, o_za, tcg_env, o_zr, svl, svl); 362 } 363 } 364 return true; 365 } 366 367 TRANS_FEAT(MOVA_az2, aa64_sme2, do_mova_array_n, a, 2, false, false) 368 TRANS_FEAT(MOVA_az4, aa64_sme2, do_mova_array_n, a, 4, false, false) 369 TRANS_FEAT(MOVA_za2, aa64_sme2, do_mova_array_n, a, 2, true, false) 370 TRANS_FEAT(MOVA_za4, aa64_sme2, do_mova_array_n, a, 4, true, false) 371 372 TRANS_FEAT(MOVAZ_za2, aa64_sme2p1, do_mova_array_n, a, 2, true, true) 373 TRANS_FEAT(MOVAZ_za4, aa64_sme2p1, do_mova_array_n, a, 4, true, true) 374 375 static bool do_movt(DisasContext *s, arg_MOVT_rzt *a, 376 void (*func)(TCGv_i64, TCGv_ptr, tcg_target_long)) 377 { 378 if (sme2_zt0_enabled_check(s)) { 379 func(cpu_reg(s, a->rt), tcg_env, 380 offsetof(CPUARMState, za_state.zt0) + a->off * 8); 381 } 382 return true; 383 } 384 385 TRANS_FEAT(MOVT_rzt, aa64_sme2, do_movt, a, tcg_gen_ld_i64) 386 TRANS_FEAT(MOVT_ztr, aa64_sme2, do_movt, a, tcg_gen_st_i64) 387 388 static bool trans_LDST1(DisasContext *s, arg_LDST1 *a) 389 { 390 typedef void GenLdSt1(TCGv_env, TCGv_ptr, TCGv_ptr, TCGv, TCGv_i32); 391 392 /* 393 * Indexed by [esz][be][v][mte][st], which is (except for load/store) 394 * also the order in which the elements appear in the function names, 395 * and so how we must concatenate the pieces. 396 */ 397 398 #define FN_LS(F) { gen_helper_sme_ld1##F, gen_helper_sme_st1##F } 399 #define FN_MTE(F) { FN_LS(F), FN_LS(F##_mte) } 400 #define FN_HV(F) { FN_MTE(F##_h), FN_MTE(F##_v) } 401 #define FN_END(L, B) { FN_HV(L), FN_HV(B) } 402 403 static GenLdSt1 * const fns[5][2][2][2][2] = { 404 FN_END(b, b), 405 FN_END(h_le, h_be), 406 FN_END(s_le, s_be), 407 FN_END(d_le, d_be), 408 FN_END(q_le, q_be), 409 }; 410 411 #undef FN_LS 412 #undef FN_MTE 413 #undef FN_HV 414 #undef FN_END 415 416 TCGv_ptr t_za, t_pg; 417 TCGv_i64 addr; 418 uint32_t desc; 419 bool be = s->be_data == MO_BE; 420 bool mte = s->mte_active[0]; 421 422 if (!dc_isar_feature(aa64_sme, s)) { 423 return false; 424 } 425 if (!sme_smza_enabled_check(s)) { 426 return true; 427 } 428 429 t_za = get_tile_rowcol(s, a->esz, a->rs, a->za, a->off, 1, 0, a->v); 430 t_pg = pred_full_reg_ptr(s, a->pg); 431 addr = tcg_temp_new_i64(); 432 433 tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), a->esz); 434 tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn)); 435 436 if (!mte) { 437 addr = clean_data_tbi(s, addr); 438 } 439 440 desc = make_svemte_desc(s, streaming_vec_reg_size(s), 1, a->esz, a->st, 0); 441 442 fns[a->esz][be][a->v][mte][a->st](tcg_env, t_za, t_pg, addr, 443 tcg_constant_i32(desc)); 444 return true; 445 } 446 447 typedef void GenLdStR(DisasContext *, TCGv_ptr, int, int, int, int, MemOp); 448 449 static bool do_ldst_r(DisasContext *s, arg_ldstr *a, GenLdStR *fn) 450 { 451 if (sme_za_enabled_check(s)) { 452 int svl = streaming_vec_reg_size(s); 453 int imm = a->imm; 454 TCGv_ptr base = get_zarray(s, a->rv, imm, 1, 0); 455 456 fn(s, base, 0, svl, a->rn, imm * svl, 457 s->align_mem ? MO_ALIGN_16 : MO_UNALN); 458 } 459 return true; 460 } 461 462 TRANS_FEAT(LDR, aa64_sme, do_ldst_r, a, gen_sve_ldr) 463 TRANS_FEAT(STR, aa64_sme, do_ldst_r, a, gen_sve_str) 464 465 static bool do_ldst_zt0(DisasContext *s, arg_ldstzt0 *a, GenLdStR *fn) 466 { 467 if (sme2_zt0_enabled_check(s)) { 468 fn(s, tcg_env, offsetof(CPUARMState, za_state.zt0), 469 sizeof_field(CPUARMState, za_state.zt0), a->rn, 0, 470 s->align_mem ? MO_ALIGN_16 : MO_UNALN); 471 } 472 return true; 473 } 474 475 TRANS_FEAT(LDR_zt0, aa64_sme2, do_ldst_zt0, a, gen_sve_ldr) 476 TRANS_FEAT(STR_zt0, aa64_sme2, do_ldst_zt0, a, gen_sve_str) 477 478 static bool do_adda(DisasContext *s, arg_adda *a, MemOp esz, 479 gen_helper_gvec_4 *fn) 480 { 481 int svl = streaming_vec_reg_size(s); 482 uint32_t desc = simd_desc(svl, svl, 0); 483 TCGv_ptr za, zn, pn, pm; 484 485 if (!sme_smza_enabled_check(s)) { 486 return true; 487 } 488 489 za = get_tile(s, esz, a->zad); 490 zn = vec_full_reg_ptr(s, a->zn); 491 pn = pred_full_reg_ptr(s, a->pn); 492 pm = pred_full_reg_ptr(s, a->pm); 493 494 fn(za, zn, pn, pm, tcg_constant_i32(desc)); 495 return true; 496 } 497 498 TRANS_FEAT(ADDHA_s, aa64_sme, do_adda, a, MO_32, gen_helper_sme_addha_s) 499 TRANS_FEAT(ADDVA_s, aa64_sme, do_adda, a, MO_32, gen_helper_sme_addva_s) 500 TRANS_FEAT(ADDHA_d, aa64_sme_i16i64, do_adda, a, MO_64, gen_helper_sme_addha_d) 501 TRANS_FEAT(ADDVA_d, aa64_sme_i16i64, do_adda, a, MO_64, gen_helper_sme_addva_d) 502 503 static bool do_outprod(DisasContext *s, arg_op *a, MemOp esz, 504 gen_helper_gvec_5 *fn) 505 { 506 int svl = streaming_vec_reg_size(s); 507 uint32_t desc = simd_desc(svl, svl, a->sub); 508 TCGv_ptr za, zn, zm, pn, pm; 509 510 if (!sme_smza_enabled_check(s)) { 511 return true; 512 } 513 514 za = get_tile(s, esz, a->zad); 515 zn = vec_full_reg_ptr(s, a->zn); 516 zm = vec_full_reg_ptr(s, a->zm); 517 pn = pred_full_reg_ptr(s, a->pn); 518 pm = pred_full_reg_ptr(s, a->pm); 519 520 fn(za, zn, zm, pn, pm, tcg_constant_i32(desc)); 521 return true; 522 } 523 524 static bool do_outprod_fpst(DisasContext *s, arg_op *a, MemOp esz, 525 ARMFPStatusFlavour e_fpst, 526 gen_helper_gvec_5_ptr *fn) 527 { 528 int svl = streaming_vec_reg_size(s); 529 uint32_t desc = simd_desc(svl, svl, 0); 530 TCGv_ptr za, zn, zm, pn, pm, fpst; 531 532 if (!sme_smza_enabled_check(s)) { 533 return true; 534 } 535 536 za = get_tile(s, esz, a->zad); 537 zn = vec_full_reg_ptr(s, a->zn); 538 zm = vec_full_reg_ptr(s, a->zm); 539 pn = pred_full_reg_ptr(s, a->pn); 540 pm = pred_full_reg_ptr(s, a->pm); 541 fpst = fpstatus_ptr(e_fpst); 542 543 fn(za, zn, zm, pn, pm, fpst, tcg_constant_i32(desc)); 544 return true; 545 } 546 547 static bool do_outprod_env(DisasContext *s, arg_op *a, MemOp esz, 548 gen_helper_gvec_5_ptr *fn) 549 { 550 int svl = streaming_vec_reg_size(s); 551 uint32_t desc = simd_desc(svl, svl, 0); 552 TCGv_ptr za, zn, zm, pn, pm; 553 554 if (!sme_smza_enabled_check(s)) { 555 return true; 556 } 557 558 za = get_tile(s, esz, a->zad); 559 zn = vec_full_reg_ptr(s, a->zn); 560 zm = vec_full_reg_ptr(s, a->zm); 561 pn = pred_full_reg_ptr(s, a->pn); 562 pm = pred_full_reg_ptr(s, a->pm); 563 564 fn(za, zn, zm, pn, pm, tcg_env, tcg_constant_i32(desc)); 565 return true; 566 } 567 568 TRANS_FEAT(FMOPA_w_h, aa64_sme, do_outprod_env, a, MO_32, 569 !a->sub ? gen_helper_sme_fmopa_w_h 570 : !s->fpcr_ah ? gen_helper_sme_fmops_w_h 571 : gen_helper_sme_ah_fmops_w_h) 572 TRANS_FEAT(FMOPA_h, aa64_sme_f16f16, do_outprod_fpst, a, MO_16, FPST_ZA_F16, 573 !a->sub ? gen_helper_sme_fmopa_h 574 : !s->fpcr_ah ? gen_helper_sme_fmops_h 575 : gen_helper_sme_ah_fmops_h) 576 TRANS_FEAT(FMOPA_s, aa64_sme, do_outprod_fpst, a, MO_32, FPST_ZA, 577 !a->sub ? gen_helper_sme_fmopa_s 578 : !s->fpcr_ah ? gen_helper_sme_fmops_s 579 : gen_helper_sme_ah_fmops_s) 580 TRANS_FEAT(FMOPA_d, aa64_sme_f64f64, do_outprod_fpst, a, MO_64, FPST_ZA, 581 !a->sub ? gen_helper_sme_fmopa_d 582 : !s->fpcr_ah ? gen_helper_sme_fmops_d 583 : gen_helper_sme_ah_fmops_d) 584 585 TRANS_FEAT(BFMOPA, aa64_sme_b16b16, do_outprod_fpst, a, MO_16, FPST_ZA, 586 !a->sub ? gen_helper_sme_bfmopa 587 : !s->fpcr_ah ? gen_helper_sme_bfmops 588 : gen_helper_sme_ah_bfmops) 589 590 TRANS_FEAT(BFMOPA_w, aa64_sme, do_outprod_env, a, MO_32, 591 !a->sub ? gen_helper_sme_bfmopa_w 592 : !s->fpcr_ah ? gen_helper_sme_bfmops_w 593 : gen_helper_sme_ah_bfmops_w) 594 595 TRANS_FEAT(SMOPA_s, aa64_sme, do_outprod, a, MO_32, gen_helper_sme_smopa_s) 596 TRANS_FEAT(UMOPA_s, aa64_sme, do_outprod, a, MO_32, gen_helper_sme_umopa_s) 597 TRANS_FEAT(SUMOPA_s, aa64_sme, do_outprod, a, MO_32, gen_helper_sme_sumopa_s) 598 TRANS_FEAT(USMOPA_s, aa64_sme, do_outprod, a, MO_32, gen_helper_sme_usmopa_s) 599 600 TRANS_FEAT(SMOPA_d, aa64_sme_i16i64, do_outprod, a, MO_64, gen_helper_sme_smopa_d) 601 TRANS_FEAT(UMOPA_d, aa64_sme_i16i64, do_outprod, a, MO_64, gen_helper_sme_umopa_d) 602 TRANS_FEAT(SUMOPA_d, aa64_sme_i16i64, do_outprod, a, MO_64, gen_helper_sme_sumopa_d) 603 TRANS_FEAT(USMOPA_d, aa64_sme_i16i64, do_outprod, a, MO_64, gen_helper_sme_usmopa_d) 604 605 TRANS_FEAT(BMOPA, aa64_sme2, do_outprod, a, MO_32, gen_helper_sme2_bmopa_s) 606 TRANS_FEAT(SMOPA2_s, aa64_sme2, do_outprod, a, MO_32, gen_helper_sme2_smopa2_s) 607 TRANS_FEAT(UMOPA2_s, aa64_sme2, do_outprod, a, MO_32, gen_helper_sme2_umopa2_s) 608 609 static bool do_z2z_n1(DisasContext *s, arg_z2z_en *a, GVecGen3Fn *fn) 610 { 611 int esz, dn, vsz, mofs, n; 612 bool overlap = false; 613 614 if (!sme_sm_enabled_check(s)) { 615 return true; 616 } 617 618 esz = a->esz; 619 n = a->n; 620 dn = a->zdn; 621 mofs = vec_full_reg_offset(s, a->zm); 622 vsz = streaming_vec_reg_size(s); 623 624 for (int i = 0; i < n; i++) { 625 int dofs = vec_full_reg_offset(s, dn + i); 626 if (dofs == mofs) { 627 overlap = true; 628 } else { 629 fn(esz, dofs, dofs, mofs, vsz, vsz); 630 } 631 } 632 if (overlap) { 633 fn(esz, mofs, mofs, mofs, vsz, vsz); 634 } 635 return true; 636 } 637 638 static void gen_sme2_srshl(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs, 639 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz) 640 { 641 static gen_helper_gvec_3 * const fns[] = { 642 gen_helper_gvec_srshl_b, gen_helper_sme2_srshl_h, 643 gen_helper_sme2_srshl_s, gen_helper_sme2_srshl_d, 644 }; 645 tcg_debug_assert(vece <= MO_64); 646 tcg_gen_gvec_3_ool(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, 0, fns[vece]); 647 } 648 649 static void gen_sme2_urshl(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs, 650 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz) 651 { 652 static gen_helper_gvec_3 * const fns[] = { 653 gen_helper_gvec_urshl_b, gen_helper_sme2_urshl_h, 654 gen_helper_sme2_urshl_s, gen_helper_sme2_urshl_d, 655 }; 656 tcg_debug_assert(vece <= MO_64); 657 tcg_gen_gvec_3_ool(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, 0, fns[vece]); 658 } 659 660 TRANS_FEAT(ADD_n1, aa64_sme2, do_z2z_n1, a, tcg_gen_gvec_add) 661 TRANS_FEAT(SMAX_n1, aa64_sme2, do_z2z_n1, a, tcg_gen_gvec_smax) 662 TRANS_FEAT(SMIN_n1, aa64_sme2, do_z2z_n1, a, tcg_gen_gvec_smin) 663 TRANS_FEAT(UMAX_n1, aa64_sme2, do_z2z_n1, a, tcg_gen_gvec_umax) 664 TRANS_FEAT(UMIN_n1, aa64_sme2, do_z2z_n1, a, tcg_gen_gvec_umin) 665 TRANS_FEAT(SRSHL_n1, aa64_sme2, do_z2z_n1, a, gen_sme2_srshl) 666 TRANS_FEAT(URSHL_n1, aa64_sme2, do_z2z_n1, a, gen_sme2_urshl) 667 TRANS_FEAT(SQDMULH_n1, aa64_sme2, do_z2z_n1, a, gen_gvec_sve2_sqdmulh) 668 669 static bool do_z2z_nn(DisasContext *s, arg_z2z_en *a, GVecGen3Fn *fn) 670 { 671 int esz, dn, dm, vsz, n; 672 673 if (!sme_sm_enabled_check(s)) { 674 return true; 675 } 676 677 esz = a->esz; 678 n = a->n; 679 dn = a->zdn; 680 dm = a->zm; 681 vsz = streaming_vec_reg_size(s); 682 683 for (int i = 0; i < n; i++) { 684 int dofs = vec_full_reg_offset(s, dn + i); 685 int mofs = vec_full_reg_offset(s, dm + i); 686 687 fn(esz, dofs, dofs, mofs, vsz, vsz); 688 } 689 return true; 690 } 691 692 TRANS_FEAT(SMAX_nn, aa64_sme2, do_z2z_nn, a, tcg_gen_gvec_smax) 693 TRANS_FEAT(SMIN_nn, aa64_sme2, do_z2z_nn, a, tcg_gen_gvec_smin) 694 TRANS_FEAT(UMAX_nn, aa64_sme2, do_z2z_nn, a, tcg_gen_gvec_umax) 695 TRANS_FEAT(UMIN_nn, aa64_sme2, do_z2z_nn, a, tcg_gen_gvec_umin) 696 TRANS_FEAT(SRSHL_nn, aa64_sme2, do_z2z_nn, a, gen_sme2_srshl) 697 TRANS_FEAT(URSHL_nn, aa64_sme2, do_z2z_nn, a, gen_sme2_urshl) 698 TRANS_FEAT(SQDMULH_nn, aa64_sme2, do_z2z_nn, a, gen_gvec_sve2_sqdmulh) 699 700 static bool do_z2z_n1_fpst(DisasContext *s, arg_z2z_en *a, 701 gen_helper_gvec_3_ptr * const fns[4]) 702 { 703 int esz = a->esz, n, dn, vsz, mofs; 704 bool overlap = false; 705 gen_helper_gvec_3_ptr *fn; 706 TCGv_ptr fpst; 707 708 /* These insns use MO_8 to encode BFloat16. */ 709 if (esz == MO_8 && !dc_isar_feature(aa64_sme_b16b16, s)) { 710 return false; 711 } 712 if (!sme_sm_enabled_check(s)) { 713 return true; 714 } 715 716 fpst = fpstatus_ptr(esz == MO_16 ? FPST_A64_F16 : FPST_A64); 717 fn = fns[esz]; 718 n = a->n; 719 dn = a->zdn; 720 mofs = vec_full_reg_offset(s, a->zm); 721 vsz = streaming_vec_reg_size(s); 722 723 for (int i = 0; i < n; i++) { 724 int dofs = vec_full_reg_offset(s, dn + i); 725 if (dofs == mofs) { 726 overlap = true; 727 } else { 728 tcg_gen_gvec_3_ptr(dofs, dofs, mofs, fpst, vsz, vsz, 0, fn); 729 } 730 } 731 if (overlap) { 732 tcg_gen_gvec_3_ptr(mofs, mofs, mofs, fpst, vsz, vsz, 0, fn); 733 } 734 return true; 735 } 736 737 static bool do_z2z_nn_fpst(DisasContext *s, arg_z2z_en *a, 738 gen_helper_gvec_3_ptr * const fns[4]) 739 { 740 int esz = a->esz, n, dn, dm, vsz; 741 gen_helper_gvec_3_ptr *fn; 742 TCGv_ptr fpst; 743 744 if (esz == MO_8 && !dc_isar_feature(aa64_sme_b16b16, s)) { 745 return false; 746 } 747 if (!sme_sm_enabled_check(s)) { 748 return true; 749 } 750 751 fpst = fpstatus_ptr(esz == MO_16 ? FPST_A64_F16 : FPST_A64); 752 fn = fns[esz]; 753 n = a->n; 754 dn = a->zdn; 755 dm = a->zm; 756 vsz = streaming_vec_reg_size(s); 757 758 for (int i = 0; i < n; i++) { 759 int dofs = vec_full_reg_offset(s, dn + i); 760 int mofs = vec_full_reg_offset(s, dm + i); 761 762 tcg_gen_gvec_3_ptr(dofs, dofs, mofs, fpst, vsz, vsz, 0, fn); 763 } 764 return true; 765 } 766 767 static gen_helper_gvec_3_ptr * const f_vector_fmax[2][4] = { 768 { gen_helper_gvec_fmax_b16, 769 gen_helper_gvec_fmax_h, 770 gen_helper_gvec_fmax_s, 771 gen_helper_gvec_fmax_d }, 772 { gen_helper_gvec_ah_fmax_b16, 773 gen_helper_gvec_ah_fmax_h, 774 gen_helper_gvec_ah_fmax_s, 775 gen_helper_gvec_ah_fmax_d }, 776 }; 777 TRANS_FEAT(FMAX_n1, aa64_sme2, do_z2z_n1_fpst, a, f_vector_fmax[s->fpcr_ah]) 778 TRANS_FEAT(FMAX_nn, aa64_sme2, do_z2z_nn_fpst, a, f_vector_fmax[s->fpcr_ah]) 779 780 static gen_helper_gvec_3_ptr * const f_vector_fmin[2][4] = { 781 { gen_helper_gvec_fmin_b16, 782 gen_helper_gvec_fmin_h, 783 gen_helper_gvec_fmin_s, 784 gen_helper_gvec_fmin_d }, 785 { gen_helper_gvec_ah_fmin_b16, 786 gen_helper_gvec_ah_fmin_h, 787 gen_helper_gvec_ah_fmin_s, 788 gen_helper_gvec_ah_fmin_d }, 789 }; 790 TRANS_FEAT(FMIN_n1, aa64_sme2, do_z2z_n1_fpst, a, f_vector_fmin[s->fpcr_ah]) 791 TRANS_FEAT(FMIN_nn, aa64_sme2, do_z2z_nn_fpst, a, f_vector_fmin[s->fpcr_ah]) 792 793 static gen_helper_gvec_3_ptr * const f_vector_fmaxnm[4] = { 794 gen_helper_gvec_fmaxnum_b16, 795 gen_helper_gvec_fmaxnum_h, 796 gen_helper_gvec_fmaxnum_s, 797 gen_helper_gvec_fmaxnum_d, 798 }; 799 TRANS_FEAT(FMAXNM_n1, aa64_sme2, do_z2z_n1_fpst, a, f_vector_fmaxnm) 800 TRANS_FEAT(FMAXNM_nn, aa64_sme2, do_z2z_nn_fpst, a, f_vector_fmaxnm) 801 802 static gen_helper_gvec_3_ptr * const f_vector_fminnm[4] = { 803 gen_helper_gvec_fminnum_b16, 804 gen_helper_gvec_fminnum_h, 805 gen_helper_gvec_fminnum_s, 806 gen_helper_gvec_fminnum_d, 807 }; 808 TRANS_FEAT(FMINNM_n1, aa64_sme2, do_z2z_n1_fpst, a, f_vector_fminnm) 809 TRANS_FEAT(FMINNM_nn, aa64_sme2, do_z2z_nn_fpst, a, f_vector_fminnm) 810 811 /* Add/Sub vector Z[m] to each Z[n*N] with result in ZA[d*N]. */ 812 static bool do_azz_n1(DisasContext *s, arg_azz_n *a, int esz, 813 GVecGen3FnVar *fn) 814 { 815 TCGv_ptr t_za; 816 int svl, n, o_zm; 817 818 if (!sme_smza_enabled_check(s)) { 819 return true; 820 } 821 822 n = a->n; 823 t_za = get_zarray(s, a->rv, a->off, n, 0); 824 o_zm = vec_full_reg_offset(s, a->zm); 825 svl = streaming_vec_reg_size(s); 826 827 for (int i = 0; i < n; ++i) { 828 int o_za = (svl / n * sizeof(ARMVectorReg)) * i; 829 int o_zn = vec_full_reg_offset(s, (a->zn + i) % 32); 830 831 fn(esz, t_za, o_za, tcg_env, o_zn, tcg_env, o_zm, svl, svl); 832 } 833 return true; 834 } 835 836 TRANS_FEAT(ADD_azz_n1_s, aa64_sme2, do_azz_n1, a, MO_32, tcg_gen_gvec_add_var) 837 TRANS_FEAT(SUB_azz_n1_s, aa64_sme2, do_azz_n1, a, MO_32, tcg_gen_gvec_sub_var) 838 TRANS_FEAT(ADD_azz_n1_d, aa64_sme2_i16i64, do_azz_n1, a, MO_64, tcg_gen_gvec_add_var) 839 TRANS_FEAT(SUB_azz_n1_d, aa64_sme2_i16i64, do_azz_n1, a, MO_64, tcg_gen_gvec_sub_var) 840 841 /* Add/Sub each vector Z[m*N] to each Z[n*N] with result in ZA[d*N]. */ 842 static bool do_azz_nn(DisasContext *s, arg_azz_n *a, int esz, 843 GVecGen3FnVar *fn) 844 { 845 TCGv_ptr t_za; 846 int svl, n; 847 848 if (!sme_smza_enabled_check(s)) { 849 return true; 850 } 851 852 n = a->n; 853 t_za = get_zarray(s, a->rv, a->off, n, 1); 854 svl = streaming_vec_reg_size(s); 855 856 for (int i = 0; i < n; ++i) { 857 int o_za = (svl / n * sizeof(ARMVectorReg)) * i; 858 int o_zn = vec_full_reg_offset(s, a->zn + i); 859 int o_zm = vec_full_reg_offset(s, a->zm + i); 860 861 fn(esz, t_za, o_za, tcg_env, o_zn, tcg_env, o_zm, svl, svl); 862 } 863 return true; 864 } 865 866 TRANS_FEAT(ADD_azz_nn_s, aa64_sme2, do_azz_nn, a, MO_32, tcg_gen_gvec_add_var) 867 TRANS_FEAT(SUB_azz_nn_s, aa64_sme2, do_azz_nn, a, MO_32, tcg_gen_gvec_sub_var) 868 TRANS_FEAT(ADD_azz_nn_d, aa64_sme2_i16i64, do_azz_nn, a, MO_64, tcg_gen_gvec_add_var) 869 TRANS_FEAT(SUB_azz_nn_d, aa64_sme2_i16i64, do_azz_nn, a, MO_64, tcg_gen_gvec_sub_var) 870 871 /* Add/Sub each ZA[d*N] += Z[m*N] */ 872 static bool do_aaz(DisasContext *s, arg_az_n *a, int esz, GVecGen3FnVar *fn) 873 { 874 TCGv_ptr t_za; 875 int svl, n; 876 877 if (!sme_smza_enabled_check(s)) { 878 return true; 879 } 880 881 n = a->n; 882 t_za = get_zarray(s, a->rv, a->off, n, 0); 883 svl = streaming_vec_reg_size(s); 884 885 for (int i = 0; i < n; ++i) { 886 int o_za = (svl / n * sizeof(ARMVectorReg)) * i; 887 int o_zm = vec_full_reg_offset(s, a->zm + i); 888 889 fn(esz, t_za, o_za, t_za, o_za, tcg_env, o_zm, svl, svl); 890 } 891 return true; 892 } 893 894 TRANS_FEAT(ADD_aaz_s, aa64_sme2, do_aaz, a, MO_32, tcg_gen_gvec_add_var) 895 TRANS_FEAT(SUB_aaz_s, aa64_sme2, do_aaz, a, MO_32, tcg_gen_gvec_sub_var) 896 TRANS_FEAT(ADD_aaz_d, aa64_sme2_i16i64, do_aaz, a, MO_64, tcg_gen_gvec_add_var) 897 TRANS_FEAT(SUB_aaz_d, aa64_sme2_i16i64, do_aaz, a, MO_64, tcg_gen_gvec_sub_var) 898 899 /* 900 * Expand array multi-vector single (n1), array multi-vector (nn), 901 * and array multi-vector indexed (nx), for floating-point accumulate. 902 * multi: true for nn, false for n1. 903 * fpst: >= 0 to set ptr argument for FPST_*, < 0 for ENV. 904 * data: stuff for simd_data, including any index. 905 */ 906 #define FPST_ENV -1 907 908 static bool do_azz_fp(DisasContext *s, int nreg, int nsel, 909 int rv, int off, int zn, int zm, 910 int data, int shsel, bool multi, int fpst, 911 gen_helper_gvec_3_ptr *fn) 912 { 913 if (sme_smza_enabled_check(s)) { 914 int svl = streaming_vec_reg_size(s); 915 int vstride = svl / nreg; 916 TCGv_ptr t_za = get_zarray(s, rv, off, nreg, nsel); 917 TCGv_ptr t, ptr; 918 919 if (fpst >= 0) { 920 ptr = fpstatus_ptr(fpst); 921 } else { 922 ptr = tcg_env; 923 } 924 t = tcg_temp_new_ptr(); 925 926 for (int r = 0; r < nreg; ++r) { 927 TCGv_ptr t_zn = vec_full_reg_ptr(s, zn); 928 TCGv_ptr t_zm = vec_full_reg_ptr(s, zm); 929 930 for (int i = 0; i < nsel; ++i) { 931 int o_za = (r * vstride + i) * sizeof(ARMVectorReg); 932 int desc = simd_desc(svl, svl, data | (i << shsel)); 933 934 tcg_gen_addi_ptr(t, t_za, o_za); 935 fn(t, t_zn, t_zm, ptr, tcg_constant_i32(desc)); 936 } 937 938 /* 939 * For multiple-and-single vectors, Zn may wrap. 940 * For multiple vectors, both Zn and Zm are aligned. 941 */ 942 zn = (zn + 1) % 32; 943 zm += multi; 944 } 945 } 946 return true; 947 } 948 949 static bool do_azz_acc_fp(DisasContext *s, int nreg, int nsel, 950 int rv, int off, int zn, int zm, 951 int data, int shsel, bool multi, int fpst, 952 gen_helper_gvec_4_ptr *fn) 953 { 954 if (sme_smza_enabled_check(s)) { 955 int svl = streaming_vec_reg_size(s); 956 int vstride = svl / nreg; 957 TCGv_ptr t_za = get_zarray(s, rv, off, nreg, nsel); 958 TCGv_ptr t, ptr; 959 960 if (fpst >= 0) { 961 ptr = fpstatus_ptr(fpst); 962 } else { 963 ptr = tcg_env; 964 } 965 t = tcg_temp_new_ptr(); 966 967 for (int r = 0; r < nreg; ++r) { 968 TCGv_ptr t_zn = vec_full_reg_ptr(s, zn); 969 TCGv_ptr t_zm = vec_full_reg_ptr(s, zm); 970 971 for (int i = 0; i < nsel; ++i) { 972 int o_za = (r * vstride + i) * sizeof(ARMVectorReg); 973 int desc = simd_desc(svl, svl, data | (i << shsel)); 974 975 tcg_gen_addi_ptr(t, t_za, o_za); 976 fn(t, t_zn, t_zm, t, ptr, tcg_constant_i32(desc)); 977 } 978 979 /* 980 * For multiple-and-single vectors, Zn may wrap. 981 * For multiple vectors, both Zn and Zm are aligned. 982 */ 983 zn = (zn + 1) % 32; 984 zm += multi; 985 } 986 } 987 return true; 988 } 989 990 static bool do_fmlal(DisasContext *s, arg_azz_n *a, bool sub, bool multi) 991 { 992 return do_azz_acc_fp(s, a->n, 2, a->rv, a->off, a->zn, a->zm, 993 (1 << 2) | sub, 1, 994 multi, FPST_ENV, gen_helper_sve2_fmlal_zzzw_s); 995 } 996 997 TRANS_FEAT(FMLAL_n1, aa64_sme2, do_fmlal, a, false, false) 998 TRANS_FEAT(FMLSL_n1, aa64_sme2, do_fmlal, a, true, false) 999 TRANS_FEAT(FMLAL_nn, aa64_sme2, do_fmlal, a, false, true) 1000 TRANS_FEAT(FMLSL_nn, aa64_sme2, do_fmlal, a, true, true) 1001 1002 static bool do_fmlal_nx(DisasContext *s, arg_azx_n *a, bool sub) 1003 { 1004 return do_azz_acc_fp(s, a->n, 2, a->rv, a->off, a->zn, a->zm, 1005 (a->idx << 3) | (1 << 2) | sub, 1, 1006 false, FPST_ENV, gen_helper_sve2_fmlal_zzxw_s); 1007 } 1008 1009 TRANS_FEAT(FMLAL_nx, aa64_sme2, do_fmlal_nx, a, false) 1010 TRANS_FEAT(FMLSL_nx, aa64_sme2, do_fmlal_nx, a, true) 1011 1012 static bool do_bfmlal(DisasContext *s, arg_azz_n *a, bool sub, bool multi) 1013 { 1014 return do_azz_acc_fp(s, a->n, 2, a->rv, a->off, a->zn, a->zm, 1015 0, 0, multi, FPST_ZA, 1016 (!sub ? gen_helper_gvec_bfmlal 1017 : s->fpcr_ah ? gen_helper_gvec_ah_bfmlsl 1018 : gen_helper_gvec_bfmlsl)); 1019 } 1020 1021 TRANS_FEAT(BFMLAL_n1, aa64_sme2, do_bfmlal, a, false, false) 1022 TRANS_FEAT(BFMLSL_n1, aa64_sme2, do_bfmlal, a, true, false) 1023 TRANS_FEAT(BFMLAL_nn, aa64_sme2, do_bfmlal, a, false, true) 1024 TRANS_FEAT(BFMLSL_nn, aa64_sme2, do_bfmlal, a, true, true) 1025 1026 static bool do_bfmlal_nx(DisasContext *s, arg_azx_n *a, bool sub) 1027 { 1028 return do_azz_acc_fp(s, a->n, 2, a->rv, a->off, a->zn, a->zm, 1029 a->idx << 1, 0, false, FPST_ZA, 1030 !sub ? gen_helper_gvec_bfmlal_idx 1031 : s->fpcr_ah ? gen_helper_gvec_ah_bfmlsl_idx 1032 : gen_helper_gvec_bfmlsl_idx); 1033 } 1034 1035 TRANS_FEAT(BFMLAL_nx, aa64_sme2, do_bfmlal_nx, a, false) 1036 TRANS_FEAT(BFMLSL_nx, aa64_sme2, do_bfmlal_nx, a, true) 1037 1038 static bool do_fdot(DisasContext *s, arg_azz_n *a, bool multi) 1039 { 1040 return do_azz_acc_fp(s, a->n, 1, a->rv, a->off, a->zn, a->zm, 1, 0, 1041 multi, FPST_ENV, gen_helper_sme2_fdot_h); 1042 } 1043 1044 TRANS_FEAT(FDOT_n1, aa64_sme2, do_fdot, a, false) 1045 TRANS_FEAT(FDOT_nn, aa64_sme2, do_fdot, a, true) 1046 1047 static bool do_fdot_nx(DisasContext *s, arg_azx_n *a) 1048 { 1049 return do_azz_acc_fp(s, a->n, 1, a->rv, a->off, a->zn, a->zm, 1050 a->idx | (1 << 2), 0, false, FPST_ENV, 1051 gen_helper_sme2_fdot_idx_h); 1052 } 1053 1054 TRANS_FEAT(FDOT_nx, aa64_sme2, do_fdot_nx, a) 1055 1056 static bool do_bfdot(DisasContext *s, arg_azz_n *a, bool multi) 1057 { 1058 return do_azz_acc_fp(s, a->n, 1, a->rv, a->off, a->zn, a->zm, 0, 0, 1059 multi, FPST_ENV, gen_helper_gvec_bfdot); 1060 } 1061 1062 TRANS_FEAT(BFDOT_n1, aa64_sme2, do_bfdot, a, false) 1063 TRANS_FEAT(BFDOT_nn, aa64_sme2, do_bfdot, a, true) 1064 1065 static bool do_bfdot_nx(DisasContext *s, arg_azx_n *a) 1066 { 1067 return do_azz_acc_fp(s, a->n, 1, a->rv, a->off, a->zn, a->zm, a->idx, 0, 1068 false, FPST_ENV, gen_helper_gvec_bfdot_idx); 1069 } 1070 1071 TRANS_FEAT(BFDOT_nx, aa64_sme2, do_bfdot_nx, a) 1072 1073 static bool do_vdot(DisasContext *s, arg_azx_n *a, gen_helper_gvec_4_ptr *fn) 1074 { 1075 if (sme_smza_enabled_check(s)) { 1076 int svl = streaming_vec_reg_size(s); 1077 int vstride = svl / 2; 1078 TCGv_ptr t_za = get_zarray(s, a->rv, a->off, 2, 1); 1079 TCGv_ptr t_zn = vec_full_reg_ptr(s, a->zn); 1080 TCGv_ptr t_zm = vec_full_reg_ptr(s, a->zm); 1081 TCGv_ptr t = tcg_temp_new_ptr(); 1082 1083 for (int i = 0; i < 2; ++i) { 1084 int o_za = i * vstride * sizeof(ARMVectorReg); 1085 int desc = simd_desc(svl, svl, a->idx | (i << 2)); 1086 1087 tcg_gen_addi_ptr(t, t_za, o_za); 1088 fn(t, t_zn, t_zm, t, tcg_env, tcg_constant_i32(desc)); 1089 } 1090 } 1091 return true; 1092 } 1093 1094 TRANS_FEAT(FVDOT, aa64_sme, do_vdot, a, gen_helper_sme2_fvdot_idx_h) 1095 TRANS_FEAT(BFVDOT, aa64_sme, do_vdot, a, gen_helper_sme2_bfvdot_idx) 1096 1097 static bool do_fmla(DisasContext *s, arg_azz_n *a, bool multi, 1098 ARMFPStatusFlavour fpst, gen_helper_gvec_3_ptr *fn) 1099 { 1100 return do_azz_fp(s, a->n, 1, a->rv, a->off, a->zn, a->zm, 1101 0, 0, multi, fpst, fn); 1102 } 1103 1104 TRANS_FEAT(FMLA_n1_h, aa64_sme_f16f16, do_fmla, a, false, FPST_ZA_F16, 1105 gen_helper_gvec_vfma_h) 1106 TRANS_FEAT(FMLS_n1_h, aa64_sme_f16f16, do_fmla, a, false, FPST_ZA_F16, 1107 s->fpcr_ah ? gen_helper_gvec_ah_vfms_h : gen_helper_gvec_vfms_h) 1108 TRANS_FEAT(FMLA_nn_h, aa64_sme_f16f16, do_fmla, a, true, FPST_ZA_F16, 1109 gen_helper_gvec_vfma_h) 1110 TRANS_FEAT(FMLS_nn_h, aa64_sme_f16f16, do_fmla, a, true, FPST_ZA_F16, 1111 s->fpcr_ah ? gen_helper_gvec_ah_vfms_h : gen_helper_gvec_vfms_h) 1112 1113 TRANS_FEAT(FMLA_n1_s, aa64_sme2, do_fmla, a, false, FPST_ZA, 1114 gen_helper_gvec_vfma_s) 1115 TRANS_FEAT(FMLS_n1_s, aa64_sme2, do_fmla, a, false, FPST_ZA, 1116 s->fpcr_ah ? gen_helper_gvec_ah_vfms_s : gen_helper_gvec_vfms_s) 1117 TRANS_FEAT(FMLA_nn_s, aa64_sme2, do_fmla, a, true, FPST_ZA, 1118 gen_helper_gvec_vfma_s) 1119 TRANS_FEAT(FMLS_nn_s, aa64_sme2, do_fmla, a, true, FPST_ZA, 1120 s->fpcr_ah ? gen_helper_gvec_ah_vfms_s : gen_helper_gvec_vfms_s) 1121 1122 TRANS_FEAT(FMLA_n1_d, aa64_sme2_f64f64, do_fmla, a, false, FPST_ZA, 1123 gen_helper_gvec_vfma_d) 1124 TRANS_FEAT(FMLS_n1_d, aa64_sme2_f64f64, do_fmla, a, false, FPST_ZA, 1125 s->fpcr_ah ? gen_helper_gvec_ah_vfms_d : gen_helper_gvec_vfms_d) 1126 TRANS_FEAT(FMLA_nn_d, aa64_sme2_f64f64, do_fmla, a, true, FPST_ZA, 1127 gen_helper_gvec_vfma_d) 1128 TRANS_FEAT(FMLS_nn_d, aa64_sme2_f64f64, do_fmla, a, true, FPST_ZA, 1129 s->fpcr_ah ? gen_helper_gvec_ah_vfms_d : gen_helper_gvec_vfms_d) 1130 1131 TRANS_FEAT(BFMLA_n1, aa64_sme_b16b16, do_fmla, a, false, FPST_ZA, 1132 gen_helper_gvec_bfmla) 1133 TRANS_FEAT(BFMLS_n1, aa64_sme_b16b16, do_fmla, a, false, FPST_ZA, 1134 s->fpcr_ah ? gen_helper_gvec_ah_bfmls : gen_helper_gvec_bfmls) 1135 TRANS_FEAT(BFMLA_nn, aa64_sme_b16b16, do_fmla, a, true, FPST_ZA, 1136 gen_helper_gvec_bfmla) 1137 TRANS_FEAT(BFMLS_nn, aa64_sme_b16b16, do_fmla, a, true, FPST_ZA, 1138 s->fpcr_ah ? gen_helper_gvec_ah_bfmls : gen_helper_gvec_bfmls) 1139 1140 static bool do_fmla_nx(DisasContext *s, arg_azx_n *a, 1141 ARMFPStatusFlavour fpst, gen_helper_gvec_4_ptr *fn) 1142 { 1143 return do_azz_acc_fp(s, a->n, 1, a->rv, a->off, a->zn, a->zm, 1144 a->idx, 0, false, fpst, fn); 1145 } 1146 1147 TRANS_FEAT(FMLA_nx_h, aa64_sme_f16f16, do_fmla_nx, a, FPST_ZA_F16, 1148 gen_helper_gvec_fmla_idx_h) 1149 TRANS_FEAT(FMLS_nx_h, aa64_sme_f16f16, do_fmla_nx, a, FPST_ZA_F16, 1150 s->fpcr_ah ? gen_helper_gvec_ah_fmls_idx_h : gen_helper_gvec_fmls_idx_h) 1151 TRANS_FEAT(FMLA_nx_s, aa64_sme2, do_fmla_nx, a, FPST_ZA, 1152 gen_helper_gvec_fmla_idx_s) 1153 TRANS_FEAT(FMLS_nx_s, aa64_sme2, do_fmla_nx, a, FPST_ZA, 1154 s->fpcr_ah ? gen_helper_gvec_ah_fmls_idx_s : gen_helper_gvec_fmls_idx_s) 1155 TRANS_FEAT(FMLA_nx_d, aa64_sme2_f64f64, do_fmla_nx, a, FPST_ZA, 1156 gen_helper_gvec_fmla_idx_d) 1157 TRANS_FEAT(FMLS_nx_d, aa64_sme2_f64f64, do_fmla_nx, a, FPST_ZA, 1158 s->fpcr_ah ? gen_helper_gvec_ah_fmls_idx_d : gen_helper_gvec_fmls_idx_d) 1159 1160 TRANS_FEAT(BFMLA_nx, aa64_sme_b16b16, do_fmla_nx, a, FPST_ZA, 1161 gen_helper_gvec_bfmla_idx) 1162 TRANS_FEAT(BFMLS_nx, aa64_sme_b16b16, do_fmla_nx, a, FPST_ZA, 1163 s->fpcr_ah ? gen_helper_gvec_ah_bfmls_idx : gen_helper_gvec_bfmls_idx) 1164 1165 static bool do_faddsub(DisasContext *s, arg_az_n *a, ARMFPStatusFlavour fpst, 1166 gen_helper_gvec_3_ptr *fn) 1167 { 1168 if (sme_smza_enabled_check(s)) { 1169 int svl = streaming_vec_reg_size(s); 1170 int n = a->n; 1171 int zm = a->zm; 1172 int vstride = svl / n; 1173 TCGv_ptr t_za = get_zarray(s, a->rv, a->off, n, 0); 1174 TCGv_ptr ptr = fpstatus_ptr(fpst); 1175 TCGv_ptr t = tcg_temp_new_ptr(); 1176 1177 for (int r = 0; r < n; ++r) { 1178 TCGv_ptr t_zm = vec_full_reg_ptr(s, zm + r); 1179 int o_za = r * vstride * sizeof(ARMVectorReg); 1180 int desc = simd_desc(svl, svl, 0); 1181 1182 tcg_gen_addi_ptr(t, t_za, o_za); 1183 fn(t, t, t_zm, ptr, tcg_constant_i32(desc)); 1184 } 1185 } 1186 return true; 1187 } 1188 1189 TRANS_FEAT(FADD_nn_h, aa64_sme_f16f16, do_faddsub, a, 1190 FPST_ZA_F16, gen_helper_gvec_fadd_h) 1191 TRANS_FEAT(FSUB_nn_h, aa64_sme_f16f16, do_faddsub, a, 1192 FPST_ZA_F16, gen_helper_gvec_fsub_h) 1193 1194 TRANS_FEAT(FADD_nn_s, aa64_sme2, do_faddsub, a, 1195 FPST_ZA, gen_helper_gvec_fadd_s) 1196 TRANS_FEAT(FSUB_nn_s, aa64_sme2, do_faddsub, a, 1197 FPST_ZA, gen_helper_gvec_fsub_s) 1198 1199 TRANS_FEAT(FADD_nn_d, aa64_sme2_f64f64, do_faddsub, a, 1200 FPST_ZA, gen_helper_gvec_fadd_d) 1201 TRANS_FEAT(FSUB_nn_d, aa64_sme2_f64f64, do_faddsub, a, 1202 FPST_ZA, gen_helper_gvec_fsub_d) 1203 1204 TRANS_FEAT(BFADD_nn, aa64_sme_b16b16, do_faddsub, a, 1205 FPST_ZA, gen_helper_gvec_bfadd) 1206 TRANS_FEAT(BFSUB_nn, aa64_sme_b16b16, do_faddsub, a, 1207 FPST_ZA, gen_helper_gvec_bfsub) 1208 1209 /* 1210 * Expand array multi-vector single (n1), array multi-vector (nn), 1211 * and array multi-vector indexed (nx), for integer accumulate. 1212 * multi: true for nn, false for n1. 1213 * data: stuff for simd_data, including any index. 1214 */ 1215 static bool do_azz_acc(DisasContext *s, int nreg, int nsel, 1216 int rv, int off, int zn, int zm, 1217 int data, int shsel, bool multi, 1218 gen_helper_gvec_4 *fn) 1219 { 1220 if (sme_smza_enabled_check(s)) { 1221 int svl = streaming_vec_reg_size(s); 1222 int vstride = svl / nreg; 1223 TCGv_ptr t_za = get_zarray(s, rv, off, nreg, nsel); 1224 TCGv_ptr t = tcg_temp_new_ptr(); 1225 1226 for (int r = 0; r < nreg; ++r) { 1227 TCGv_ptr t_zn = vec_full_reg_ptr(s, zn); 1228 TCGv_ptr t_zm = vec_full_reg_ptr(s, zm); 1229 1230 for (int i = 0; i < nsel; ++i) { 1231 int o_za = (r * vstride + i) * sizeof(ARMVectorReg); 1232 int desc = simd_desc(svl, svl, data | (i << shsel)); 1233 1234 tcg_gen_addi_ptr(t, t_za, o_za); 1235 fn(t, t_zn, t_zm, t, tcg_constant_i32(desc)); 1236 } 1237 1238 /* 1239 * For multiple-and-single vectors, Zn may wrap. 1240 * For multiple vectors, both Zn and Zm are aligned. 1241 */ 1242 zn = (zn + 1) % 32; 1243 zm += multi; 1244 } 1245 } 1246 return true; 1247 } 1248 1249 static bool do_dot(DisasContext *s, arg_azz_n *a, bool multi, 1250 gen_helper_gvec_4 *fn) 1251 { 1252 return do_azz_acc(s, a->n, 1, a->rv, a->off, a->zn, a->zm, 1253 0, 0, multi, fn); 1254 } 1255 1256 static void gen_helper_gvec_sudot_4b(TCGv_ptr d, TCGv_ptr n, TCGv_ptr m, 1257 TCGv_ptr a, TCGv_i32 desc) 1258 { 1259 gen_helper_gvec_usdot_4b(d, m, n, a, desc); 1260 } 1261 1262 TRANS_FEAT(USDOT_n1, aa64_sme2, do_dot, a, false, gen_helper_gvec_usdot_4b) 1263 TRANS_FEAT(SUDOT_n1, aa64_sme2, do_dot, a, false, gen_helper_gvec_sudot_4b) 1264 TRANS_FEAT(SDOT_n1_2h, aa64_sme2, do_dot, a, false, gen_helper_gvec_sdot_2h) 1265 TRANS_FEAT(UDOT_n1_2h, aa64_sme2, do_dot, a, false, gen_helper_gvec_udot_2h) 1266 TRANS_FEAT(SDOT_n1_4b, aa64_sme2, do_dot, a, false, gen_helper_gvec_sdot_4b) 1267 TRANS_FEAT(UDOT_n1_4b, aa64_sme2, do_dot, a, false, gen_helper_gvec_udot_4b) 1268 TRANS_FEAT(SDOT_n1_4h, aa64_sme2_i16i64, do_dot, a, false, gen_helper_gvec_sdot_4h) 1269 TRANS_FEAT(UDOT_n1_4h, aa64_sme2_i16i64, do_dot, a, false, gen_helper_gvec_udot_4h) 1270 1271 TRANS_FEAT(USDOT_nn, aa64_sme2, do_dot, a, true, gen_helper_gvec_usdot_4b) 1272 TRANS_FEAT(SDOT_nn_2h, aa64_sme2, do_dot, a, true, gen_helper_gvec_sdot_2h) 1273 TRANS_FEAT(UDOT_nn_2h, aa64_sme2, do_dot, a, true, gen_helper_gvec_udot_2h) 1274 TRANS_FEAT(SDOT_nn_4b, aa64_sme2, do_dot, a, true, gen_helper_gvec_sdot_4b) 1275 TRANS_FEAT(UDOT_nn_4b, aa64_sme2, do_dot, a, true, gen_helper_gvec_udot_4b) 1276 TRANS_FEAT(SDOT_nn_4h, aa64_sme2_i16i64, do_dot, a, true, gen_helper_gvec_sdot_4h) 1277 TRANS_FEAT(UDOT_nn_4h, aa64_sme2_i16i64, do_dot, a, true, gen_helper_gvec_udot_4h) 1278 1279 static bool do_dot_nx(DisasContext *s, arg_azx_n *a, gen_helper_gvec_4 *fn) 1280 { 1281 return do_azz_acc(s, a->n, 1, a->rv, a->off, a->zn, a->zm, 1282 a->idx, 0, false, fn); 1283 } 1284 1285 TRANS_FEAT(USDOT_nx, aa64_sme2, do_dot_nx, a, gen_helper_gvec_usdot_idx_4b) 1286 TRANS_FEAT(SUDOT_nx, aa64_sme2, do_dot_nx, a, gen_helper_gvec_sudot_idx_4b) 1287 TRANS_FEAT(SDOT_nx_2h, aa64_sme2, do_dot_nx, a, gen_helper_gvec_sdot_idx_2h) 1288 TRANS_FEAT(UDOT_nx_2h, aa64_sme2, do_dot_nx, a, gen_helper_gvec_udot_idx_2h) 1289 TRANS_FEAT(SDOT_nx_4b, aa64_sme2, do_dot_nx, a, gen_helper_gvec_sdot_idx_4b) 1290 TRANS_FEAT(UDOT_nx_4b, aa64_sme2, do_dot_nx, a, gen_helper_gvec_udot_idx_4b) 1291 TRANS_FEAT(SDOT_nx_4h, aa64_sme2_i16i64, do_dot_nx, a, gen_helper_gvec_sdot_idx_4h) 1292 TRANS_FEAT(UDOT_nx_4h, aa64_sme2_i16i64, do_dot_nx, a, gen_helper_gvec_udot_idx_4h) 1293 1294 static bool do_vdot_nx(DisasContext *s, arg_azx_n *a, gen_helper_gvec_3 *fn) 1295 { 1296 if (sme_smza_enabled_check(s)) { 1297 int svl = streaming_vec_reg_size(s); 1298 fn(get_zarray(s, a->rv, a->off, a->n, 0), 1299 vec_full_reg_ptr(s, a->zn), 1300 vec_full_reg_ptr(s, a->zm), 1301 tcg_constant_i32(simd_desc(svl, svl, a->idx))); 1302 } 1303 return true; 1304 } 1305 1306 TRANS_FEAT(SVDOT_nx_2h, aa64_sme2, do_vdot_nx, a, gen_helper_sme2_svdot_idx_2h) 1307 TRANS_FEAT(SVDOT_nx_4b, aa64_sme2, do_vdot_nx, a, gen_helper_sme2_svdot_idx_4b) 1308 TRANS_FEAT(SVDOT_nx_4h, aa64_sme2, do_vdot_nx, a, gen_helper_sme2_svdot_idx_4h) 1309 1310 TRANS_FEAT(UVDOT_nx_2h, aa64_sme2, do_vdot_nx, a, gen_helper_sme2_uvdot_idx_2h) 1311 TRANS_FEAT(UVDOT_nx_4b, aa64_sme2, do_vdot_nx, a, gen_helper_sme2_uvdot_idx_4b) 1312 TRANS_FEAT(UVDOT_nx_4h, aa64_sme2, do_vdot_nx, a, gen_helper_sme2_uvdot_idx_4h) 1313 1314 TRANS_FEAT(SUVDOT_nx_4b, aa64_sme2, do_vdot_nx, a, gen_helper_sme2_suvdot_idx_4b) 1315 TRANS_FEAT(USVDOT_nx_4b, aa64_sme2, do_vdot_nx, a, gen_helper_sme2_usvdot_idx_4b) 1316 1317 static bool do_smlal(DisasContext *s, arg_azz_n *a, bool multi, 1318 gen_helper_gvec_4 *fn) 1319 { 1320 return do_azz_acc(s, a->n, 2, a->rv, a->off, a->zn, a->zm, 1321 0, 0, multi, fn); 1322 } 1323 1324 TRANS_FEAT(SMLAL_n1, aa64_sme2, do_smlal, a, false, gen_helper_sve2_smlal_zzzw_s) 1325 TRANS_FEAT(SMLSL_n1, aa64_sme2, do_smlal, a, false, gen_helper_sve2_smlsl_zzzw_s) 1326 TRANS_FEAT(UMLAL_n1, aa64_sme2, do_smlal, a, false, gen_helper_sve2_umlal_zzzw_s) 1327 TRANS_FEAT(UMLSL_n1, aa64_sme2, do_smlal, a, false, gen_helper_sve2_umlsl_zzzw_s) 1328 1329 TRANS_FEAT(SMLAL_nn, aa64_sme2, do_smlal, a, true, gen_helper_sve2_smlal_zzzw_s) 1330 TRANS_FEAT(SMLSL_nn, aa64_sme2, do_smlal, a, true, gen_helper_sve2_smlsl_zzzw_s) 1331 TRANS_FEAT(UMLAL_nn, aa64_sme2, do_smlal, a, true, gen_helper_sve2_umlal_zzzw_s) 1332 TRANS_FEAT(UMLSL_nn, aa64_sme2, do_smlal, a, true, gen_helper_sve2_umlsl_zzzw_s) 1333 1334 static bool do_smlal_nx(DisasContext *s, arg_azx_n *a, 1335 gen_helper_gvec_4 *fn) 1336 { 1337 return do_azz_acc(s, a->n, 2, a->rv, a->off, a->zn, a->zm, 1338 a->idx << 1, 0, false, fn); 1339 } 1340 1341 TRANS_FEAT(SMLAL_nx, aa64_sme2, do_smlal_nx, a, gen_helper_sve2_smlal_idx_s) 1342 TRANS_FEAT(SMLSL_nx, aa64_sme2, do_smlal_nx, a, gen_helper_sve2_smlsl_idx_s) 1343 TRANS_FEAT(UMLAL_nx, aa64_sme2, do_smlal_nx, a, gen_helper_sve2_umlal_idx_s) 1344 TRANS_FEAT(UMLSL_nx, aa64_sme2, do_smlal_nx, a, gen_helper_sve2_umlsl_idx_s) 1345 1346 static bool do_smlall(DisasContext *s, arg_azz_n *a, bool multi, 1347 gen_helper_gvec_4 *fn) 1348 { 1349 return do_azz_acc(s, a->n, 4, a->rv, a->off, a->zn, a->zm, 1350 0, 0, multi, fn); 1351 } 1352 1353 static void gen_helper_sme2_sumlall_s(TCGv_ptr d, TCGv_ptr n, TCGv_ptr m, 1354 TCGv_ptr a, TCGv_i32 desc) 1355 { 1356 gen_helper_sme2_usmlall_s(d, m, n, a, desc); 1357 } 1358 1359 TRANS_FEAT(SMLALL_n1_s, aa64_sme2, do_smlall, a, false, gen_helper_sme2_smlall_s) 1360 TRANS_FEAT(SMLSLL_n1_s, aa64_sme2, do_smlall, a, false, gen_helper_sme2_smlsll_s) 1361 TRANS_FEAT(UMLALL_n1_s, aa64_sme2, do_smlall, a, false, gen_helper_sme2_umlall_s) 1362 TRANS_FEAT(UMLSLL_n1_s, aa64_sme2, do_smlall, a, false, gen_helper_sme2_umlsll_s) 1363 TRANS_FEAT(USMLALL_n1_s, aa64_sme2, do_smlall, a, false, gen_helper_sme2_usmlall_s) 1364 TRANS_FEAT(SUMLALL_n1_s, aa64_sme2, do_smlall, a, false, gen_helper_sme2_sumlall_s) 1365 1366 TRANS_FEAT(SMLALL_n1_d, aa64_sme2_i16i64, do_smlall, a, false, gen_helper_sme2_smlall_d) 1367 TRANS_FEAT(SMLSLL_n1_d, aa64_sme2_i16i64, do_smlall, a, false, gen_helper_sme2_smlsll_d) 1368 TRANS_FEAT(UMLALL_n1_d, aa64_sme2_i16i64, do_smlall, a, false, gen_helper_sme2_umlall_d) 1369 TRANS_FEAT(UMLSLL_n1_d, aa64_sme2_i16i64, do_smlall, a, false, gen_helper_sme2_umlsll_d) 1370 1371 TRANS_FEAT(SMLALL_nn_s, aa64_sme2, do_smlall, a, true, gen_helper_sme2_smlall_s) 1372 TRANS_FEAT(SMLSLL_nn_s, aa64_sme2, do_smlall, a, true, gen_helper_sme2_smlsll_s) 1373 TRANS_FEAT(UMLALL_nn_s, aa64_sme2, do_smlall, a, true, gen_helper_sme2_umlall_s) 1374 TRANS_FEAT(UMLSLL_nn_s, aa64_sme2, do_smlall, a, true, gen_helper_sme2_umlsll_s) 1375 TRANS_FEAT(USMLALL_nn_s, aa64_sme2, do_smlall, a, true, gen_helper_sme2_usmlall_s) 1376 1377 TRANS_FEAT(SMLALL_nn_d, aa64_sme2_i16i64, do_smlall, a, true, gen_helper_sme2_smlall_d) 1378 TRANS_FEAT(SMLSLL_nn_d, aa64_sme2_i16i64, do_smlall, a, true, gen_helper_sme2_smlsll_d) 1379 TRANS_FEAT(UMLALL_nn_d, aa64_sme2_i16i64, do_smlall, a, true, gen_helper_sme2_umlall_d) 1380 TRANS_FEAT(UMLSLL_nn_d, aa64_sme2_i16i64, do_smlall, a, true, gen_helper_sme2_umlsll_d) 1381 1382 static bool do_smlall_nx(DisasContext *s, arg_azx_n *a, 1383 gen_helper_gvec_4 *fn) 1384 { 1385 return do_azz_acc(s, a->n, 4, a->rv, a->off, a->zn, a->zm, 1386 a->idx << 2, 0, false, fn); 1387 } 1388 1389 TRANS_FEAT(SMLALL_nx_s, aa64_sme2, do_smlall_nx, a, gen_helper_sme2_smlall_idx_s) 1390 TRANS_FEAT(SMLSLL_nx_s, aa64_sme2, do_smlall_nx, a, gen_helper_sme2_smlsll_idx_s) 1391 TRANS_FEAT(UMLALL_nx_s, aa64_sme2, do_smlall_nx, a, gen_helper_sme2_umlall_idx_s) 1392 TRANS_FEAT(UMLSLL_nx_s, aa64_sme2, do_smlall_nx, a, gen_helper_sme2_umlsll_idx_s) 1393 TRANS_FEAT(USMLALL_nx_s, aa64_sme2, do_smlall_nx, a, gen_helper_sme2_usmlall_idx_s) 1394 TRANS_FEAT(SUMLALL_nx_s, aa64_sme2, do_smlall_nx, a, gen_helper_sme2_sumlall_idx_s) 1395 1396 TRANS_FEAT(SMLALL_nx_d, aa64_sme2_i16i64, do_smlall_nx, a, gen_helper_sme2_smlall_idx_d) 1397 TRANS_FEAT(SMLSLL_nx_d, aa64_sme2_i16i64, do_smlall_nx, a, gen_helper_sme2_smlsll_idx_d) 1398 TRANS_FEAT(UMLALL_nx_d, aa64_sme2_i16i64, do_smlall_nx, a, gen_helper_sme2_umlall_idx_d) 1399 TRANS_FEAT(UMLSLL_nx_d, aa64_sme2_i16i64, do_smlall_nx, a, gen_helper_sme2_umlsll_idx_d) 1400 1401 static bool do_zz_fpst(DisasContext *s, arg_zz_n *a, int data, 1402 ARMFPStatusFlavour type, gen_helper_gvec_2_ptr *fn) 1403 { 1404 if (sme_sm_enabled_check(s)) { 1405 int svl = streaming_vec_reg_size(s); 1406 TCGv_ptr fpst = fpstatus_ptr(type); 1407 1408 for (int i = 0, n = a->n; i < n; ++i) { 1409 tcg_gen_gvec_2_ptr(vec_full_reg_offset(s, a->zd + i), 1410 vec_full_reg_offset(s, a->zn + i), 1411 fpst, svl, svl, data, fn); 1412 } 1413 } 1414 return true; 1415 } 1416 1417 TRANS_FEAT(BFCVT, aa64_sme2, do_zz_fpst, a, 0, 1418 FPST_A64, gen_helper_sme2_bfcvt) 1419 TRANS_FEAT(BFCVTN, aa64_sme2, do_zz_fpst, a, 0, 1420 FPST_A64, gen_helper_sme2_bfcvtn) 1421 TRANS_FEAT(FCVT_n, aa64_sme2, do_zz_fpst, a, 0, 1422 FPST_A64, gen_helper_sme2_fcvt_n) 1423 TRANS_FEAT(FCVTN, aa64_sme2, do_zz_fpst, a, 0, 1424 FPST_A64, gen_helper_sme2_fcvtn) 1425 1426 TRANS_FEAT(FCVT_w, aa64_sme_f16f16, do_zz_fpst, a, 0, 1427 FPST_A64_F16, gen_helper_sme2_fcvt_w) 1428 TRANS_FEAT(FCVTL, aa64_sme_f16f16, do_zz_fpst, a, 0, 1429 FPST_A64_F16, gen_helper_sme2_fcvtl) 1430 1431 TRANS_FEAT(FCVTZS, aa64_sme2, do_zz_fpst, a, 0, 1432 FPST_A64, gen_helper_gvec_vcvt_rz_fs) 1433 TRANS_FEAT(FCVTZU, aa64_sme2, do_zz_fpst, a, 0, 1434 FPST_A64, gen_helper_gvec_vcvt_rz_fu) 1435 1436 TRANS_FEAT(SCVTF, aa64_sme2, do_zz_fpst, a, 0, 1437 FPST_A64, gen_helper_sme2_scvtf) 1438 TRANS_FEAT(UCVTF, aa64_sme2, do_zz_fpst, a, 0, 1439 FPST_A64, gen_helper_sme2_ucvtf) 1440 1441 TRANS_FEAT(FRINTN, aa64_sme2, do_zz_fpst, a, float_round_nearest_even, 1442 FPST_A64, gen_helper_gvec_vrint_rm_s) 1443 TRANS_FEAT(FRINTP, aa64_sme2, do_zz_fpst, a, float_round_up, 1444 FPST_A64, gen_helper_gvec_vrint_rm_s) 1445 TRANS_FEAT(FRINTM, aa64_sme2, do_zz_fpst, a, float_round_down, 1446 FPST_A64, gen_helper_gvec_vrint_rm_s) 1447 TRANS_FEAT(FRINTA, aa64_sme2, do_zz_fpst, a, float_round_ties_away, 1448 FPST_A64, gen_helper_gvec_vrint_rm_s) 1449 1450 static bool do_zz(DisasContext *s, arg_zz_n *a, int data, 1451 gen_helper_gvec_2 *fn) 1452 { 1453 if (sme_sm_enabled_check(s)) { 1454 int svl = streaming_vec_reg_size(s); 1455 1456 for (int i = 0, n = a->n; i < n; ++i) { 1457 tcg_gen_gvec_2_ool(vec_full_reg_offset(s, a->zd + i), 1458 vec_full_reg_offset(s, a->zn + i), 1459 svl, svl, data, fn); 1460 } 1461 } 1462 return true; 1463 } 1464 1465 TRANS_FEAT(SQCVT_sh, aa64_sme2, do_zz, a, 0, gen_helper_sme2_sqcvt_sh) 1466 TRANS_FEAT(UQCVT_sh, aa64_sme2, do_zz, a, 0, gen_helper_sme2_uqcvt_sh) 1467 TRANS_FEAT(SQCVTU_sh, aa64_sme2, do_zz, a, 0, gen_helper_sme2_sqcvtu_sh) 1468 1469 TRANS_FEAT(SQCVT_sb, aa64_sme2, do_zz, a, 0, gen_helper_sme2_sqcvt_sb) 1470 TRANS_FEAT(UQCVT_sb, aa64_sme2, do_zz, a, 0, gen_helper_sme2_uqcvt_sb) 1471 TRANS_FEAT(SQCVTU_sb, aa64_sme2, do_zz, a, 0, gen_helper_sme2_sqcvtu_sb) 1472 1473 TRANS_FEAT(SQCVT_dh, aa64_sme2, do_zz, a, 0, gen_helper_sme2_sqcvt_dh) 1474 TRANS_FEAT(UQCVT_dh, aa64_sme2, do_zz, a, 0, gen_helper_sme2_uqcvt_dh) 1475 TRANS_FEAT(SQCVTU_dh, aa64_sme2, do_zz, a, 0, gen_helper_sme2_sqcvtu_dh) 1476 1477 TRANS_FEAT(SQCVTN_sb, aa64_sme2, do_zz, a, 0, gen_helper_sme2_sqcvtn_sb) 1478 TRANS_FEAT(UQCVTN_sb, aa64_sme2, do_zz, a, 0, gen_helper_sme2_uqcvtn_sb) 1479 TRANS_FEAT(SQCVTUN_sb, aa64_sme2, do_zz, a, 0, gen_helper_sme2_sqcvtun_sb) 1480 1481 TRANS_FEAT(SQCVTN_dh, aa64_sme2, do_zz, a, 0, gen_helper_sme2_sqcvtn_dh) 1482 TRANS_FEAT(UQCVTN_dh, aa64_sme2, do_zz, a, 0, gen_helper_sme2_uqcvtn_dh) 1483 TRANS_FEAT(SQCVTUN_dh, aa64_sme2, do_zz, a, 0, gen_helper_sme2_sqcvtun_dh) 1484 1485 TRANS_FEAT(SUNPK_2bh, aa64_sme2, do_zz, a, 0, gen_helper_sme2_sunpk2_bh) 1486 TRANS_FEAT(SUNPK_2hs, aa64_sme2, do_zz, a, 0, gen_helper_sme2_sunpk2_hs) 1487 TRANS_FEAT(SUNPK_2sd, aa64_sme2, do_zz, a, 0, gen_helper_sme2_sunpk2_sd) 1488 1489 TRANS_FEAT(SUNPK_4bh, aa64_sme2, do_zz, a, 0, gen_helper_sme2_sunpk4_bh) 1490 TRANS_FEAT(SUNPK_4hs, aa64_sme2, do_zz, a, 0, gen_helper_sme2_sunpk4_hs) 1491 TRANS_FEAT(SUNPK_4sd, aa64_sme2, do_zz, a, 0, gen_helper_sme2_sunpk4_sd) 1492 1493 TRANS_FEAT(UUNPK_2bh, aa64_sme2, do_zz, a, 0, gen_helper_sme2_uunpk2_bh) 1494 TRANS_FEAT(UUNPK_2hs, aa64_sme2, do_zz, a, 0, gen_helper_sme2_uunpk2_hs) 1495 TRANS_FEAT(UUNPK_2sd, aa64_sme2, do_zz, a, 0, gen_helper_sme2_uunpk2_sd) 1496 1497 TRANS_FEAT(UUNPK_4bh, aa64_sme2, do_zz, a, 0, gen_helper_sme2_uunpk4_bh) 1498 TRANS_FEAT(UUNPK_4hs, aa64_sme2, do_zz, a, 0, gen_helper_sme2_uunpk4_hs) 1499 TRANS_FEAT(UUNPK_4sd, aa64_sme2, do_zz, a, 0, gen_helper_sme2_uunpk4_sd) 1500 1501 static bool do_zipuzp_4(DisasContext *s, arg_zz_e *a, 1502 gen_helper_gvec_2 * const fn[5]) 1503 { 1504 int bytes_per_op = 4 << a->esz; 1505 1506 /* Both MO_64 and MO_128 can fail the size test. */ 1507 if (s->max_svl < bytes_per_op) { 1508 unallocated_encoding(s); 1509 } else if (sme_sm_enabled_check(s)) { 1510 int svl = streaming_vec_reg_size(s); 1511 if (svl < bytes_per_op) { 1512 unallocated_encoding(s); 1513 } else { 1514 tcg_gen_gvec_2_ool(vec_full_reg_offset(s, a->zd), 1515 vec_full_reg_offset(s, a->zn), 1516 svl, svl, 0, fn[a->esz]); 1517 } 1518 } 1519 return true; 1520 } 1521 1522 static gen_helper_gvec_2 * const zip4_fns[] = { 1523 gen_helper_sme2_zip4_b, 1524 gen_helper_sme2_zip4_h, 1525 gen_helper_sme2_zip4_s, 1526 gen_helper_sme2_zip4_d, 1527 gen_helper_sme2_zip4_q, 1528 }; 1529 TRANS_FEAT(ZIP_4, aa64_sme2, do_zipuzp_4, a, zip4_fns) 1530 1531 static gen_helper_gvec_2 * const uzp4_fns[] = { 1532 gen_helper_sme2_uzp4_b, 1533 gen_helper_sme2_uzp4_h, 1534 gen_helper_sme2_uzp4_s, 1535 gen_helper_sme2_uzp4_d, 1536 gen_helper_sme2_uzp4_q, 1537 }; 1538 TRANS_FEAT(UZP_4, aa64_sme2, do_zipuzp_4, a, uzp4_fns) 1539 1540 static bool do_zz_rshr(DisasContext *s, arg_rshr *a, gen_helper_gvec_2 *fn) 1541 { 1542 if (sve_access_check(s)) { 1543 int vl = vec_full_reg_size(s); 1544 tcg_gen_gvec_2_ool(vec_full_reg_offset(s, a->zd), 1545 vec_full_reg_offset(s, a->zn), 1546 vl, vl, a->shift, fn); 1547 } 1548 return true; 1549 } 1550 1551 TRANS_FEAT(SQRSHR_sh, aa64_sme2, do_zz_rshr, a, gen_helper_sme2_sqrshr_sh) 1552 TRANS_FEAT(UQRSHR_sh, aa64_sme2, do_zz_rshr, a, gen_helper_sme2_uqrshr_sh) 1553 TRANS_FEAT(SQRSHRU_sh, aa64_sme2, do_zz_rshr, a, gen_helper_sme2_sqrshru_sh) 1554 1555 TRANS_FEAT(SQRSHR_sb, aa64_sme2, do_zz_rshr, a, gen_helper_sme2_sqrshr_sb) 1556 TRANS_FEAT(SQRSHR_dh, aa64_sme2, do_zz_rshr, a, gen_helper_sme2_sqrshr_dh) 1557 TRANS_FEAT(UQRSHR_sb, aa64_sme2, do_zz_rshr, a, gen_helper_sme2_uqrshr_sb) 1558 TRANS_FEAT(UQRSHR_dh, aa64_sme2, do_zz_rshr, a, gen_helper_sme2_uqrshr_dh) 1559 TRANS_FEAT(SQRSHRU_sb, aa64_sme2, do_zz_rshr, a, gen_helper_sme2_sqrshru_sb) 1560 TRANS_FEAT(SQRSHRU_dh, aa64_sme2, do_zz_rshr, a, gen_helper_sme2_sqrshru_dh) 1561 1562 TRANS_FEAT(SQRSHRN_sh, aa64_sme2_or_sve2p1, do_zz_rshr, a, gen_helper_sme2_sqrshrn_sh) 1563 TRANS_FEAT(UQRSHRN_sh, aa64_sme2_or_sve2p1, do_zz_rshr, a, gen_helper_sme2_uqrshrn_sh) 1564 TRANS_FEAT(SQRSHRUN_sh, aa64_sme2_or_sve2p1, do_zz_rshr, a, gen_helper_sme2_sqrshrun_sh) 1565 1566 TRANS_FEAT(SQRSHRN_sb, aa64_sme2, do_zz_rshr, a, gen_helper_sme2_sqrshrn_sb) 1567 TRANS_FEAT(SQRSHRN_dh, aa64_sme2, do_zz_rshr, a, gen_helper_sme2_sqrshrn_dh) 1568 TRANS_FEAT(UQRSHRN_sb, aa64_sme2, do_zz_rshr, a, gen_helper_sme2_uqrshrn_sb) 1569 TRANS_FEAT(UQRSHRN_dh, aa64_sme2, do_zz_rshr, a, gen_helper_sme2_uqrshrn_dh) 1570 TRANS_FEAT(SQRSHRUN_sb, aa64_sme2, do_zz_rshr, a, gen_helper_sme2_sqrshrun_sb) 1571 TRANS_FEAT(SQRSHRUN_dh, aa64_sme2, do_zz_rshr, a, gen_helper_sme2_sqrshrun_dh) 1572 1573 static bool do_zipuzp_2(DisasContext *s, arg_zzz_e *a, 1574 gen_helper_gvec_3 * const fn[5]) 1575 { 1576 int bytes_per_op = 2 << a->esz; 1577 1578 /* MO_128 can fail the size test. */ 1579 if (s->max_svl < bytes_per_op) { 1580 unallocated_encoding(s); 1581 } else if (sme_sm_enabled_check(s)) { 1582 int svl = streaming_vec_reg_size(s); 1583 if (svl < bytes_per_op) { 1584 unallocated_encoding(s); 1585 } else { 1586 tcg_gen_gvec_3_ool(vec_full_reg_offset(s, a->zd), 1587 vec_full_reg_offset(s, a->zn), 1588 vec_full_reg_offset(s, a->zm), 1589 svl, svl, 0, fn[a->esz]); 1590 } 1591 } 1592 return true; 1593 } 1594 1595 static gen_helper_gvec_3 * const zip2_fns[] = { 1596 gen_helper_sme2_zip2_b, 1597 gen_helper_sme2_zip2_h, 1598 gen_helper_sme2_zip2_s, 1599 gen_helper_sme2_zip2_d, 1600 gen_helper_sme2_zip2_q, 1601 }; 1602 TRANS_FEAT(ZIP_2, aa64_sme2, do_zipuzp_2, a, zip2_fns) 1603 1604 static gen_helper_gvec_3 * const uzp2_fns[] = { 1605 gen_helper_sme2_uzp2_b, 1606 gen_helper_sme2_uzp2_h, 1607 gen_helper_sme2_uzp2_s, 1608 gen_helper_sme2_uzp2_d, 1609 gen_helper_sme2_uzp2_q, 1610 }; 1611 TRANS_FEAT(UZP_2, aa64_sme2, do_zipuzp_2, a, uzp2_fns) 1612 1613 static bool trans_FCLAMP(DisasContext *s, arg_zzz_en *a) 1614 { 1615 static gen_helper_gvec_3_ptr * const fn[] = { 1616 gen_helper_sme2_bfclamp, 1617 gen_helper_sme2_fclamp_h, 1618 gen_helper_sme2_fclamp_s, 1619 gen_helper_sme2_fclamp_d, 1620 }; 1621 TCGv_ptr fpst; 1622 int vl; 1623 1624 if (!dc_isar_feature(aa64_sme2, s)) { 1625 return false; 1626 } 1627 /* This insn uses MO_8 to encode BFloat16. */ 1628 if (a->esz == MO_8 && !dc_isar_feature(aa64_sme_b16b16, s)) { 1629 return false; 1630 } 1631 if (!sme_sm_enabled_check(s)) { 1632 return true; 1633 } 1634 1635 fpst = fpstatus_ptr(a->esz == MO_16 ? FPST_A64_F16 : FPST_A64); 1636 vl = vec_full_reg_size(s); 1637 1638 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, a->zd), 1639 vec_full_reg_offset(s, a->zn), 1640 vec_full_reg_offset(s, a->zm), 1641 fpst, vl, vl, a->n, fn[a->esz]); 1642 return true; 1643 } 1644 1645 static bool do_clamp(DisasContext *s, arg_zzz_en *a, 1646 gen_helper_gvec_3 * const fn[4]) 1647 { 1648 int vl; 1649 1650 if (!dc_isar_feature(aa64_sme2, s)) { 1651 return false; 1652 } 1653 if (!sme_sm_enabled_check(s)) { 1654 return true; 1655 } 1656 1657 /* 1658 * Clamp is just a min+max, easily supported by most host 1659 * vector operations -- we already have such an expansion in 1660 * translate-sve.c for a single output. 1661 * TODO: Add support in gvec for multiple simultaneous output, 1662 * and/or copy to temporary upon overlap. 1663 */ 1664 vl = vec_full_reg_size(s); 1665 tcg_gen_gvec_3_ool(vec_full_reg_offset(s, a->zd), 1666 vec_full_reg_offset(s, a->zn), 1667 vec_full_reg_offset(s, a->zm), 1668 vl, vl, a->n, fn[a->esz]); 1669 return true; 1670 } 1671 1672 static gen_helper_gvec_3 * const sclamp_fns[] = { 1673 gen_helper_sme2_sclamp_b, 1674 gen_helper_sme2_sclamp_h, 1675 gen_helper_sme2_sclamp_s, 1676 gen_helper_sme2_sclamp_d, 1677 }; 1678 TRANS(SCLAMP, do_clamp, a, sclamp_fns) 1679 1680 static gen_helper_gvec_3 * const uclamp_fns[] = { 1681 gen_helper_sme2_uclamp_b, 1682 gen_helper_sme2_uclamp_h, 1683 gen_helper_sme2_uclamp_s, 1684 gen_helper_sme2_uclamp_d, 1685 }; 1686 TRANS(UCLAMP, do_clamp, a, uclamp_fns) 1687 1688 static bool trans_SEL(DisasContext *s, arg_SEL *a) 1689 { 1690 typedef void sme_sel_fn(TCGv_ptr, TCGv_ptr, TCGv_ptr, TCGv_i32, TCGv_i32); 1691 static sme_sel_fn * const fns[4] = { 1692 gen_helper_sme2_sel_b, gen_helper_sme2_sel_h, 1693 gen_helper_sme2_sel_s, gen_helper_sme2_sel_d 1694 }; 1695 1696 if (!dc_isar_feature(aa64_sme2, s)) { 1697 return false; 1698 } 1699 if (sme_sm_enabled_check(s)) { 1700 int svl = streaming_vec_reg_size(s); 1701 uint32_t desc = simd_desc(svl, svl, a->n); 1702 TCGv_ptr t_d = tcg_temp_new_ptr(); 1703 TCGv_ptr t_n = tcg_temp_new_ptr(); 1704 TCGv_ptr t_m = tcg_temp_new_ptr(); 1705 TCGv_i32 png = tcg_temp_new_i32(); 1706 1707 tcg_gen_addi_ptr(t_d, tcg_env, vec_full_reg_offset(s, a->zd)); 1708 tcg_gen_addi_ptr(t_n, tcg_env, vec_full_reg_offset(s, a->zn)); 1709 tcg_gen_addi_ptr(t_m, tcg_env, vec_full_reg_offset(s, a->zm)); 1710 1711 tcg_gen_ld16u_i32(png, tcg_env, pred_full_reg_offset(s, a->pg) 1712 ^ (HOST_BIG_ENDIAN ? 6 : 0)); 1713 1714 fns[a->esz](t_d, t_n, t_m, png, tcg_constant_i32(desc)); 1715 } 1716 return true; 1717 } 1718 1719 static bool do_lut(DisasContext *s, arg_lut *a, 1720 gen_helper_gvec_2_ptr *fn, bool strided) 1721 { 1722 if (sme_sm_enabled_check(s) && sme2_zt0_enabled_check(s)) { 1723 int svl = streaming_vec_reg_size(s); 1724 tcg_gen_gvec_2_ptr(vec_full_reg_offset(s, a->zd), 1725 vec_full_reg_offset(s, a->zn), 1726 tcg_env, svl, svl, strided | (a->idx << 1), fn); 1727 } 1728 return true; 1729 } 1730 1731 TRANS_FEAT(LUTI2_c_1b, aa64_sme2, do_lut, a, gen_helper_sme2_luti2_1b, false) 1732 TRANS_FEAT(LUTI2_c_1h, aa64_sme2, do_lut, a, gen_helper_sme2_luti2_1h, false) 1733 TRANS_FEAT(LUTI2_c_1s, aa64_sme2, do_lut, a, gen_helper_sme2_luti2_1s, false) 1734 1735 TRANS_FEAT(LUTI2_c_2b, aa64_sme2, do_lut, a, gen_helper_sme2_luti2_2b, false) 1736 TRANS_FEAT(LUTI2_c_2h, aa64_sme2, do_lut, a, gen_helper_sme2_luti2_2h, false) 1737 TRANS_FEAT(LUTI2_c_2s, aa64_sme2, do_lut, a, gen_helper_sme2_luti2_2s, false) 1738 1739 TRANS_FEAT(LUTI2_c_4b, aa64_sme2, do_lut, a, gen_helper_sme2_luti2_4b, false) 1740 TRANS_FEAT(LUTI2_c_4h, aa64_sme2, do_lut, a, gen_helper_sme2_luti2_4h, false) 1741 TRANS_FEAT(LUTI2_c_4s, aa64_sme2, do_lut, a, gen_helper_sme2_luti2_4s, false) 1742 1743 TRANS_FEAT(LUTI4_c_1b, aa64_sme2, do_lut, a, gen_helper_sme2_luti4_1b, false) 1744 TRANS_FEAT(LUTI4_c_1h, aa64_sme2, do_lut, a, gen_helper_sme2_luti4_1h, false) 1745 TRANS_FEAT(LUTI4_c_1s, aa64_sme2, do_lut, a, gen_helper_sme2_luti4_1s, false) 1746 1747 TRANS_FEAT(LUTI4_c_2b, aa64_sme2, do_lut, a, gen_helper_sme2_luti4_2b, false) 1748 TRANS_FEAT(LUTI4_c_2h, aa64_sme2, do_lut, a, gen_helper_sme2_luti4_2h, false) 1749 TRANS_FEAT(LUTI4_c_2s, aa64_sme2, do_lut, a, gen_helper_sme2_luti4_2s, false) 1750 1751 TRANS_FEAT(LUTI4_c_4h, aa64_sme2, do_lut, a, gen_helper_sme2_luti4_4h, false) 1752 TRANS_FEAT(LUTI4_c_4s, aa64_sme2, do_lut, a, gen_helper_sme2_luti4_4s, false) 1753 1754 static bool do_lut_s4(DisasContext *s, arg_lut *a, gen_helper_gvec_2_ptr *fn) 1755 { 1756 return !(a->zd & 0b01100) && do_lut(s, a, fn, true); 1757 } 1758 1759 static bool do_lut_s8(DisasContext *s, arg_lut *a, gen_helper_gvec_2_ptr *fn) 1760 { 1761 return !(a->zd & 0b01000) && do_lut(s, a, fn, true); 1762 } 1763 1764 TRANS_FEAT(LUTI2_s_2b, aa64_sme2p1, do_lut_s8, a, gen_helper_sme2_luti2_2b) 1765 TRANS_FEAT(LUTI2_s_2h, aa64_sme2p1, do_lut_s8, a, gen_helper_sme2_luti2_2h) 1766 1767 TRANS_FEAT(LUTI2_s_4b, aa64_sme2p1, do_lut_s4, a, gen_helper_sme2_luti2_4b) 1768 TRANS_FEAT(LUTI2_s_4h, aa64_sme2p1, do_lut_s4, a, gen_helper_sme2_luti2_4h) 1769 1770 TRANS_FEAT(LUTI4_s_2b, aa64_sme2p1, do_lut_s8, a, gen_helper_sme2_luti4_2b) 1771 TRANS_FEAT(LUTI4_s_2h, aa64_sme2p1, do_lut_s8, a, gen_helper_sme2_luti4_2h) 1772 1773 TRANS_FEAT(LUTI4_s_4h, aa64_sme2p1, do_lut_s4, a, gen_helper_sme2_luti4_4h) 1774