1 /* 2 * AArch64 SVE translation 3 * 4 * Copyright (c) 2018 Linaro, Ltd 5 * 6 * This library is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Lesser General Public 8 * License as published by the Free Software Foundation; either 9 * version 2.1 of the License, or (at your option) any later version. 10 * 11 * This library is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * Lesser General Public License for more details. 15 * 16 * You should have received a copy of the GNU Lesser General Public 17 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 18 */ 19 20 #include "qemu/osdep.h" 21 #include "cpu.h" 22 #include "exec/exec-all.h" 23 #include "tcg/tcg-op.h" 24 #include "tcg/tcg-op-gvec.h" 25 #include "tcg/tcg-gvec-desc.h" 26 #include "qemu/log.h" 27 #include "arm_ldst.h" 28 #include "translate.h" 29 #include "internals.h" 30 #include "exec/helper-proto.h" 31 #include "exec/helper-gen.h" 32 #include "exec/log.h" 33 #include "translate-a64.h" 34 #include "fpu/softfloat.h" 35 36 37 typedef void GVecGen2sFn(unsigned, uint32_t, uint32_t, 38 TCGv_i64, uint32_t, uint32_t); 39 40 typedef void gen_helper_gvec_flags_3(TCGv_i32, TCGv_ptr, TCGv_ptr, 41 TCGv_ptr, TCGv_i32); 42 typedef void gen_helper_gvec_flags_4(TCGv_i32, TCGv_ptr, TCGv_ptr, 43 TCGv_ptr, TCGv_ptr, TCGv_i32); 44 45 typedef void gen_helper_gvec_mem(TCGv_env, TCGv_ptr, TCGv_i64, TCGv_i32); 46 typedef void gen_helper_gvec_mem_scatter(TCGv_env, TCGv_ptr, TCGv_ptr, 47 TCGv_ptr, TCGv_i64, TCGv_i32); 48 49 /* 50 * Helpers for extracting complex instruction fields. 51 */ 52 53 /* See e.g. ASR (immediate, predicated). 54 * Returns -1 for unallocated encoding; diagnose later. 55 */ 56 static int tszimm_esz(DisasContext *s, int x) 57 { 58 x >>= 3; /* discard imm3 */ 59 return 31 - clz32(x); 60 } 61 62 static int tszimm_shr(DisasContext *s, int x) 63 { 64 return (16 << tszimm_esz(s, x)) - x; 65 } 66 67 /* See e.g. LSL (immediate, predicated). */ 68 static int tszimm_shl(DisasContext *s, int x) 69 { 70 return x - (8 << tszimm_esz(s, x)); 71 } 72 73 /* The SH bit is in bit 8. Extract the low 8 and shift. */ 74 static inline int expand_imm_sh8s(DisasContext *s, int x) 75 { 76 return (int8_t)x << (x & 0x100 ? 8 : 0); 77 } 78 79 static inline int expand_imm_sh8u(DisasContext *s, int x) 80 { 81 return (uint8_t)x << (x & 0x100 ? 8 : 0); 82 } 83 84 /* Convert a 2-bit memory size (msz) to a 4-bit data type (dtype) 85 * with unsigned data. C.f. SVE Memory Contiguous Load Group. 86 */ 87 static inline int msz_dtype(DisasContext *s, int msz) 88 { 89 static const uint8_t dtype[4] = { 0, 5, 10, 15 }; 90 return dtype[msz]; 91 } 92 93 /* 94 * Include the generated decoder. 95 */ 96 97 #include "decode-sve.c.inc" 98 99 /* 100 * Implement all of the translator functions referenced by the decoder. 101 */ 102 103 /* Invoke an out-of-line helper on 2 Zregs. */ 104 static bool gen_gvec_ool_zz(DisasContext *s, gen_helper_gvec_2 *fn, 105 int rd, int rn, int data) 106 { 107 if (fn == NULL) { 108 return false; 109 } 110 if (sve_access_check(s)) { 111 unsigned vsz = vec_full_reg_size(s); 112 tcg_gen_gvec_2_ool(vec_full_reg_offset(s, rd), 113 vec_full_reg_offset(s, rn), 114 vsz, vsz, data, fn); 115 } 116 return true; 117 } 118 119 static bool gen_gvec_fpst_zz(DisasContext *s, gen_helper_gvec_2_ptr *fn, 120 int rd, int rn, int data, 121 ARMFPStatusFlavour flavour) 122 { 123 if (fn == NULL) { 124 return false; 125 } 126 if (sve_access_check(s)) { 127 unsigned vsz = vec_full_reg_size(s); 128 TCGv_ptr status = fpstatus_ptr(flavour); 129 130 tcg_gen_gvec_2_ptr(vec_full_reg_offset(s, rd), 131 vec_full_reg_offset(s, rn), 132 status, vsz, vsz, data, fn); 133 } 134 return true; 135 } 136 137 static bool gen_gvec_fpst_arg_zz(DisasContext *s, gen_helper_gvec_2_ptr *fn, 138 arg_rr_esz *a, int data) 139 { 140 return gen_gvec_fpst_zz(s, fn, a->rd, a->rn, data, 141 a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR); 142 } 143 144 /* Invoke an out-of-line helper on 3 Zregs. */ 145 static bool gen_gvec_ool_zzz(DisasContext *s, gen_helper_gvec_3 *fn, 146 int rd, int rn, int rm, int data) 147 { 148 if (fn == NULL) { 149 return false; 150 } 151 if (sve_access_check(s)) { 152 unsigned vsz = vec_full_reg_size(s); 153 tcg_gen_gvec_3_ool(vec_full_reg_offset(s, rd), 154 vec_full_reg_offset(s, rn), 155 vec_full_reg_offset(s, rm), 156 vsz, vsz, data, fn); 157 } 158 return true; 159 } 160 161 static bool gen_gvec_ool_arg_zzz(DisasContext *s, gen_helper_gvec_3 *fn, 162 arg_rrr_esz *a, int data) 163 { 164 return gen_gvec_ool_zzz(s, fn, a->rd, a->rn, a->rm, data); 165 } 166 167 /* Invoke an out-of-line helper on 3 Zregs, plus float_status. */ 168 static bool gen_gvec_fpst_zzz(DisasContext *s, gen_helper_gvec_3_ptr *fn, 169 int rd, int rn, int rm, 170 int data, ARMFPStatusFlavour flavour) 171 { 172 if (fn == NULL) { 173 return false; 174 } 175 if (sve_access_check(s)) { 176 unsigned vsz = vec_full_reg_size(s); 177 TCGv_ptr status = fpstatus_ptr(flavour); 178 179 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd), 180 vec_full_reg_offset(s, rn), 181 vec_full_reg_offset(s, rm), 182 status, vsz, vsz, data, fn); 183 } 184 return true; 185 } 186 187 static bool gen_gvec_fpst_arg_zzz(DisasContext *s, gen_helper_gvec_3_ptr *fn, 188 arg_rrr_esz *a, int data) 189 { 190 return gen_gvec_fpst_zzz(s, fn, a->rd, a->rn, a->rm, data, 191 a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR); 192 } 193 194 /* Invoke an out-of-line helper on 4 Zregs. */ 195 static bool gen_gvec_ool_zzzz(DisasContext *s, gen_helper_gvec_4 *fn, 196 int rd, int rn, int rm, int ra, int data) 197 { 198 if (fn == NULL) { 199 return false; 200 } 201 if (sve_access_check(s)) { 202 unsigned vsz = vec_full_reg_size(s); 203 tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd), 204 vec_full_reg_offset(s, rn), 205 vec_full_reg_offset(s, rm), 206 vec_full_reg_offset(s, ra), 207 vsz, vsz, data, fn); 208 } 209 return true; 210 } 211 212 static bool gen_gvec_ool_arg_zzzz(DisasContext *s, gen_helper_gvec_4 *fn, 213 arg_rrrr_esz *a, int data) 214 { 215 return gen_gvec_ool_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, data); 216 } 217 218 static bool gen_gvec_ool_arg_zzxz(DisasContext *s, gen_helper_gvec_4 *fn, 219 arg_rrxr_esz *a) 220 { 221 return gen_gvec_ool_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, a->index); 222 } 223 224 /* Invoke an out-of-line helper on 4 Zregs, plus a pointer. */ 225 static bool gen_gvec_ptr_zzzz(DisasContext *s, gen_helper_gvec_4_ptr *fn, 226 int rd, int rn, int rm, int ra, 227 int data, TCGv_ptr ptr) 228 { 229 if (fn == NULL) { 230 return false; 231 } 232 if (sve_access_check(s)) { 233 unsigned vsz = vec_full_reg_size(s); 234 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, rd), 235 vec_full_reg_offset(s, rn), 236 vec_full_reg_offset(s, rm), 237 vec_full_reg_offset(s, ra), 238 ptr, vsz, vsz, data, fn); 239 } 240 return true; 241 } 242 243 static bool gen_gvec_fpst_zzzz(DisasContext *s, gen_helper_gvec_4_ptr *fn, 244 int rd, int rn, int rm, int ra, 245 int data, ARMFPStatusFlavour flavour) 246 { 247 TCGv_ptr status = fpstatus_ptr(flavour); 248 bool ret = gen_gvec_ptr_zzzz(s, fn, rd, rn, rm, ra, data, status); 249 return ret; 250 } 251 252 /* Invoke an out-of-line helper on 4 Zregs, 1 Preg, plus fpst. */ 253 static bool gen_gvec_fpst_zzzzp(DisasContext *s, gen_helper_gvec_5_ptr *fn, 254 int rd, int rn, int rm, int ra, int pg, 255 int data, ARMFPStatusFlavour flavour) 256 { 257 if (fn == NULL) { 258 return false; 259 } 260 if (sve_access_check(s)) { 261 unsigned vsz = vec_full_reg_size(s); 262 TCGv_ptr status = fpstatus_ptr(flavour); 263 264 tcg_gen_gvec_5_ptr(vec_full_reg_offset(s, rd), 265 vec_full_reg_offset(s, rn), 266 vec_full_reg_offset(s, rm), 267 vec_full_reg_offset(s, ra), 268 pred_full_reg_offset(s, pg), 269 status, vsz, vsz, data, fn); 270 } 271 return true; 272 } 273 274 /* Invoke an out-of-line helper on 2 Zregs and a predicate. */ 275 static bool gen_gvec_ool_zzp(DisasContext *s, gen_helper_gvec_3 *fn, 276 int rd, int rn, int pg, int data) 277 { 278 if (fn == NULL) { 279 return false; 280 } 281 if (sve_access_check(s)) { 282 unsigned vsz = vec_full_reg_size(s); 283 tcg_gen_gvec_3_ool(vec_full_reg_offset(s, rd), 284 vec_full_reg_offset(s, rn), 285 pred_full_reg_offset(s, pg), 286 vsz, vsz, data, fn); 287 } 288 return true; 289 } 290 291 static bool gen_gvec_ool_arg_zpz(DisasContext *s, gen_helper_gvec_3 *fn, 292 arg_rpr_esz *a, int data) 293 { 294 return gen_gvec_ool_zzp(s, fn, a->rd, a->rn, a->pg, data); 295 } 296 297 static bool gen_gvec_ool_arg_zpzi(DisasContext *s, gen_helper_gvec_3 *fn, 298 arg_rpri_esz *a) 299 { 300 return gen_gvec_ool_zzp(s, fn, a->rd, a->rn, a->pg, a->imm); 301 } 302 303 static bool gen_gvec_fpst_zzp(DisasContext *s, gen_helper_gvec_3_ptr *fn, 304 int rd, int rn, int pg, int data, 305 ARMFPStatusFlavour flavour) 306 { 307 if (fn == NULL) { 308 return false; 309 } 310 if (sve_access_check(s)) { 311 unsigned vsz = vec_full_reg_size(s); 312 TCGv_ptr status = fpstatus_ptr(flavour); 313 314 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd), 315 vec_full_reg_offset(s, rn), 316 pred_full_reg_offset(s, pg), 317 status, vsz, vsz, data, fn); 318 } 319 return true; 320 } 321 322 static bool gen_gvec_fpst_arg_zpz(DisasContext *s, gen_helper_gvec_3_ptr *fn, 323 arg_rpr_esz *a, int data, 324 ARMFPStatusFlavour flavour) 325 { 326 return gen_gvec_fpst_zzp(s, fn, a->rd, a->rn, a->pg, data, flavour); 327 } 328 329 /* Invoke an out-of-line helper on 3 Zregs and a predicate. */ 330 static bool gen_gvec_ool_zzzp(DisasContext *s, gen_helper_gvec_4 *fn, 331 int rd, int rn, int rm, int pg, int data) 332 { 333 if (fn == NULL) { 334 return false; 335 } 336 if (sve_access_check(s)) { 337 unsigned vsz = vec_full_reg_size(s); 338 tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd), 339 vec_full_reg_offset(s, rn), 340 vec_full_reg_offset(s, rm), 341 pred_full_reg_offset(s, pg), 342 vsz, vsz, data, fn); 343 } 344 return true; 345 } 346 347 static bool gen_gvec_ool_arg_zpzz(DisasContext *s, gen_helper_gvec_4 *fn, 348 arg_rprr_esz *a, int data) 349 { 350 return gen_gvec_ool_zzzp(s, fn, a->rd, a->rn, a->rm, a->pg, data); 351 } 352 353 /* Invoke an out-of-line helper on 3 Zregs and a predicate. */ 354 static bool gen_gvec_fpst_zzzp(DisasContext *s, gen_helper_gvec_4_ptr *fn, 355 int rd, int rn, int rm, int pg, int data, 356 ARMFPStatusFlavour flavour) 357 { 358 if (fn == NULL) { 359 return false; 360 } 361 if (sve_access_check(s)) { 362 unsigned vsz = vec_full_reg_size(s); 363 TCGv_ptr status = fpstatus_ptr(flavour); 364 365 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, rd), 366 vec_full_reg_offset(s, rn), 367 vec_full_reg_offset(s, rm), 368 pred_full_reg_offset(s, pg), 369 status, vsz, vsz, data, fn); 370 } 371 return true; 372 } 373 374 static bool gen_gvec_fpst_arg_zpzz(DisasContext *s, gen_helper_gvec_4_ptr *fn, 375 arg_rprr_esz *a) 376 { 377 return gen_gvec_fpst_zzzp(s, fn, a->rd, a->rn, a->rm, a->pg, 0, 378 a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR); 379 } 380 381 /* Invoke a vector expander on two Zregs and an immediate. */ 382 static bool gen_gvec_fn_zzi(DisasContext *s, GVecGen2iFn *gvec_fn, 383 int esz, int rd, int rn, uint64_t imm) 384 { 385 if (gvec_fn == NULL) { 386 return false; 387 } 388 if (sve_access_check(s)) { 389 unsigned vsz = vec_full_reg_size(s); 390 gvec_fn(esz, vec_full_reg_offset(s, rd), 391 vec_full_reg_offset(s, rn), imm, vsz, vsz); 392 } 393 return true; 394 } 395 396 static bool gen_gvec_fn_arg_zzi(DisasContext *s, GVecGen2iFn *gvec_fn, 397 arg_rri_esz *a) 398 { 399 if (a->esz < 0) { 400 /* Invalid tsz encoding -- see tszimm_esz. */ 401 return false; 402 } 403 return gen_gvec_fn_zzi(s, gvec_fn, a->esz, a->rd, a->rn, a->imm); 404 } 405 406 /* Invoke a vector expander on three Zregs. */ 407 static bool gen_gvec_fn_zzz(DisasContext *s, GVecGen3Fn *gvec_fn, 408 int esz, int rd, int rn, int rm) 409 { 410 if (gvec_fn == NULL) { 411 return false; 412 } 413 if (sve_access_check(s)) { 414 unsigned vsz = vec_full_reg_size(s); 415 gvec_fn(esz, vec_full_reg_offset(s, rd), 416 vec_full_reg_offset(s, rn), 417 vec_full_reg_offset(s, rm), vsz, vsz); 418 } 419 return true; 420 } 421 422 static bool gen_gvec_fn_arg_zzz(DisasContext *s, GVecGen3Fn *fn, 423 arg_rrr_esz *a) 424 { 425 return gen_gvec_fn_zzz(s, fn, a->esz, a->rd, a->rn, a->rm); 426 } 427 428 /* Invoke a vector expander on four Zregs. */ 429 static bool gen_gvec_fn_arg_zzzz(DisasContext *s, GVecGen4Fn *gvec_fn, 430 arg_rrrr_esz *a) 431 { 432 if (gvec_fn == NULL) { 433 return false; 434 } 435 if (sve_access_check(s)) { 436 unsigned vsz = vec_full_reg_size(s); 437 gvec_fn(a->esz, vec_full_reg_offset(s, a->rd), 438 vec_full_reg_offset(s, a->rn), 439 vec_full_reg_offset(s, a->rm), 440 vec_full_reg_offset(s, a->ra), vsz, vsz); 441 } 442 return true; 443 } 444 445 /* Invoke a vector move on two Zregs. */ 446 static bool do_mov_z(DisasContext *s, int rd, int rn) 447 { 448 if (sve_access_check(s)) { 449 unsigned vsz = vec_full_reg_size(s); 450 tcg_gen_gvec_mov(MO_8, vec_full_reg_offset(s, rd), 451 vec_full_reg_offset(s, rn), vsz, vsz); 452 } 453 return true; 454 } 455 456 /* Initialize a Zreg with replications of a 64-bit immediate. */ 457 static void do_dupi_z(DisasContext *s, int rd, uint64_t word) 458 { 459 unsigned vsz = vec_full_reg_size(s); 460 tcg_gen_gvec_dup_imm(MO_64, vec_full_reg_offset(s, rd), vsz, vsz, word); 461 } 462 463 /* Invoke a vector expander on three Pregs. */ 464 static bool gen_gvec_fn_ppp(DisasContext *s, GVecGen3Fn *gvec_fn, 465 int rd, int rn, int rm) 466 { 467 if (sve_access_check(s)) { 468 unsigned psz = pred_gvec_reg_size(s); 469 gvec_fn(MO_64, pred_full_reg_offset(s, rd), 470 pred_full_reg_offset(s, rn), 471 pred_full_reg_offset(s, rm), psz, psz); 472 } 473 return true; 474 } 475 476 /* Invoke a vector move on two Pregs. */ 477 static bool do_mov_p(DisasContext *s, int rd, int rn) 478 { 479 if (sve_access_check(s)) { 480 unsigned psz = pred_gvec_reg_size(s); 481 tcg_gen_gvec_mov(MO_8, pred_full_reg_offset(s, rd), 482 pred_full_reg_offset(s, rn), psz, psz); 483 } 484 return true; 485 } 486 487 /* Set the cpu flags as per a return from an SVE helper. */ 488 static void do_pred_flags(TCGv_i32 t) 489 { 490 tcg_gen_mov_i32(cpu_NF, t); 491 tcg_gen_andi_i32(cpu_ZF, t, 2); 492 tcg_gen_andi_i32(cpu_CF, t, 1); 493 tcg_gen_movi_i32(cpu_VF, 0); 494 } 495 496 /* Subroutines computing the ARM PredTest psuedofunction. */ 497 static void do_predtest1(TCGv_i64 d, TCGv_i64 g) 498 { 499 TCGv_i32 t = tcg_temp_new_i32(); 500 501 gen_helper_sve_predtest1(t, d, g); 502 do_pred_flags(t); 503 } 504 505 static void do_predtest(DisasContext *s, int dofs, int gofs, int words) 506 { 507 TCGv_ptr dptr = tcg_temp_new_ptr(); 508 TCGv_ptr gptr = tcg_temp_new_ptr(); 509 TCGv_i32 t = tcg_temp_new_i32(); 510 511 tcg_gen_addi_ptr(dptr, cpu_env, dofs); 512 tcg_gen_addi_ptr(gptr, cpu_env, gofs); 513 514 gen_helper_sve_predtest(t, dptr, gptr, tcg_constant_i32(words)); 515 516 do_pred_flags(t); 517 } 518 519 /* For each element size, the bits within a predicate word that are active. */ 520 const uint64_t pred_esz_masks[5] = { 521 0xffffffffffffffffull, 0x5555555555555555ull, 522 0x1111111111111111ull, 0x0101010101010101ull, 523 0x0001000100010001ull, 524 }; 525 526 static bool trans_INVALID(DisasContext *s, arg_INVALID *a) 527 { 528 unallocated_encoding(s); 529 return true; 530 } 531 532 /* 533 *** SVE Logical - Unpredicated Group 534 */ 535 536 TRANS_FEAT(AND_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_and, a) 537 TRANS_FEAT(ORR_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_or, a) 538 TRANS_FEAT(EOR_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_xor, a) 539 TRANS_FEAT(BIC_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_andc, a) 540 541 static void gen_xar8_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, int64_t sh) 542 { 543 TCGv_i64 t = tcg_temp_new_i64(); 544 uint64_t mask = dup_const(MO_8, 0xff >> sh); 545 546 tcg_gen_xor_i64(t, n, m); 547 tcg_gen_shri_i64(d, t, sh); 548 tcg_gen_shli_i64(t, t, 8 - sh); 549 tcg_gen_andi_i64(d, d, mask); 550 tcg_gen_andi_i64(t, t, ~mask); 551 tcg_gen_or_i64(d, d, t); 552 } 553 554 static void gen_xar16_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, int64_t sh) 555 { 556 TCGv_i64 t = tcg_temp_new_i64(); 557 uint64_t mask = dup_const(MO_16, 0xffff >> sh); 558 559 tcg_gen_xor_i64(t, n, m); 560 tcg_gen_shri_i64(d, t, sh); 561 tcg_gen_shli_i64(t, t, 16 - sh); 562 tcg_gen_andi_i64(d, d, mask); 563 tcg_gen_andi_i64(t, t, ~mask); 564 tcg_gen_or_i64(d, d, t); 565 } 566 567 static void gen_xar_i32(TCGv_i32 d, TCGv_i32 n, TCGv_i32 m, int32_t sh) 568 { 569 tcg_gen_xor_i32(d, n, m); 570 tcg_gen_rotri_i32(d, d, sh); 571 } 572 573 static void gen_xar_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, int64_t sh) 574 { 575 tcg_gen_xor_i64(d, n, m); 576 tcg_gen_rotri_i64(d, d, sh); 577 } 578 579 static void gen_xar_vec(unsigned vece, TCGv_vec d, TCGv_vec n, 580 TCGv_vec m, int64_t sh) 581 { 582 tcg_gen_xor_vec(vece, d, n, m); 583 tcg_gen_rotri_vec(vece, d, d, sh); 584 } 585 586 void gen_gvec_xar(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs, 587 uint32_t rm_ofs, int64_t shift, 588 uint32_t opr_sz, uint32_t max_sz) 589 { 590 static const TCGOpcode vecop[] = { INDEX_op_rotli_vec, 0 }; 591 static const GVecGen3i ops[4] = { 592 { .fni8 = gen_xar8_i64, 593 .fniv = gen_xar_vec, 594 .fno = gen_helper_sve2_xar_b, 595 .opt_opc = vecop, 596 .vece = MO_8 }, 597 { .fni8 = gen_xar16_i64, 598 .fniv = gen_xar_vec, 599 .fno = gen_helper_sve2_xar_h, 600 .opt_opc = vecop, 601 .vece = MO_16 }, 602 { .fni4 = gen_xar_i32, 603 .fniv = gen_xar_vec, 604 .fno = gen_helper_sve2_xar_s, 605 .opt_opc = vecop, 606 .vece = MO_32 }, 607 { .fni8 = gen_xar_i64, 608 .fniv = gen_xar_vec, 609 .fno = gen_helper_gvec_xar_d, 610 .opt_opc = vecop, 611 .vece = MO_64 } 612 }; 613 int esize = 8 << vece; 614 615 /* The SVE2 range is 1 .. esize; the AdvSIMD range is 0 .. esize-1. */ 616 tcg_debug_assert(shift >= 0); 617 tcg_debug_assert(shift <= esize); 618 shift &= esize - 1; 619 620 if (shift == 0) { 621 /* xar with no rotate devolves to xor. */ 622 tcg_gen_gvec_xor(vece, rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz); 623 } else { 624 tcg_gen_gvec_3i(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, 625 shift, &ops[vece]); 626 } 627 } 628 629 static bool trans_XAR(DisasContext *s, arg_rrri_esz *a) 630 { 631 if (a->esz < 0 || !dc_isar_feature(aa64_sve2, s)) { 632 return false; 633 } 634 if (sve_access_check(s)) { 635 unsigned vsz = vec_full_reg_size(s); 636 gen_gvec_xar(a->esz, vec_full_reg_offset(s, a->rd), 637 vec_full_reg_offset(s, a->rn), 638 vec_full_reg_offset(s, a->rm), a->imm, vsz, vsz); 639 } 640 return true; 641 } 642 643 static void gen_eor3_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 k) 644 { 645 tcg_gen_xor_i64(d, n, m); 646 tcg_gen_xor_i64(d, d, k); 647 } 648 649 static void gen_eor3_vec(unsigned vece, TCGv_vec d, TCGv_vec n, 650 TCGv_vec m, TCGv_vec k) 651 { 652 tcg_gen_xor_vec(vece, d, n, m); 653 tcg_gen_xor_vec(vece, d, d, k); 654 } 655 656 static void gen_eor3(unsigned vece, uint32_t d, uint32_t n, uint32_t m, 657 uint32_t a, uint32_t oprsz, uint32_t maxsz) 658 { 659 static const GVecGen4 op = { 660 .fni8 = gen_eor3_i64, 661 .fniv = gen_eor3_vec, 662 .fno = gen_helper_sve2_eor3, 663 .vece = MO_64, 664 .prefer_i64 = TCG_TARGET_REG_BITS == 64, 665 }; 666 tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op); 667 } 668 669 TRANS_FEAT(EOR3, aa64_sve2, gen_gvec_fn_arg_zzzz, gen_eor3, a) 670 671 static void gen_bcax_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 k) 672 { 673 tcg_gen_andc_i64(d, m, k); 674 tcg_gen_xor_i64(d, d, n); 675 } 676 677 static void gen_bcax_vec(unsigned vece, TCGv_vec d, TCGv_vec n, 678 TCGv_vec m, TCGv_vec k) 679 { 680 tcg_gen_andc_vec(vece, d, m, k); 681 tcg_gen_xor_vec(vece, d, d, n); 682 } 683 684 static void gen_bcax(unsigned vece, uint32_t d, uint32_t n, uint32_t m, 685 uint32_t a, uint32_t oprsz, uint32_t maxsz) 686 { 687 static const GVecGen4 op = { 688 .fni8 = gen_bcax_i64, 689 .fniv = gen_bcax_vec, 690 .fno = gen_helper_sve2_bcax, 691 .vece = MO_64, 692 .prefer_i64 = TCG_TARGET_REG_BITS == 64, 693 }; 694 tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op); 695 } 696 697 TRANS_FEAT(BCAX, aa64_sve2, gen_gvec_fn_arg_zzzz, gen_bcax, a) 698 699 static void gen_bsl(unsigned vece, uint32_t d, uint32_t n, uint32_t m, 700 uint32_t a, uint32_t oprsz, uint32_t maxsz) 701 { 702 /* BSL differs from the generic bitsel in argument ordering. */ 703 tcg_gen_gvec_bitsel(vece, d, a, n, m, oprsz, maxsz); 704 } 705 706 TRANS_FEAT(BSL, aa64_sve2, gen_gvec_fn_arg_zzzz, gen_bsl, a) 707 708 static void gen_bsl1n_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 k) 709 { 710 tcg_gen_andc_i64(n, k, n); 711 tcg_gen_andc_i64(m, m, k); 712 tcg_gen_or_i64(d, n, m); 713 } 714 715 static void gen_bsl1n_vec(unsigned vece, TCGv_vec d, TCGv_vec n, 716 TCGv_vec m, TCGv_vec k) 717 { 718 if (TCG_TARGET_HAS_bitsel_vec) { 719 tcg_gen_not_vec(vece, n, n); 720 tcg_gen_bitsel_vec(vece, d, k, n, m); 721 } else { 722 tcg_gen_andc_vec(vece, n, k, n); 723 tcg_gen_andc_vec(vece, m, m, k); 724 tcg_gen_or_vec(vece, d, n, m); 725 } 726 } 727 728 static void gen_bsl1n(unsigned vece, uint32_t d, uint32_t n, uint32_t m, 729 uint32_t a, uint32_t oprsz, uint32_t maxsz) 730 { 731 static const GVecGen4 op = { 732 .fni8 = gen_bsl1n_i64, 733 .fniv = gen_bsl1n_vec, 734 .fno = gen_helper_sve2_bsl1n, 735 .vece = MO_64, 736 .prefer_i64 = TCG_TARGET_REG_BITS == 64, 737 }; 738 tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op); 739 } 740 741 TRANS_FEAT(BSL1N, aa64_sve2, gen_gvec_fn_arg_zzzz, gen_bsl1n, a) 742 743 static void gen_bsl2n_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 k) 744 { 745 /* 746 * Z[dn] = (n & k) | (~m & ~k) 747 * = | ~(m | k) 748 */ 749 tcg_gen_and_i64(n, n, k); 750 if (TCG_TARGET_HAS_orc_i64) { 751 tcg_gen_or_i64(m, m, k); 752 tcg_gen_orc_i64(d, n, m); 753 } else { 754 tcg_gen_nor_i64(m, m, k); 755 tcg_gen_or_i64(d, n, m); 756 } 757 } 758 759 static void gen_bsl2n_vec(unsigned vece, TCGv_vec d, TCGv_vec n, 760 TCGv_vec m, TCGv_vec k) 761 { 762 if (TCG_TARGET_HAS_bitsel_vec) { 763 tcg_gen_not_vec(vece, m, m); 764 tcg_gen_bitsel_vec(vece, d, k, n, m); 765 } else { 766 tcg_gen_and_vec(vece, n, n, k); 767 tcg_gen_or_vec(vece, m, m, k); 768 tcg_gen_orc_vec(vece, d, n, m); 769 } 770 } 771 772 static void gen_bsl2n(unsigned vece, uint32_t d, uint32_t n, uint32_t m, 773 uint32_t a, uint32_t oprsz, uint32_t maxsz) 774 { 775 static const GVecGen4 op = { 776 .fni8 = gen_bsl2n_i64, 777 .fniv = gen_bsl2n_vec, 778 .fno = gen_helper_sve2_bsl2n, 779 .vece = MO_64, 780 .prefer_i64 = TCG_TARGET_REG_BITS == 64, 781 }; 782 tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op); 783 } 784 785 TRANS_FEAT(BSL2N, aa64_sve2, gen_gvec_fn_arg_zzzz, gen_bsl2n, a) 786 787 static void gen_nbsl_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 k) 788 { 789 tcg_gen_and_i64(n, n, k); 790 tcg_gen_andc_i64(m, m, k); 791 tcg_gen_nor_i64(d, n, m); 792 } 793 794 static void gen_nbsl_vec(unsigned vece, TCGv_vec d, TCGv_vec n, 795 TCGv_vec m, TCGv_vec k) 796 { 797 tcg_gen_bitsel_vec(vece, d, k, n, m); 798 tcg_gen_not_vec(vece, d, d); 799 } 800 801 static void gen_nbsl(unsigned vece, uint32_t d, uint32_t n, uint32_t m, 802 uint32_t a, uint32_t oprsz, uint32_t maxsz) 803 { 804 static const GVecGen4 op = { 805 .fni8 = gen_nbsl_i64, 806 .fniv = gen_nbsl_vec, 807 .fno = gen_helper_sve2_nbsl, 808 .vece = MO_64, 809 .prefer_i64 = TCG_TARGET_REG_BITS == 64, 810 }; 811 tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op); 812 } 813 814 TRANS_FEAT(NBSL, aa64_sve2, gen_gvec_fn_arg_zzzz, gen_nbsl, a) 815 816 /* 817 *** SVE Integer Arithmetic - Unpredicated Group 818 */ 819 820 TRANS_FEAT(ADD_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_add, a) 821 TRANS_FEAT(SUB_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_sub, a) 822 TRANS_FEAT(SQADD_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_ssadd, a) 823 TRANS_FEAT(SQSUB_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_sssub, a) 824 TRANS_FEAT(UQADD_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_usadd, a) 825 TRANS_FEAT(UQSUB_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_ussub, a) 826 827 /* 828 *** SVE Integer Arithmetic - Binary Predicated Group 829 */ 830 831 /* Select active elememnts from Zn and inactive elements from Zm, 832 * storing the result in Zd. 833 */ 834 static bool do_sel_z(DisasContext *s, int rd, int rn, int rm, int pg, int esz) 835 { 836 static gen_helper_gvec_4 * const fns[4] = { 837 gen_helper_sve_sel_zpzz_b, gen_helper_sve_sel_zpzz_h, 838 gen_helper_sve_sel_zpzz_s, gen_helper_sve_sel_zpzz_d 839 }; 840 return gen_gvec_ool_zzzp(s, fns[esz], rd, rn, rm, pg, 0); 841 } 842 843 #define DO_ZPZZ(NAME, FEAT, name) \ 844 static gen_helper_gvec_4 * const name##_zpzz_fns[4] = { \ 845 gen_helper_##name##_zpzz_b, gen_helper_##name##_zpzz_h, \ 846 gen_helper_##name##_zpzz_s, gen_helper_##name##_zpzz_d, \ 847 }; \ 848 TRANS_FEAT(NAME, FEAT, gen_gvec_ool_arg_zpzz, \ 849 name##_zpzz_fns[a->esz], a, 0) 850 851 DO_ZPZZ(AND_zpzz, aa64_sve, sve_and) 852 DO_ZPZZ(EOR_zpzz, aa64_sve, sve_eor) 853 DO_ZPZZ(ORR_zpzz, aa64_sve, sve_orr) 854 DO_ZPZZ(BIC_zpzz, aa64_sve, sve_bic) 855 856 DO_ZPZZ(ADD_zpzz, aa64_sve, sve_add) 857 DO_ZPZZ(SUB_zpzz, aa64_sve, sve_sub) 858 859 DO_ZPZZ(SMAX_zpzz, aa64_sve, sve_smax) 860 DO_ZPZZ(UMAX_zpzz, aa64_sve, sve_umax) 861 DO_ZPZZ(SMIN_zpzz, aa64_sve, sve_smin) 862 DO_ZPZZ(UMIN_zpzz, aa64_sve, sve_umin) 863 DO_ZPZZ(SABD_zpzz, aa64_sve, sve_sabd) 864 DO_ZPZZ(UABD_zpzz, aa64_sve, sve_uabd) 865 866 DO_ZPZZ(MUL_zpzz, aa64_sve, sve_mul) 867 DO_ZPZZ(SMULH_zpzz, aa64_sve, sve_smulh) 868 DO_ZPZZ(UMULH_zpzz, aa64_sve, sve_umulh) 869 870 DO_ZPZZ(ASR_zpzz, aa64_sve, sve_asr) 871 DO_ZPZZ(LSR_zpzz, aa64_sve, sve_lsr) 872 DO_ZPZZ(LSL_zpzz, aa64_sve, sve_lsl) 873 874 static gen_helper_gvec_4 * const sdiv_fns[4] = { 875 NULL, NULL, gen_helper_sve_sdiv_zpzz_s, gen_helper_sve_sdiv_zpzz_d 876 }; 877 TRANS_FEAT(SDIV_zpzz, aa64_sve, gen_gvec_ool_arg_zpzz, sdiv_fns[a->esz], a, 0) 878 879 static gen_helper_gvec_4 * const udiv_fns[4] = { 880 NULL, NULL, gen_helper_sve_udiv_zpzz_s, gen_helper_sve_udiv_zpzz_d 881 }; 882 TRANS_FEAT(UDIV_zpzz, aa64_sve, gen_gvec_ool_arg_zpzz, udiv_fns[a->esz], a, 0) 883 884 TRANS_FEAT(SEL_zpzz, aa64_sve, do_sel_z, a->rd, a->rn, a->rm, a->pg, a->esz) 885 886 /* 887 *** SVE Integer Arithmetic - Unary Predicated Group 888 */ 889 890 #define DO_ZPZ(NAME, FEAT, name) \ 891 static gen_helper_gvec_3 * const name##_fns[4] = { \ 892 gen_helper_##name##_b, gen_helper_##name##_h, \ 893 gen_helper_##name##_s, gen_helper_##name##_d, \ 894 }; \ 895 TRANS_FEAT(NAME, FEAT, gen_gvec_ool_arg_zpz, name##_fns[a->esz], a, 0) 896 897 DO_ZPZ(CLS, aa64_sve, sve_cls) 898 DO_ZPZ(CLZ, aa64_sve, sve_clz) 899 DO_ZPZ(CNT_zpz, aa64_sve, sve_cnt_zpz) 900 DO_ZPZ(CNOT, aa64_sve, sve_cnot) 901 DO_ZPZ(NOT_zpz, aa64_sve, sve_not_zpz) 902 DO_ZPZ(ABS, aa64_sve, sve_abs) 903 DO_ZPZ(NEG, aa64_sve, sve_neg) 904 DO_ZPZ(RBIT, aa64_sve, sve_rbit) 905 906 static gen_helper_gvec_3 * const fabs_fns[4] = { 907 NULL, gen_helper_sve_fabs_h, 908 gen_helper_sve_fabs_s, gen_helper_sve_fabs_d, 909 }; 910 TRANS_FEAT(FABS, aa64_sve, gen_gvec_ool_arg_zpz, fabs_fns[a->esz], a, 0) 911 912 static gen_helper_gvec_3 * const fneg_fns[4] = { 913 NULL, gen_helper_sve_fneg_h, 914 gen_helper_sve_fneg_s, gen_helper_sve_fneg_d, 915 }; 916 TRANS_FEAT(FNEG, aa64_sve, gen_gvec_ool_arg_zpz, fneg_fns[a->esz], a, 0) 917 918 static gen_helper_gvec_3 * const sxtb_fns[4] = { 919 NULL, gen_helper_sve_sxtb_h, 920 gen_helper_sve_sxtb_s, gen_helper_sve_sxtb_d, 921 }; 922 TRANS_FEAT(SXTB, aa64_sve, gen_gvec_ool_arg_zpz, sxtb_fns[a->esz], a, 0) 923 924 static gen_helper_gvec_3 * const uxtb_fns[4] = { 925 NULL, gen_helper_sve_uxtb_h, 926 gen_helper_sve_uxtb_s, gen_helper_sve_uxtb_d, 927 }; 928 TRANS_FEAT(UXTB, aa64_sve, gen_gvec_ool_arg_zpz, uxtb_fns[a->esz], a, 0) 929 930 static gen_helper_gvec_3 * const sxth_fns[4] = { 931 NULL, NULL, gen_helper_sve_sxth_s, gen_helper_sve_sxth_d 932 }; 933 TRANS_FEAT(SXTH, aa64_sve, gen_gvec_ool_arg_zpz, sxth_fns[a->esz], a, 0) 934 935 static gen_helper_gvec_3 * const uxth_fns[4] = { 936 NULL, NULL, gen_helper_sve_uxth_s, gen_helper_sve_uxth_d 937 }; 938 TRANS_FEAT(UXTH, aa64_sve, gen_gvec_ool_arg_zpz, uxth_fns[a->esz], a, 0) 939 940 TRANS_FEAT(SXTW, aa64_sve, gen_gvec_ool_arg_zpz, 941 a->esz == 3 ? gen_helper_sve_sxtw_d : NULL, a, 0) 942 TRANS_FEAT(UXTW, aa64_sve, gen_gvec_ool_arg_zpz, 943 a->esz == 3 ? gen_helper_sve_uxtw_d : NULL, a, 0) 944 945 /* 946 *** SVE Integer Reduction Group 947 */ 948 949 typedef void gen_helper_gvec_reduc(TCGv_i64, TCGv_ptr, TCGv_ptr, TCGv_i32); 950 static bool do_vpz_ool(DisasContext *s, arg_rpr_esz *a, 951 gen_helper_gvec_reduc *fn) 952 { 953 unsigned vsz = vec_full_reg_size(s); 954 TCGv_ptr t_zn, t_pg; 955 TCGv_i32 desc; 956 TCGv_i64 temp; 957 958 if (fn == NULL) { 959 return false; 960 } 961 if (!sve_access_check(s)) { 962 return true; 963 } 964 965 desc = tcg_constant_i32(simd_desc(vsz, vsz, 0)); 966 temp = tcg_temp_new_i64(); 967 t_zn = tcg_temp_new_ptr(); 968 t_pg = tcg_temp_new_ptr(); 969 970 tcg_gen_addi_ptr(t_zn, cpu_env, vec_full_reg_offset(s, a->rn)); 971 tcg_gen_addi_ptr(t_pg, cpu_env, pred_full_reg_offset(s, a->pg)); 972 fn(temp, t_zn, t_pg, desc); 973 974 write_fp_dreg(s, a->rd, temp); 975 return true; 976 } 977 978 #define DO_VPZ(NAME, name) \ 979 static gen_helper_gvec_reduc * const name##_fns[4] = { \ 980 gen_helper_sve_##name##_b, gen_helper_sve_##name##_h, \ 981 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \ 982 }; \ 983 TRANS_FEAT(NAME, aa64_sve, do_vpz_ool, a, name##_fns[a->esz]) 984 985 DO_VPZ(ORV, orv) 986 DO_VPZ(ANDV, andv) 987 DO_VPZ(EORV, eorv) 988 989 DO_VPZ(UADDV, uaddv) 990 DO_VPZ(SMAXV, smaxv) 991 DO_VPZ(UMAXV, umaxv) 992 DO_VPZ(SMINV, sminv) 993 DO_VPZ(UMINV, uminv) 994 995 static gen_helper_gvec_reduc * const saddv_fns[4] = { 996 gen_helper_sve_saddv_b, gen_helper_sve_saddv_h, 997 gen_helper_sve_saddv_s, NULL 998 }; 999 TRANS_FEAT(SADDV, aa64_sve, do_vpz_ool, a, saddv_fns[a->esz]) 1000 1001 #undef DO_VPZ 1002 1003 /* 1004 *** SVE Shift by Immediate - Predicated Group 1005 */ 1006 1007 /* 1008 * Copy Zn into Zd, storing zeros into inactive elements. 1009 * If invert, store zeros into the active elements. 1010 */ 1011 static bool do_movz_zpz(DisasContext *s, int rd, int rn, int pg, 1012 int esz, bool invert) 1013 { 1014 static gen_helper_gvec_3 * const fns[4] = { 1015 gen_helper_sve_movz_b, gen_helper_sve_movz_h, 1016 gen_helper_sve_movz_s, gen_helper_sve_movz_d, 1017 }; 1018 return gen_gvec_ool_zzp(s, fns[esz], rd, rn, pg, invert); 1019 } 1020 1021 static bool do_shift_zpzi(DisasContext *s, arg_rpri_esz *a, bool asr, 1022 gen_helper_gvec_3 * const fns[4]) 1023 { 1024 int max; 1025 1026 if (a->esz < 0) { 1027 /* Invalid tsz encoding -- see tszimm_esz. */ 1028 return false; 1029 } 1030 1031 /* 1032 * Shift by element size is architecturally valid. 1033 * For arithmetic right-shift, it's the same as by one less. 1034 * For logical shifts and ASRD, it is a zeroing operation. 1035 */ 1036 max = 8 << a->esz; 1037 if (a->imm >= max) { 1038 if (asr) { 1039 a->imm = max - 1; 1040 } else { 1041 return do_movz_zpz(s, a->rd, a->rd, a->pg, a->esz, true); 1042 } 1043 } 1044 return gen_gvec_ool_arg_zpzi(s, fns[a->esz], a); 1045 } 1046 1047 static gen_helper_gvec_3 * const asr_zpzi_fns[4] = { 1048 gen_helper_sve_asr_zpzi_b, gen_helper_sve_asr_zpzi_h, 1049 gen_helper_sve_asr_zpzi_s, gen_helper_sve_asr_zpzi_d, 1050 }; 1051 TRANS_FEAT(ASR_zpzi, aa64_sve, do_shift_zpzi, a, true, asr_zpzi_fns) 1052 1053 static gen_helper_gvec_3 * const lsr_zpzi_fns[4] = { 1054 gen_helper_sve_lsr_zpzi_b, gen_helper_sve_lsr_zpzi_h, 1055 gen_helper_sve_lsr_zpzi_s, gen_helper_sve_lsr_zpzi_d, 1056 }; 1057 TRANS_FEAT(LSR_zpzi, aa64_sve, do_shift_zpzi, a, false, lsr_zpzi_fns) 1058 1059 static gen_helper_gvec_3 * const lsl_zpzi_fns[4] = { 1060 gen_helper_sve_lsl_zpzi_b, gen_helper_sve_lsl_zpzi_h, 1061 gen_helper_sve_lsl_zpzi_s, gen_helper_sve_lsl_zpzi_d, 1062 }; 1063 TRANS_FEAT(LSL_zpzi, aa64_sve, do_shift_zpzi, a, false, lsl_zpzi_fns) 1064 1065 static gen_helper_gvec_3 * const asrd_fns[4] = { 1066 gen_helper_sve_asrd_b, gen_helper_sve_asrd_h, 1067 gen_helper_sve_asrd_s, gen_helper_sve_asrd_d, 1068 }; 1069 TRANS_FEAT(ASRD, aa64_sve, do_shift_zpzi, a, false, asrd_fns) 1070 1071 static gen_helper_gvec_3 * const sqshl_zpzi_fns[4] = { 1072 gen_helper_sve2_sqshl_zpzi_b, gen_helper_sve2_sqshl_zpzi_h, 1073 gen_helper_sve2_sqshl_zpzi_s, gen_helper_sve2_sqshl_zpzi_d, 1074 }; 1075 TRANS_FEAT(SQSHL_zpzi, aa64_sve2, gen_gvec_ool_arg_zpzi, 1076 a->esz < 0 ? NULL : sqshl_zpzi_fns[a->esz], a) 1077 1078 static gen_helper_gvec_3 * const uqshl_zpzi_fns[4] = { 1079 gen_helper_sve2_uqshl_zpzi_b, gen_helper_sve2_uqshl_zpzi_h, 1080 gen_helper_sve2_uqshl_zpzi_s, gen_helper_sve2_uqshl_zpzi_d, 1081 }; 1082 TRANS_FEAT(UQSHL_zpzi, aa64_sve2, gen_gvec_ool_arg_zpzi, 1083 a->esz < 0 ? NULL : uqshl_zpzi_fns[a->esz], a) 1084 1085 static gen_helper_gvec_3 * const srshr_fns[4] = { 1086 gen_helper_sve2_srshr_b, gen_helper_sve2_srshr_h, 1087 gen_helper_sve2_srshr_s, gen_helper_sve2_srshr_d, 1088 }; 1089 TRANS_FEAT(SRSHR, aa64_sve2, gen_gvec_ool_arg_zpzi, 1090 a->esz < 0 ? NULL : srshr_fns[a->esz], a) 1091 1092 static gen_helper_gvec_3 * const urshr_fns[4] = { 1093 gen_helper_sve2_urshr_b, gen_helper_sve2_urshr_h, 1094 gen_helper_sve2_urshr_s, gen_helper_sve2_urshr_d, 1095 }; 1096 TRANS_FEAT(URSHR, aa64_sve2, gen_gvec_ool_arg_zpzi, 1097 a->esz < 0 ? NULL : urshr_fns[a->esz], a) 1098 1099 static gen_helper_gvec_3 * const sqshlu_fns[4] = { 1100 gen_helper_sve2_sqshlu_b, gen_helper_sve2_sqshlu_h, 1101 gen_helper_sve2_sqshlu_s, gen_helper_sve2_sqshlu_d, 1102 }; 1103 TRANS_FEAT(SQSHLU, aa64_sve2, gen_gvec_ool_arg_zpzi, 1104 a->esz < 0 ? NULL : sqshlu_fns[a->esz], a) 1105 1106 /* 1107 *** SVE Bitwise Shift - Predicated Group 1108 */ 1109 1110 #define DO_ZPZW(NAME, name) \ 1111 static gen_helper_gvec_4 * const name##_zpzw_fns[4] = { \ 1112 gen_helper_sve_##name##_zpzw_b, gen_helper_sve_##name##_zpzw_h, \ 1113 gen_helper_sve_##name##_zpzw_s, NULL \ 1114 }; \ 1115 TRANS_FEAT(NAME##_zpzw, aa64_sve, gen_gvec_ool_arg_zpzz, \ 1116 a->esz < 0 ? NULL : name##_zpzw_fns[a->esz], a, 0) 1117 1118 DO_ZPZW(ASR, asr) 1119 DO_ZPZW(LSR, lsr) 1120 DO_ZPZW(LSL, lsl) 1121 1122 #undef DO_ZPZW 1123 1124 /* 1125 *** SVE Bitwise Shift - Unpredicated Group 1126 */ 1127 1128 static bool do_shift_imm(DisasContext *s, arg_rri_esz *a, bool asr, 1129 void (*gvec_fn)(unsigned, uint32_t, uint32_t, 1130 int64_t, uint32_t, uint32_t)) 1131 { 1132 if (a->esz < 0) { 1133 /* Invalid tsz encoding -- see tszimm_esz. */ 1134 return false; 1135 } 1136 if (sve_access_check(s)) { 1137 unsigned vsz = vec_full_reg_size(s); 1138 /* Shift by element size is architecturally valid. For 1139 arithmetic right-shift, it's the same as by one less. 1140 Otherwise it is a zeroing operation. */ 1141 if (a->imm >= 8 << a->esz) { 1142 if (asr) { 1143 a->imm = (8 << a->esz) - 1; 1144 } else { 1145 do_dupi_z(s, a->rd, 0); 1146 return true; 1147 } 1148 } 1149 gvec_fn(a->esz, vec_full_reg_offset(s, a->rd), 1150 vec_full_reg_offset(s, a->rn), a->imm, vsz, vsz); 1151 } 1152 return true; 1153 } 1154 1155 TRANS_FEAT(ASR_zzi, aa64_sve, do_shift_imm, a, true, tcg_gen_gvec_sari) 1156 TRANS_FEAT(LSR_zzi, aa64_sve, do_shift_imm, a, false, tcg_gen_gvec_shri) 1157 TRANS_FEAT(LSL_zzi, aa64_sve, do_shift_imm, a, false, tcg_gen_gvec_shli) 1158 1159 #define DO_ZZW(NAME, name) \ 1160 static gen_helper_gvec_3 * const name##_zzw_fns[4] = { \ 1161 gen_helper_sve_##name##_zzw_b, gen_helper_sve_##name##_zzw_h, \ 1162 gen_helper_sve_##name##_zzw_s, NULL \ 1163 }; \ 1164 TRANS_FEAT(NAME, aa64_sve, gen_gvec_ool_arg_zzz, \ 1165 name##_zzw_fns[a->esz], a, 0) 1166 1167 DO_ZZW(ASR_zzw, asr) 1168 DO_ZZW(LSR_zzw, lsr) 1169 DO_ZZW(LSL_zzw, lsl) 1170 1171 #undef DO_ZZW 1172 1173 /* 1174 *** SVE Integer Multiply-Add Group 1175 */ 1176 1177 static bool do_zpzzz_ool(DisasContext *s, arg_rprrr_esz *a, 1178 gen_helper_gvec_5 *fn) 1179 { 1180 if (sve_access_check(s)) { 1181 unsigned vsz = vec_full_reg_size(s); 1182 tcg_gen_gvec_5_ool(vec_full_reg_offset(s, a->rd), 1183 vec_full_reg_offset(s, a->ra), 1184 vec_full_reg_offset(s, a->rn), 1185 vec_full_reg_offset(s, a->rm), 1186 pred_full_reg_offset(s, a->pg), 1187 vsz, vsz, 0, fn); 1188 } 1189 return true; 1190 } 1191 1192 static gen_helper_gvec_5 * const mla_fns[4] = { 1193 gen_helper_sve_mla_b, gen_helper_sve_mla_h, 1194 gen_helper_sve_mla_s, gen_helper_sve_mla_d, 1195 }; 1196 TRANS_FEAT(MLA, aa64_sve, do_zpzzz_ool, a, mla_fns[a->esz]) 1197 1198 static gen_helper_gvec_5 * const mls_fns[4] = { 1199 gen_helper_sve_mls_b, gen_helper_sve_mls_h, 1200 gen_helper_sve_mls_s, gen_helper_sve_mls_d, 1201 }; 1202 TRANS_FEAT(MLS, aa64_sve, do_zpzzz_ool, a, mls_fns[a->esz]) 1203 1204 /* 1205 *** SVE Index Generation Group 1206 */ 1207 1208 static bool do_index(DisasContext *s, int esz, int rd, 1209 TCGv_i64 start, TCGv_i64 incr) 1210 { 1211 unsigned vsz; 1212 TCGv_i32 desc; 1213 TCGv_ptr t_zd; 1214 1215 if (!sve_access_check(s)) { 1216 return true; 1217 } 1218 1219 vsz = vec_full_reg_size(s); 1220 desc = tcg_constant_i32(simd_desc(vsz, vsz, 0)); 1221 t_zd = tcg_temp_new_ptr(); 1222 1223 tcg_gen_addi_ptr(t_zd, cpu_env, vec_full_reg_offset(s, rd)); 1224 if (esz == 3) { 1225 gen_helper_sve_index_d(t_zd, start, incr, desc); 1226 } else { 1227 typedef void index_fn(TCGv_ptr, TCGv_i32, TCGv_i32, TCGv_i32); 1228 static index_fn * const fns[3] = { 1229 gen_helper_sve_index_b, 1230 gen_helper_sve_index_h, 1231 gen_helper_sve_index_s, 1232 }; 1233 TCGv_i32 s32 = tcg_temp_new_i32(); 1234 TCGv_i32 i32 = tcg_temp_new_i32(); 1235 1236 tcg_gen_extrl_i64_i32(s32, start); 1237 tcg_gen_extrl_i64_i32(i32, incr); 1238 fns[esz](t_zd, s32, i32, desc); 1239 } 1240 return true; 1241 } 1242 1243 TRANS_FEAT(INDEX_ii, aa64_sve, do_index, a->esz, a->rd, 1244 tcg_constant_i64(a->imm1), tcg_constant_i64(a->imm2)) 1245 TRANS_FEAT(INDEX_ir, aa64_sve, do_index, a->esz, a->rd, 1246 tcg_constant_i64(a->imm), cpu_reg(s, a->rm)) 1247 TRANS_FEAT(INDEX_ri, aa64_sve, do_index, a->esz, a->rd, 1248 cpu_reg(s, a->rn), tcg_constant_i64(a->imm)) 1249 TRANS_FEAT(INDEX_rr, aa64_sve, do_index, a->esz, a->rd, 1250 cpu_reg(s, a->rn), cpu_reg(s, a->rm)) 1251 1252 /* 1253 *** SVE Stack Allocation Group 1254 */ 1255 1256 static bool trans_ADDVL(DisasContext *s, arg_ADDVL *a) 1257 { 1258 if (!dc_isar_feature(aa64_sve, s)) { 1259 return false; 1260 } 1261 if (sve_access_check(s)) { 1262 TCGv_i64 rd = cpu_reg_sp(s, a->rd); 1263 TCGv_i64 rn = cpu_reg_sp(s, a->rn); 1264 tcg_gen_addi_i64(rd, rn, a->imm * vec_full_reg_size(s)); 1265 } 1266 return true; 1267 } 1268 1269 static bool trans_ADDSVL(DisasContext *s, arg_ADDSVL *a) 1270 { 1271 if (!dc_isar_feature(aa64_sme, s)) { 1272 return false; 1273 } 1274 if (sme_enabled_check(s)) { 1275 TCGv_i64 rd = cpu_reg_sp(s, a->rd); 1276 TCGv_i64 rn = cpu_reg_sp(s, a->rn); 1277 tcg_gen_addi_i64(rd, rn, a->imm * streaming_vec_reg_size(s)); 1278 } 1279 return true; 1280 } 1281 1282 static bool trans_ADDPL(DisasContext *s, arg_ADDPL *a) 1283 { 1284 if (!dc_isar_feature(aa64_sve, s)) { 1285 return false; 1286 } 1287 if (sve_access_check(s)) { 1288 TCGv_i64 rd = cpu_reg_sp(s, a->rd); 1289 TCGv_i64 rn = cpu_reg_sp(s, a->rn); 1290 tcg_gen_addi_i64(rd, rn, a->imm * pred_full_reg_size(s)); 1291 } 1292 return true; 1293 } 1294 1295 static bool trans_ADDSPL(DisasContext *s, arg_ADDSPL *a) 1296 { 1297 if (!dc_isar_feature(aa64_sme, s)) { 1298 return false; 1299 } 1300 if (sme_enabled_check(s)) { 1301 TCGv_i64 rd = cpu_reg_sp(s, a->rd); 1302 TCGv_i64 rn = cpu_reg_sp(s, a->rn); 1303 tcg_gen_addi_i64(rd, rn, a->imm * streaming_pred_reg_size(s)); 1304 } 1305 return true; 1306 } 1307 1308 static bool trans_RDVL(DisasContext *s, arg_RDVL *a) 1309 { 1310 if (!dc_isar_feature(aa64_sve, s)) { 1311 return false; 1312 } 1313 if (sve_access_check(s)) { 1314 TCGv_i64 reg = cpu_reg(s, a->rd); 1315 tcg_gen_movi_i64(reg, a->imm * vec_full_reg_size(s)); 1316 } 1317 return true; 1318 } 1319 1320 static bool trans_RDSVL(DisasContext *s, arg_RDSVL *a) 1321 { 1322 if (!dc_isar_feature(aa64_sme, s)) { 1323 return false; 1324 } 1325 if (sme_enabled_check(s)) { 1326 TCGv_i64 reg = cpu_reg(s, a->rd); 1327 tcg_gen_movi_i64(reg, a->imm * streaming_vec_reg_size(s)); 1328 } 1329 return true; 1330 } 1331 1332 /* 1333 *** SVE Compute Vector Address Group 1334 */ 1335 1336 static bool do_adr(DisasContext *s, arg_rrri *a, gen_helper_gvec_3 *fn) 1337 { 1338 return gen_gvec_ool_zzz(s, fn, a->rd, a->rn, a->rm, a->imm); 1339 } 1340 1341 TRANS_FEAT_NONSTREAMING(ADR_p32, aa64_sve, do_adr, a, gen_helper_sve_adr_p32) 1342 TRANS_FEAT_NONSTREAMING(ADR_p64, aa64_sve, do_adr, a, gen_helper_sve_adr_p64) 1343 TRANS_FEAT_NONSTREAMING(ADR_s32, aa64_sve, do_adr, a, gen_helper_sve_adr_s32) 1344 TRANS_FEAT_NONSTREAMING(ADR_u32, aa64_sve, do_adr, a, gen_helper_sve_adr_u32) 1345 1346 /* 1347 *** SVE Integer Misc - Unpredicated Group 1348 */ 1349 1350 static gen_helper_gvec_2 * const fexpa_fns[4] = { 1351 NULL, gen_helper_sve_fexpa_h, 1352 gen_helper_sve_fexpa_s, gen_helper_sve_fexpa_d, 1353 }; 1354 TRANS_FEAT_NONSTREAMING(FEXPA, aa64_sve, gen_gvec_ool_zz, 1355 fexpa_fns[a->esz], a->rd, a->rn, 0) 1356 1357 static gen_helper_gvec_3 * const ftssel_fns[4] = { 1358 NULL, gen_helper_sve_ftssel_h, 1359 gen_helper_sve_ftssel_s, gen_helper_sve_ftssel_d, 1360 }; 1361 TRANS_FEAT_NONSTREAMING(FTSSEL, aa64_sve, gen_gvec_ool_arg_zzz, 1362 ftssel_fns[a->esz], a, 0) 1363 1364 /* 1365 *** SVE Predicate Logical Operations Group 1366 */ 1367 1368 static bool do_pppp_flags(DisasContext *s, arg_rprr_s *a, 1369 const GVecGen4 *gvec_op) 1370 { 1371 if (!sve_access_check(s)) { 1372 return true; 1373 } 1374 1375 unsigned psz = pred_gvec_reg_size(s); 1376 int dofs = pred_full_reg_offset(s, a->rd); 1377 int nofs = pred_full_reg_offset(s, a->rn); 1378 int mofs = pred_full_reg_offset(s, a->rm); 1379 int gofs = pred_full_reg_offset(s, a->pg); 1380 1381 if (!a->s) { 1382 tcg_gen_gvec_4(dofs, nofs, mofs, gofs, psz, psz, gvec_op); 1383 return true; 1384 } 1385 1386 if (psz == 8) { 1387 /* Do the operation and the flags generation in temps. */ 1388 TCGv_i64 pd = tcg_temp_new_i64(); 1389 TCGv_i64 pn = tcg_temp_new_i64(); 1390 TCGv_i64 pm = tcg_temp_new_i64(); 1391 TCGv_i64 pg = tcg_temp_new_i64(); 1392 1393 tcg_gen_ld_i64(pn, cpu_env, nofs); 1394 tcg_gen_ld_i64(pm, cpu_env, mofs); 1395 tcg_gen_ld_i64(pg, cpu_env, gofs); 1396 1397 gvec_op->fni8(pd, pn, pm, pg); 1398 tcg_gen_st_i64(pd, cpu_env, dofs); 1399 1400 do_predtest1(pd, pg); 1401 } else { 1402 /* The operation and flags generation is large. The computation 1403 * of the flags depends on the original contents of the guarding 1404 * predicate. If the destination overwrites the guarding predicate, 1405 * then the easiest way to get this right is to save a copy. 1406 */ 1407 int tofs = gofs; 1408 if (a->rd == a->pg) { 1409 tofs = offsetof(CPUARMState, vfp.preg_tmp); 1410 tcg_gen_gvec_mov(0, tofs, gofs, psz, psz); 1411 } 1412 1413 tcg_gen_gvec_4(dofs, nofs, mofs, gofs, psz, psz, gvec_op); 1414 do_predtest(s, dofs, tofs, psz / 8); 1415 } 1416 return true; 1417 } 1418 1419 static void gen_and_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg) 1420 { 1421 tcg_gen_and_i64(pd, pn, pm); 1422 tcg_gen_and_i64(pd, pd, pg); 1423 } 1424 1425 static void gen_and_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn, 1426 TCGv_vec pm, TCGv_vec pg) 1427 { 1428 tcg_gen_and_vec(vece, pd, pn, pm); 1429 tcg_gen_and_vec(vece, pd, pd, pg); 1430 } 1431 1432 static bool trans_AND_pppp(DisasContext *s, arg_rprr_s *a) 1433 { 1434 static const GVecGen4 op = { 1435 .fni8 = gen_and_pg_i64, 1436 .fniv = gen_and_pg_vec, 1437 .fno = gen_helper_sve_and_pppp, 1438 .prefer_i64 = TCG_TARGET_REG_BITS == 64, 1439 }; 1440 1441 if (!dc_isar_feature(aa64_sve, s)) { 1442 return false; 1443 } 1444 if (!a->s) { 1445 if (a->rn == a->rm) { 1446 if (a->pg == a->rn) { 1447 return do_mov_p(s, a->rd, a->rn); 1448 } 1449 return gen_gvec_fn_ppp(s, tcg_gen_gvec_and, a->rd, a->rn, a->pg); 1450 } else if (a->pg == a->rn || a->pg == a->rm) { 1451 return gen_gvec_fn_ppp(s, tcg_gen_gvec_and, a->rd, a->rn, a->rm); 1452 } 1453 } 1454 return do_pppp_flags(s, a, &op); 1455 } 1456 1457 static void gen_bic_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg) 1458 { 1459 tcg_gen_andc_i64(pd, pn, pm); 1460 tcg_gen_and_i64(pd, pd, pg); 1461 } 1462 1463 static void gen_bic_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn, 1464 TCGv_vec pm, TCGv_vec pg) 1465 { 1466 tcg_gen_andc_vec(vece, pd, pn, pm); 1467 tcg_gen_and_vec(vece, pd, pd, pg); 1468 } 1469 1470 static bool trans_BIC_pppp(DisasContext *s, arg_rprr_s *a) 1471 { 1472 static const GVecGen4 op = { 1473 .fni8 = gen_bic_pg_i64, 1474 .fniv = gen_bic_pg_vec, 1475 .fno = gen_helper_sve_bic_pppp, 1476 .prefer_i64 = TCG_TARGET_REG_BITS == 64, 1477 }; 1478 1479 if (!dc_isar_feature(aa64_sve, s)) { 1480 return false; 1481 } 1482 if (!a->s && a->pg == a->rn) { 1483 return gen_gvec_fn_ppp(s, tcg_gen_gvec_andc, a->rd, a->rn, a->rm); 1484 } 1485 return do_pppp_flags(s, a, &op); 1486 } 1487 1488 static void gen_eor_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg) 1489 { 1490 tcg_gen_xor_i64(pd, pn, pm); 1491 tcg_gen_and_i64(pd, pd, pg); 1492 } 1493 1494 static void gen_eor_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn, 1495 TCGv_vec pm, TCGv_vec pg) 1496 { 1497 tcg_gen_xor_vec(vece, pd, pn, pm); 1498 tcg_gen_and_vec(vece, pd, pd, pg); 1499 } 1500 1501 static bool trans_EOR_pppp(DisasContext *s, arg_rprr_s *a) 1502 { 1503 static const GVecGen4 op = { 1504 .fni8 = gen_eor_pg_i64, 1505 .fniv = gen_eor_pg_vec, 1506 .fno = gen_helper_sve_eor_pppp, 1507 .prefer_i64 = TCG_TARGET_REG_BITS == 64, 1508 }; 1509 1510 if (!dc_isar_feature(aa64_sve, s)) { 1511 return false; 1512 } 1513 /* Alias NOT (predicate) is EOR Pd.B, Pg/Z, Pn.B, Pg.B */ 1514 if (!a->s && a->pg == a->rm) { 1515 return gen_gvec_fn_ppp(s, tcg_gen_gvec_andc, a->rd, a->pg, a->rn); 1516 } 1517 return do_pppp_flags(s, a, &op); 1518 } 1519 1520 static bool trans_SEL_pppp(DisasContext *s, arg_rprr_s *a) 1521 { 1522 if (a->s || !dc_isar_feature(aa64_sve, s)) { 1523 return false; 1524 } 1525 if (sve_access_check(s)) { 1526 unsigned psz = pred_gvec_reg_size(s); 1527 tcg_gen_gvec_bitsel(MO_8, pred_full_reg_offset(s, a->rd), 1528 pred_full_reg_offset(s, a->pg), 1529 pred_full_reg_offset(s, a->rn), 1530 pred_full_reg_offset(s, a->rm), psz, psz); 1531 } 1532 return true; 1533 } 1534 1535 static void gen_orr_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg) 1536 { 1537 tcg_gen_or_i64(pd, pn, pm); 1538 tcg_gen_and_i64(pd, pd, pg); 1539 } 1540 1541 static void gen_orr_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn, 1542 TCGv_vec pm, TCGv_vec pg) 1543 { 1544 tcg_gen_or_vec(vece, pd, pn, pm); 1545 tcg_gen_and_vec(vece, pd, pd, pg); 1546 } 1547 1548 static bool trans_ORR_pppp(DisasContext *s, arg_rprr_s *a) 1549 { 1550 static const GVecGen4 op = { 1551 .fni8 = gen_orr_pg_i64, 1552 .fniv = gen_orr_pg_vec, 1553 .fno = gen_helper_sve_orr_pppp, 1554 .prefer_i64 = TCG_TARGET_REG_BITS == 64, 1555 }; 1556 1557 if (!dc_isar_feature(aa64_sve, s)) { 1558 return false; 1559 } 1560 if (!a->s && a->pg == a->rn && a->rn == a->rm) { 1561 return do_mov_p(s, a->rd, a->rn); 1562 } 1563 return do_pppp_flags(s, a, &op); 1564 } 1565 1566 static void gen_orn_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg) 1567 { 1568 tcg_gen_orc_i64(pd, pn, pm); 1569 tcg_gen_and_i64(pd, pd, pg); 1570 } 1571 1572 static void gen_orn_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn, 1573 TCGv_vec pm, TCGv_vec pg) 1574 { 1575 tcg_gen_orc_vec(vece, pd, pn, pm); 1576 tcg_gen_and_vec(vece, pd, pd, pg); 1577 } 1578 1579 static bool trans_ORN_pppp(DisasContext *s, arg_rprr_s *a) 1580 { 1581 static const GVecGen4 op = { 1582 .fni8 = gen_orn_pg_i64, 1583 .fniv = gen_orn_pg_vec, 1584 .fno = gen_helper_sve_orn_pppp, 1585 .prefer_i64 = TCG_TARGET_REG_BITS == 64, 1586 }; 1587 1588 if (!dc_isar_feature(aa64_sve, s)) { 1589 return false; 1590 } 1591 return do_pppp_flags(s, a, &op); 1592 } 1593 1594 static void gen_nor_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg) 1595 { 1596 tcg_gen_or_i64(pd, pn, pm); 1597 tcg_gen_andc_i64(pd, pg, pd); 1598 } 1599 1600 static void gen_nor_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn, 1601 TCGv_vec pm, TCGv_vec pg) 1602 { 1603 tcg_gen_or_vec(vece, pd, pn, pm); 1604 tcg_gen_andc_vec(vece, pd, pg, pd); 1605 } 1606 1607 static bool trans_NOR_pppp(DisasContext *s, arg_rprr_s *a) 1608 { 1609 static const GVecGen4 op = { 1610 .fni8 = gen_nor_pg_i64, 1611 .fniv = gen_nor_pg_vec, 1612 .fno = gen_helper_sve_nor_pppp, 1613 .prefer_i64 = TCG_TARGET_REG_BITS == 64, 1614 }; 1615 1616 if (!dc_isar_feature(aa64_sve, s)) { 1617 return false; 1618 } 1619 return do_pppp_flags(s, a, &op); 1620 } 1621 1622 static void gen_nand_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg) 1623 { 1624 tcg_gen_and_i64(pd, pn, pm); 1625 tcg_gen_andc_i64(pd, pg, pd); 1626 } 1627 1628 static void gen_nand_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn, 1629 TCGv_vec pm, TCGv_vec pg) 1630 { 1631 tcg_gen_and_vec(vece, pd, pn, pm); 1632 tcg_gen_andc_vec(vece, pd, pg, pd); 1633 } 1634 1635 static bool trans_NAND_pppp(DisasContext *s, arg_rprr_s *a) 1636 { 1637 static const GVecGen4 op = { 1638 .fni8 = gen_nand_pg_i64, 1639 .fniv = gen_nand_pg_vec, 1640 .fno = gen_helper_sve_nand_pppp, 1641 .prefer_i64 = TCG_TARGET_REG_BITS == 64, 1642 }; 1643 1644 if (!dc_isar_feature(aa64_sve, s)) { 1645 return false; 1646 } 1647 return do_pppp_flags(s, a, &op); 1648 } 1649 1650 /* 1651 *** SVE Predicate Misc Group 1652 */ 1653 1654 static bool trans_PTEST(DisasContext *s, arg_PTEST *a) 1655 { 1656 if (!dc_isar_feature(aa64_sve, s)) { 1657 return false; 1658 } 1659 if (sve_access_check(s)) { 1660 int nofs = pred_full_reg_offset(s, a->rn); 1661 int gofs = pred_full_reg_offset(s, a->pg); 1662 int words = DIV_ROUND_UP(pred_full_reg_size(s), 8); 1663 1664 if (words == 1) { 1665 TCGv_i64 pn = tcg_temp_new_i64(); 1666 TCGv_i64 pg = tcg_temp_new_i64(); 1667 1668 tcg_gen_ld_i64(pn, cpu_env, nofs); 1669 tcg_gen_ld_i64(pg, cpu_env, gofs); 1670 do_predtest1(pn, pg); 1671 } else { 1672 do_predtest(s, nofs, gofs, words); 1673 } 1674 } 1675 return true; 1676 } 1677 1678 /* See the ARM pseudocode DecodePredCount. */ 1679 static unsigned decode_pred_count(unsigned fullsz, int pattern, int esz) 1680 { 1681 unsigned elements = fullsz >> esz; 1682 unsigned bound; 1683 1684 switch (pattern) { 1685 case 0x0: /* POW2 */ 1686 return pow2floor(elements); 1687 case 0x1: /* VL1 */ 1688 case 0x2: /* VL2 */ 1689 case 0x3: /* VL3 */ 1690 case 0x4: /* VL4 */ 1691 case 0x5: /* VL5 */ 1692 case 0x6: /* VL6 */ 1693 case 0x7: /* VL7 */ 1694 case 0x8: /* VL8 */ 1695 bound = pattern; 1696 break; 1697 case 0x9: /* VL16 */ 1698 case 0xa: /* VL32 */ 1699 case 0xb: /* VL64 */ 1700 case 0xc: /* VL128 */ 1701 case 0xd: /* VL256 */ 1702 bound = 16 << (pattern - 9); 1703 break; 1704 case 0x1d: /* MUL4 */ 1705 return elements - elements % 4; 1706 case 0x1e: /* MUL3 */ 1707 return elements - elements % 3; 1708 case 0x1f: /* ALL */ 1709 return elements; 1710 default: /* #uimm5 */ 1711 return 0; 1712 } 1713 return elements >= bound ? bound : 0; 1714 } 1715 1716 /* This handles all of the predicate initialization instructions, 1717 * PTRUE, PFALSE, SETFFR. For PFALSE, we will have set PAT == 32 1718 * so that decode_pred_count returns 0. For SETFFR, we will have 1719 * set RD == 16 == FFR. 1720 */ 1721 static bool do_predset(DisasContext *s, int esz, int rd, int pat, bool setflag) 1722 { 1723 if (!sve_access_check(s)) { 1724 return true; 1725 } 1726 1727 unsigned fullsz = vec_full_reg_size(s); 1728 unsigned ofs = pred_full_reg_offset(s, rd); 1729 unsigned numelem, setsz, i; 1730 uint64_t word, lastword; 1731 TCGv_i64 t; 1732 1733 numelem = decode_pred_count(fullsz, pat, esz); 1734 1735 /* Determine what we must store into each bit, and how many. */ 1736 if (numelem == 0) { 1737 lastword = word = 0; 1738 setsz = fullsz; 1739 } else { 1740 setsz = numelem << esz; 1741 lastword = word = pred_esz_masks[esz]; 1742 if (setsz % 64) { 1743 lastword &= MAKE_64BIT_MASK(0, setsz % 64); 1744 } 1745 } 1746 1747 t = tcg_temp_new_i64(); 1748 if (fullsz <= 64) { 1749 tcg_gen_movi_i64(t, lastword); 1750 tcg_gen_st_i64(t, cpu_env, ofs); 1751 goto done; 1752 } 1753 1754 if (word == lastword) { 1755 unsigned maxsz = size_for_gvec(fullsz / 8); 1756 unsigned oprsz = size_for_gvec(setsz / 8); 1757 1758 if (oprsz * 8 == setsz) { 1759 tcg_gen_gvec_dup_imm(MO_64, ofs, oprsz, maxsz, word); 1760 goto done; 1761 } 1762 } 1763 1764 setsz /= 8; 1765 fullsz /= 8; 1766 1767 tcg_gen_movi_i64(t, word); 1768 for (i = 0; i < QEMU_ALIGN_DOWN(setsz, 8); i += 8) { 1769 tcg_gen_st_i64(t, cpu_env, ofs + i); 1770 } 1771 if (lastword != word) { 1772 tcg_gen_movi_i64(t, lastword); 1773 tcg_gen_st_i64(t, cpu_env, ofs + i); 1774 i += 8; 1775 } 1776 if (i < fullsz) { 1777 tcg_gen_movi_i64(t, 0); 1778 for (; i < fullsz; i += 8) { 1779 tcg_gen_st_i64(t, cpu_env, ofs + i); 1780 } 1781 } 1782 1783 done: 1784 /* PTRUES */ 1785 if (setflag) { 1786 tcg_gen_movi_i32(cpu_NF, -(word != 0)); 1787 tcg_gen_movi_i32(cpu_CF, word == 0); 1788 tcg_gen_movi_i32(cpu_VF, 0); 1789 tcg_gen_mov_i32(cpu_ZF, cpu_NF); 1790 } 1791 return true; 1792 } 1793 1794 TRANS_FEAT(PTRUE, aa64_sve, do_predset, a->esz, a->rd, a->pat, a->s) 1795 1796 /* Note pat == 31 is #all, to set all elements. */ 1797 TRANS_FEAT_NONSTREAMING(SETFFR, aa64_sve, 1798 do_predset, 0, FFR_PRED_NUM, 31, false) 1799 1800 /* Note pat == 32 is #unimp, to set no elements. */ 1801 TRANS_FEAT(PFALSE, aa64_sve, do_predset, 0, a->rd, 32, false) 1802 1803 static bool trans_RDFFR_p(DisasContext *s, arg_RDFFR_p *a) 1804 { 1805 /* The path through do_pppp_flags is complicated enough to want to avoid 1806 * duplication. Frob the arguments into the form of a predicated AND. 1807 */ 1808 arg_rprr_s alt_a = { 1809 .rd = a->rd, .pg = a->pg, .s = a->s, 1810 .rn = FFR_PRED_NUM, .rm = FFR_PRED_NUM, 1811 }; 1812 1813 s->is_nonstreaming = true; 1814 return trans_AND_pppp(s, &alt_a); 1815 } 1816 1817 TRANS_FEAT_NONSTREAMING(RDFFR, aa64_sve, do_mov_p, a->rd, FFR_PRED_NUM) 1818 TRANS_FEAT_NONSTREAMING(WRFFR, aa64_sve, do_mov_p, FFR_PRED_NUM, a->rn) 1819 1820 static bool do_pfirst_pnext(DisasContext *s, arg_rr_esz *a, 1821 void (*gen_fn)(TCGv_i32, TCGv_ptr, 1822 TCGv_ptr, TCGv_i32)) 1823 { 1824 if (!sve_access_check(s)) { 1825 return true; 1826 } 1827 1828 TCGv_ptr t_pd = tcg_temp_new_ptr(); 1829 TCGv_ptr t_pg = tcg_temp_new_ptr(); 1830 TCGv_i32 t; 1831 unsigned desc = 0; 1832 1833 desc = FIELD_DP32(desc, PREDDESC, OPRSZ, pred_full_reg_size(s)); 1834 desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz); 1835 1836 tcg_gen_addi_ptr(t_pd, cpu_env, pred_full_reg_offset(s, a->rd)); 1837 tcg_gen_addi_ptr(t_pg, cpu_env, pred_full_reg_offset(s, a->rn)); 1838 t = tcg_temp_new_i32(); 1839 1840 gen_fn(t, t_pd, t_pg, tcg_constant_i32(desc)); 1841 1842 do_pred_flags(t); 1843 return true; 1844 } 1845 1846 TRANS_FEAT(PFIRST, aa64_sve, do_pfirst_pnext, a, gen_helper_sve_pfirst) 1847 TRANS_FEAT(PNEXT, aa64_sve, do_pfirst_pnext, a, gen_helper_sve_pnext) 1848 1849 /* 1850 *** SVE Element Count Group 1851 */ 1852 1853 /* Perform an inline saturating addition of a 32-bit value within 1854 * a 64-bit register. The second operand is known to be positive, 1855 * which halves the comparisions we must perform to bound the result. 1856 */ 1857 static void do_sat_addsub_32(TCGv_i64 reg, TCGv_i64 val, bool u, bool d) 1858 { 1859 int64_t ibound; 1860 1861 /* Use normal 64-bit arithmetic to detect 32-bit overflow. */ 1862 if (u) { 1863 tcg_gen_ext32u_i64(reg, reg); 1864 } else { 1865 tcg_gen_ext32s_i64(reg, reg); 1866 } 1867 if (d) { 1868 tcg_gen_sub_i64(reg, reg, val); 1869 ibound = (u ? 0 : INT32_MIN); 1870 tcg_gen_smax_i64(reg, reg, tcg_constant_i64(ibound)); 1871 } else { 1872 tcg_gen_add_i64(reg, reg, val); 1873 ibound = (u ? UINT32_MAX : INT32_MAX); 1874 tcg_gen_smin_i64(reg, reg, tcg_constant_i64(ibound)); 1875 } 1876 } 1877 1878 /* Similarly with 64-bit values. */ 1879 static void do_sat_addsub_64(TCGv_i64 reg, TCGv_i64 val, bool u, bool d) 1880 { 1881 TCGv_i64 t0 = tcg_temp_new_i64(); 1882 TCGv_i64 t2; 1883 1884 if (u) { 1885 if (d) { 1886 tcg_gen_sub_i64(t0, reg, val); 1887 t2 = tcg_constant_i64(0); 1888 tcg_gen_movcond_i64(TCG_COND_LTU, reg, reg, val, t2, t0); 1889 } else { 1890 tcg_gen_add_i64(t0, reg, val); 1891 t2 = tcg_constant_i64(-1); 1892 tcg_gen_movcond_i64(TCG_COND_LTU, reg, t0, reg, t2, t0); 1893 } 1894 } else { 1895 TCGv_i64 t1 = tcg_temp_new_i64(); 1896 if (d) { 1897 /* Detect signed overflow for subtraction. */ 1898 tcg_gen_xor_i64(t0, reg, val); 1899 tcg_gen_sub_i64(t1, reg, val); 1900 tcg_gen_xor_i64(reg, reg, t1); 1901 tcg_gen_and_i64(t0, t0, reg); 1902 1903 /* Bound the result. */ 1904 tcg_gen_movi_i64(reg, INT64_MIN); 1905 t2 = tcg_constant_i64(0); 1906 tcg_gen_movcond_i64(TCG_COND_LT, reg, t0, t2, reg, t1); 1907 } else { 1908 /* Detect signed overflow for addition. */ 1909 tcg_gen_xor_i64(t0, reg, val); 1910 tcg_gen_add_i64(reg, reg, val); 1911 tcg_gen_xor_i64(t1, reg, val); 1912 tcg_gen_andc_i64(t0, t1, t0); 1913 1914 /* Bound the result. */ 1915 tcg_gen_movi_i64(t1, INT64_MAX); 1916 t2 = tcg_constant_i64(0); 1917 tcg_gen_movcond_i64(TCG_COND_LT, reg, t0, t2, t1, reg); 1918 } 1919 } 1920 } 1921 1922 /* Similarly with a vector and a scalar operand. */ 1923 static void do_sat_addsub_vec(DisasContext *s, int esz, int rd, int rn, 1924 TCGv_i64 val, bool u, bool d) 1925 { 1926 unsigned vsz = vec_full_reg_size(s); 1927 TCGv_ptr dptr, nptr; 1928 TCGv_i32 t32, desc; 1929 TCGv_i64 t64; 1930 1931 dptr = tcg_temp_new_ptr(); 1932 nptr = tcg_temp_new_ptr(); 1933 tcg_gen_addi_ptr(dptr, cpu_env, vec_full_reg_offset(s, rd)); 1934 tcg_gen_addi_ptr(nptr, cpu_env, vec_full_reg_offset(s, rn)); 1935 desc = tcg_constant_i32(simd_desc(vsz, vsz, 0)); 1936 1937 switch (esz) { 1938 case MO_8: 1939 t32 = tcg_temp_new_i32(); 1940 tcg_gen_extrl_i64_i32(t32, val); 1941 if (d) { 1942 tcg_gen_neg_i32(t32, t32); 1943 } 1944 if (u) { 1945 gen_helper_sve_uqaddi_b(dptr, nptr, t32, desc); 1946 } else { 1947 gen_helper_sve_sqaddi_b(dptr, nptr, t32, desc); 1948 } 1949 break; 1950 1951 case MO_16: 1952 t32 = tcg_temp_new_i32(); 1953 tcg_gen_extrl_i64_i32(t32, val); 1954 if (d) { 1955 tcg_gen_neg_i32(t32, t32); 1956 } 1957 if (u) { 1958 gen_helper_sve_uqaddi_h(dptr, nptr, t32, desc); 1959 } else { 1960 gen_helper_sve_sqaddi_h(dptr, nptr, t32, desc); 1961 } 1962 break; 1963 1964 case MO_32: 1965 t64 = tcg_temp_new_i64(); 1966 if (d) { 1967 tcg_gen_neg_i64(t64, val); 1968 } else { 1969 tcg_gen_mov_i64(t64, val); 1970 } 1971 if (u) { 1972 gen_helper_sve_uqaddi_s(dptr, nptr, t64, desc); 1973 } else { 1974 gen_helper_sve_sqaddi_s(dptr, nptr, t64, desc); 1975 } 1976 break; 1977 1978 case MO_64: 1979 if (u) { 1980 if (d) { 1981 gen_helper_sve_uqsubi_d(dptr, nptr, val, desc); 1982 } else { 1983 gen_helper_sve_uqaddi_d(dptr, nptr, val, desc); 1984 } 1985 } else if (d) { 1986 t64 = tcg_temp_new_i64(); 1987 tcg_gen_neg_i64(t64, val); 1988 gen_helper_sve_sqaddi_d(dptr, nptr, t64, desc); 1989 } else { 1990 gen_helper_sve_sqaddi_d(dptr, nptr, val, desc); 1991 } 1992 break; 1993 1994 default: 1995 g_assert_not_reached(); 1996 } 1997 } 1998 1999 static bool trans_CNT_r(DisasContext *s, arg_CNT_r *a) 2000 { 2001 if (!dc_isar_feature(aa64_sve, s)) { 2002 return false; 2003 } 2004 if (sve_access_check(s)) { 2005 unsigned fullsz = vec_full_reg_size(s); 2006 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz); 2007 tcg_gen_movi_i64(cpu_reg(s, a->rd), numelem * a->imm); 2008 } 2009 return true; 2010 } 2011 2012 static bool trans_INCDEC_r(DisasContext *s, arg_incdec_cnt *a) 2013 { 2014 if (!dc_isar_feature(aa64_sve, s)) { 2015 return false; 2016 } 2017 if (sve_access_check(s)) { 2018 unsigned fullsz = vec_full_reg_size(s); 2019 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz); 2020 int inc = numelem * a->imm * (a->d ? -1 : 1); 2021 TCGv_i64 reg = cpu_reg(s, a->rd); 2022 2023 tcg_gen_addi_i64(reg, reg, inc); 2024 } 2025 return true; 2026 } 2027 2028 static bool trans_SINCDEC_r_32(DisasContext *s, arg_incdec_cnt *a) 2029 { 2030 if (!dc_isar_feature(aa64_sve, s)) { 2031 return false; 2032 } 2033 if (!sve_access_check(s)) { 2034 return true; 2035 } 2036 2037 unsigned fullsz = vec_full_reg_size(s); 2038 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz); 2039 int inc = numelem * a->imm; 2040 TCGv_i64 reg = cpu_reg(s, a->rd); 2041 2042 /* Use normal 64-bit arithmetic to detect 32-bit overflow. */ 2043 if (inc == 0) { 2044 if (a->u) { 2045 tcg_gen_ext32u_i64(reg, reg); 2046 } else { 2047 tcg_gen_ext32s_i64(reg, reg); 2048 } 2049 } else { 2050 do_sat_addsub_32(reg, tcg_constant_i64(inc), a->u, a->d); 2051 } 2052 return true; 2053 } 2054 2055 static bool trans_SINCDEC_r_64(DisasContext *s, arg_incdec_cnt *a) 2056 { 2057 if (!dc_isar_feature(aa64_sve, s)) { 2058 return false; 2059 } 2060 if (!sve_access_check(s)) { 2061 return true; 2062 } 2063 2064 unsigned fullsz = vec_full_reg_size(s); 2065 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz); 2066 int inc = numelem * a->imm; 2067 TCGv_i64 reg = cpu_reg(s, a->rd); 2068 2069 if (inc != 0) { 2070 do_sat_addsub_64(reg, tcg_constant_i64(inc), a->u, a->d); 2071 } 2072 return true; 2073 } 2074 2075 static bool trans_INCDEC_v(DisasContext *s, arg_incdec2_cnt *a) 2076 { 2077 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) { 2078 return false; 2079 } 2080 2081 unsigned fullsz = vec_full_reg_size(s); 2082 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz); 2083 int inc = numelem * a->imm; 2084 2085 if (inc != 0) { 2086 if (sve_access_check(s)) { 2087 tcg_gen_gvec_adds(a->esz, vec_full_reg_offset(s, a->rd), 2088 vec_full_reg_offset(s, a->rn), 2089 tcg_constant_i64(a->d ? -inc : inc), 2090 fullsz, fullsz); 2091 } 2092 } else { 2093 do_mov_z(s, a->rd, a->rn); 2094 } 2095 return true; 2096 } 2097 2098 static bool trans_SINCDEC_v(DisasContext *s, arg_incdec2_cnt *a) 2099 { 2100 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) { 2101 return false; 2102 } 2103 2104 unsigned fullsz = vec_full_reg_size(s); 2105 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz); 2106 int inc = numelem * a->imm; 2107 2108 if (inc != 0) { 2109 if (sve_access_check(s)) { 2110 do_sat_addsub_vec(s, a->esz, a->rd, a->rn, 2111 tcg_constant_i64(inc), a->u, a->d); 2112 } 2113 } else { 2114 do_mov_z(s, a->rd, a->rn); 2115 } 2116 return true; 2117 } 2118 2119 /* 2120 *** SVE Bitwise Immediate Group 2121 */ 2122 2123 static bool do_zz_dbm(DisasContext *s, arg_rr_dbm *a, GVecGen2iFn *gvec_fn) 2124 { 2125 uint64_t imm; 2126 if (!logic_imm_decode_wmask(&imm, extract32(a->dbm, 12, 1), 2127 extract32(a->dbm, 0, 6), 2128 extract32(a->dbm, 6, 6))) { 2129 return false; 2130 } 2131 return gen_gvec_fn_zzi(s, gvec_fn, MO_64, a->rd, a->rn, imm); 2132 } 2133 2134 TRANS_FEAT(AND_zzi, aa64_sve, do_zz_dbm, a, tcg_gen_gvec_andi) 2135 TRANS_FEAT(ORR_zzi, aa64_sve, do_zz_dbm, a, tcg_gen_gvec_ori) 2136 TRANS_FEAT(EOR_zzi, aa64_sve, do_zz_dbm, a, tcg_gen_gvec_xori) 2137 2138 static bool trans_DUPM(DisasContext *s, arg_DUPM *a) 2139 { 2140 uint64_t imm; 2141 2142 if (!dc_isar_feature(aa64_sve, s)) { 2143 return false; 2144 } 2145 if (!logic_imm_decode_wmask(&imm, extract32(a->dbm, 12, 1), 2146 extract32(a->dbm, 0, 6), 2147 extract32(a->dbm, 6, 6))) { 2148 return false; 2149 } 2150 if (sve_access_check(s)) { 2151 do_dupi_z(s, a->rd, imm); 2152 } 2153 return true; 2154 } 2155 2156 /* 2157 *** SVE Integer Wide Immediate - Predicated Group 2158 */ 2159 2160 /* Implement all merging copies. This is used for CPY (immediate), 2161 * FCPY, CPY (scalar), CPY (SIMD&FP scalar). 2162 */ 2163 static void do_cpy_m(DisasContext *s, int esz, int rd, int rn, int pg, 2164 TCGv_i64 val) 2165 { 2166 typedef void gen_cpy(TCGv_ptr, TCGv_ptr, TCGv_ptr, TCGv_i64, TCGv_i32); 2167 static gen_cpy * const fns[4] = { 2168 gen_helper_sve_cpy_m_b, gen_helper_sve_cpy_m_h, 2169 gen_helper_sve_cpy_m_s, gen_helper_sve_cpy_m_d, 2170 }; 2171 unsigned vsz = vec_full_reg_size(s); 2172 TCGv_i32 desc = tcg_constant_i32(simd_desc(vsz, vsz, 0)); 2173 TCGv_ptr t_zd = tcg_temp_new_ptr(); 2174 TCGv_ptr t_zn = tcg_temp_new_ptr(); 2175 TCGv_ptr t_pg = tcg_temp_new_ptr(); 2176 2177 tcg_gen_addi_ptr(t_zd, cpu_env, vec_full_reg_offset(s, rd)); 2178 tcg_gen_addi_ptr(t_zn, cpu_env, vec_full_reg_offset(s, rn)); 2179 tcg_gen_addi_ptr(t_pg, cpu_env, pred_full_reg_offset(s, pg)); 2180 2181 fns[esz](t_zd, t_zn, t_pg, val, desc); 2182 } 2183 2184 static bool trans_FCPY(DisasContext *s, arg_FCPY *a) 2185 { 2186 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) { 2187 return false; 2188 } 2189 if (sve_access_check(s)) { 2190 /* Decode the VFP immediate. */ 2191 uint64_t imm = vfp_expand_imm(a->esz, a->imm); 2192 do_cpy_m(s, a->esz, a->rd, a->rn, a->pg, tcg_constant_i64(imm)); 2193 } 2194 return true; 2195 } 2196 2197 static bool trans_CPY_m_i(DisasContext *s, arg_rpri_esz *a) 2198 { 2199 if (!dc_isar_feature(aa64_sve, s)) { 2200 return false; 2201 } 2202 if (sve_access_check(s)) { 2203 do_cpy_m(s, a->esz, a->rd, a->rn, a->pg, tcg_constant_i64(a->imm)); 2204 } 2205 return true; 2206 } 2207 2208 static bool trans_CPY_z_i(DisasContext *s, arg_CPY_z_i *a) 2209 { 2210 static gen_helper_gvec_2i * const fns[4] = { 2211 gen_helper_sve_cpy_z_b, gen_helper_sve_cpy_z_h, 2212 gen_helper_sve_cpy_z_s, gen_helper_sve_cpy_z_d, 2213 }; 2214 2215 if (!dc_isar_feature(aa64_sve, s)) { 2216 return false; 2217 } 2218 if (sve_access_check(s)) { 2219 unsigned vsz = vec_full_reg_size(s); 2220 tcg_gen_gvec_2i_ool(vec_full_reg_offset(s, a->rd), 2221 pred_full_reg_offset(s, a->pg), 2222 tcg_constant_i64(a->imm), 2223 vsz, vsz, 0, fns[a->esz]); 2224 } 2225 return true; 2226 } 2227 2228 /* 2229 *** SVE Permute Extract Group 2230 */ 2231 2232 static bool do_EXT(DisasContext *s, int rd, int rn, int rm, int imm) 2233 { 2234 if (!sve_access_check(s)) { 2235 return true; 2236 } 2237 2238 unsigned vsz = vec_full_reg_size(s); 2239 unsigned n_ofs = imm >= vsz ? 0 : imm; 2240 unsigned n_siz = vsz - n_ofs; 2241 unsigned d = vec_full_reg_offset(s, rd); 2242 unsigned n = vec_full_reg_offset(s, rn); 2243 unsigned m = vec_full_reg_offset(s, rm); 2244 2245 /* Use host vector move insns if we have appropriate sizes 2246 * and no unfortunate overlap. 2247 */ 2248 if (m != d 2249 && n_ofs == size_for_gvec(n_ofs) 2250 && n_siz == size_for_gvec(n_siz) 2251 && (d != n || n_siz <= n_ofs)) { 2252 tcg_gen_gvec_mov(0, d, n + n_ofs, n_siz, n_siz); 2253 if (n_ofs != 0) { 2254 tcg_gen_gvec_mov(0, d + n_siz, m, n_ofs, n_ofs); 2255 } 2256 } else { 2257 tcg_gen_gvec_3_ool(d, n, m, vsz, vsz, n_ofs, gen_helper_sve_ext); 2258 } 2259 return true; 2260 } 2261 2262 TRANS_FEAT(EXT, aa64_sve, do_EXT, a->rd, a->rn, a->rm, a->imm) 2263 TRANS_FEAT(EXT_sve2, aa64_sve2, do_EXT, a->rd, a->rn, (a->rn + 1) % 32, a->imm) 2264 2265 /* 2266 *** SVE Permute - Unpredicated Group 2267 */ 2268 2269 static bool trans_DUP_s(DisasContext *s, arg_DUP_s *a) 2270 { 2271 if (!dc_isar_feature(aa64_sve, s)) { 2272 return false; 2273 } 2274 if (sve_access_check(s)) { 2275 unsigned vsz = vec_full_reg_size(s); 2276 tcg_gen_gvec_dup_i64(a->esz, vec_full_reg_offset(s, a->rd), 2277 vsz, vsz, cpu_reg_sp(s, a->rn)); 2278 } 2279 return true; 2280 } 2281 2282 static bool trans_DUP_x(DisasContext *s, arg_DUP_x *a) 2283 { 2284 if (!dc_isar_feature(aa64_sve, s)) { 2285 return false; 2286 } 2287 if ((a->imm & 0x1f) == 0) { 2288 return false; 2289 } 2290 if (sve_access_check(s)) { 2291 unsigned vsz = vec_full_reg_size(s); 2292 unsigned dofs = vec_full_reg_offset(s, a->rd); 2293 unsigned esz, index; 2294 2295 esz = ctz32(a->imm); 2296 index = a->imm >> (esz + 1); 2297 2298 if ((index << esz) < vsz) { 2299 unsigned nofs = vec_reg_offset(s, a->rn, index, esz); 2300 tcg_gen_gvec_dup_mem(esz, dofs, nofs, vsz, vsz); 2301 } else { 2302 /* 2303 * While dup_mem handles 128-bit elements, dup_imm does not. 2304 * Thankfully element size doesn't matter for splatting zero. 2305 */ 2306 tcg_gen_gvec_dup_imm(MO_64, dofs, vsz, vsz, 0); 2307 } 2308 } 2309 return true; 2310 } 2311 2312 static void do_insr_i64(DisasContext *s, arg_rrr_esz *a, TCGv_i64 val) 2313 { 2314 typedef void gen_insr(TCGv_ptr, TCGv_ptr, TCGv_i64, TCGv_i32); 2315 static gen_insr * const fns[4] = { 2316 gen_helper_sve_insr_b, gen_helper_sve_insr_h, 2317 gen_helper_sve_insr_s, gen_helper_sve_insr_d, 2318 }; 2319 unsigned vsz = vec_full_reg_size(s); 2320 TCGv_i32 desc = tcg_constant_i32(simd_desc(vsz, vsz, 0)); 2321 TCGv_ptr t_zd = tcg_temp_new_ptr(); 2322 TCGv_ptr t_zn = tcg_temp_new_ptr(); 2323 2324 tcg_gen_addi_ptr(t_zd, cpu_env, vec_full_reg_offset(s, a->rd)); 2325 tcg_gen_addi_ptr(t_zn, cpu_env, vec_full_reg_offset(s, a->rn)); 2326 2327 fns[a->esz](t_zd, t_zn, val, desc); 2328 } 2329 2330 static bool trans_INSR_f(DisasContext *s, arg_rrr_esz *a) 2331 { 2332 if (!dc_isar_feature(aa64_sve, s)) { 2333 return false; 2334 } 2335 if (sve_access_check(s)) { 2336 TCGv_i64 t = tcg_temp_new_i64(); 2337 tcg_gen_ld_i64(t, cpu_env, vec_reg_offset(s, a->rm, 0, MO_64)); 2338 do_insr_i64(s, a, t); 2339 } 2340 return true; 2341 } 2342 2343 static bool trans_INSR_r(DisasContext *s, arg_rrr_esz *a) 2344 { 2345 if (!dc_isar_feature(aa64_sve, s)) { 2346 return false; 2347 } 2348 if (sve_access_check(s)) { 2349 do_insr_i64(s, a, cpu_reg(s, a->rm)); 2350 } 2351 return true; 2352 } 2353 2354 static gen_helper_gvec_2 * const rev_fns[4] = { 2355 gen_helper_sve_rev_b, gen_helper_sve_rev_h, 2356 gen_helper_sve_rev_s, gen_helper_sve_rev_d 2357 }; 2358 TRANS_FEAT(REV_v, aa64_sve, gen_gvec_ool_zz, rev_fns[a->esz], a->rd, a->rn, 0) 2359 2360 static gen_helper_gvec_3 * const sve_tbl_fns[4] = { 2361 gen_helper_sve_tbl_b, gen_helper_sve_tbl_h, 2362 gen_helper_sve_tbl_s, gen_helper_sve_tbl_d 2363 }; 2364 TRANS_FEAT(TBL, aa64_sve, gen_gvec_ool_arg_zzz, sve_tbl_fns[a->esz], a, 0) 2365 2366 static gen_helper_gvec_4 * const sve2_tbl_fns[4] = { 2367 gen_helper_sve2_tbl_b, gen_helper_sve2_tbl_h, 2368 gen_helper_sve2_tbl_s, gen_helper_sve2_tbl_d 2369 }; 2370 TRANS_FEAT(TBL_sve2, aa64_sve2, gen_gvec_ool_zzzz, sve2_tbl_fns[a->esz], 2371 a->rd, a->rn, (a->rn + 1) % 32, a->rm, 0) 2372 2373 static gen_helper_gvec_3 * const tbx_fns[4] = { 2374 gen_helper_sve2_tbx_b, gen_helper_sve2_tbx_h, 2375 gen_helper_sve2_tbx_s, gen_helper_sve2_tbx_d 2376 }; 2377 TRANS_FEAT(TBX, aa64_sve2, gen_gvec_ool_arg_zzz, tbx_fns[a->esz], a, 0) 2378 2379 static bool trans_UNPK(DisasContext *s, arg_UNPK *a) 2380 { 2381 static gen_helper_gvec_2 * const fns[4][2] = { 2382 { NULL, NULL }, 2383 { gen_helper_sve_sunpk_h, gen_helper_sve_uunpk_h }, 2384 { gen_helper_sve_sunpk_s, gen_helper_sve_uunpk_s }, 2385 { gen_helper_sve_sunpk_d, gen_helper_sve_uunpk_d }, 2386 }; 2387 2388 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) { 2389 return false; 2390 } 2391 if (sve_access_check(s)) { 2392 unsigned vsz = vec_full_reg_size(s); 2393 tcg_gen_gvec_2_ool(vec_full_reg_offset(s, a->rd), 2394 vec_full_reg_offset(s, a->rn) 2395 + (a->h ? vsz / 2 : 0), 2396 vsz, vsz, 0, fns[a->esz][a->u]); 2397 } 2398 return true; 2399 } 2400 2401 /* 2402 *** SVE Permute - Predicates Group 2403 */ 2404 2405 static bool do_perm_pred3(DisasContext *s, arg_rrr_esz *a, bool high_odd, 2406 gen_helper_gvec_3 *fn) 2407 { 2408 if (!sve_access_check(s)) { 2409 return true; 2410 } 2411 2412 unsigned vsz = pred_full_reg_size(s); 2413 2414 TCGv_ptr t_d = tcg_temp_new_ptr(); 2415 TCGv_ptr t_n = tcg_temp_new_ptr(); 2416 TCGv_ptr t_m = tcg_temp_new_ptr(); 2417 uint32_t desc = 0; 2418 2419 desc = FIELD_DP32(desc, PREDDESC, OPRSZ, vsz); 2420 desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz); 2421 desc = FIELD_DP32(desc, PREDDESC, DATA, high_odd); 2422 2423 tcg_gen_addi_ptr(t_d, cpu_env, pred_full_reg_offset(s, a->rd)); 2424 tcg_gen_addi_ptr(t_n, cpu_env, pred_full_reg_offset(s, a->rn)); 2425 tcg_gen_addi_ptr(t_m, cpu_env, pred_full_reg_offset(s, a->rm)); 2426 2427 fn(t_d, t_n, t_m, tcg_constant_i32(desc)); 2428 return true; 2429 } 2430 2431 static bool do_perm_pred2(DisasContext *s, arg_rr_esz *a, bool high_odd, 2432 gen_helper_gvec_2 *fn) 2433 { 2434 if (!sve_access_check(s)) { 2435 return true; 2436 } 2437 2438 unsigned vsz = pred_full_reg_size(s); 2439 TCGv_ptr t_d = tcg_temp_new_ptr(); 2440 TCGv_ptr t_n = tcg_temp_new_ptr(); 2441 uint32_t desc = 0; 2442 2443 tcg_gen_addi_ptr(t_d, cpu_env, pred_full_reg_offset(s, a->rd)); 2444 tcg_gen_addi_ptr(t_n, cpu_env, pred_full_reg_offset(s, a->rn)); 2445 2446 desc = FIELD_DP32(desc, PREDDESC, OPRSZ, vsz); 2447 desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz); 2448 desc = FIELD_DP32(desc, PREDDESC, DATA, high_odd); 2449 2450 fn(t_d, t_n, tcg_constant_i32(desc)); 2451 return true; 2452 } 2453 2454 TRANS_FEAT(ZIP1_p, aa64_sve, do_perm_pred3, a, 0, gen_helper_sve_zip_p) 2455 TRANS_FEAT(ZIP2_p, aa64_sve, do_perm_pred3, a, 1, gen_helper_sve_zip_p) 2456 TRANS_FEAT(UZP1_p, aa64_sve, do_perm_pred3, a, 0, gen_helper_sve_uzp_p) 2457 TRANS_FEAT(UZP2_p, aa64_sve, do_perm_pred3, a, 1, gen_helper_sve_uzp_p) 2458 TRANS_FEAT(TRN1_p, aa64_sve, do_perm_pred3, a, 0, gen_helper_sve_trn_p) 2459 TRANS_FEAT(TRN2_p, aa64_sve, do_perm_pred3, a, 1, gen_helper_sve_trn_p) 2460 2461 TRANS_FEAT(REV_p, aa64_sve, do_perm_pred2, a, 0, gen_helper_sve_rev_p) 2462 TRANS_FEAT(PUNPKLO, aa64_sve, do_perm_pred2, a, 0, gen_helper_sve_punpk_p) 2463 TRANS_FEAT(PUNPKHI, aa64_sve, do_perm_pred2, a, 1, gen_helper_sve_punpk_p) 2464 2465 /* 2466 *** SVE Permute - Interleaving Group 2467 */ 2468 2469 static gen_helper_gvec_3 * const zip_fns[4] = { 2470 gen_helper_sve_zip_b, gen_helper_sve_zip_h, 2471 gen_helper_sve_zip_s, gen_helper_sve_zip_d, 2472 }; 2473 TRANS_FEAT(ZIP1_z, aa64_sve, gen_gvec_ool_arg_zzz, 2474 zip_fns[a->esz], a, 0) 2475 TRANS_FEAT(ZIP2_z, aa64_sve, gen_gvec_ool_arg_zzz, 2476 zip_fns[a->esz], a, vec_full_reg_size(s) / 2) 2477 2478 TRANS_FEAT(ZIP1_q, aa64_sve_f64mm, gen_gvec_ool_arg_zzz, 2479 gen_helper_sve2_zip_q, a, 0) 2480 TRANS_FEAT(ZIP2_q, aa64_sve_f64mm, gen_gvec_ool_arg_zzz, 2481 gen_helper_sve2_zip_q, a, 2482 QEMU_ALIGN_DOWN(vec_full_reg_size(s), 32) / 2) 2483 2484 static gen_helper_gvec_3 * const uzp_fns[4] = { 2485 gen_helper_sve_uzp_b, gen_helper_sve_uzp_h, 2486 gen_helper_sve_uzp_s, gen_helper_sve_uzp_d, 2487 }; 2488 2489 TRANS_FEAT(UZP1_z, aa64_sve, gen_gvec_ool_arg_zzz, 2490 uzp_fns[a->esz], a, 0) 2491 TRANS_FEAT(UZP2_z, aa64_sve, gen_gvec_ool_arg_zzz, 2492 uzp_fns[a->esz], a, 1 << a->esz) 2493 2494 TRANS_FEAT(UZP1_q, aa64_sve_f64mm, gen_gvec_ool_arg_zzz, 2495 gen_helper_sve2_uzp_q, a, 0) 2496 TRANS_FEAT(UZP2_q, aa64_sve_f64mm, gen_gvec_ool_arg_zzz, 2497 gen_helper_sve2_uzp_q, a, 16) 2498 2499 static gen_helper_gvec_3 * const trn_fns[4] = { 2500 gen_helper_sve_trn_b, gen_helper_sve_trn_h, 2501 gen_helper_sve_trn_s, gen_helper_sve_trn_d, 2502 }; 2503 2504 TRANS_FEAT(TRN1_z, aa64_sve, gen_gvec_ool_arg_zzz, 2505 trn_fns[a->esz], a, 0) 2506 TRANS_FEAT(TRN2_z, aa64_sve, gen_gvec_ool_arg_zzz, 2507 trn_fns[a->esz], a, 1 << a->esz) 2508 2509 TRANS_FEAT(TRN1_q, aa64_sve_f64mm, gen_gvec_ool_arg_zzz, 2510 gen_helper_sve2_trn_q, a, 0) 2511 TRANS_FEAT(TRN2_q, aa64_sve_f64mm, gen_gvec_ool_arg_zzz, 2512 gen_helper_sve2_trn_q, a, 16) 2513 2514 /* 2515 *** SVE Permute Vector - Predicated Group 2516 */ 2517 2518 static gen_helper_gvec_3 * const compact_fns[4] = { 2519 NULL, NULL, gen_helper_sve_compact_s, gen_helper_sve_compact_d 2520 }; 2521 TRANS_FEAT_NONSTREAMING(COMPACT, aa64_sve, gen_gvec_ool_arg_zpz, 2522 compact_fns[a->esz], a, 0) 2523 2524 /* Call the helper that computes the ARM LastActiveElement pseudocode 2525 * function, scaled by the element size. This includes the not found 2526 * indication; e.g. not found for esz=3 is -8. 2527 */ 2528 static void find_last_active(DisasContext *s, TCGv_i32 ret, int esz, int pg) 2529 { 2530 /* Predicate sizes may be smaller and cannot use simd_desc. We cannot 2531 * round up, as we do elsewhere, because we need the exact size. 2532 */ 2533 TCGv_ptr t_p = tcg_temp_new_ptr(); 2534 unsigned desc = 0; 2535 2536 desc = FIELD_DP32(desc, PREDDESC, OPRSZ, pred_full_reg_size(s)); 2537 desc = FIELD_DP32(desc, PREDDESC, ESZ, esz); 2538 2539 tcg_gen_addi_ptr(t_p, cpu_env, pred_full_reg_offset(s, pg)); 2540 2541 gen_helper_sve_last_active_element(ret, t_p, tcg_constant_i32(desc)); 2542 } 2543 2544 /* Increment LAST to the offset of the next element in the vector, 2545 * wrapping around to 0. 2546 */ 2547 static void incr_last_active(DisasContext *s, TCGv_i32 last, int esz) 2548 { 2549 unsigned vsz = vec_full_reg_size(s); 2550 2551 tcg_gen_addi_i32(last, last, 1 << esz); 2552 if (is_power_of_2(vsz)) { 2553 tcg_gen_andi_i32(last, last, vsz - 1); 2554 } else { 2555 TCGv_i32 max = tcg_constant_i32(vsz); 2556 TCGv_i32 zero = tcg_constant_i32(0); 2557 tcg_gen_movcond_i32(TCG_COND_GEU, last, last, max, zero, last); 2558 } 2559 } 2560 2561 /* If LAST < 0, set LAST to the offset of the last element in the vector. */ 2562 static void wrap_last_active(DisasContext *s, TCGv_i32 last, int esz) 2563 { 2564 unsigned vsz = vec_full_reg_size(s); 2565 2566 if (is_power_of_2(vsz)) { 2567 tcg_gen_andi_i32(last, last, vsz - 1); 2568 } else { 2569 TCGv_i32 max = tcg_constant_i32(vsz - (1 << esz)); 2570 TCGv_i32 zero = tcg_constant_i32(0); 2571 tcg_gen_movcond_i32(TCG_COND_LT, last, last, zero, max, last); 2572 } 2573 } 2574 2575 /* Load an unsigned element of ESZ from BASE+OFS. */ 2576 static TCGv_i64 load_esz(TCGv_ptr base, int ofs, int esz) 2577 { 2578 TCGv_i64 r = tcg_temp_new_i64(); 2579 2580 switch (esz) { 2581 case 0: 2582 tcg_gen_ld8u_i64(r, base, ofs); 2583 break; 2584 case 1: 2585 tcg_gen_ld16u_i64(r, base, ofs); 2586 break; 2587 case 2: 2588 tcg_gen_ld32u_i64(r, base, ofs); 2589 break; 2590 case 3: 2591 tcg_gen_ld_i64(r, base, ofs); 2592 break; 2593 default: 2594 g_assert_not_reached(); 2595 } 2596 return r; 2597 } 2598 2599 /* Load an unsigned element of ESZ from RM[LAST]. */ 2600 static TCGv_i64 load_last_active(DisasContext *s, TCGv_i32 last, 2601 int rm, int esz) 2602 { 2603 TCGv_ptr p = tcg_temp_new_ptr(); 2604 2605 /* Convert offset into vector into offset into ENV. 2606 * The final adjustment for the vector register base 2607 * is added via constant offset to the load. 2608 */ 2609 #if HOST_BIG_ENDIAN 2610 /* Adjust for element ordering. See vec_reg_offset. */ 2611 if (esz < 3) { 2612 tcg_gen_xori_i32(last, last, 8 - (1 << esz)); 2613 } 2614 #endif 2615 tcg_gen_ext_i32_ptr(p, last); 2616 tcg_gen_add_ptr(p, p, cpu_env); 2617 2618 return load_esz(p, vec_full_reg_offset(s, rm), esz); 2619 } 2620 2621 /* Compute CLAST for a Zreg. */ 2622 static bool do_clast_vector(DisasContext *s, arg_rprr_esz *a, bool before) 2623 { 2624 TCGv_i32 last; 2625 TCGLabel *over; 2626 TCGv_i64 ele; 2627 unsigned vsz, esz = a->esz; 2628 2629 if (!sve_access_check(s)) { 2630 return true; 2631 } 2632 2633 last = tcg_temp_new_i32(); 2634 over = gen_new_label(); 2635 2636 find_last_active(s, last, esz, a->pg); 2637 2638 /* There is of course no movcond for a 2048-bit vector, 2639 * so we must branch over the actual store. 2640 */ 2641 tcg_gen_brcondi_i32(TCG_COND_LT, last, 0, over); 2642 2643 if (!before) { 2644 incr_last_active(s, last, esz); 2645 } 2646 2647 ele = load_last_active(s, last, a->rm, esz); 2648 2649 vsz = vec_full_reg_size(s); 2650 tcg_gen_gvec_dup_i64(esz, vec_full_reg_offset(s, a->rd), vsz, vsz, ele); 2651 2652 /* If this insn used MOVPRFX, we may need a second move. */ 2653 if (a->rd != a->rn) { 2654 TCGLabel *done = gen_new_label(); 2655 tcg_gen_br(done); 2656 2657 gen_set_label(over); 2658 do_mov_z(s, a->rd, a->rn); 2659 2660 gen_set_label(done); 2661 } else { 2662 gen_set_label(over); 2663 } 2664 return true; 2665 } 2666 2667 TRANS_FEAT(CLASTA_z, aa64_sve, do_clast_vector, a, false) 2668 TRANS_FEAT(CLASTB_z, aa64_sve, do_clast_vector, a, true) 2669 2670 /* Compute CLAST for a scalar. */ 2671 static void do_clast_scalar(DisasContext *s, int esz, int pg, int rm, 2672 bool before, TCGv_i64 reg_val) 2673 { 2674 TCGv_i32 last = tcg_temp_new_i32(); 2675 TCGv_i64 ele, cmp; 2676 2677 find_last_active(s, last, esz, pg); 2678 2679 /* Extend the original value of last prior to incrementing. */ 2680 cmp = tcg_temp_new_i64(); 2681 tcg_gen_ext_i32_i64(cmp, last); 2682 2683 if (!before) { 2684 incr_last_active(s, last, esz); 2685 } 2686 2687 /* The conceit here is that while last < 0 indicates not found, after 2688 * adjusting for cpu_env->vfp.zregs[rm], it is still a valid address 2689 * from which we can load garbage. We then discard the garbage with 2690 * a conditional move. 2691 */ 2692 ele = load_last_active(s, last, rm, esz); 2693 2694 tcg_gen_movcond_i64(TCG_COND_GE, reg_val, cmp, tcg_constant_i64(0), 2695 ele, reg_val); 2696 } 2697 2698 /* Compute CLAST for a Vreg. */ 2699 static bool do_clast_fp(DisasContext *s, arg_rpr_esz *a, bool before) 2700 { 2701 if (sve_access_check(s)) { 2702 int esz = a->esz; 2703 int ofs = vec_reg_offset(s, a->rd, 0, esz); 2704 TCGv_i64 reg = load_esz(cpu_env, ofs, esz); 2705 2706 do_clast_scalar(s, esz, a->pg, a->rn, before, reg); 2707 write_fp_dreg(s, a->rd, reg); 2708 } 2709 return true; 2710 } 2711 2712 TRANS_FEAT(CLASTA_v, aa64_sve, do_clast_fp, a, false) 2713 TRANS_FEAT(CLASTB_v, aa64_sve, do_clast_fp, a, true) 2714 2715 /* Compute CLAST for a Xreg. */ 2716 static bool do_clast_general(DisasContext *s, arg_rpr_esz *a, bool before) 2717 { 2718 TCGv_i64 reg; 2719 2720 if (!sve_access_check(s)) { 2721 return true; 2722 } 2723 2724 reg = cpu_reg(s, a->rd); 2725 switch (a->esz) { 2726 case 0: 2727 tcg_gen_ext8u_i64(reg, reg); 2728 break; 2729 case 1: 2730 tcg_gen_ext16u_i64(reg, reg); 2731 break; 2732 case 2: 2733 tcg_gen_ext32u_i64(reg, reg); 2734 break; 2735 case 3: 2736 break; 2737 default: 2738 g_assert_not_reached(); 2739 } 2740 2741 do_clast_scalar(s, a->esz, a->pg, a->rn, before, reg); 2742 return true; 2743 } 2744 2745 TRANS_FEAT(CLASTA_r, aa64_sve, do_clast_general, a, false) 2746 TRANS_FEAT(CLASTB_r, aa64_sve, do_clast_general, a, true) 2747 2748 /* Compute LAST for a scalar. */ 2749 static TCGv_i64 do_last_scalar(DisasContext *s, int esz, 2750 int pg, int rm, bool before) 2751 { 2752 TCGv_i32 last = tcg_temp_new_i32(); 2753 2754 find_last_active(s, last, esz, pg); 2755 if (before) { 2756 wrap_last_active(s, last, esz); 2757 } else { 2758 incr_last_active(s, last, esz); 2759 } 2760 2761 return load_last_active(s, last, rm, esz); 2762 } 2763 2764 /* Compute LAST for a Vreg. */ 2765 static bool do_last_fp(DisasContext *s, arg_rpr_esz *a, bool before) 2766 { 2767 if (sve_access_check(s)) { 2768 TCGv_i64 val = do_last_scalar(s, a->esz, a->pg, a->rn, before); 2769 write_fp_dreg(s, a->rd, val); 2770 } 2771 return true; 2772 } 2773 2774 TRANS_FEAT(LASTA_v, aa64_sve, do_last_fp, a, false) 2775 TRANS_FEAT(LASTB_v, aa64_sve, do_last_fp, a, true) 2776 2777 /* Compute LAST for a Xreg. */ 2778 static bool do_last_general(DisasContext *s, arg_rpr_esz *a, bool before) 2779 { 2780 if (sve_access_check(s)) { 2781 TCGv_i64 val = do_last_scalar(s, a->esz, a->pg, a->rn, before); 2782 tcg_gen_mov_i64(cpu_reg(s, a->rd), val); 2783 } 2784 return true; 2785 } 2786 2787 TRANS_FEAT(LASTA_r, aa64_sve, do_last_general, a, false) 2788 TRANS_FEAT(LASTB_r, aa64_sve, do_last_general, a, true) 2789 2790 static bool trans_CPY_m_r(DisasContext *s, arg_rpr_esz *a) 2791 { 2792 if (!dc_isar_feature(aa64_sve, s)) { 2793 return false; 2794 } 2795 if (sve_access_check(s)) { 2796 do_cpy_m(s, a->esz, a->rd, a->rd, a->pg, cpu_reg_sp(s, a->rn)); 2797 } 2798 return true; 2799 } 2800 2801 static bool trans_CPY_m_v(DisasContext *s, arg_rpr_esz *a) 2802 { 2803 if (!dc_isar_feature(aa64_sve, s)) { 2804 return false; 2805 } 2806 if (sve_access_check(s)) { 2807 int ofs = vec_reg_offset(s, a->rn, 0, a->esz); 2808 TCGv_i64 t = load_esz(cpu_env, ofs, a->esz); 2809 do_cpy_m(s, a->esz, a->rd, a->rd, a->pg, t); 2810 } 2811 return true; 2812 } 2813 2814 static gen_helper_gvec_3 * const revb_fns[4] = { 2815 NULL, gen_helper_sve_revb_h, 2816 gen_helper_sve_revb_s, gen_helper_sve_revb_d, 2817 }; 2818 TRANS_FEAT(REVB, aa64_sve, gen_gvec_ool_arg_zpz, revb_fns[a->esz], a, 0) 2819 2820 static gen_helper_gvec_3 * const revh_fns[4] = { 2821 NULL, NULL, gen_helper_sve_revh_s, gen_helper_sve_revh_d, 2822 }; 2823 TRANS_FEAT(REVH, aa64_sve, gen_gvec_ool_arg_zpz, revh_fns[a->esz], a, 0) 2824 2825 TRANS_FEAT(REVW, aa64_sve, gen_gvec_ool_arg_zpz, 2826 a->esz == 3 ? gen_helper_sve_revw_d : NULL, a, 0) 2827 2828 TRANS_FEAT(REVD, aa64_sme, gen_gvec_ool_arg_zpz, gen_helper_sme_revd_q, a, 0) 2829 2830 TRANS_FEAT(SPLICE, aa64_sve, gen_gvec_ool_arg_zpzz, 2831 gen_helper_sve_splice, a, a->esz) 2832 2833 TRANS_FEAT(SPLICE_sve2, aa64_sve2, gen_gvec_ool_zzzp, gen_helper_sve_splice, 2834 a->rd, a->rn, (a->rn + 1) % 32, a->pg, a->esz) 2835 2836 /* 2837 *** SVE Integer Compare - Vectors Group 2838 */ 2839 2840 static bool do_ppzz_flags(DisasContext *s, arg_rprr_esz *a, 2841 gen_helper_gvec_flags_4 *gen_fn) 2842 { 2843 TCGv_ptr pd, zn, zm, pg; 2844 unsigned vsz; 2845 TCGv_i32 t; 2846 2847 if (gen_fn == NULL) { 2848 return false; 2849 } 2850 if (!sve_access_check(s)) { 2851 return true; 2852 } 2853 2854 vsz = vec_full_reg_size(s); 2855 t = tcg_temp_new_i32(); 2856 pd = tcg_temp_new_ptr(); 2857 zn = tcg_temp_new_ptr(); 2858 zm = tcg_temp_new_ptr(); 2859 pg = tcg_temp_new_ptr(); 2860 2861 tcg_gen_addi_ptr(pd, cpu_env, pred_full_reg_offset(s, a->rd)); 2862 tcg_gen_addi_ptr(zn, cpu_env, vec_full_reg_offset(s, a->rn)); 2863 tcg_gen_addi_ptr(zm, cpu_env, vec_full_reg_offset(s, a->rm)); 2864 tcg_gen_addi_ptr(pg, cpu_env, pred_full_reg_offset(s, a->pg)); 2865 2866 gen_fn(t, pd, zn, zm, pg, tcg_constant_i32(simd_desc(vsz, vsz, 0))); 2867 2868 do_pred_flags(t); 2869 return true; 2870 } 2871 2872 #define DO_PPZZ(NAME, name) \ 2873 static gen_helper_gvec_flags_4 * const name##_ppzz_fns[4] = { \ 2874 gen_helper_sve_##name##_ppzz_b, gen_helper_sve_##name##_ppzz_h, \ 2875 gen_helper_sve_##name##_ppzz_s, gen_helper_sve_##name##_ppzz_d, \ 2876 }; \ 2877 TRANS_FEAT(NAME##_ppzz, aa64_sve, do_ppzz_flags, \ 2878 a, name##_ppzz_fns[a->esz]) 2879 2880 DO_PPZZ(CMPEQ, cmpeq) 2881 DO_PPZZ(CMPNE, cmpne) 2882 DO_PPZZ(CMPGT, cmpgt) 2883 DO_PPZZ(CMPGE, cmpge) 2884 DO_PPZZ(CMPHI, cmphi) 2885 DO_PPZZ(CMPHS, cmphs) 2886 2887 #undef DO_PPZZ 2888 2889 #define DO_PPZW(NAME, name) \ 2890 static gen_helper_gvec_flags_4 * const name##_ppzw_fns[4] = { \ 2891 gen_helper_sve_##name##_ppzw_b, gen_helper_sve_##name##_ppzw_h, \ 2892 gen_helper_sve_##name##_ppzw_s, NULL \ 2893 }; \ 2894 TRANS_FEAT(NAME##_ppzw, aa64_sve, do_ppzz_flags, \ 2895 a, name##_ppzw_fns[a->esz]) 2896 2897 DO_PPZW(CMPEQ, cmpeq) 2898 DO_PPZW(CMPNE, cmpne) 2899 DO_PPZW(CMPGT, cmpgt) 2900 DO_PPZW(CMPGE, cmpge) 2901 DO_PPZW(CMPHI, cmphi) 2902 DO_PPZW(CMPHS, cmphs) 2903 DO_PPZW(CMPLT, cmplt) 2904 DO_PPZW(CMPLE, cmple) 2905 DO_PPZW(CMPLO, cmplo) 2906 DO_PPZW(CMPLS, cmpls) 2907 2908 #undef DO_PPZW 2909 2910 /* 2911 *** SVE Integer Compare - Immediate Groups 2912 */ 2913 2914 static bool do_ppzi_flags(DisasContext *s, arg_rpri_esz *a, 2915 gen_helper_gvec_flags_3 *gen_fn) 2916 { 2917 TCGv_ptr pd, zn, pg; 2918 unsigned vsz; 2919 TCGv_i32 t; 2920 2921 if (gen_fn == NULL) { 2922 return false; 2923 } 2924 if (!sve_access_check(s)) { 2925 return true; 2926 } 2927 2928 vsz = vec_full_reg_size(s); 2929 t = tcg_temp_new_i32(); 2930 pd = tcg_temp_new_ptr(); 2931 zn = tcg_temp_new_ptr(); 2932 pg = tcg_temp_new_ptr(); 2933 2934 tcg_gen_addi_ptr(pd, cpu_env, pred_full_reg_offset(s, a->rd)); 2935 tcg_gen_addi_ptr(zn, cpu_env, vec_full_reg_offset(s, a->rn)); 2936 tcg_gen_addi_ptr(pg, cpu_env, pred_full_reg_offset(s, a->pg)); 2937 2938 gen_fn(t, pd, zn, pg, tcg_constant_i32(simd_desc(vsz, vsz, a->imm))); 2939 2940 do_pred_flags(t); 2941 return true; 2942 } 2943 2944 #define DO_PPZI(NAME, name) \ 2945 static gen_helper_gvec_flags_3 * const name##_ppzi_fns[4] = { \ 2946 gen_helper_sve_##name##_ppzi_b, gen_helper_sve_##name##_ppzi_h, \ 2947 gen_helper_sve_##name##_ppzi_s, gen_helper_sve_##name##_ppzi_d, \ 2948 }; \ 2949 TRANS_FEAT(NAME##_ppzi, aa64_sve, do_ppzi_flags, a, \ 2950 name##_ppzi_fns[a->esz]) 2951 2952 DO_PPZI(CMPEQ, cmpeq) 2953 DO_PPZI(CMPNE, cmpne) 2954 DO_PPZI(CMPGT, cmpgt) 2955 DO_PPZI(CMPGE, cmpge) 2956 DO_PPZI(CMPHI, cmphi) 2957 DO_PPZI(CMPHS, cmphs) 2958 DO_PPZI(CMPLT, cmplt) 2959 DO_PPZI(CMPLE, cmple) 2960 DO_PPZI(CMPLO, cmplo) 2961 DO_PPZI(CMPLS, cmpls) 2962 2963 #undef DO_PPZI 2964 2965 /* 2966 *** SVE Partition Break Group 2967 */ 2968 2969 static bool do_brk3(DisasContext *s, arg_rprr_s *a, 2970 gen_helper_gvec_4 *fn, gen_helper_gvec_flags_4 *fn_s) 2971 { 2972 if (!sve_access_check(s)) { 2973 return true; 2974 } 2975 2976 unsigned vsz = pred_full_reg_size(s); 2977 2978 /* Predicate sizes may be smaller and cannot use simd_desc. */ 2979 TCGv_ptr d = tcg_temp_new_ptr(); 2980 TCGv_ptr n = tcg_temp_new_ptr(); 2981 TCGv_ptr m = tcg_temp_new_ptr(); 2982 TCGv_ptr g = tcg_temp_new_ptr(); 2983 TCGv_i32 desc = tcg_constant_i32(FIELD_DP32(0, PREDDESC, OPRSZ, vsz)); 2984 2985 tcg_gen_addi_ptr(d, cpu_env, pred_full_reg_offset(s, a->rd)); 2986 tcg_gen_addi_ptr(n, cpu_env, pred_full_reg_offset(s, a->rn)); 2987 tcg_gen_addi_ptr(m, cpu_env, pred_full_reg_offset(s, a->rm)); 2988 tcg_gen_addi_ptr(g, cpu_env, pred_full_reg_offset(s, a->pg)); 2989 2990 if (a->s) { 2991 TCGv_i32 t = tcg_temp_new_i32(); 2992 fn_s(t, d, n, m, g, desc); 2993 do_pred_flags(t); 2994 } else { 2995 fn(d, n, m, g, desc); 2996 } 2997 return true; 2998 } 2999 3000 static bool do_brk2(DisasContext *s, arg_rpr_s *a, 3001 gen_helper_gvec_3 *fn, gen_helper_gvec_flags_3 *fn_s) 3002 { 3003 if (!sve_access_check(s)) { 3004 return true; 3005 } 3006 3007 unsigned vsz = pred_full_reg_size(s); 3008 3009 /* Predicate sizes may be smaller and cannot use simd_desc. */ 3010 TCGv_ptr d = tcg_temp_new_ptr(); 3011 TCGv_ptr n = tcg_temp_new_ptr(); 3012 TCGv_ptr g = tcg_temp_new_ptr(); 3013 TCGv_i32 desc = tcg_constant_i32(FIELD_DP32(0, PREDDESC, OPRSZ, vsz)); 3014 3015 tcg_gen_addi_ptr(d, cpu_env, pred_full_reg_offset(s, a->rd)); 3016 tcg_gen_addi_ptr(n, cpu_env, pred_full_reg_offset(s, a->rn)); 3017 tcg_gen_addi_ptr(g, cpu_env, pred_full_reg_offset(s, a->pg)); 3018 3019 if (a->s) { 3020 TCGv_i32 t = tcg_temp_new_i32(); 3021 fn_s(t, d, n, g, desc); 3022 do_pred_flags(t); 3023 } else { 3024 fn(d, n, g, desc); 3025 } 3026 return true; 3027 } 3028 3029 TRANS_FEAT(BRKPA, aa64_sve, do_brk3, a, 3030 gen_helper_sve_brkpa, gen_helper_sve_brkpas) 3031 TRANS_FEAT(BRKPB, aa64_sve, do_brk3, a, 3032 gen_helper_sve_brkpb, gen_helper_sve_brkpbs) 3033 3034 TRANS_FEAT(BRKA_m, aa64_sve, do_brk2, a, 3035 gen_helper_sve_brka_m, gen_helper_sve_brkas_m) 3036 TRANS_FEAT(BRKB_m, aa64_sve, do_brk2, a, 3037 gen_helper_sve_brkb_m, gen_helper_sve_brkbs_m) 3038 3039 TRANS_FEAT(BRKA_z, aa64_sve, do_brk2, a, 3040 gen_helper_sve_brka_z, gen_helper_sve_brkas_z) 3041 TRANS_FEAT(BRKB_z, aa64_sve, do_brk2, a, 3042 gen_helper_sve_brkb_z, gen_helper_sve_brkbs_z) 3043 3044 TRANS_FEAT(BRKN, aa64_sve, do_brk2, a, 3045 gen_helper_sve_brkn, gen_helper_sve_brkns) 3046 3047 /* 3048 *** SVE Predicate Count Group 3049 */ 3050 3051 static void do_cntp(DisasContext *s, TCGv_i64 val, int esz, int pn, int pg) 3052 { 3053 unsigned psz = pred_full_reg_size(s); 3054 3055 if (psz <= 8) { 3056 uint64_t psz_mask; 3057 3058 tcg_gen_ld_i64(val, cpu_env, pred_full_reg_offset(s, pn)); 3059 if (pn != pg) { 3060 TCGv_i64 g = tcg_temp_new_i64(); 3061 tcg_gen_ld_i64(g, cpu_env, pred_full_reg_offset(s, pg)); 3062 tcg_gen_and_i64(val, val, g); 3063 } 3064 3065 /* Reduce the pred_esz_masks value simply to reduce the 3066 * size of the code generated here. 3067 */ 3068 psz_mask = MAKE_64BIT_MASK(0, psz * 8); 3069 tcg_gen_andi_i64(val, val, pred_esz_masks[esz] & psz_mask); 3070 3071 tcg_gen_ctpop_i64(val, val); 3072 } else { 3073 TCGv_ptr t_pn = tcg_temp_new_ptr(); 3074 TCGv_ptr t_pg = tcg_temp_new_ptr(); 3075 unsigned desc = 0; 3076 3077 desc = FIELD_DP32(desc, PREDDESC, OPRSZ, psz); 3078 desc = FIELD_DP32(desc, PREDDESC, ESZ, esz); 3079 3080 tcg_gen_addi_ptr(t_pn, cpu_env, pred_full_reg_offset(s, pn)); 3081 tcg_gen_addi_ptr(t_pg, cpu_env, pred_full_reg_offset(s, pg)); 3082 3083 gen_helper_sve_cntp(val, t_pn, t_pg, tcg_constant_i32(desc)); 3084 } 3085 } 3086 3087 static bool trans_CNTP(DisasContext *s, arg_CNTP *a) 3088 { 3089 if (!dc_isar_feature(aa64_sve, s)) { 3090 return false; 3091 } 3092 if (sve_access_check(s)) { 3093 do_cntp(s, cpu_reg(s, a->rd), a->esz, a->rn, a->pg); 3094 } 3095 return true; 3096 } 3097 3098 static bool trans_INCDECP_r(DisasContext *s, arg_incdec_pred *a) 3099 { 3100 if (!dc_isar_feature(aa64_sve, s)) { 3101 return false; 3102 } 3103 if (sve_access_check(s)) { 3104 TCGv_i64 reg = cpu_reg(s, a->rd); 3105 TCGv_i64 val = tcg_temp_new_i64(); 3106 3107 do_cntp(s, val, a->esz, a->pg, a->pg); 3108 if (a->d) { 3109 tcg_gen_sub_i64(reg, reg, val); 3110 } else { 3111 tcg_gen_add_i64(reg, reg, val); 3112 } 3113 } 3114 return true; 3115 } 3116 3117 static bool trans_INCDECP_z(DisasContext *s, arg_incdec2_pred *a) 3118 { 3119 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) { 3120 return false; 3121 } 3122 if (sve_access_check(s)) { 3123 unsigned vsz = vec_full_reg_size(s); 3124 TCGv_i64 val = tcg_temp_new_i64(); 3125 GVecGen2sFn *gvec_fn = a->d ? tcg_gen_gvec_subs : tcg_gen_gvec_adds; 3126 3127 do_cntp(s, val, a->esz, a->pg, a->pg); 3128 gvec_fn(a->esz, vec_full_reg_offset(s, a->rd), 3129 vec_full_reg_offset(s, a->rn), val, vsz, vsz); 3130 } 3131 return true; 3132 } 3133 3134 static bool trans_SINCDECP_r_32(DisasContext *s, arg_incdec_pred *a) 3135 { 3136 if (!dc_isar_feature(aa64_sve, s)) { 3137 return false; 3138 } 3139 if (sve_access_check(s)) { 3140 TCGv_i64 reg = cpu_reg(s, a->rd); 3141 TCGv_i64 val = tcg_temp_new_i64(); 3142 3143 do_cntp(s, val, a->esz, a->pg, a->pg); 3144 do_sat_addsub_32(reg, val, a->u, a->d); 3145 } 3146 return true; 3147 } 3148 3149 static bool trans_SINCDECP_r_64(DisasContext *s, arg_incdec_pred *a) 3150 { 3151 if (!dc_isar_feature(aa64_sve, s)) { 3152 return false; 3153 } 3154 if (sve_access_check(s)) { 3155 TCGv_i64 reg = cpu_reg(s, a->rd); 3156 TCGv_i64 val = tcg_temp_new_i64(); 3157 3158 do_cntp(s, val, a->esz, a->pg, a->pg); 3159 do_sat_addsub_64(reg, val, a->u, a->d); 3160 } 3161 return true; 3162 } 3163 3164 static bool trans_SINCDECP_z(DisasContext *s, arg_incdec2_pred *a) 3165 { 3166 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) { 3167 return false; 3168 } 3169 if (sve_access_check(s)) { 3170 TCGv_i64 val = tcg_temp_new_i64(); 3171 do_cntp(s, val, a->esz, a->pg, a->pg); 3172 do_sat_addsub_vec(s, a->esz, a->rd, a->rn, val, a->u, a->d); 3173 } 3174 return true; 3175 } 3176 3177 /* 3178 *** SVE Integer Compare Scalars Group 3179 */ 3180 3181 static bool trans_CTERM(DisasContext *s, arg_CTERM *a) 3182 { 3183 if (!dc_isar_feature(aa64_sve, s)) { 3184 return false; 3185 } 3186 if (!sve_access_check(s)) { 3187 return true; 3188 } 3189 3190 TCGCond cond = (a->ne ? TCG_COND_NE : TCG_COND_EQ); 3191 TCGv_i64 rn = read_cpu_reg(s, a->rn, a->sf); 3192 TCGv_i64 rm = read_cpu_reg(s, a->rm, a->sf); 3193 TCGv_i64 cmp = tcg_temp_new_i64(); 3194 3195 tcg_gen_setcond_i64(cond, cmp, rn, rm); 3196 tcg_gen_extrl_i64_i32(cpu_NF, cmp); 3197 3198 /* VF = !NF & !CF. */ 3199 tcg_gen_xori_i32(cpu_VF, cpu_NF, 1); 3200 tcg_gen_andc_i32(cpu_VF, cpu_VF, cpu_CF); 3201 3202 /* Both NF and VF actually look at bit 31. */ 3203 tcg_gen_neg_i32(cpu_NF, cpu_NF); 3204 tcg_gen_neg_i32(cpu_VF, cpu_VF); 3205 return true; 3206 } 3207 3208 static bool trans_WHILE(DisasContext *s, arg_WHILE *a) 3209 { 3210 TCGv_i64 op0, op1, t0, t1, tmax; 3211 TCGv_i32 t2; 3212 TCGv_ptr ptr; 3213 unsigned vsz = vec_full_reg_size(s); 3214 unsigned desc = 0; 3215 TCGCond cond; 3216 uint64_t maxval; 3217 /* Note that GE/HS has a->eq == 0 and GT/HI has a->eq == 1. */ 3218 bool eq = a->eq == a->lt; 3219 3220 /* The greater-than conditions are all SVE2. */ 3221 if (a->lt 3222 ? !dc_isar_feature(aa64_sve, s) 3223 : !dc_isar_feature(aa64_sve2, s)) { 3224 return false; 3225 } 3226 if (!sve_access_check(s)) { 3227 return true; 3228 } 3229 3230 op0 = read_cpu_reg(s, a->rn, 1); 3231 op1 = read_cpu_reg(s, a->rm, 1); 3232 3233 if (!a->sf) { 3234 if (a->u) { 3235 tcg_gen_ext32u_i64(op0, op0); 3236 tcg_gen_ext32u_i64(op1, op1); 3237 } else { 3238 tcg_gen_ext32s_i64(op0, op0); 3239 tcg_gen_ext32s_i64(op1, op1); 3240 } 3241 } 3242 3243 /* For the helper, compress the different conditions into a computation 3244 * of how many iterations for which the condition is true. 3245 */ 3246 t0 = tcg_temp_new_i64(); 3247 t1 = tcg_temp_new_i64(); 3248 3249 if (a->lt) { 3250 tcg_gen_sub_i64(t0, op1, op0); 3251 if (a->u) { 3252 maxval = a->sf ? UINT64_MAX : UINT32_MAX; 3253 cond = eq ? TCG_COND_LEU : TCG_COND_LTU; 3254 } else { 3255 maxval = a->sf ? INT64_MAX : INT32_MAX; 3256 cond = eq ? TCG_COND_LE : TCG_COND_LT; 3257 } 3258 } else { 3259 tcg_gen_sub_i64(t0, op0, op1); 3260 if (a->u) { 3261 maxval = 0; 3262 cond = eq ? TCG_COND_GEU : TCG_COND_GTU; 3263 } else { 3264 maxval = a->sf ? INT64_MIN : INT32_MIN; 3265 cond = eq ? TCG_COND_GE : TCG_COND_GT; 3266 } 3267 } 3268 3269 tmax = tcg_constant_i64(vsz >> a->esz); 3270 if (eq) { 3271 /* Equality means one more iteration. */ 3272 tcg_gen_addi_i64(t0, t0, 1); 3273 3274 /* 3275 * For the less-than while, if op1 is maxval (and the only time 3276 * the addition above could overflow), then we produce an all-true 3277 * predicate by setting the count to the vector length. This is 3278 * because the pseudocode is described as an increment + compare 3279 * loop, and the maximum integer would always compare true. 3280 * Similarly, the greater-than while has the same issue with the 3281 * minimum integer due to the decrement + compare loop. 3282 */ 3283 tcg_gen_movi_i64(t1, maxval); 3284 tcg_gen_movcond_i64(TCG_COND_EQ, t0, op1, t1, tmax, t0); 3285 } 3286 3287 /* Bound to the maximum. */ 3288 tcg_gen_umin_i64(t0, t0, tmax); 3289 3290 /* Set the count to zero if the condition is false. */ 3291 tcg_gen_movi_i64(t1, 0); 3292 tcg_gen_movcond_i64(cond, t0, op0, op1, t0, t1); 3293 3294 /* Since we're bounded, pass as a 32-bit type. */ 3295 t2 = tcg_temp_new_i32(); 3296 tcg_gen_extrl_i64_i32(t2, t0); 3297 3298 /* Scale elements to bits. */ 3299 tcg_gen_shli_i32(t2, t2, a->esz); 3300 3301 desc = FIELD_DP32(desc, PREDDESC, OPRSZ, vsz / 8); 3302 desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz); 3303 3304 ptr = tcg_temp_new_ptr(); 3305 tcg_gen_addi_ptr(ptr, cpu_env, pred_full_reg_offset(s, a->rd)); 3306 3307 if (a->lt) { 3308 gen_helper_sve_whilel(t2, ptr, t2, tcg_constant_i32(desc)); 3309 } else { 3310 gen_helper_sve_whileg(t2, ptr, t2, tcg_constant_i32(desc)); 3311 } 3312 do_pred_flags(t2); 3313 return true; 3314 } 3315 3316 static bool trans_WHILE_ptr(DisasContext *s, arg_WHILE_ptr *a) 3317 { 3318 TCGv_i64 op0, op1, diff, t1, tmax; 3319 TCGv_i32 t2; 3320 TCGv_ptr ptr; 3321 unsigned vsz = vec_full_reg_size(s); 3322 unsigned desc = 0; 3323 3324 if (!dc_isar_feature(aa64_sve2, s)) { 3325 return false; 3326 } 3327 if (!sve_access_check(s)) { 3328 return true; 3329 } 3330 3331 op0 = read_cpu_reg(s, a->rn, 1); 3332 op1 = read_cpu_reg(s, a->rm, 1); 3333 3334 tmax = tcg_constant_i64(vsz); 3335 diff = tcg_temp_new_i64(); 3336 3337 if (a->rw) { 3338 /* WHILERW */ 3339 /* diff = abs(op1 - op0), noting that op0/1 are unsigned. */ 3340 t1 = tcg_temp_new_i64(); 3341 tcg_gen_sub_i64(diff, op0, op1); 3342 tcg_gen_sub_i64(t1, op1, op0); 3343 tcg_gen_movcond_i64(TCG_COND_GEU, diff, op0, op1, diff, t1); 3344 /* Round down to a multiple of ESIZE. */ 3345 tcg_gen_andi_i64(diff, diff, -1 << a->esz); 3346 /* If op1 == op0, diff == 0, and the condition is always true. */ 3347 tcg_gen_movcond_i64(TCG_COND_EQ, diff, op0, op1, tmax, diff); 3348 } else { 3349 /* WHILEWR */ 3350 tcg_gen_sub_i64(diff, op1, op0); 3351 /* Round down to a multiple of ESIZE. */ 3352 tcg_gen_andi_i64(diff, diff, -1 << a->esz); 3353 /* If op0 >= op1, diff <= 0, the condition is always true. */ 3354 tcg_gen_movcond_i64(TCG_COND_GEU, diff, op0, op1, tmax, diff); 3355 } 3356 3357 /* Bound to the maximum. */ 3358 tcg_gen_umin_i64(diff, diff, tmax); 3359 3360 /* Since we're bounded, pass as a 32-bit type. */ 3361 t2 = tcg_temp_new_i32(); 3362 tcg_gen_extrl_i64_i32(t2, diff); 3363 3364 desc = FIELD_DP32(desc, PREDDESC, OPRSZ, vsz / 8); 3365 desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz); 3366 3367 ptr = tcg_temp_new_ptr(); 3368 tcg_gen_addi_ptr(ptr, cpu_env, pred_full_reg_offset(s, a->rd)); 3369 3370 gen_helper_sve_whilel(t2, ptr, t2, tcg_constant_i32(desc)); 3371 do_pred_flags(t2); 3372 return true; 3373 } 3374 3375 /* 3376 *** SVE Integer Wide Immediate - Unpredicated Group 3377 */ 3378 3379 static bool trans_FDUP(DisasContext *s, arg_FDUP *a) 3380 { 3381 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) { 3382 return false; 3383 } 3384 if (sve_access_check(s)) { 3385 unsigned vsz = vec_full_reg_size(s); 3386 int dofs = vec_full_reg_offset(s, a->rd); 3387 uint64_t imm; 3388 3389 /* Decode the VFP immediate. */ 3390 imm = vfp_expand_imm(a->esz, a->imm); 3391 tcg_gen_gvec_dup_imm(a->esz, dofs, vsz, vsz, imm); 3392 } 3393 return true; 3394 } 3395 3396 static bool trans_DUP_i(DisasContext *s, arg_DUP_i *a) 3397 { 3398 if (!dc_isar_feature(aa64_sve, s)) { 3399 return false; 3400 } 3401 if (sve_access_check(s)) { 3402 unsigned vsz = vec_full_reg_size(s); 3403 int dofs = vec_full_reg_offset(s, a->rd); 3404 tcg_gen_gvec_dup_imm(a->esz, dofs, vsz, vsz, a->imm); 3405 } 3406 return true; 3407 } 3408 3409 TRANS_FEAT(ADD_zzi, aa64_sve, gen_gvec_fn_arg_zzi, tcg_gen_gvec_addi, a) 3410 3411 static bool trans_SUB_zzi(DisasContext *s, arg_rri_esz *a) 3412 { 3413 a->imm = -a->imm; 3414 return trans_ADD_zzi(s, a); 3415 } 3416 3417 static bool trans_SUBR_zzi(DisasContext *s, arg_rri_esz *a) 3418 { 3419 static const TCGOpcode vecop_list[] = { INDEX_op_sub_vec, 0 }; 3420 static const GVecGen2s op[4] = { 3421 { .fni8 = tcg_gen_vec_sub8_i64, 3422 .fniv = tcg_gen_sub_vec, 3423 .fno = gen_helper_sve_subri_b, 3424 .opt_opc = vecop_list, 3425 .vece = MO_8, 3426 .scalar_first = true }, 3427 { .fni8 = tcg_gen_vec_sub16_i64, 3428 .fniv = tcg_gen_sub_vec, 3429 .fno = gen_helper_sve_subri_h, 3430 .opt_opc = vecop_list, 3431 .vece = MO_16, 3432 .scalar_first = true }, 3433 { .fni4 = tcg_gen_sub_i32, 3434 .fniv = tcg_gen_sub_vec, 3435 .fno = gen_helper_sve_subri_s, 3436 .opt_opc = vecop_list, 3437 .vece = MO_32, 3438 .scalar_first = true }, 3439 { .fni8 = tcg_gen_sub_i64, 3440 .fniv = tcg_gen_sub_vec, 3441 .fno = gen_helper_sve_subri_d, 3442 .opt_opc = vecop_list, 3443 .prefer_i64 = TCG_TARGET_REG_BITS == 64, 3444 .vece = MO_64, 3445 .scalar_first = true } 3446 }; 3447 3448 if (!dc_isar_feature(aa64_sve, s)) { 3449 return false; 3450 } 3451 if (sve_access_check(s)) { 3452 unsigned vsz = vec_full_reg_size(s); 3453 tcg_gen_gvec_2s(vec_full_reg_offset(s, a->rd), 3454 vec_full_reg_offset(s, a->rn), 3455 vsz, vsz, tcg_constant_i64(a->imm), &op[a->esz]); 3456 } 3457 return true; 3458 } 3459 3460 TRANS_FEAT(MUL_zzi, aa64_sve, gen_gvec_fn_arg_zzi, tcg_gen_gvec_muli, a) 3461 3462 static bool do_zzi_sat(DisasContext *s, arg_rri_esz *a, bool u, bool d) 3463 { 3464 if (sve_access_check(s)) { 3465 do_sat_addsub_vec(s, a->esz, a->rd, a->rn, 3466 tcg_constant_i64(a->imm), u, d); 3467 } 3468 return true; 3469 } 3470 3471 TRANS_FEAT(SQADD_zzi, aa64_sve, do_zzi_sat, a, false, false) 3472 TRANS_FEAT(UQADD_zzi, aa64_sve, do_zzi_sat, a, true, false) 3473 TRANS_FEAT(SQSUB_zzi, aa64_sve, do_zzi_sat, a, false, true) 3474 TRANS_FEAT(UQSUB_zzi, aa64_sve, do_zzi_sat, a, true, true) 3475 3476 static bool do_zzi_ool(DisasContext *s, arg_rri_esz *a, gen_helper_gvec_2i *fn) 3477 { 3478 if (sve_access_check(s)) { 3479 unsigned vsz = vec_full_reg_size(s); 3480 tcg_gen_gvec_2i_ool(vec_full_reg_offset(s, a->rd), 3481 vec_full_reg_offset(s, a->rn), 3482 tcg_constant_i64(a->imm), vsz, vsz, 0, fn); 3483 } 3484 return true; 3485 } 3486 3487 #define DO_ZZI(NAME, name) \ 3488 static gen_helper_gvec_2i * const name##i_fns[4] = { \ 3489 gen_helper_sve_##name##i_b, gen_helper_sve_##name##i_h, \ 3490 gen_helper_sve_##name##i_s, gen_helper_sve_##name##i_d, \ 3491 }; \ 3492 TRANS_FEAT(NAME##_zzi, aa64_sve, do_zzi_ool, a, name##i_fns[a->esz]) 3493 3494 DO_ZZI(SMAX, smax) 3495 DO_ZZI(UMAX, umax) 3496 DO_ZZI(SMIN, smin) 3497 DO_ZZI(UMIN, umin) 3498 3499 #undef DO_ZZI 3500 3501 static gen_helper_gvec_4 * const dot_fns[2][2] = { 3502 { gen_helper_gvec_sdot_b, gen_helper_gvec_sdot_h }, 3503 { gen_helper_gvec_udot_b, gen_helper_gvec_udot_h } 3504 }; 3505 TRANS_FEAT(DOT_zzzz, aa64_sve, gen_gvec_ool_zzzz, 3506 dot_fns[a->u][a->sz], a->rd, a->rn, a->rm, a->ra, 0) 3507 3508 /* 3509 * SVE Multiply - Indexed 3510 */ 3511 3512 TRANS_FEAT(SDOT_zzxw_s, aa64_sve, gen_gvec_ool_arg_zzxz, 3513 gen_helper_gvec_sdot_idx_b, a) 3514 TRANS_FEAT(SDOT_zzxw_d, aa64_sve, gen_gvec_ool_arg_zzxz, 3515 gen_helper_gvec_sdot_idx_h, a) 3516 TRANS_FEAT(UDOT_zzxw_s, aa64_sve, gen_gvec_ool_arg_zzxz, 3517 gen_helper_gvec_udot_idx_b, a) 3518 TRANS_FEAT(UDOT_zzxw_d, aa64_sve, gen_gvec_ool_arg_zzxz, 3519 gen_helper_gvec_udot_idx_h, a) 3520 3521 TRANS_FEAT(SUDOT_zzxw_s, aa64_sve_i8mm, gen_gvec_ool_arg_zzxz, 3522 gen_helper_gvec_sudot_idx_b, a) 3523 TRANS_FEAT(USDOT_zzxw_s, aa64_sve_i8mm, gen_gvec_ool_arg_zzxz, 3524 gen_helper_gvec_usdot_idx_b, a) 3525 3526 #define DO_SVE2_RRX(NAME, FUNC) \ 3527 TRANS_FEAT(NAME, aa64_sve, gen_gvec_ool_zzz, FUNC, \ 3528 a->rd, a->rn, a->rm, a->index) 3529 3530 DO_SVE2_RRX(MUL_zzx_h, gen_helper_gvec_mul_idx_h) 3531 DO_SVE2_RRX(MUL_zzx_s, gen_helper_gvec_mul_idx_s) 3532 DO_SVE2_RRX(MUL_zzx_d, gen_helper_gvec_mul_idx_d) 3533 3534 DO_SVE2_RRX(SQDMULH_zzx_h, gen_helper_sve2_sqdmulh_idx_h) 3535 DO_SVE2_RRX(SQDMULH_zzx_s, gen_helper_sve2_sqdmulh_idx_s) 3536 DO_SVE2_RRX(SQDMULH_zzx_d, gen_helper_sve2_sqdmulh_idx_d) 3537 3538 DO_SVE2_RRX(SQRDMULH_zzx_h, gen_helper_sve2_sqrdmulh_idx_h) 3539 DO_SVE2_RRX(SQRDMULH_zzx_s, gen_helper_sve2_sqrdmulh_idx_s) 3540 DO_SVE2_RRX(SQRDMULH_zzx_d, gen_helper_sve2_sqrdmulh_idx_d) 3541 3542 #undef DO_SVE2_RRX 3543 3544 #define DO_SVE2_RRX_TB(NAME, FUNC, TOP) \ 3545 TRANS_FEAT(NAME, aa64_sve, gen_gvec_ool_zzz, FUNC, \ 3546 a->rd, a->rn, a->rm, (a->index << 1) | TOP) 3547 3548 DO_SVE2_RRX_TB(SQDMULLB_zzx_s, gen_helper_sve2_sqdmull_idx_s, false) 3549 DO_SVE2_RRX_TB(SQDMULLB_zzx_d, gen_helper_sve2_sqdmull_idx_d, false) 3550 DO_SVE2_RRX_TB(SQDMULLT_zzx_s, gen_helper_sve2_sqdmull_idx_s, true) 3551 DO_SVE2_RRX_TB(SQDMULLT_zzx_d, gen_helper_sve2_sqdmull_idx_d, true) 3552 3553 DO_SVE2_RRX_TB(SMULLB_zzx_s, gen_helper_sve2_smull_idx_s, false) 3554 DO_SVE2_RRX_TB(SMULLB_zzx_d, gen_helper_sve2_smull_idx_d, false) 3555 DO_SVE2_RRX_TB(SMULLT_zzx_s, gen_helper_sve2_smull_idx_s, true) 3556 DO_SVE2_RRX_TB(SMULLT_zzx_d, gen_helper_sve2_smull_idx_d, true) 3557 3558 DO_SVE2_RRX_TB(UMULLB_zzx_s, gen_helper_sve2_umull_idx_s, false) 3559 DO_SVE2_RRX_TB(UMULLB_zzx_d, gen_helper_sve2_umull_idx_d, false) 3560 DO_SVE2_RRX_TB(UMULLT_zzx_s, gen_helper_sve2_umull_idx_s, true) 3561 DO_SVE2_RRX_TB(UMULLT_zzx_d, gen_helper_sve2_umull_idx_d, true) 3562 3563 #undef DO_SVE2_RRX_TB 3564 3565 #define DO_SVE2_RRXR(NAME, FUNC) \ 3566 TRANS_FEAT(NAME, aa64_sve2, gen_gvec_ool_arg_zzxz, FUNC, a) 3567 3568 DO_SVE2_RRXR(MLA_zzxz_h, gen_helper_gvec_mla_idx_h) 3569 DO_SVE2_RRXR(MLA_zzxz_s, gen_helper_gvec_mla_idx_s) 3570 DO_SVE2_RRXR(MLA_zzxz_d, gen_helper_gvec_mla_idx_d) 3571 3572 DO_SVE2_RRXR(MLS_zzxz_h, gen_helper_gvec_mls_idx_h) 3573 DO_SVE2_RRXR(MLS_zzxz_s, gen_helper_gvec_mls_idx_s) 3574 DO_SVE2_RRXR(MLS_zzxz_d, gen_helper_gvec_mls_idx_d) 3575 3576 DO_SVE2_RRXR(SQRDMLAH_zzxz_h, gen_helper_sve2_sqrdmlah_idx_h) 3577 DO_SVE2_RRXR(SQRDMLAH_zzxz_s, gen_helper_sve2_sqrdmlah_idx_s) 3578 DO_SVE2_RRXR(SQRDMLAH_zzxz_d, gen_helper_sve2_sqrdmlah_idx_d) 3579 3580 DO_SVE2_RRXR(SQRDMLSH_zzxz_h, gen_helper_sve2_sqrdmlsh_idx_h) 3581 DO_SVE2_RRXR(SQRDMLSH_zzxz_s, gen_helper_sve2_sqrdmlsh_idx_s) 3582 DO_SVE2_RRXR(SQRDMLSH_zzxz_d, gen_helper_sve2_sqrdmlsh_idx_d) 3583 3584 #undef DO_SVE2_RRXR 3585 3586 #define DO_SVE2_RRXR_TB(NAME, FUNC, TOP) \ 3587 TRANS_FEAT(NAME, aa64_sve2, gen_gvec_ool_zzzz, FUNC, \ 3588 a->rd, a->rn, a->rm, a->ra, (a->index << 1) | TOP) 3589 3590 DO_SVE2_RRXR_TB(SQDMLALB_zzxw_s, gen_helper_sve2_sqdmlal_idx_s, false) 3591 DO_SVE2_RRXR_TB(SQDMLALB_zzxw_d, gen_helper_sve2_sqdmlal_idx_d, false) 3592 DO_SVE2_RRXR_TB(SQDMLALT_zzxw_s, gen_helper_sve2_sqdmlal_idx_s, true) 3593 DO_SVE2_RRXR_TB(SQDMLALT_zzxw_d, gen_helper_sve2_sqdmlal_idx_d, true) 3594 3595 DO_SVE2_RRXR_TB(SQDMLSLB_zzxw_s, gen_helper_sve2_sqdmlsl_idx_s, false) 3596 DO_SVE2_RRXR_TB(SQDMLSLB_zzxw_d, gen_helper_sve2_sqdmlsl_idx_d, false) 3597 DO_SVE2_RRXR_TB(SQDMLSLT_zzxw_s, gen_helper_sve2_sqdmlsl_idx_s, true) 3598 DO_SVE2_RRXR_TB(SQDMLSLT_zzxw_d, gen_helper_sve2_sqdmlsl_idx_d, true) 3599 3600 DO_SVE2_RRXR_TB(SMLALB_zzxw_s, gen_helper_sve2_smlal_idx_s, false) 3601 DO_SVE2_RRXR_TB(SMLALB_zzxw_d, gen_helper_sve2_smlal_idx_d, false) 3602 DO_SVE2_RRXR_TB(SMLALT_zzxw_s, gen_helper_sve2_smlal_idx_s, true) 3603 DO_SVE2_RRXR_TB(SMLALT_zzxw_d, gen_helper_sve2_smlal_idx_d, true) 3604 3605 DO_SVE2_RRXR_TB(UMLALB_zzxw_s, gen_helper_sve2_umlal_idx_s, false) 3606 DO_SVE2_RRXR_TB(UMLALB_zzxw_d, gen_helper_sve2_umlal_idx_d, false) 3607 DO_SVE2_RRXR_TB(UMLALT_zzxw_s, gen_helper_sve2_umlal_idx_s, true) 3608 DO_SVE2_RRXR_TB(UMLALT_zzxw_d, gen_helper_sve2_umlal_idx_d, true) 3609 3610 DO_SVE2_RRXR_TB(SMLSLB_zzxw_s, gen_helper_sve2_smlsl_idx_s, false) 3611 DO_SVE2_RRXR_TB(SMLSLB_zzxw_d, gen_helper_sve2_smlsl_idx_d, false) 3612 DO_SVE2_RRXR_TB(SMLSLT_zzxw_s, gen_helper_sve2_smlsl_idx_s, true) 3613 DO_SVE2_RRXR_TB(SMLSLT_zzxw_d, gen_helper_sve2_smlsl_idx_d, true) 3614 3615 DO_SVE2_RRXR_TB(UMLSLB_zzxw_s, gen_helper_sve2_umlsl_idx_s, false) 3616 DO_SVE2_RRXR_TB(UMLSLB_zzxw_d, gen_helper_sve2_umlsl_idx_d, false) 3617 DO_SVE2_RRXR_TB(UMLSLT_zzxw_s, gen_helper_sve2_umlsl_idx_s, true) 3618 DO_SVE2_RRXR_TB(UMLSLT_zzxw_d, gen_helper_sve2_umlsl_idx_d, true) 3619 3620 #undef DO_SVE2_RRXR_TB 3621 3622 #define DO_SVE2_RRXR_ROT(NAME, FUNC) \ 3623 TRANS_FEAT(NAME, aa64_sve2, gen_gvec_ool_zzzz, FUNC, \ 3624 a->rd, a->rn, a->rm, a->ra, (a->index << 2) | a->rot) 3625 3626 DO_SVE2_RRXR_ROT(CMLA_zzxz_h, gen_helper_sve2_cmla_idx_h) 3627 DO_SVE2_RRXR_ROT(CMLA_zzxz_s, gen_helper_sve2_cmla_idx_s) 3628 3629 DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_h, gen_helper_sve2_sqrdcmlah_idx_h) 3630 DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_s, gen_helper_sve2_sqrdcmlah_idx_s) 3631 3632 DO_SVE2_RRXR_ROT(CDOT_zzxw_s, gen_helper_sve2_cdot_idx_s) 3633 DO_SVE2_RRXR_ROT(CDOT_zzxw_d, gen_helper_sve2_cdot_idx_d) 3634 3635 #undef DO_SVE2_RRXR_ROT 3636 3637 /* 3638 *** SVE Floating Point Multiply-Add Indexed Group 3639 */ 3640 3641 static bool do_FMLA_zzxz(DisasContext *s, arg_rrxr_esz *a, bool sub) 3642 { 3643 static gen_helper_gvec_4_ptr * const fns[4] = { 3644 NULL, 3645 gen_helper_gvec_fmla_idx_h, 3646 gen_helper_gvec_fmla_idx_s, 3647 gen_helper_gvec_fmla_idx_d, 3648 }; 3649 return gen_gvec_fpst_zzzz(s, fns[a->esz], a->rd, a->rn, a->rm, a->ra, 3650 (a->index << 1) | sub, 3651 a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR); 3652 } 3653 3654 TRANS_FEAT(FMLA_zzxz, aa64_sve, do_FMLA_zzxz, a, false) 3655 TRANS_FEAT(FMLS_zzxz, aa64_sve, do_FMLA_zzxz, a, true) 3656 3657 /* 3658 *** SVE Floating Point Multiply Indexed Group 3659 */ 3660 3661 static gen_helper_gvec_3_ptr * const fmul_idx_fns[4] = { 3662 NULL, gen_helper_gvec_fmul_idx_h, 3663 gen_helper_gvec_fmul_idx_s, gen_helper_gvec_fmul_idx_d, 3664 }; 3665 TRANS_FEAT(FMUL_zzx, aa64_sve, gen_gvec_fpst_zzz, 3666 fmul_idx_fns[a->esz], a->rd, a->rn, a->rm, a->index, 3667 a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR) 3668 3669 /* 3670 *** SVE Floating Point Fast Reduction Group 3671 */ 3672 3673 typedef void gen_helper_fp_reduce(TCGv_i64, TCGv_ptr, TCGv_ptr, 3674 TCGv_ptr, TCGv_i32); 3675 3676 static bool do_reduce(DisasContext *s, arg_rpr_esz *a, 3677 gen_helper_fp_reduce *fn) 3678 { 3679 unsigned vsz, p2vsz; 3680 TCGv_i32 t_desc; 3681 TCGv_ptr t_zn, t_pg, status; 3682 TCGv_i64 temp; 3683 3684 if (fn == NULL) { 3685 return false; 3686 } 3687 if (!sve_access_check(s)) { 3688 return true; 3689 } 3690 3691 vsz = vec_full_reg_size(s); 3692 p2vsz = pow2ceil(vsz); 3693 t_desc = tcg_constant_i32(simd_desc(vsz, vsz, p2vsz)); 3694 temp = tcg_temp_new_i64(); 3695 t_zn = tcg_temp_new_ptr(); 3696 t_pg = tcg_temp_new_ptr(); 3697 3698 tcg_gen_addi_ptr(t_zn, cpu_env, vec_full_reg_offset(s, a->rn)); 3699 tcg_gen_addi_ptr(t_pg, cpu_env, pred_full_reg_offset(s, a->pg)); 3700 status = fpstatus_ptr(a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR); 3701 3702 fn(temp, t_zn, t_pg, status, t_desc); 3703 3704 write_fp_dreg(s, a->rd, temp); 3705 return true; 3706 } 3707 3708 #define DO_VPZ(NAME, name) \ 3709 static gen_helper_fp_reduce * const name##_fns[4] = { \ 3710 NULL, gen_helper_sve_##name##_h, \ 3711 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \ 3712 }; \ 3713 TRANS_FEAT(NAME, aa64_sve, do_reduce, a, name##_fns[a->esz]) 3714 3715 DO_VPZ(FADDV, faddv) 3716 DO_VPZ(FMINNMV, fminnmv) 3717 DO_VPZ(FMAXNMV, fmaxnmv) 3718 DO_VPZ(FMINV, fminv) 3719 DO_VPZ(FMAXV, fmaxv) 3720 3721 #undef DO_VPZ 3722 3723 /* 3724 *** SVE Floating Point Unary Operations - Unpredicated Group 3725 */ 3726 3727 static gen_helper_gvec_2_ptr * const frecpe_fns[] = { 3728 NULL, gen_helper_gvec_frecpe_h, 3729 gen_helper_gvec_frecpe_s, gen_helper_gvec_frecpe_d, 3730 }; 3731 TRANS_FEAT(FRECPE, aa64_sve, gen_gvec_fpst_arg_zz, frecpe_fns[a->esz], a, 0) 3732 3733 static gen_helper_gvec_2_ptr * const frsqrte_fns[] = { 3734 NULL, gen_helper_gvec_frsqrte_h, 3735 gen_helper_gvec_frsqrte_s, gen_helper_gvec_frsqrte_d, 3736 }; 3737 TRANS_FEAT(FRSQRTE, aa64_sve, gen_gvec_fpst_arg_zz, frsqrte_fns[a->esz], a, 0) 3738 3739 /* 3740 *** SVE Floating Point Compare with Zero Group 3741 */ 3742 3743 static bool do_ppz_fp(DisasContext *s, arg_rpr_esz *a, 3744 gen_helper_gvec_3_ptr *fn) 3745 { 3746 if (fn == NULL) { 3747 return false; 3748 } 3749 if (sve_access_check(s)) { 3750 unsigned vsz = vec_full_reg_size(s); 3751 TCGv_ptr status = 3752 fpstatus_ptr(a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR); 3753 3754 tcg_gen_gvec_3_ptr(pred_full_reg_offset(s, a->rd), 3755 vec_full_reg_offset(s, a->rn), 3756 pred_full_reg_offset(s, a->pg), 3757 status, vsz, vsz, 0, fn); 3758 } 3759 return true; 3760 } 3761 3762 #define DO_PPZ(NAME, name) \ 3763 static gen_helper_gvec_3_ptr * const name##_fns[] = { \ 3764 NULL, gen_helper_sve_##name##_h, \ 3765 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \ 3766 }; \ 3767 TRANS_FEAT(NAME, aa64_sve, do_ppz_fp, a, name##_fns[a->esz]) 3768 3769 DO_PPZ(FCMGE_ppz0, fcmge0) 3770 DO_PPZ(FCMGT_ppz0, fcmgt0) 3771 DO_PPZ(FCMLE_ppz0, fcmle0) 3772 DO_PPZ(FCMLT_ppz0, fcmlt0) 3773 DO_PPZ(FCMEQ_ppz0, fcmeq0) 3774 DO_PPZ(FCMNE_ppz0, fcmne0) 3775 3776 #undef DO_PPZ 3777 3778 /* 3779 *** SVE floating-point trig multiply-add coefficient 3780 */ 3781 3782 static gen_helper_gvec_3_ptr * const ftmad_fns[4] = { 3783 NULL, gen_helper_sve_ftmad_h, 3784 gen_helper_sve_ftmad_s, gen_helper_sve_ftmad_d, 3785 }; 3786 TRANS_FEAT_NONSTREAMING(FTMAD, aa64_sve, gen_gvec_fpst_zzz, 3787 ftmad_fns[a->esz], a->rd, a->rn, a->rm, a->imm, 3788 a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR) 3789 3790 /* 3791 *** SVE Floating Point Accumulating Reduction Group 3792 */ 3793 3794 static bool trans_FADDA(DisasContext *s, arg_rprr_esz *a) 3795 { 3796 typedef void fadda_fn(TCGv_i64, TCGv_i64, TCGv_ptr, 3797 TCGv_ptr, TCGv_ptr, TCGv_i32); 3798 static fadda_fn * const fns[3] = { 3799 gen_helper_sve_fadda_h, 3800 gen_helper_sve_fadda_s, 3801 gen_helper_sve_fadda_d, 3802 }; 3803 unsigned vsz = vec_full_reg_size(s); 3804 TCGv_ptr t_rm, t_pg, t_fpst; 3805 TCGv_i64 t_val; 3806 TCGv_i32 t_desc; 3807 3808 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) { 3809 return false; 3810 } 3811 s->is_nonstreaming = true; 3812 if (!sve_access_check(s)) { 3813 return true; 3814 } 3815 3816 t_val = load_esz(cpu_env, vec_reg_offset(s, a->rn, 0, a->esz), a->esz); 3817 t_rm = tcg_temp_new_ptr(); 3818 t_pg = tcg_temp_new_ptr(); 3819 tcg_gen_addi_ptr(t_rm, cpu_env, vec_full_reg_offset(s, a->rm)); 3820 tcg_gen_addi_ptr(t_pg, cpu_env, pred_full_reg_offset(s, a->pg)); 3821 t_fpst = fpstatus_ptr(a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR); 3822 t_desc = tcg_constant_i32(simd_desc(vsz, vsz, 0)); 3823 3824 fns[a->esz - 1](t_val, t_val, t_rm, t_pg, t_fpst, t_desc); 3825 3826 write_fp_dreg(s, a->rd, t_val); 3827 return true; 3828 } 3829 3830 /* 3831 *** SVE Floating Point Arithmetic - Unpredicated Group 3832 */ 3833 3834 #define DO_FP3(NAME, name) \ 3835 static gen_helper_gvec_3_ptr * const name##_fns[4] = { \ 3836 NULL, gen_helper_gvec_##name##_h, \ 3837 gen_helper_gvec_##name##_s, gen_helper_gvec_##name##_d \ 3838 }; \ 3839 TRANS_FEAT(NAME, aa64_sve, gen_gvec_fpst_arg_zzz, name##_fns[a->esz], a, 0) 3840 3841 DO_FP3(FADD_zzz, fadd) 3842 DO_FP3(FSUB_zzz, fsub) 3843 DO_FP3(FMUL_zzz, fmul) 3844 DO_FP3(FRECPS, recps) 3845 DO_FP3(FRSQRTS, rsqrts) 3846 3847 #undef DO_FP3 3848 3849 static gen_helper_gvec_3_ptr * const ftsmul_fns[4] = { 3850 NULL, gen_helper_gvec_ftsmul_h, 3851 gen_helper_gvec_ftsmul_s, gen_helper_gvec_ftsmul_d 3852 }; 3853 TRANS_FEAT_NONSTREAMING(FTSMUL, aa64_sve, gen_gvec_fpst_arg_zzz, 3854 ftsmul_fns[a->esz], a, 0) 3855 3856 /* 3857 *** SVE Floating Point Arithmetic - Predicated Group 3858 */ 3859 3860 #define DO_ZPZZ_FP(NAME, FEAT, name) \ 3861 static gen_helper_gvec_4_ptr * const name##_zpzz_fns[4] = { \ 3862 NULL, gen_helper_##name##_h, \ 3863 gen_helper_##name##_s, gen_helper_##name##_d \ 3864 }; \ 3865 TRANS_FEAT(NAME, FEAT, gen_gvec_fpst_arg_zpzz, name##_zpzz_fns[a->esz], a) 3866 3867 DO_ZPZZ_FP(FADD_zpzz, aa64_sve, sve_fadd) 3868 DO_ZPZZ_FP(FSUB_zpzz, aa64_sve, sve_fsub) 3869 DO_ZPZZ_FP(FMUL_zpzz, aa64_sve, sve_fmul) 3870 DO_ZPZZ_FP(FMIN_zpzz, aa64_sve, sve_fmin) 3871 DO_ZPZZ_FP(FMAX_zpzz, aa64_sve, sve_fmax) 3872 DO_ZPZZ_FP(FMINNM_zpzz, aa64_sve, sve_fminnum) 3873 DO_ZPZZ_FP(FMAXNM_zpzz, aa64_sve, sve_fmaxnum) 3874 DO_ZPZZ_FP(FABD, aa64_sve, sve_fabd) 3875 DO_ZPZZ_FP(FSCALE, aa64_sve, sve_fscalbn) 3876 DO_ZPZZ_FP(FDIV, aa64_sve, sve_fdiv) 3877 DO_ZPZZ_FP(FMULX, aa64_sve, sve_fmulx) 3878 3879 typedef void gen_helper_sve_fp2scalar(TCGv_ptr, TCGv_ptr, TCGv_ptr, 3880 TCGv_i64, TCGv_ptr, TCGv_i32); 3881 3882 static void do_fp_scalar(DisasContext *s, int zd, int zn, int pg, bool is_fp16, 3883 TCGv_i64 scalar, gen_helper_sve_fp2scalar *fn) 3884 { 3885 unsigned vsz = vec_full_reg_size(s); 3886 TCGv_ptr t_zd, t_zn, t_pg, status; 3887 TCGv_i32 desc; 3888 3889 t_zd = tcg_temp_new_ptr(); 3890 t_zn = tcg_temp_new_ptr(); 3891 t_pg = tcg_temp_new_ptr(); 3892 tcg_gen_addi_ptr(t_zd, cpu_env, vec_full_reg_offset(s, zd)); 3893 tcg_gen_addi_ptr(t_zn, cpu_env, vec_full_reg_offset(s, zn)); 3894 tcg_gen_addi_ptr(t_pg, cpu_env, pred_full_reg_offset(s, pg)); 3895 3896 status = fpstatus_ptr(is_fp16 ? FPST_FPCR_F16 : FPST_FPCR); 3897 desc = tcg_constant_i32(simd_desc(vsz, vsz, 0)); 3898 fn(t_zd, t_zn, t_pg, scalar, status, desc); 3899 } 3900 3901 static bool do_fp_imm(DisasContext *s, arg_rpri_esz *a, uint64_t imm, 3902 gen_helper_sve_fp2scalar *fn) 3903 { 3904 if (fn == NULL) { 3905 return false; 3906 } 3907 if (sve_access_check(s)) { 3908 do_fp_scalar(s, a->rd, a->rn, a->pg, a->esz == MO_16, 3909 tcg_constant_i64(imm), fn); 3910 } 3911 return true; 3912 } 3913 3914 #define DO_FP_IMM(NAME, name, const0, const1) \ 3915 static gen_helper_sve_fp2scalar * const name##_fns[4] = { \ 3916 NULL, gen_helper_sve_##name##_h, \ 3917 gen_helper_sve_##name##_s, \ 3918 gen_helper_sve_##name##_d \ 3919 }; \ 3920 static uint64_t const name##_const[4][2] = { \ 3921 { -1, -1 }, \ 3922 { float16_##const0, float16_##const1 }, \ 3923 { float32_##const0, float32_##const1 }, \ 3924 { float64_##const0, float64_##const1 }, \ 3925 }; \ 3926 TRANS_FEAT(NAME##_zpzi, aa64_sve, do_fp_imm, a, \ 3927 name##_const[a->esz][a->imm], name##_fns[a->esz]) 3928 3929 DO_FP_IMM(FADD, fadds, half, one) 3930 DO_FP_IMM(FSUB, fsubs, half, one) 3931 DO_FP_IMM(FMUL, fmuls, half, two) 3932 DO_FP_IMM(FSUBR, fsubrs, half, one) 3933 DO_FP_IMM(FMAXNM, fmaxnms, zero, one) 3934 DO_FP_IMM(FMINNM, fminnms, zero, one) 3935 DO_FP_IMM(FMAX, fmaxs, zero, one) 3936 DO_FP_IMM(FMIN, fmins, zero, one) 3937 3938 #undef DO_FP_IMM 3939 3940 static bool do_fp_cmp(DisasContext *s, arg_rprr_esz *a, 3941 gen_helper_gvec_4_ptr *fn) 3942 { 3943 if (fn == NULL) { 3944 return false; 3945 } 3946 if (sve_access_check(s)) { 3947 unsigned vsz = vec_full_reg_size(s); 3948 TCGv_ptr status = fpstatus_ptr(a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR); 3949 tcg_gen_gvec_4_ptr(pred_full_reg_offset(s, a->rd), 3950 vec_full_reg_offset(s, a->rn), 3951 vec_full_reg_offset(s, a->rm), 3952 pred_full_reg_offset(s, a->pg), 3953 status, vsz, vsz, 0, fn); 3954 } 3955 return true; 3956 } 3957 3958 #define DO_FPCMP(NAME, name) \ 3959 static gen_helper_gvec_4_ptr * const name##_fns[4] = { \ 3960 NULL, gen_helper_sve_##name##_h, \ 3961 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \ 3962 }; \ 3963 TRANS_FEAT(NAME##_ppzz, aa64_sve, do_fp_cmp, a, name##_fns[a->esz]) 3964 3965 DO_FPCMP(FCMGE, fcmge) 3966 DO_FPCMP(FCMGT, fcmgt) 3967 DO_FPCMP(FCMEQ, fcmeq) 3968 DO_FPCMP(FCMNE, fcmne) 3969 DO_FPCMP(FCMUO, fcmuo) 3970 DO_FPCMP(FACGE, facge) 3971 DO_FPCMP(FACGT, facgt) 3972 3973 #undef DO_FPCMP 3974 3975 static gen_helper_gvec_4_ptr * const fcadd_fns[] = { 3976 NULL, gen_helper_sve_fcadd_h, 3977 gen_helper_sve_fcadd_s, gen_helper_sve_fcadd_d, 3978 }; 3979 TRANS_FEAT(FCADD, aa64_sve, gen_gvec_fpst_zzzp, fcadd_fns[a->esz], 3980 a->rd, a->rn, a->rm, a->pg, a->rot, 3981 a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR) 3982 3983 #define DO_FMLA(NAME, name) \ 3984 static gen_helper_gvec_5_ptr * const name##_fns[4] = { \ 3985 NULL, gen_helper_sve_##name##_h, \ 3986 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \ 3987 }; \ 3988 TRANS_FEAT(NAME, aa64_sve, gen_gvec_fpst_zzzzp, name##_fns[a->esz], \ 3989 a->rd, a->rn, a->rm, a->ra, a->pg, 0, \ 3990 a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR) 3991 3992 DO_FMLA(FMLA_zpzzz, fmla_zpzzz) 3993 DO_FMLA(FMLS_zpzzz, fmls_zpzzz) 3994 DO_FMLA(FNMLA_zpzzz, fnmla_zpzzz) 3995 DO_FMLA(FNMLS_zpzzz, fnmls_zpzzz) 3996 3997 #undef DO_FMLA 3998 3999 static gen_helper_gvec_5_ptr * const fcmla_fns[4] = { 4000 NULL, gen_helper_sve_fcmla_zpzzz_h, 4001 gen_helper_sve_fcmla_zpzzz_s, gen_helper_sve_fcmla_zpzzz_d, 4002 }; 4003 TRANS_FEAT(FCMLA_zpzzz, aa64_sve, gen_gvec_fpst_zzzzp, fcmla_fns[a->esz], 4004 a->rd, a->rn, a->rm, a->ra, a->pg, a->rot, 4005 a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR) 4006 4007 static gen_helper_gvec_4_ptr * const fcmla_idx_fns[4] = { 4008 NULL, gen_helper_gvec_fcmlah_idx, gen_helper_gvec_fcmlas_idx, NULL 4009 }; 4010 TRANS_FEAT(FCMLA_zzxz, aa64_sve, gen_gvec_fpst_zzzz, fcmla_idx_fns[a->esz], 4011 a->rd, a->rn, a->rm, a->ra, a->index * 4 + a->rot, 4012 a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR) 4013 4014 /* 4015 *** SVE Floating Point Unary Operations Predicated Group 4016 */ 4017 4018 TRANS_FEAT(FCVT_sh, aa64_sve, gen_gvec_fpst_arg_zpz, 4019 gen_helper_sve_fcvt_sh, a, 0, FPST_FPCR) 4020 TRANS_FEAT(FCVT_hs, aa64_sve, gen_gvec_fpst_arg_zpz, 4021 gen_helper_sve_fcvt_hs, a, 0, FPST_FPCR) 4022 4023 TRANS_FEAT(BFCVT, aa64_sve_bf16, gen_gvec_fpst_arg_zpz, 4024 gen_helper_sve_bfcvt, a, 0, FPST_FPCR) 4025 4026 TRANS_FEAT(FCVT_dh, aa64_sve, gen_gvec_fpst_arg_zpz, 4027 gen_helper_sve_fcvt_dh, a, 0, FPST_FPCR) 4028 TRANS_FEAT(FCVT_hd, aa64_sve, gen_gvec_fpst_arg_zpz, 4029 gen_helper_sve_fcvt_hd, a, 0, FPST_FPCR) 4030 TRANS_FEAT(FCVT_ds, aa64_sve, gen_gvec_fpst_arg_zpz, 4031 gen_helper_sve_fcvt_ds, a, 0, FPST_FPCR) 4032 TRANS_FEAT(FCVT_sd, aa64_sve, gen_gvec_fpst_arg_zpz, 4033 gen_helper_sve_fcvt_sd, a, 0, FPST_FPCR) 4034 4035 TRANS_FEAT(FCVTZS_hh, aa64_sve, gen_gvec_fpst_arg_zpz, 4036 gen_helper_sve_fcvtzs_hh, a, 0, FPST_FPCR_F16) 4037 TRANS_FEAT(FCVTZU_hh, aa64_sve, gen_gvec_fpst_arg_zpz, 4038 gen_helper_sve_fcvtzu_hh, a, 0, FPST_FPCR_F16) 4039 TRANS_FEAT(FCVTZS_hs, aa64_sve, gen_gvec_fpst_arg_zpz, 4040 gen_helper_sve_fcvtzs_hs, a, 0, FPST_FPCR_F16) 4041 TRANS_FEAT(FCVTZU_hs, aa64_sve, gen_gvec_fpst_arg_zpz, 4042 gen_helper_sve_fcvtzu_hs, a, 0, FPST_FPCR_F16) 4043 TRANS_FEAT(FCVTZS_hd, aa64_sve, gen_gvec_fpst_arg_zpz, 4044 gen_helper_sve_fcvtzs_hd, a, 0, FPST_FPCR_F16) 4045 TRANS_FEAT(FCVTZU_hd, aa64_sve, gen_gvec_fpst_arg_zpz, 4046 gen_helper_sve_fcvtzu_hd, a, 0, FPST_FPCR_F16) 4047 4048 TRANS_FEAT(FCVTZS_ss, aa64_sve, gen_gvec_fpst_arg_zpz, 4049 gen_helper_sve_fcvtzs_ss, a, 0, FPST_FPCR) 4050 TRANS_FEAT(FCVTZU_ss, aa64_sve, gen_gvec_fpst_arg_zpz, 4051 gen_helper_sve_fcvtzu_ss, a, 0, FPST_FPCR) 4052 TRANS_FEAT(FCVTZS_sd, aa64_sve, gen_gvec_fpst_arg_zpz, 4053 gen_helper_sve_fcvtzs_sd, a, 0, FPST_FPCR) 4054 TRANS_FEAT(FCVTZU_sd, aa64_sve, gen_gvec_fpst_arg_zpz, 4055 gen_helper_sve_fcvtzu_sd, a, 0, FPST_FPCR) 4056 TRANS_FEAT(FCVTZS_ds, aa64_sve, gen_gvec_fpst_arg_zpz, 4057 gen_helper_sve_fcvtzs_ds, a, 0, FPST_FPCR) 4058 TRANS_FEAT(FCVTZU_ds, aa64_sve, gen_gvec_fpst_arg_zpz, 4059 gen_helper_sve_fcvtzu_ds, a, 0, FPST_FPCR) 4060 4061 TRANS_FEAT(FCVTZS_dd, aa64_sve, gen_gvec_fpst_arg_zpz, 4062 gen_helper_sve_fcvtzs_dd, a, 0, FPST_FPCR) 4063 TRANS_FEAT(FCVTZU_dd, aa64_sve, gen_gvec_fpst_arg_zpz, 4064 gen_helper_sve_fcvtzu_dd, a, 0, FPST_FPCR) 4065 4066 static gen_helper_gvec_3_ptr * const frint_fns[] = { 4067 NULL, 4068 gen_helper_sve_frint_h, 4069 gen_helper_sve_frint_s, 4070 gen_helper_sve_frint_d 4071 }; 4072 TRANS_FEAT(FRINTI, aa64_sve, gen_gvec_fpst_arg_zpz, frint_fns[a->esz], 4073 a, 0, a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR) 4074 4075 static gen_helper_gvec_3_ptr * const frintx_fns[] = { 4076 NULL, 4077 gen_helper_sve_frintx_h, 4078 gen_helper_sve_frintx_s, 4079 gen_helper_sve_frintx_d 4080 }; 4081 TRANS_FEAT(FRINTX, aa64_sve, gen_gvec_fpst_arg_zpz, frintx_fns[a->esz], 4082 a, 0, a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR); 4083 4084 static bool do_frint_mode(DisasContext *s, arg_rpr_esz *a, 4085 ARMFPRounding mode, gen_helper_gvec_3_ptr *fn) 4086 { 4087 unsigned vsz; 4088 TCGv_i32 tmode; 4089 TCGv_ptr status; 4090 4091 if (fn == NULL) { 4092 return false; 4093 } 4094 if (!sve_access_check(s)) { 4095 return true; 4096 } 4097 4098 vsz = vec_full_reg_size(s); 4099 status = fpstatus_ptr(a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR); 4100 tmode = gen_set_rmode(mode, status); 4101 4102 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, a->rd), 4103 vec_full_reg_offset(s, a->rn), 4104 pred_full_reg_offset(s, a->pg), 4105 status, vsz, vsz, 0, fn); 4106 4107 gen_restore_rmode(tmode, status); 4108 return true; 4109 } 4110 4111 TRANS_FEAT(FRINTN, aa64_sve, do_frint_mode, a, 4112 FPROUNDING_TIEEVEN, frint_fns[a->esz]) 4113 TRANS_FEAT(FRINTP, aa64_sve, do_frint_mode, a, 4114 FPROUNDING_POSINF, frint_fns[a->esz]) 4115 TRANS_FEAT(FRINTM, aa64_sve, do_frint_mode, a, 4116 FPROUNDING_NEGINF, frint_fns[a->esz]) 4117 TRANS_FEAT(FRINTZ, aa64_sve, do_frint_mode, a, 4118 FPROUNDING_ZERO, frint_fns[a->esz]) 4119 TRANS_FEAT(FRINTA, aa64_sve, do_frint_mode, a, 4120 FPROUNDING_TIEAWAY, frint_fns[a->esz]) 4121 4122 static gen_helper_gvec_3_ptr * const frecpx_fns[] = { 4123 NULL, gen_helper_sve_frecpx_h, 4124 gen_helper_sve_frecpx_s, gen_helper_sve_frecpx_d, 4125 }; 4126 TRANS_FEAT(FRECPX, aa64_sve, gen_gvec_fpst_arg_zpz, frecpx_fns[a->esz], 4127 a, 0, a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR) 4128 4129 static gen_helper_gvec_3_ptr * const fsqrt_fns[] = { 4130 NULL, gen_helper_sve_fsqrt_h, 4131 gen_helper_sve_fsqrt_s, gen_helper_sve_fsqrt_d, 4132 }; 4133 TRANS_FEAT(FSQRT, aa64_sve, gen_gvec_fpst_arg_zpz, fsqrt_fns[a->esz], 4134 a, 0, a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR) 4135 4136 TRANS_FEAT(SCVTF_hh, aa64_sve, gen_gvec_fpst_arg_zpz, 4137 gen_helper_sve_scvt_hh, a, 0, FPST_FPCR_F16) 4138 TRANS_FEAT(SCVTF_sh, aa64_sve, gen_gvec_fpst_arg_zpz, 4139 gen_helper_sve_scvt_sh, a, 0, FPST_FPCR_F16) 4140 TRANS_FEAT(SCVTF_dh, aa64_sve, gen_gvec_fpst_arg_zpz, 4141 gen_helper_sve_scvt_dh, a, 0, FPST_FPCR_F16) 4142 4143 TRANS_FEAT(SCVTF_ss, aa64_sve, gen_gvec_fpst_arg_zpz, 4144 gen_helper_sve_scvt_ss, a, 0, FPST_FPCR) 4145 TRANS_FEAT(SCVTF_ds, aa64_sve, gen_gvec_fpst_arg_zpz, 4146 gen_helper_sve_scvt_ds, a, 0, FPST_FPCR) 4147 4148 TRANS_FEAT(SCVTF_sd, aa64_sve, gen_gvec_fpst_arg_zpz, 4149 gen_helper_sve_scvt_sd, a, 0, FPST_FPCR) 4150 TRANS_FEAT(SCVTF_dd, aa64_sve, gen_gvec_fpst_arg_zpz, 4151 gen_helper_sve_scvt_dd, a, 0, FPST_FPCR) 4152 4153 TRANS_FEAT(UCVTF_hh, aa64_sve, gen_gvec_fpst_arg_zpz, 4154 gen_helper_sve_ucvt_hh, a, 0, FPST_FPCR_F16) 4155 TRANS_FEAT(UCVTF_sh, aa64_sve, gen_gvec_fpst_arg_zpz, 4156 gen_helper_sve_ucvt_sh, a, 0, FPST_FPCR_F16) 4157 TRANS_FEAT(UCVTF_dh, aa64_sve, gen_gvec_fpst_arg_zpz, 4158 gen_helper_sve_ucvt_dh, a, 0, FPST_FPCR_F16) 4159 4160 TRANS_FEAT(UCVTF_ss, aa64_sve, gen_gvec_fpst_arg_zpz, 4161 gen_helper_sve_ucvt_ss, a, 0, FPST_FPCR) 4162 TRANS_FEAT(UCVTF_ds, aa64_sve, gen_gvec_fpst_arg_zpz, 4163 gen_helper_sve_ucvt_ds, a, 0, FPST_FPCR) 4164 TRANS_FEAT(UCVTF_sd, aa64_sve, gen_gvec_fpst_arg_zpz, 4165 gen_helper_sve_ucvt_sd, a, 0, FPST_FPCR) 4166 4167 TRANS_FEAT(UCVTF_dd, aa64_sve, gen_gvec_fpst_arg_zpz, 4168 gen_helper_sve_ucvt_dd, a, 0, FPST_FPCR) 4169 4170 /* 4171 *** SVE Memory - 32-bit Gather and Unsized Contiguous Group 4172 */ 4173 4174 /* Subroutine loading a vector register at VOFS of LEN bytes. 4175 * The load should begin at the address Rn + IMM. 4176 */ 4177 4178 void gen_sve_ldr(DisasContext *s, TCGv_ptr base, int vofs, 4179 int len, int rn, int imm) 4180 { 4181 int len_align = QEMU_ALIGN_DOWN(len, 8); 4182 int len_remain = len % 8; 4183 int nparts = len / 8 + ctpop8(len_remain); 4184 int midx = get_mem_index(s); 4185 TCGv_i64 dirty_addr, clean_addr, t0, t1; 4186 4187 dirty_addr = tcg_temp_new_i64(); 4188 tcg_gen_addi_i64(dirty_addr, cpu_reg_sp(s, rn), imm); 4189 clean_addr = gen_mte_checkN(s, dirty_addr, false, rn != 31, len); 4190 4191 /* 4192 * Note that unpredicated load/store of vector/predicate registers 4193 * are defined as a stream of bytes, which equates to little-endian 4194 * operations on larger quantities. 4195 * Attempt to keep code expansion to a minimum by limiting the 4196 * amount of unrolling done. 4197 */ 4198 if (nparts <= 4) { 4199 int i; 4200 4201 t0 = tcg_temp_new_i64(); 4202 for (i = 0; i < len_align; i += 8) { 4203 tcg_gen_qemu_ld_i64(t0, clean_addr, midx, MO_LEUQ); 4204 tcg_gen_st_i64(t0, base, vofs + i); 4205 tcg_gen_addi_i64(clean_addr, clean_addr, 8); 4206 } 4207 } else { 4208 TCGLabel *loop = gen_new_label(); 4209 TCGv_ptr tp, i = tcg_const_ptr(0); 4210 4211 gen_set_label(loop); 4212 4213 t0 = tcg_temp_new_i64(); 4214 tcg_gen_qemu_ld_i64(t0, clean_addr, midx, MO_LEUQ); 4215 tcg_gen_addi_i64(clean_addr, clean_addr, 8); 4216 4217 tp = tcg_temp_new_ptr(); 4218 tcg_gen_add_ptr(tp, base, i); 4219 tcg_gen_addi_ptr(i, i, 8); 4220 tcg_gen_st_i64(t0, tp, vofs); 4221 4222 tcg_gen_brcondi_ptr(TCG_COND_LTU, i, len_align, loop); 4223 } 4224 4225 /* 4226 * Predicate register loads can be any multiple of 2. 4227 * Note that we still store the entire 64-bit unit into cpu_env. 4228 */ 4229 if (len_remain) { 4230 t0 = tcg_temp_new_i64(); 4231 switch (len_remain) { 4232 case 2: 4233 case 4: 4234 case 8: 4235 tcg_gen_qemu_ld_i64(t0, clean_addr, midx, 4236 MO_LE | ctz32(len_remain)); 4237 break; 4238 4239 case 6: 4240 t1 = tcg_temp_new_i64(); 4241 tcg_gen_qemu_ld_i64(t0, clean_addr, midx, MO_LEUL); 4242 tcg_gen_addi_i64(clean_addr, clean_addr, 4); 4243 tcg_gen_qemu_ld_i64(t1, clean_addr, midx, MO_LEUW); 4244 tcg_gen_deposit_i64(t0, t0, t1, 32, 32); 4245 break; 4246 4247 default: 4248 g_assert_not_reached(); 4249 } 4250 tcg_gen_st_i64(t0, base, vofs + len_align); 4251 } 4252 } 4253 4254 /* Similarly for stores. */ 4255 void gen_sve_str(DisasContext *s, TCGv_ptr base, int vofs, 4256 int len, int rn, int imm) 4257 { 4258 int len_align = QEMU_ALIGN_DOWN(len, 8); 4259 int len_remain = len % 8; 4260 int nparts = len / 8 + ctpop8(len_remain); 4261 int midx = get_mem_index(s); 4262 TCGv_i64 dirty_addr, clean_addr, t0; 4263 4264 dirty_addr = tcg_temp_new_i64(); 4265 tcg_gen_addi_i64(dirty_addr, cpu_reg_sp(s, rn), imm); 4266 clean_addr = gen_mte_checkN(s, dirty_addr, false, rn != 31, len); 4267 4268 /* Note that unpredicated load/store of vector/predicate registers 4269 * are defined as a stream of bytes, which equates to little-endian 4270 * operations on larger quantities. There is no nice way to force 4271 * a little-endian store for aarch64_be-linux-user out of line. 4272 * 4273 * Attempt to keep code expansion to a minimum by limiting the 4274 * amount of unrolling done. 4275 */ 4276 if (nparts <= 4) { 4277 int i; 4278 4279 t0 = tcg_temp_new_i64(); 4280 for (i = 0; i < len_align; i += 8) { 4281 tcg_gen_ld_i64(t0, base, vofs + i); 4282 tcg_gen_qemu_st_i64(t0, clean_addr, midx, MO_LEUQ); 4283 tcg_gen_addi_i64(clean_addr, clean_addr, 8); 4284 } 4285 } else { 4286 TCGLabel *loop = gen_new_label(); 4287 TCGv_ptr tp, i = tcg_const_ptr(0); 4288 4289 gen_set_label(loop); 4290 4291 t0 = tcg_temp_new_i64(); 4292 tp = tcg_temp_new_ptr(); 4293 tcg_gen_add_ptr(tp, base, i); 4294 tcg_gen_ld_i64(t0, tp, vofs); 4295 tcg_gen_addi_ptr(i, i, 8); 4296 4297 tcg_gen_qemu_st_i64(t0, clean_addr, midx, MO_LEUQ); 4298 tcg_gen_addi_i64(clean_addr, clean_addr, 8); 4299 4300 tcg_gen_brcondi_ptr(TCG_COND_LTU, i, len_align, loop); 4301 } 4302 4303 /* Predicate register stores can be any multiple of 2. */ 4304 if (len_remain) { 4305 t0 = tcg_temp_new_i64(); 4306 tcg_gen_ld_i64(t0, base, vofs + len_align); 4307 4308 switch (len_remain) { 4309 case 2: 4310 case 4: 4311 case 8: 4312 tcg_gen_qemu_st_i64(t0, clean_addr, midx, 4313 MO_LE | ctz32(len_remain)); 4314 break; 4315 4316 case 6: 4317 tcg_gen_qemu_st_i64(t0, clean_addr, midx, MO_LEUL); 4318 tcg_gen_addi_i64(clean_addr, clean_addr, 4); 4319 tcg_gen_shri_i64(t0, t0, 32); 4320 tcg_gen_qemu_st_i64(t0, clean_addr, midx, MO_LEUW); 4321 break; 4322 4323 default: 4324 g_assert_not_reached(); 4325 } 4326 } 4327 } 4328 4329 static bool trans_LDR_zri(DisasContext *s, arg_rri *a) 4330 { 4331 if (!dc_isar_feature(aa64_sve, s)) { 4332 return false; 4333 } 4334 if (sve_access_check(s)) { 4335 int size = vec_full_reg_size(s); 4336 int off = vec_full_reg_offset(s, a->rd); 4337 gen_sve_ldr(s, cpu_env, off, size, a->rn, a->imm * size); 4338 } 4339 return true; 4340 } 4341 4342 static bool trans_LDR_pri(DisasContext *s, arg_rri *a) 4343 { 4344 if (!dc_isar_feature(aa64_sve, s)) { 4345 return false; 4346 } 4347 if (sve_access_check(s)) { 4348 int size = pred_full_reg_size(s); 4349 int off = pred_full_reg_offset(s, a->rd); 4350 gen_sve_ldr(s, cpu_env, off, size, a->rn, a->imm * size); 4351 } 4352 return true; 4353 } 4354 4355 static bool trans_STR_zri(DisasContext *s, arg_rri *a) 4356 { 4357 if (!dc_isar_feature(aa64_sve, s)) { 4358 return false; 4359 } 4360 if (sve_access_check(s)) { 4361 int size = vec_full_reg_size(s); 4362 int off = vec_full_reg_offset(s, a->rd); 4363 gen_sve_str(s, cpu_env, off, size, a->rn, a->imm * size); 4364 } 4365 return true; 4366 } 4367 4368 static bool trans_STR_pri(DisasContext *s, arg_rri *a) 4369 { 4370 if (!dc_isar_feature(aa64_sve, s)) { 4371 return false; 4372 } 4373 if (sve_access_check(s)) { 4374 int size = pred_full_reg_size(s); 4375 int off = pred_full_reg_offset(s, a->rd); 4376 gen_sve_str(s, cpu_env, off, size, a->rn, a->imm * size); 4377 } 4378 return true; 4379 } 4380 4381 /* 4382 *** SVE Memory - Contiguous Load Group 4383 */ 4384 4385 /* The memory mode of the dtype. */ 4386 static const MemOp dtype_mop[16] = { 4387 MO_UB, MO_UB, MO_UB, MO_UB, 4388 MO_SL, MO_UW, MO_UW, MO_UW, 4389 MO_SW, MO_SW, MO_UL, MO_UL, 4390 MO_SB, MO_SB, MO_SB, MO_UQ 4391 }; 4392 4393 #define dtype_msz(x) (dtype_mop[x] & MO_SIZE) 4394 4395 /* The vector element size of dtype. */ 4396 static const uint8_t dtype_esz[16] = { 4397 0, 1, 2, 3, 4398 3, 1, 2, 3, 4399 3, 2, 2, 3, 4400 3, 2, 1, 3 4401 }; 4402 4403 static void do_mem_zpa(DisasContext *s, int zt, int pg, TCGv_i64 addr, 4404 int dtype, uint32_t mte_n, bool is_write, 4405 gen_helper_gvec_mem *fn) 4406 { 4407 unsigned vsz = vec_full_reg_size(s); 4408 TCGv_ptr t_pg; 4409 int desc = 0; 4410 4411 /* 4412 * For e.g. LD4, there are not enough arguments to pass all 4 4413 * registers as pointers, so encode the regno into the data field. 4414 * For consistency, do this even for LD1. 4415 */ 4416 if (s->mte_active[0]) { 4417 int msz = dtype_msz(dtype); 4418 4419 desc = FIELD_DP32(desc, MTEDESC, MIDX, get_mem_index(s)); 4420 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid); 4421 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma); 4422 desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write); 4423 desc = FIELD_DP32(desc, MTEDESC, SIZEM1, (mte_n << msz) - 1); 4424 desc <<= SVE_MTEDESC_SHIFT; 4425 } else { 4426 addr = clean_data_tbi(s, addr); 4427 } 4428 4429 desc = simd_desc(vsz, vsz, zt | desc); 4430 t_pg = tcg_temp_new_ptr(); 4431 4432 tcg_gen_addi_ptr(t_pg, cpu_env, pred_full_reg_offset(s, pg)); 4433 fn(cpu_env, t_pg, addr, tcg_constant_i32(desc)); 4434 } 4435 4436 /* Indexed by [mte][be][dtype][nreg] */ 4437 static gen_helper_gvec_mem * const ldr_fns[2][2][16][4] = { 4438 { /* mte inactive, little-endian */ 4439 { { gen_helper_sve_ld1bb_r, gen_helper_sve_ld2bb_r, 4440 gen_helper_sve_ld3bb_r, gen_helper_sve_ld4bb_r }, 4441 { gen_helper_sve_ld1bhu_r, NULL, NULL, NULL }, 4442 { gen_helper_sve_ld1bsu_r, NULL, NULL, NULL }, 4443 { gen_helper_sve_ld1bdu_r, NULL, NULL, NULL }, 4444 4445 { gen_helper_sve_ld1sds_le_r, NULL, NULL, NULL }, 4446 { gen_helper_sve_ld1hh_le_r, gen_helper_sve_ld2hh_le_r, 4447 gen_helper_sve_ld3hh_le_r, gen_helper_sve_ld4hh_le_r }, 4448 { gen_helper_sve_ld1hsu_le_r, NULL, NULL, NULL }, 4449 { gen_helper_sve_ld1hdu_le_r, NULL, NULL, NULL }, 4450 4451 { gen_helper_sve_ld1hds_le_r, NULL, NULL, NULL }, 4452 { gen_helper_sve_ld1hss_le_r, NULL, NULL, NULL }, 4453 { gen_helper_sve_ld1ss_le_r, gen_helper_sve_ld2ss_le_r, 4454 gen_helper_sve_ld3ss_le_r, gen_helper_sve_ld4ss_le_r }, 4455 { gen_helper_sve_ld1sdu_le_r, NULL, NULL, NULL }, 4456 4457 { gen_helper_sve_ld1bds_r, NULL, NULL, NULL }, 4458 { gen_helper_sve_ld1bss_r, NULL, NULL, NULL }, 4459 { gen_helper_sve_ld1bhs_r, NULL, NULL, NULL }, 4460 { gen_helper_sve_ld1dd_le_r, gen_helper_sve_ld2dd_le_r, 4461 gen_helper_sve_ld3dd_le_r, gen_helper_sve_ld4dd_le_r } }, 4462 4463 /* mte inactive, big-endian */ 4464 { { gen_helper_sve_ld1bb_r, gen_helper_sve_ld2bb_r, 4465 gen_helper_sve_ld3bb_r, gen_helper_sve_ld4bb_r }, 4466 { gen_helper_sve_ld1bhu_r, NULL, NULL, NULL }, 4467 { gen_helper_sve_ld1bsu_r, NULL, NULL, NULL }, 4468 { gen_helper_sve_ld1bdu_r, NULL, NULL, NULL }, 4469 4470 { gen_helper_sve_ld1sds_be_r, NULL, NULL, NULL }, 4471 { gen_helper_sve_ld1hh_be_r, gen_helper_sve_ld2hh_be_r, 4472 gen_helper_sve_ld3hh_be_r, gen_helper_sve_ld4hh_be_r }, 4473 { gen_helper_sve_ld1hsu_be_r, NULL, NULL, NULL }, 4474 { gen_helper_sve_ld1hdu_be_r, NULL, NULL, NULL }, 4475 4476 { gen_helper_sve_ld1hds_be_r, NULL, NULL, NULL }, 4477 { gen_helper_sve_ld1hss_be_r, NULL, NULL, NULL }, 4478 { gen_helper_sve_ld1ss_be_r, gen_helper_sve_ld2ss_be_r, 4479 gen_helper_sve_ld3ss_be_r, gen_helper_sve_ld4ss_be_r }, 4480 { gen_helper_sve_ld1sdu_be_r, NULL, NULL, NULL }, 4481 4482 { gen_helper_sve_ld1bds_r, NULL, NULL, NULL }, 4483 { gen_helper_sve_ld1bss_r, NULL, NULL, NULL }, 4484 { gen_helper_sve_ld1bhs_r, NULL, NULL, NULL }, 4485 { gen_helper_sve_ld1dd_be_r, gen_helper_sve_ld2dd_be_r, 4486 gen_helper_sve_ld3dd_be_r, gen_helper_sve_ld4dd_be_r } } }, 4487 4488 { /* mte active, little-endian */ 4489 { { gen_helper_sve_ld1bb_r_mte, 4490 gen_helper_sve_ld2bb_r_mte, 4491 gen_helper_sve_ld3bb_r_mte, 4492 gen_helper_sve_ld4bb_r_mte }, 4493 { gen_helper_sve_ld1bhu_r_mte, NULL, NULL, NULL }, 4494 { gen_helper_sve_ld1bsu_r_mte, NULL, NULL, NULL }, 4495 { gen_helper_sve_ld1bdu_r_mte, NULL, NULL, NULL }, 4496 4497 { gen_helper_sve_ld1sds_le_r_mte, NULL, NULL, NULL }, 4498 { gen_helper_sve_ld1hh_le_r_mte, 4499 gen_helper_sve_ld2hh_le_r_mte, 4500 gen_helper_sve_ld3hh_le_r_mte, 4501 gen_helper_sve_ld4hh_le_r_mte }, 4502 { gen_helper_sve_ld1hsu_le_r_mte, NULL, NULL, NULL }, 4503 { gen_helper_sve_ld1hdu_le_r_mte, NULL, NULL, NULL }, 4504 4505 { gen_helper_sve_ld1hds_le_r_mte, NULL, NULL, NULL }, 4506 { gen_helper_sve_ld1hss_le_r_mte, NULL, NULL, NULL }, 4507 { gen_helper_sve_ld1ss_le_r_mte, 4508 gen_helper_sve_ld2ss_le_r_mte, 4509 gen_helper_sve_ld3ss_le_r_mte, 4510 gen_helper_sve_ld4ss_le_r_mte }, 4511 { gen_helper_sve_ld1sdu_le_r_mte, NULL, NULL, NULL }, 4512 4513 { gen_helper_sve_ld1bds_r_mte, NULL, NULL, NULL }, 4514 { gen_helper_sve_ld1bss_r_mte, NULL, NULL, NULL }, 4515 { gen_helper_sve_ld1bhs_r_mte, NULL, NULL, NULL }, 4516 { gen_helper_sve_ld1dd_le_r_mte, 4517 gen_helper_sve_ld2dd_le_r_mte, 4518 gen_helper_sve_ld3dd_le_r_mte, 4519 gen_helper_sve_ld4dd_le_r_mte } }, 4520 4521 /* mte active, big-endian */ 4522 { { gen_helper_sve_ld1bb_r_mte, 4523 gen_helper_sve_ld2bb_r_mte, 4524 gen_helper_sve_ld3bb_r_mte, 4525 gen_helper_sve_ld4bb_r_mte }, 4526 { gen_helper_sve_ld1bhu_r_mte, NULL, NULL, NULL }, 4527 { gen_helper_sve_ld1bsu_r_mte, NULL, NULL, NULL }, 4528 { gen_helper_sve_ld1bdu_r_mte, NULL, NULL, NULL }, 4529 4530 { gen_helper_sve_ld1sds_be_r_mte, NULL, NULL, NULL }, 4531 { gen_helper_sve_ld1hh_be_r_mte, 4532 gen_helper_sve_ld2hh_be_r_mte, 4533 gen_helper_sve_ld3hh_be_r_mte, 4534 gen_helper_sve_ld4hh_be_r_mte }, 4535 { gen_helper_sve_ld1hsu_be_r_mte, NULL, NULL, NULL }, 4536 { gen_helper_sve_ld1hdu_be_r_mte, NULL, NULL, NULL }, 4537 4538 { gen_helper_sve_ld1hds_be_r_mte, NULL, NULL, NULL }, 4539 { gen_helper_sve_ld1hss_be_r_mte, NULL, NULL, NULL }, 4540 { gen_helper_sve_ld1ss_be_r_mte, 4541 gen_helper_sve_ld2ss_be_r_mte, 4542 gen_helper_sve_ld3ss_be_r_mte, 4543 gen_helper_sve_ld4ss_be_r_mte }, 4544 { gen_helper_sve_ld1sdu_be_r_mte, NULL, NULL, NULL }, 4545 4546 { gen_helper_sve_ld1bds_r_mte, NULL, NULL, NULL }, 4547 { gen_helper_sve_ld1bss_r_mte, NULL, NULL, NULL }, 4548 { gen_helper_sve_ld1bhs_r_mte, NULL, NULL, NULL }, 4549 { gen_helper_sve_ld1dd_be_r_mte, 4550 gen_helper_sve_ld2dd_be_r_mte, 4551 gen_helper_sve_ld3dd_be_r_mte, 4552 gen_helper_sve_ld4dd_be_r_mte } } }, 4553 }; 4554 4555 static void do_ld_zpa(DisasContext *s, int zt, int pg, 4556 TCGv_i64 addr, int dtype, int nreg) 4557 { 4558 gen_helper_gvec_mem *fn 4559 = ldr_fns[s->mte_active[0]][s->be_data == MO_BE][dtype][nreg]; 4560 4561 /* 4562 * While there are holes in the table, they are not 4563 * accessible via the instruction encoding. 4564 */ 4565 assert(fn != NULL); 4566 do_mem_zpa(s, zt, pg, addr, dtype, nreg, false, fn); 4567 } 4568 4569 static bool trans_LD_zprr(DisasContext *s, arg_rprr_load *a) 4570 { 4571 if (a->rm == 31 || !dc_isar_feature(aa64_sve, s)) { 4572 return false; 4573 } 4574 if (sve_access_check(s)) { 4575 TCGv_i64 addr = tcg_temp_new_i64(); 4576 tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), dtype_msz(a->dtype)); 4577 tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn)); 4578 do_ld_zpa(s, a->rd, a->pg, addr, a->dtype, a->nreg); 4579 } 4580 return true; 4581 } 4582 4583 static bool trans_LD_zpri(DisasContext *s, arg_rpri_load *a) 4584 { 4585 if (!dc_isar_feature(aa64_sve, s)) { 4586 return false; 4587 } 4588 if (sve_access_check(s)) { 4589 int vsz = vec_full_reg_size(s); 4590 int elements = vsz >> dtype_esz[a->dtype]; 4591 TCGv_i64 addr = tcg_temp_new_i64(); 4592 4593 tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn), 4594 (a->imm * elements * (a->nreg + 1)) 4595 << dtype_msz(a->dtype)); 4596 do_ld_zpa(s, a->rd, a->pg, addr, a->dtype, a->nreg); 4597 } 4598 return true; 4599 } 4600 4601 static bool trans_LDFF1_zprr(DisasContext *s, arg_rprr_load *a) 4602 { 4603 static gen_helper_gvec_mem * const fns[2][2][16] = { 4604 { /* mte inactive, little-endian */ 4605 { gen_helper_sve_ldff1bb_r, 4606 gen_helper_sve_ldff1bhu_r, 4607 gen_helper_sve_ldff1bsu_r, 4608 gen_helper_sve_ldff1bdu_r, 4609 4610 gen_helper_sve_ldff1sds_le_r, 4611 gen_helper_sve_ldff1hh_le_r, 4612 gen_helper_sve_ldff1hsu_le_r, 4613 gen_helper_sve_ldff1hdu_le_r, 4614 4615 gen_helper_sve_ldff1hds_le_r, 4616 gen_helper_sve_ldff1hss_le_r, 4617 gen_helper_sve_ldff1ss_le_r, 4618 gen_helper_sve_ldff1sdu_le_r, 4619 4620 gen_helper_sve_ldff1bds_r, 4621 gen_helper_sve_ldff1bss_r, 4622 gen_helper_sve_ldff1bhs_r, 4623 gen_helper_sve_ldff1dd_le_r }, 4624 4625 /* mte inactive, big-endian */ 4626 { gen_helper_sve_ldff1bb_r, 4627 gen_helper_sve_ldff1bhu_r, 4628 gen_helper_sve_ldff1bsu_r, 4629 gen_helper_sve_ldff1bdu_r, 4630 4631 gen_helper_sve_ldff1sds_be_r, 4632 gen_helper_sve_ldff1hh_be_r, 4633 gen_helper_sve_ldff1hsu_be_r, 4634 gen_helper_sve_ldff1hdu_be_r, 4635 4636 gen_helper_sve_ldff1hds_be_r, 4637 gen_helper_sve_ldff1hss_be_r, 4638 gen_helper_sve_ldff1ss_be_r, 4639 gen_helper_sve_ldff1sdu_be_r, 4640 4641 gen_helper_sve_ldff1bds_r, 4642 gen_helper_sve_ldff1bss_r, 4643 gen_helper_sve_ldff1bhs_r, 4644 gen_helper_sve_ldff1dd_be_r } }, 4645 4646 { /* mte active, little-endian */ 4647 { gen_helper_sve_ldff1bb_r_mte, 4648 gen_helper_sve_ldff1bhu_r_mte, 4649 gen_helper_sve_ldff1bsu_r_mte, 4650 gen_helper_sve_ldff1bdu_r_mte, 4651 4652 gen_helper_sve_ldff1sds_le_r_mte, 4653 gen_helper_sve_ldff1hh_le_r_mte, 4654 gen_helper_sve_ldff1hsu_le_r_mte, 4655 gen_helper_sve_ldff1hdu_le_r_mte, 4656 4657 gen_helper_sve_ldff1hds_le_r_mte, 4658 gen_helper_sve_ldff1hss_le_r_mte, 4659 gen_helper_sve_ldff1ss_le_r_mte, 4660 gen_helper_sve_ldff1sdu_le_r_mte, 4661 4662 gen_helper_sve_ldff1bds_r_mte, 4663 gen_helper_sve_ldff1bss_r_mte, 4664 gen_helper_sve_ldff1bhs_r_mte, 4665 gen_helper_sve_ldff1dd_le_r_mte }, 4666 4667 /* mte active, big-endian */ 4668 { gen_helper_sve_ldff1bb_r_mte, 4669 gen_helper_sve_ldff1bhu_r_mte, 4670 gen_helper_sve_ldff1bsu_r_mte, 4671 gen_helper_sve_ldff1bdu_r_mte, 4672 4673 gen_helper_sve_ldff1sds_be_r_mte, 4674 gen_helper_sve_ldff1hh_be_r_mte, 4675 gen_helper_sve_ldff1hsu_be_r_mte, 4676 gen_helper_sve_ldff1hdu_be_r_mte, 4677 4678 gen_helper_sve_ldff1hds_be_r_mte, 4679 gen_helper_sve_ldff1hss_be_r_mte, 4680 gen_helper_sve_ldff1ss_be_r_mte, 4681 gen_helper_sve_ldff1sdu_be_r_mte, 4682 4683 gen_helper_sve_ldff1bds_r_mte, 4684 gen_helper_sve_ldff1bss_r_mte, 4685 gen_helper_sve_ldff1bhs_r_mte, 4686 gen_helper_sve_ldff1dd_be_r_mte } }, 4687 }; 4688 4689 if (!dc_isar_feature(aa64_sve, s)) { 4690 return false; 4691 } 4692 s->is_nonstreaming = true; 4693 if (sve_access_check(s)) { 4694 TCGv_i64 addr = tcg_temp_new_i64(); 4695 tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), dtype_msz(a->dtype)); 4696 tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn)); 4697 do_mem_zpa(s, a->rd, a->pg, addr, a->dtype, 1, false, 4698 fns[s->mte_active[0]][s->be_data == MO_BE][a->dtype]); 4699 } 4700 return true; 4701 } 4702 4703 static bool trans_LDNF1_zpri(DisasContext *s, arg_rpri_load *a) 4704 { 4705 static gen_helper_gvec_mem * const fns[2][2][16] = { 4706 { /* mte inactive, little-endian */ 4707 { gen_helper_sve_ldnf1bb_r, 4708 gen_helper_sve_ldnf1bhu_r, 4709 gen_helper_sve_ldnf1bsu_r, 4710 gen_helper_sve_ldnf1bdu_r, 4711 4712 gen_helper_sve_ldnf1sds_le_r, 4713 gen_helper_sve_ldnf1hh_le_r, 4714 gen_helper_sve_ldnf1hsu_le_r, 4715 gen_helper_sve_ldnf1hdu_le_r, 4716 4717 gen_helper_sve_ldnf1hds_le_r, 4718 gen_helper_sve_ldnf1hss_le_r, 4719 gen_helper_sve_ldnf1ss_le_r, 4720 gen_helper_sve_ldnf1sdu_le_r, 4721 4722 gen_helper_sve_ldnf1bds_r, 4723 gen_helper_sve_ldnf1bss_r, 4724 gen_helper_sve_ldnf1bhs_r, 4725 gen_helper_sve_ldnf1dd_le_r }, 4726 4727 /* mte inactive, big-endian */ 4728 { gen_helper_sve_ldnf1bb_r, 4729 gen_helper_sve_ldnf1bhu_r, 4730 gen_helper_sve_ldnf1bsu_r, 4731 gen_helper_sve_ldnf1bdu_r, 4732 4733 gen_helper_sve_ldnf1sds_be_r, 4734 gen_helper_sve_ldnf1hh_be_r, 4735 gen_helper_sve_ldnf1hsu_be_r, 4736 gen_helper_sve_ldnf1hdu_be_r, 4737 4738 gen_helper_sve_ldnf1hds_be_r, 4739 gen_helper_sve_ldnf1hss_be_r, 4740 gen_helper_sve_ldnf1ss_be_r, 4741 gen_helper_sve_ldnf1sdu_be_r, 4742 4743 gen_helper_sve_ldnf1bds_r, 4744 gen_helper_sve_ldnf1bss_r, 4745 gen_helper_sve_ldnf1bhs_r, 4746 gen_helper_sve_ldnf1dd_be_r } }, 4747 4748 { /* mte inactive, little-endian */ 4749 { gen_helper_sve_ldnf1bb_r_mte, 4750 gen_helper_sve_ldnf1bhu_r_mte, 4751 gen_helper_sve_ldnf1bsu_r_mte, 4752 gen_helper_sve_ldnf1bdu_r_mte, 4753 4754 gen_helper_sve_ldnf1sds_le_r_mte, 4755 gen_helper_sve_ldnf1hh_le_r_mte, 4756 gen_helper_sve_ldnf1hsu_le_r_mte, 4757 gen_helper_sve_ldnf1hdu_le_r_mte, 4758 4759 gen_helper_sve_ldnf1hds_le_r_mte, 4760 gen_helper_sve_ldnf1hss_le_r_mte, 4761 gen_helper_sve_ldnf1ss_le_r_mte, 4762 gen_helper_sve_ldnf1sdu_le_r_mte, 4763 4764 gen_helper_sve_ldnf1bds_r_mte, 4765 gen_helper_sve_ldnf1bss_r_mte, 4766 gen_helper_sve_ldnf1bhs_r_mte, 4767 gen_helper_sve_ldnf1dd_le_r_mte }, 4768 4769 /* mte inactive, big-endian */ 4770 { gen_helper_sve_ldnf1bb_r_mte, 4771 gen_helper_sve_ldnf1bhu_r_mte, 4772 gen_helper_sve_ldnf1bsu_r_mte, 4773 gen_helper_sve_ldnf1bdu_r_mte, 4774 4775 gen_helper_sve_ldnf1sds_be_r_mte, 4776 gen_helper_sve_ldnf1hh_be_r_mte, 4777 gen_helper_sve_ldnf1hsu_be_r_mte, 4778 gen_helper_sve_ldnf1hdu_be_r_mte, 4779 4780 gen_helper_sve_ldnf1hds_be_r_mte, 4781 gen_helper_sve_ldnf1hss_be_r_mte, 4782 gen_helper_sve_ldnf1ss_be_r_mte, 4783 gen_helper_sve_ldnf1sdu_be_r_mte, 4784 4785 gen_helper_sve_ldnf1bds_r_mte, 4786 gen_helper_sve_ldnf1bss_r_mte, 4787 gen_helper_sve_ldnf1bhs_r_mte, 4788 gen_helper_sve_ldnf1dd_be_r_mte } }, 4789 }; 4790 4791 if (!dc_isar_feature(aa64_sve, s)) { 4792 return false; 4793 } 4794 s->is_nonstreaming = true; 4795 if (sve_access_check(s)) { 4796 int vsz = vec_full_reg_size(s); 4797 int elements = vsz >> dtype_esz[a->dtype]; 4798 int off = (a->imm * elements) << dtype_msz(a->dtype); 4799 TCGv_i64 addr = tcg_temp_new_i64(); 4800 4801 tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn), off); 4802 do_mem_zpa(s, a->rd, a->pg, addr, a->dtype, 1, false, 4803 fns[s->mte_active[0]][s->be_data == MO_BE][a->dtype]); 4804 } 4805 return true; 4806 } 4807 4808 static void do_ldrq(DisasContext *s, int zt, int pg, TCGv_i64 addr, int dtype) 4809 { 4810 unsigned vsz = vec_full_reg_size(s); 4811 TCGv_ptr t_pg; 4812 int poff; 4813 4814 /* Load the first quadword using the normal predicated load helpers. */ 4815 poff = pred_full_reg_offset(s, pg); 4816 if (vsz > 16) { 4817 /* 4818 * Zero-extend the first 16 bits of the predicate into a temporary. 4819 * This avoids triggering an assert making sure we don't have bits 4820 * set within a predicate beyond VQ, but we have lowered VQ to 1 4821 * for this load operation. 4822 */ 4823 TCGv_i64 tmp = tcg_temp_new_i64(); 4824 #if HOST_BIG_ENDIAN 4825 poff += 6; 4826 #endif 4827 tcg_gen_ld16u_i64(tmp, cpu_env, poff); 4828 4829 poff = offsetof(CPUARMState, vfp.preg_tmp); 4830 tcg_gen_st_i64(tmp, cpu_env, poff); 4831 } 4832 4833 t_pg = tcg_temp_new_ptr(); 4834 tcg_gen_addi_ptr(t_pg, cpu_env, poff); 4835 4836 gen_helper_gvec_mem *fn 4837 = ldr_fns[s->mte_active[0]][s->be_data == MO_BE][dtype][0]; 4838 fn(cpu_env, t_pg, addr, tcg_constant_i32(simd_desc(16, 16, zt))); 4839 4840 /* Replicate that first quadword. */ 4841 if (vsz > 16) { 4842 int doff = vec_full_reg_offset(s, zt); 4843 tcg_gen_gvec_dup_mem(4, doff + 16, doff, vsz - 16, vsz - 16); 4844 } 4845 } 4846 4847 static bool trans_LD1RQ_zprr(DisasContext *s, arg_rprr_load *a) 4848 { 4849 if (a->rm == 31 || !dc_isar_feature(aa64_sve, s)) { 4850 return false; 4851 } 4852 if (sve_access_check(s)) { 4853 int msz = dtype_msz(a->dtype); 4854 TCGv_i64 addr = tcg_temp_new_i64(); 4855 tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), msz); 4856 tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn)); 4857 do_ldrq(s, a->rd, a->pg, addr, a->dtype); 4858 } 4859 return true; 4860 } 4861 4862 static bool trans_LD1RQ_zpri(DisasContext *s, arg_rpri_load *a) 4863 { 4864 if (!dc_isar_feature(aa64_sve, s)) { 4865 return false; 4866 } 4867 if (sve_access_check(s)) { 4868 TCGv_i64 addr = tcg_temp_new_i64(); 4869 tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn), a->imm * 16); 4870 do_ldrq(s, a->rd, a->pg, addr, a->dtype); 4871 } 4872 return true; 4873 } 4874 4875 static void do_ldro(DisasContext *s, int zt, int pg, TCGv_i64 addr, int dtype) 4876 { 4877 unsigned vsz = vec_full_reg_size(s); 4878 unsigned vsz_r32; 4879 TCGv_ptr t_pg; 4880 int poff, doff; 4881 4882 if (vsz < 32) { 4883 /* 4884 * Note that this UNDEFINED check comes after CheckSVEEnabled() 4885 * in the ARM pseudocode, which is the sve_access_check() done 4886 * in our caller. We should not now return false from the caller. 4887 */ 4888 unallocated_encoding(s); 4889 return; 4890 } 4891 4892 /* Load the first octaword using the normal predicated load helpers. */ 4893 4894 poff = pred_full_reg_offset(s, pg); 4895 if (vsz > 32) { 4896 /* 4897 * Zero-extend the first 32 bits of the predicate into a temporary. 4898 * This avoids triggering an assert making sure we don't have bits 4899 * set within a predicate beyond VQ, but we have lowered VQ to 2 4900 * for this load operation. 4901 */ 4902 TCGv_i64 tmp = tcg_temp_new_i64(); 4903 #if HOST_BIG_ENDIAN 4904 poff += 4; 4905 #endif 4906 tcg_gen_ld32u_i64(tmp, cpu_env, poff); 4907 4908 poff = offsetof(CPUARMState, vfp.preg_tmp); 4909 tcg_gen_st_i64(tmp, cpu_env, poff); 4910 } 4911 4912 t_pg = tcg_temp_new_ptr(); 4913 tcg_gen_addi_ptr(t_pg, cpu_env, poff); 4914 4915 gen_helper_gvec_mem *fn 4916 = ldr_fns[s->mte_active[0]][s->be_data == MO_BE][dtype][0]; 4917 fn(cpu_env, t_pg, addr, tcg_constant_i32(simd_desc(32, 32, zt))); 4918 4919 /* 4920 * Replicate that first octaword. 4921 * The replication happens in units of 32; if the full vector size 4922 * is not a multiple of 32, the final bits are zeroed. 4923 */ 4924 doff = vec_full_reg_offset(s, zt); 4925 vsz_r32 = QEMU_ALIGN_DOWN(vsz, 32); 4926 if (vsz >= 64) { 4927 tcg_gen_gvec_dup_mem(5, doff + 32, doff, vsz_r32 - 32, vsz_r32 - 32); 4928 } 4929 vsz -= vsz_r32; 4930 if (vsz) { 4931 tcg_gen_gvec_dup_imm(MO_64, doff + vsz_r32, vsz, vsz, 0); 4932 } 4933 } 4934 4935 static bool trans_LD1RO_zprr(DisasContext *s, arg_rprr_load *a) 4936 { 4937 if (!dc_isar_feature(aa64_sve_f64mm, s)) { 4938 return false; 4939 } 4940 if (a->rm == 31) { 4941 return false; 4942 } 4943 s->is_nonstreaming = true; 4944 if (sve_access_check(s)) { 4945 TCGv_i64 addr = tcg_temp_new_i64(); 4946 tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), dtype_msz(a->dtype)); 4947 tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn)); 4948 do_ldro(s, a->rd, a->pg, addr, a->dtype); 4949 } 4950 return true; 4951 } 4952 4953 static bool trans_LD1RO_zpri(DisasContext *s, arg_rpri_load *a) 4954 { 4955 if (!dc_isar_feature(aa64_sve_f64mm, s)) { 4956 return false; 4957 } 4958 s->is_nonstreaming = true; 4959 if (sve_access_check(s)) { 4960 TCGv_i64 addr = tcg_temp_new_i64(); 4961 tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn), a->imm * 32); 4962 do_ldro(s, a->rd, a->pg, addr, a->dtype); 4963 } 4964 return true; 4965 } 4966 4967 /* Load and broadcast element. */ 4968 static bool trans_LD1R_zpri(DisasContext *s, arg_rpri_load *a) 4969 { 4970 unsigned vsz = vec_full_reg_size(s); 4971 unsigned psz = pred_full_reg_size(s); 4972 unsigned esz = dtype_esz[a->dtype]; 4973 unsigned msz = dtype_msz(a->dtype); 4974 TCGLabel *over; 4975 TCGv_i64 temp, clean_addr; 4976 4977 if (!dc_isar_feature(aa64_sve, s)) { 4978 return false; 4979 } 4980 if (!sve_access_check(s)) { 4981 return true; 4982 } 4983 4984 over = gen_new_label(); 4985 4986 /* If the guarding predicate has no bits set, no load occurs. */ 4987 if (psz <= 8) { 4988 /* Reduce the pred_esz_masks value simply to reduce the 4989 * size of the code generated here. 4990 */ 4991 uint64_t psz_mask = MAKE_64BIT_MASK(0, psz * 8); 4992 temp = tcg_temp_new_i64(); 4993 tcg_gen_ld_i64(temp, cpu_env, pred_full_reg_offset(s, a->pg)); 4994 tcg_gen_andi_i64(temp, temp, pred_esz_masks[esz] & psz_mask); 4995 tcg_gen_brcondi_i64(TCG_COND_EQ, temp, 0, over); 4996 } else { 4997 TCGv_i32 t32 = tcg_temp_new_i32(); 4998 find_last_active(s, t32, esz, a->pg); 4999 tcg_gen_brcondi_i32(TCG_COND_LT, t32, 0, over); 5000 } 5001 5002 /* Load the data. */ 5003 temp = tcg_temp_new_i64(); 5004 tcg_gen_addi_i64(temp, cpu_reg_sp(s, a->rn), a->imm << msz); 5005 clean_addr = gen_mte_check1(s, temp, false, true, msz); 5006 5007 tcg_gen_qemu_ld_i64(temp, clean_addr, get_mem_index(s), 5008 finalize_memop(s, dtype_mop[a->dtype])); 5009 5010 /* Broadcast to *all* elements. */ 5011 tcg_gen_gvec_dup_i64(esz, vec_full_reg_offset(s, a->rd), 5012 vsz, vsz, temp); 5013 5014 /* Zero the inactive elements. */ 5015 gen_set_label(over); 5016 return do_movz_zpz(s, a->rd, a->rd, a->pg, esz, false); 5017 } 5018 5019 static void do_st_zpa(DisasContext *s, int zt, int pg, TCGv_i64 addr, 5020 int msz, int esz, int nreg) 5021 { 5022 static gen_helper_gvec_mem * const fn_single[2][2][4][4] = { 5023 { { { gen_helper_sve_st1bb_r, 5024 gen_helper_sve_st1bh_r, 5025 gen_helper_sve_st1bs_r, 5026 gen_helper_sve_st1bd_r }, 5027 { NULL, 5028 gen_helper_sve_st1hh_le_r, 5029 gen_helper_sve_st1hs_le_r, 5030 gen_helper_sve_st1hd_le_r }, 5031 { NULL, NULL, 5032 gen_helper_sve_st1ss_le_r, 5033 gen_helper_sve_st1sd_le_r }, 5034 { NULL, NULL, NULL, 5035 gen_helper_sve_st1dd_le_r } }, 5036 { { gen_helper_sve_st1bb_r, 5037 gen_helper_sve_st1bh_r, 5038 gen_helper_sve_st1bs_r, 5039 gen_helper_sve_st1bd_r }, 5040 { NULL, 5041 gen_helper_sve_st1hh_be_r, 5042 gen_helper_sve_st1hs_be_r, 5043 gen_helper_sve_st1hd_be_r }, 5044 { NULL, NULL, 5045 gen_helper_sve_st1ss_be_r, 5046 gen_helper_sve_st1sd_be_r }, 5047 { NULL, NULL, NULL, 5048 gen_helper_sve_st1dd_be_r } } }, 5049 5050 { { { gen_helper_sve_st1bb_r_mte, 5051 gen_helper_sve_st1bh_r_mte, 5052 gen_helper_sve_st1bs_r_mte, 5053 gen_helper_sve_st1bd_r_mte }, 5054 { NULL, 5055 gen_helper_sve_st1hh_le_r_mte, 5056 gen_helper_sve_st1hs_le_r_mte, 5057 gen_helper_sve_st1hd_le_r_mte }, 5058 { NULL, NULL, 5059 gen_helper_sve_st1ss_le_r_mte, 5060 gen_helper_sve_st1sd_le_r_mte }, 5061 { NULL, NULL, NULL, 5062 gen_helper_sve_st1dd_le_r_mte } }, 5063 { { gen_helper_sve_st1bb_r_mte, 5064 gen_helper_sve_st1bh_r_mte, 5065 gen_helper_sve_st1bs_r_mte, 5066 gen_helper_sve_st1bd_r_mte }, 5067 { NULL, 5068 gen_helper_sve_st1hh_be_r_mte, 5069 gen_helper_sve_st1hs_be_r_mte, 5070 gen_helper_sve_st1hd_be_r_mte }, 5071 { NULL, NULL, 5072 gen_helper_sve_st1ss_be_r_mte, 5073 gen_helper_sve_st1sd_be_r_mte }, 5074 { NULL, NULL, NULL, 5075 gen_helper_sve_st1dd_be_r_mte } } }, 5076 }; 5077 static gen_helper_gvec_mem * const fn_multiple[2][2][3][4] = { 5078 { { { gen_helper_sve_st2bb_r, 5079 gen_helper_sve_st2hh_le_r, 5080 gen_helper_sve_st2ss_le_r, 5081 gen_helper_sve_st2dd_le_r }, 5082 { gen_helper_sve_st3bb_r, 5083 gen_helper_sve_st3hh_le_r, 5084 gen_helper_sve_st3ss_le_r, 5085 gen_helper_sve_st3dd_le_r }, 5086 { gen_helper_sve_st4bb_r, 5087 gen_helper_sve_st4hh_le_r, 5088 gen_helper_sve_st4ss_le_r, 5089 gen_helper_sve_st4dd_le_r } }, 5090 { { gen_helper_sve_st2bb_r, 5091 gen_helper_sve_st2hh_be_r, 5092 gen_helper_sve_st2ss_be_r, 5093 gen_helper_sve_st2dd_be_r }, 5094 { gen_helper_sve_st3bb_r, 5095 gen_helper_sve_st3hh_be_r, 5096 gen_helper_sve_st3ss_be_r, 5097 gen_helper_sve_st3dd_be_r }, 5098 { gen_helper_sve_st4bb_r, 5099 gen_helper_sve_st4hh_be_r, 5100 gen_helper_sve_st4ss_be_r, 5101 gen_helper_sve_st4dd_be_r } } }, 5102 { { { gen_helper_sve_st2bb_r_mte, 5103 gen_helper_sve_st2hh_le_r_mte, 5104 gen_helper_sve_st2ss_le_r_mte, 5105 gen_helper_sve_st2dd_le_r_mte }, 5106 { gen_helper_sve_st3bb_r_mte, 5107 gen_helper_sve_st3hh_le_r_mte, 5108 gen_helper_sve_st3ss_le_r_mte, 5109 gen_helper_sve_st3dd_le_r_mte }, 5110 { gen_helper_sve_st4bb_r_mte, 5111 gen_helper_sve_st4hh_le_r_mte, 5112 gen_helper_sve_st4ss_le_r_mte, 5113 gen_helper_sve_st4dd_le_r_mte } }, 5114 { { gen_helper_sve_st2bb_r_mte, 5115 gen_helper_sve_st2hh_be_r_mte, 5116 gen_helper_sve_st2ss_be_r_mte, 5117 gen_helper_sve_st2dd_be_r_mte }, 5118 { gen_helper_sve_st3bb_r_mte, 5119 gen_helper_sve_st3hh_be_r_mte, 5120 gen_helper_sve_st3ss_be_r_mte, 5121 gen_helper_sve_st3dd_be_r_mte }, 5122 { gen_helper_sve_st4bb_r_mte, 5123 gen_helper_sve_st4hh_be_r_mte, 5124 gen_helper_sve_st4ss_be_r_mte, 5125 gen_helper_sve_st4dd_be_r_mte } } }, 5126 }; 5127 gen_helper_gvec_mem *fn; 5128 int be = s->be_data == MO_BE; 5129 5130 if (nreg == 0) { 5131 /* ST1 */ 5132 fn = fn_single[s->mte_active[0]][be][msz][esz]; 5133 nreg = 1; 5134 } else { 5135 /* ST2, ST3, ST4 -- msz == esz, enforced by encoding */ 5136 assert(msz == esz); 5137 fn = fn_multiple[s->mte_active[0]][be][nreg - 1][msz]; 5138 } 5139 assert(fn != NULL); 5140 do_mem_zpa(s, zt, pg, addr, msz_dtype(s, msz), nreg, true, fn); 5141 } 5142 5143 static bool trans_ST_zprr(DisasContext *s, arg_rprr_store *a) 5144 { 5145 if (!dc_isar_feature(aa64_sve, s)) { 5146 return false; 5147 } 5148 if (a->rm == 31 || a->msz > a->esz) { 5149 return false; 5150 } 5151 if (sve_access_check(s)) { 5152 TCGv_i64 addr = tcg_temp_new_i64(); 5153 tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), a->msz); 5154 tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn)); 5155 do_st_zpa(s, a->rd, a->pg, addr, a->msz, a->esz, a->nreg); 5156 } 5157 return true; 5158 } 5159 5160 static bool trans_ST_zpri(DisasContext *s, arg_rpri_store *a) 5161 { 5162 if (!dc_isar_feature(aa64_sve, s)) { 5163 return false; 5164 } 5165 if (a->msz > a->esz) { 5166 return false; 5167 } 5168 if (sve_access_check(s)) { 5169 int vsz = vec_full_reg_size(s); 5170 int elements = vsz >> a->esz; 5171 TCGv_i64 addr = tcg_temp_new_i64(); 5172 5173 tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn), 5174 (a->imm * elements * (a->nreg + 1)) << a->msz); 5175 do_st_zpa(s, a->rd, a->pg, addr, a->msz, a->esz, a->nreg); 5176 } 5177 return true; 5178 } 5179 5180 /* 5181 *** SVE gather loads / scatter stores 5182 */ 5183 5184 static void do_mem_zpz(DisasContext *s, int zt, int pg, int zm, 5185 int scale, TCGv_i64 scalar, int msz, bool is_write, 5186 gen_helper_gvec_mem_scatter *fn) 5187 { 5188 unsigned vsz = vec_full_reg_size(s); 5189 TCGv_ptr t_zm = tcg_temp_new_ptr(); 5190 TCGv_ptr t_pg = tcg_temp_new_ptr(); 5191 TCGv_ptr t_zt = tcg_temp_new_ptr(); 5192 int desc = 0; 5193 5194 if (s->mte_active[0]) { 5195 desc = FIELD_DP32(desc, MTEDESC, MIDX, get_mem_index(s)); 5196 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid); 5197 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma); 5198 desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write); 5199 desc = FIELD_DP32(desc, MTEDESC, SIZEM1, (1 << msz) - 1); 5200 desc <<= SVE_MTEDESC_SHIFT; 5201 } 5202 desc = simd_desc(vsz, vsz, desc | scale); 5203 5204 tcg_gen_addi_ptr(t_pg, cpu_env, pred_full_reg_offset(s, pg)); 5205 tcg_gen_addi_ptr(t_zm, cpu_env, vec_full_reg_offset(s, zm)); 5206 tcg_gen_addi_ptr(t_zt, cpu_env, vec_full_reg_offset(s, zt)); 5207 fn(cpu_env, t_zt, t_pg, t_zm, scalar, tcg_constant_i32(desc)); 5208 } 5209 5210 /* Indexed by [mte][be][ff][xs][u][msz]. */ 5211 static gen_helper_gvec_mem_scatter * const 5212 gather_load_fn32[2][2][2][2][2][3] = { 5213 { /* MTE Inactive */ 5214 { /* Little-endian */ 5215 { { { gen_helper_sve_ldbss_zsu, 5216 gen_helper_sve_ldhss_le_zsu, 5217 NULL, }, 5218 { gen_helper_sve_ldbsu_zsu, 5219 gen_helper_sve_ldhsu_le_zsu, 5220 gen_helper_sve_ldss_le_zsu, } }, 5221 { { gen_helper_sve_ldbss_zss, 5222 gen_helper_sve_ldhss_le_zss, 5223 NULL, }, 5224 { gen_helper_sve_ldbsu_zss, 5225 gen_helper_sve_ldhsu_le_zss, 5226 gen_helper_sve_ldss_le_zss, } } }, 5227 5228 /* First-fault */ 5229 { { { gen_helper_sve_ldffbss_zsu, 5230 gen_helper_sve_ldffhss_le_zsu, 5231 NULL, }, 5232 { gen_helper_sve_ldffbsu_zsu, 5233 gen_helper_sve_ldffhsu_le_zsu, 5234 gen_helper_sve_ldffss_le_zsu, } }, 5235 { { gen_helper_sve_ldffbss_zss, 5236 gen_helper_sve_ldffhss_le_zss, 5237 NULL, }, 5238 { gen_helper_sve_ldffbsu_zss, 5239 gen_helper_sve_ldffhsu_le_zss, 5240 gen_helper_sve_ldffss_le_zss, } } } }, 5241 5242 { /* Big-endian */ 5243 { { { gen_helper_sve_ldbss_zsu, 5244 gen_helper_sve_ldhss_be_zsu, 5245 NULL, }, 5246 { gen_helper_sve_ldbsu_zsu, 5247 gen_helper_sve_ldhsu_be_zsu, 5248 gen_helper_sve_ldss_be_zsu, } }, 5249 { { gen_helper_sve_ldbss_zss, 5250 gen_helper_sve_ldhss_be_zss, 5251 NULL, }, 5252 { gen_helper_sve_ldbsu_zss, 5253 gen_helper_sve_ldhsu_be_zss, 5254 gen_helper_sve_ldss_be_zss, } } }, 5255 5256 /* First-fault */ 5257 { { { gen_helper_sve_ldffbss_zsu, 5258 gen_helper_sve_ldffhss_be_zsu, 5259 NULL, }, 5260 { gen_helper_sve_ldffbsu_zsu, 5261 gen_helper_sve_ldffhsu_be_zsu, 5262 gen_helper_sve_ldffss_be_zsu, } }, 5263 { { gen_helper_sve_ldffbss_zss, 5264 gen_helper_sve_ldffhss_be_zss, 5265 NULL, }, 5266 { gen_helper_sve_ldffbsu_zss, 5267 gen_helper_sve_ldffhsu_be_zss, 5268 gen_helper_sve_ldffss_be_zss, } } } } }, 5269 { /* MTE Active */ 5270 { /* Little-endian */ 5271 { { { gen_helper_sve_ldbss_zsu_mte, 5272 gen_helper_sve_ldhss_le_zsu_mte, 5273 NULL, }, 5274 { gen_helper_sve_ldbsu_zsu_mte, 5275 gen_helper_sve_ldhsu_le_zsu_mte, 5276 gen_helper_sve_ldss_le_zsu_mte, } }, 5277 { { gen_helper_sve_ldbss_zss_mte, 5278 gen_helper_sve_ldhss_le_zss_mte, 5279 NULL, }, 5280 { gen_helper_sve_ldbsu_zss_mte, 5281 gen_helper_sve_ldhsu_le_zss_mte, 5282 gen_helper_sve_ldss_le_zss_mte, } } }, 5283 5284 /* First-fault */ 5285 { { { gen_helper_sve_ldffbss_zsu_mte, 5286 gen_helper_sve_ldffhss_le_zsu_mte, 5287 NULL, }, 5288 { gen_helper_sve_ldffbsu_zsu_mte, 5289 gen_helper_sve_ldffhsu_le_zsu_mte, 5290 gen_helper_sve_ldffss_le_zsu_mte, } }, 5291 { { gen_helper_sve_ldffbss_zss_mte, 5292 gen_helper_sve_ldffhss_le_zss_mte, 5293 NULL, }, 5294 { gen_helper_sve_ldffbsu_zss_mte, 5295 gen_helper_sve_ldffhsu_le_zss_mte, 5296 gen_helper_sve_ldffss_le_zss_mte, } } } }, 5297 5298 { /* Big-endian */ 5299 { { { gen_helper_sve_ldbss_zsu_mte, 5300 gen_helper_sve_ldhss_be_zsu_mte, 5301 NULL, }, 5302 { gen_helper_sve_ldbsu_zsu_mte, 5303 gen_helper_sve_ldhsu_be_zsu_mte, 5304 gen_helper_sve_ldss_be_zsu_mte, } }, 5305 { { gen_helper_sve_ldbss_zss_mte, 5306 gen_helper_sve_ldhss_be_zss_mte, 5307 NULL, }, 5308 { gen_helper_sve_ldbsu_zss_mte, 5309 gen_helper_sve_ldhsu_be_zss_mte, 5310 gen_helper_sve_ldss_be_zss_mte, } } }, 5311 5312 /* First-fault */ 5313 { { { gen_helper_sve_ldffbss_zsu_mte, 5314 gen_helper_sve_ldffhss_be_zsu_mte, 5315 NULL, }, 5316 { gen_helper_sve_ldffbsu_zsu_mte, 5317 gen_helper_sve_ldffhsu_be_zsu_mte, 5318 gen_helper_sve_ldffss_be_zsu_mte, } }, 5319 { { gen_helper_sve_ldffbss_zss_mte, 5320 gen_helper_sve_ldffhss_be_zss_mte, 5321 NULL, }, 5322 { gen_helper_sve_ldffbsu_zss_mte, 5323 gen_helper_sve_ldffhsu_be_zss_mte, 5324 gen_helper_sve_ldffss_be_zss_mte, } } } } }, 5325 }; 5326 5327 /* Note that we overload xs=2 to indicate 64-bit offset. */ 5328 static gen_helper_gvec_mem_scatter * const 5329 gather_load_fn64[2][2][2][3][2][4] = { 5330 { /* MTE Inactive */ 5331 { /* Little-endian */ 5332 { { { gen_helper_sve_ldbds_zsu, 5333 gen_helper_sve_ldhds_le_zsu, 5334 gen_helper_sve_ldsds_le_zsu, 5335 NULL, }, 5336 { gen_helper_sve_ldbdu_zsu, 5337 gen_helper_sve_ldhdu_le_zsu, 5338 gen_helper_sve_ldsdu_le_zsu, 5339 gen_helper_sve_lddd_le_zsu, } }, 5340 { { gen_helper_sve_ldbds_zss, 5341 gen_helper_sve_ldhds_le_zss, 5342 gen_helper_sve_ldsds_le_zss, 5343 NULL, }, 5344 { gen_helper_sve_ldbdu_zss, 5345 gen_helper_sve_ldhdu_le_zss, 5346 gen_helper_sve_ldsdu_le_zss, 5347 gen_helper_sve_lddd_le_zss, } }, 5348 { { gen_helper_sve_ldbds_zd, 5349 gen_helper_sve_ldhds_le_zd, 5350 gen_helper_sve_ldsds_le_zd, 5351 NULL, }, 5352 { gen_helper_sve_ldbdu_zd, 5353 gen_helper_sve_ldhdu_le_zd, 5354 gen_helper_sve_ldsdu_le_zd, 5355 gen_helper_sve_lddd_le_zd, } } }, 5356 5357 /* First-fault */ 5358 { { { gen_helper_sve_ldffbds_zsu, 5359 gen_helper_sve_ldffhds_le_zsu, 5360 gen_helper_sve_ldffsds_le_zsu, 5361 NULL, }, 5362 { gen_helper_sve_ldffbdu_zsu, 5363 gen_helper_sve_ldffhdu_le_zsu, 5364 gen_helper_sve_ldffsdu_le_zsu, 5365 gen_helper_sve_ldffdd_le_zsu, } }, 5366 { { gen_helper_sve_ldffbds_zss, 5367 gen_helper_sve_ldffhds_le_zss, 5368 gen_helper_sve_ldffsds_le_zss, 5369 NULL, }, 5370 { gen_helper_sve_ldffbdu_zss, 5371 gen_helper_sve_ldffhdu_le_zss, 5372 gen_helper_sve_ldffsdu_le_zss, 5373 gen_helper_sve_ldffdd_le_zss, } }, 5374 { { gen_helper_sve_ldffbds_zd, 5375 gen_helper_sve_ldffhds_le_zd, 5376 gen_helper_sve_ldffsds_le_zd, 5377 NULL, }, 5378 { gen_helper_sve_ldffbdu_zd, 5379 gen_helper_sve_ldffhdu_le_zd, 5380 gen_helper_sve_ldffsdu_le_zd, 5381 gen_helper_sve_ldffdd_le_zd, } } } }, 5382 { /* Big-endian */ 5383 { { { gen_helper_sve_ldbds_zsu, 5384 gen_helper_sve_ldhds_be_zsu, 5385 gen_helper_sve_ldsds_be_zsu, 5386 NULL, }, 5387 { gen_helper_sve_ldbdu_zsu, 5388 gen_helper_sve_ldhdu_be_zsu, 5389 gen_helper_sve_ldsdu_be_zsu, 5390 gen_helper_sve_lddd_be_zsu, } }, 5391 { { gen_helper_sve_ldbds_zss, 5392 gen_helper_sve_ldhds_be_zss, 5393 gen_helper_sve_ldsds_be_zss, 5394 NULL, }, 5395 { gen_helper_sve_ldbdu_zss, 5396 gen_helper_sve_ldhdu_be_zss, 5397 gen_helper_sve_ldsdu_be_zss, 5398 gen_helper_sve_lddd_be_zss, } }, 5399 { { gen_helper_sve_ldbds_zd, 5400 gen_helper_sve_ldhds_be_zd, 5401 gen_helper_sve_ldsds_be_zd, 5402 NULL, }, 5403 { gen_helper_sve_ldbdu_zd, 5404 gen_helper_sve_ldhdu_be_zd, 5405 gen_helper_sve_ldsdu_be_zd, 5406 gen_helper_sve_lddd_be_zd, } } }, 5407 5408 /* First-fault */ 5409 { { { gen_helper_sve_ldffbds_zsu, 5410 gen_helper_sve_ldffhds_be_zsu, 5411 gen_helper_sve_ldffsds_be_zsu, 5412 NULL, }, 5413 { gen_helper_sve_ldffbdu_zsu, 5414 gen_helper_sve_ldffhdu_be_zsu, 5415 gen_helper_sve_ldffsdu_be_zsu, 5416 gen_helper_sve_ldffdd_be_zsu, } }, 5417 { { gen_helper_sve_ldffbds_zss, 5418 gen_helper_sve_ldffhds_be_zss, 5419 gen_helper_sve_ldffsds_be_zss, 5420 NULL, }, 5421 { gen_helper_sve_ldffbdu_zss, 5422 gen_helper_sve_ldffhdu_be_zss, 5423 gen_helper_sve_ldffsdu_be_zss, 5424 gen_helper_sve_ldffdd_be_zss, } }, 5425 { { gen_helper_sve_ldffbds_zd, 5426 gen_helper_sve_ldffhds_be_zd, 5427 gen_helper_sve_ldffsds_be_zd, 5428 NULL, }, 5429 { gen_helper_sve_ldffbdu_zd, 5430 gen_helper_sve_ldffhdu_be_zd, 5431 gen_helper_sve_ldffsdu_be_zd, 5432 gen_helper_sve_ldffdd_be_zd, } } } } }, 5433 { /* MTE Active */ 5434 { /* Little-endian */ 5435 { { { gen_helper_sve_ldbds_zsu_mte, 5436 gen_helper_sve_ldhds_le_zsu_mte, 5437 gen_helper_sve_ldsds_le_zsu_mte, 5438 NULL, }, 5439 { gen_helper_sve_ldbdu_zsu_mte, 5440 gen_helper_sve_ldhdu_le_zsu_mte, 5441 gen_helper_sve_ldsdu_le_zsu_mte, 5442 gen_helper_sve_lddd_le_zsu_mte, } }, 5443 { { gen_helper_sve_ldbds_zss_mte, 5444 gen_helper_sve_ldhds_le_zss_mte, 5445 gen_helper_sve_ldsds_le_zss_mte, 5446 NULL, }, 5447 { gen_helper_sve_ldbdu_zss_mte, 5448 gen_helper_sve_ldhdu_le_zss_mte, 5449 gen_helper_sve_ldsdu_le_zss_mte, 5450 gen_helper_sve_lddd_le_zss_mte, } }, 5451 { { gen_helper_sve_ldbds_zd_mte, 5452 gen_helper_sve_ldhds_le_zd_mte, 5453 gen_helper_sve_ldsds_le_zd_mte, 5454 NULL, }, 5455 { gen_helper_sve_ldbdu_zd_mte, 5456 gen_helper_sve_ldhdu_le_zd_mte, 5457 gen_helper_sve_ldsdu_le_zd_mte, 5458 gen_helper_sve_lddd_le_zd_mte, } } }, 5459 5460 /* First-fault */ 5461 { { { gen_helper_sve_ldffbds_zsu_mte, 5462 gen_helper_sve_ldffhds_le_zsu_mte, 5463 gen_helper_sve_ldffsds_le_zsu_mte, 5464 NULL, }, 5465 { gen_helper_sve_ldffbdu_zsu_mte, 5466 gen_helper_sve_ldffhdu_le_zsu_mte, 5467 gen_helper_sve_ldffsdu_le_zsu_mte, 5468 gen_helper_sve_ldffdd_le_zsu_mte, } }, 5469 { { gen_helper_sve_ldffbds_zss_mte, 5470 gen_helper_sve_ldffhds_le_zss_mte, 5471 gen_helper_sve_ldffsds_le_zss_mte, 5472 NULL, }, 5473 { gen_helper_sve_ldffbdu_zss_mte, 5474 gen_helper_sve_ldffhdu_le_zss_mte, 5475 gen_helper_sve_ldffsdu_le_zss_mte, 5476 gen_helper_sve_ldffdd_le_zss_mte, } }, 5477 { { gen_helper_sve_ldffbds_zd_mte, 5478 gen_helper_sve_ldffhds_le_zd_mte, 5479 gen_helper_sve_ldffsds_le_zd_mte, 5480 NULL, }, 5481 { gen_helper_sve_ldffbdu_zd_mte, 5482 gen_helper_sve_ldffhdu_le_zd_mte, 5483 gen_helper_sve_ldffsdu_le_zd_mte, 5484 gen_helper_sve_ldffdd_le_zd_mte, } } } }, 5485 { /* Big-endian */ 5486 { { { gen_helper_sve_ldbds_zsu_mte, 5487 gen_helper_sve_ldhds_be_zsu_mte, 5488 gen_helper_sve_ldsds_be_zsu_mte, 5489 NULL, }, 5490 { gen_helper_sve_ldbdu_zsu_mte, 5491 gen_helper_sve_ldhdu_be_zsu_mte, 5492 gen_helper_sve_ldsdu_be_zsu_mte, 5493 gen_helper_sve_lddd_be_zsu_mte, } }, 5494 { { gen_helper_sve_ldbds_zss_mte, 5495 gen_helper_sve_ldhds_be_zss_mte, 5496 gen_helper_sve_ldsds_be_zss_mte, 5497 NULL, }, 5498 { gen_helper_sve_ldbdu_zss_mte, 5499 gen_helper_sve_ldhdu_be_zss_mte, 5500 gen_helper_sve_ldsdu_be_zss_mte, 5501 gen_helper_sve_lddd_be_zss_mte, } }, 5502 { { gen_helper_sve_ldbds_zd_mte, 5503 gen_helper_sve_ldhds_be_zd_mte, 5504 gen_helper_sve_ldsds_be_zd_mte, 5505 NULL, }, 5506 { gen_helper_sve_ldbdu_zd_mte, 5507 gen_helper_sve_ldhdu_be_zd_mte, 5508 gen_helper_sve_ldsdu_be_zd_mte, 5509 gen_helper_sve_lddd_be_zd_mte, } } }, 5510 5511 /* First-fault */ 5512 { { { gen_helper_sve_ldffbds_zsu_mte, 5513 gen_helper_sve_ldffhds_be_zsu_mte, 5514 gen_helper_sve_ldffsds_be_zsu_mte, 5515 NULL, }, 5516 { gen_helper_sve_ldffbdu_zsu_mte, 5517 gen_helper_sve_ldffhdu_be_zsu_mte, 5518 gen_helper_sve_ldffsdu_be_zsu_mte, 5519 gen_helper_sve_ldffdd_be_zsu_mte, } }, 5520 { { gen_helper_sve_ldffbds_zss_mte, 5521 gen_helper_sve_ldffhds_be_zss_mte, 5522 gen_helper_sve_ldffsds_be_zss_mte, 5523 NULL, }, 5524 { gen_helper_sve_ldffbdu_zss_mte, 5525 gen_helper_sve_ldffhdu_be_zss_mte, 5526 gen_helper_sve_ldffsdu_be_zss_mte, 5527 gen_helper_sve_ldffdd_be_zss_mte, } }, 5528 { { gen_helper_sve_ldffbds_zd_mte, 5529 gen_helper_sve_ldffhds_be_zd_mte, 5530 gen_helper_sve_ldffsds_be_zd_mte, 5531 NULL, }, 5532 { gen_helper_sve_ldffbdu_zd_mte, 5533 gen_helper_sve_ldffhdu_be_zd_mte, 5534 gen_helper_sve_ldffsdu_be_zd_mte, 5535 gen_helper_sve_ldffdd_be_zd_mte, } } } } }, 5536 }; 5537 5538 static bool trans_LD1_zprz(DisasContext *s, arg_LD1_zprz *a) 5539 { 5540 gen_helper_gvec_mem_scatter *fn = NULL; 5541 bool be = s->be_data == MO_BE; 5542 bool mte = s->mte_active[0]; 5543 5544 if (!dc_isar_feature(aa64_sve, s)) { 5545 return false; 5546 } 5547 s->is_nonstreaming = true; 5548 if (!sve_access_check(s)) { 5549 return true; 5550 } 5551 5552 switch (a->esz) { 5553 case MO_32: 5554 fn = gather_load_fn32[mte][be][a->ff][a->xs][a->u][a->msz]; 5555 break; 5556 case MO_64: 5557 fn = gather_load_fn64[mte][be][a->ff][a->xs][a->u][a->msz]; 5558 break; 5559 } 5560 assert(fn != NULL); 5561 5562 do_mem_zpz(s, a->rd, a->pg, a->rm, a->scale * a->msz, 5563 cpu_reg_sp(s, a->rn), a->msz, false, fn); 5564 return true; 5565 } 5566 5567 static bool trans_LD1_zpiz(DisasContext *s, arg_LD1_zpiz *a) 5568 { 5569 gen_helper_gvec_mem_scatter *fn = NULL; 5570 bool be = s->be_data == MO_BE; 5571 bool mte = s->mte_active[0]; 5572 5573 if (a->esz < a->msz || (a->esz == a->msz && !a->u)) { 5574 return false; 5575 } 5576 if (!dc_isar_feature(aa64_sve, s)) { 5577 return false; 5578 } 5579 s->is_nonstreaming = true; 5580 if (!sve_access_check(s)) { 5581 return true; 5582 } 5583 5584 switch (a->esz) { 5585 case MO_32: 5586 fn = gather_load_fn32[mte][be][a->ff][0][a->u][a->msz]; 5587 break; 5588 case MO_64: 5589 fn = gather_load_fn64[mte][be][a->ff][2][a->u][a->msz]; 5590 break; 5591 } 5592 assert(fn != NULL); 5593 5594 /* Treat LD1_zpiz (zn[x] + imm) the same way as LD1_zprz (rn + zm[x]) 5595 * by loading the immediate into the scalar parameter. 5596 */ 5597 do_mem_zpz(s, a->rd, a->pg, a->rn, 0, 5598 tcg_constant_i64(a->imm << a->msz), a->msz, false, fn); 5599 return true; 5600 } 5601 5602 static bool trans_LDNT1_zprz(DisasContext *s, arg_LD1_zprz *a) 5603 { 5604 gen_helper_gvec_mem_scatter *fn = NULL; 5605 bool be = s->be_data == MO_BE; 5606 bool mte = s->mte_active[0]; 5607 5608 if (a->esz < a->msz + !a->u) { 5609 return false; 5610 } 5611 if (!dc_isar_feature(aa64_sve2, s)) { 5612 return false; 5613 } 5614 s->is_nonstreaming = true; 5615 if (!sve_access_check(s)) { 5616 return true; 5617 } 5618 5619 switch (a->esz) { 5620 case MO_32: 5621 fn = gather_load_fn32[mte][be][0][0][a->u][a->msz]; 5622 break; 5623 case MO_64: 5624 fn = gather_load_fn64[mte][be][0][2][a->u][a->msz]; 5625 break; 5626 } 5627 assert(fn != NULL); 5628 5629 do_mem_zpz(s, a->rd, a->pg, a->rn, 0, 5630 cpu_reg(s, a->rm), a->msz, false, fn); 5631 return true; 5632 } 5633 5634 /* Indexed by [mte][be][xs][msz]. */ 5635 static gen_helper_gvec_mem_scatter * const scatter_store_fn32[2][2][2][3] = { 5636 { /* MTE Inactive */ 5637 { /* Little-endian */ 5638 { gen_helper_sve_stbs_zsu, 5639 gen_helper_sve_sths_le_zsu, 5640 gen_helper_sve_stss_le_zsu, }, 5641 { gen_helper_sve_stbs_zss, 5642 gen_helper_sve_sths_le_zss, 5643 gen_helper_sve_stss_le_zss, } }, 5644 { /* Big-endian */ 5645 { gen_helper_sve_stbs_zsu, 5646 gen_helper_sve_sths_be_zsu, 5647 gen_helper_sve_stss_be_zsu, }, 5648 { gen_helper_sve_stbs_zss, 5649 gen_helper_sve_sths_be_zss, 5650 gen_helper_sve_stss_be_zss, } } }, 5651 { /* MTE Active */ 5652 { /* Little-endian */ 5653 { gen_helper_sve_stbs_zsu_mte, 5654 gen_helper_sve_sths_le_zsu_mte, 5655 gen_helper_sve_stss_le_zsu_mte, }, 5656 { gen_helper_sve_stbs_zss_mte, 5657 gen_helper_sve_sths_le_zss_mte, 5658 gen_helper_sve_stss_le_zss_mte, } }, 5659 { /* Big-endian */ 5660 { gen_helper_sve_stbs_zsu_mte, 5661 gen_helper_sve_sths_be_zsu_mte, 5662 gen_helper_sve_stss_be_zsu_mte, }, 5663 { gen_helper_sve_stbs_zss_mte, 5664 gen_helper_sve_sths_be_zss_mte, 5665 gen_helper_sve_stss_be_zss_mte, } } }, 5666 }; 5667 5668 /* Note that we overload xs=2 to indicate 64-bit offset. */ 5669 static gen_helper_gvec_mem_scatter * const scatter_store_fn64[2][2][3][4] = { 5670 { /* MTE Inactive */ 5671 { /* Little-endian */ 5672 { gen_helper_sve_stbd_zsu, 5673 gen_helper_sve_sthd_le_zsu, 5674 gen_helper_sve_stsd_le_zsu, 5675 gen_helper_sve_stdd_le_zsu, }, 5676 { gen_helper_sve_stbd_zss, 5677 gen_helper_sve_sthd_le_zss, 5678 gen_helper_sve_stsd_le_zss, 5679 gen_helper_sve_stdd_le_zss, }, 5680 { gen_helper_sve_stbd_zd, 5681 gen_helper_sve_sthd_le_zd, 5682 gen_helper_sve_stsd_le_zd, 5683 gen_helper_sve_stdd_le_zd, } }, 5684 { /* Big-endian */ 5685 { gen_helper_sve_stbd_zsu, 5686 gen_helper_sve_sthd_be_zsu, 5687 gen_helper_sve_stsd_be_zsu, 5688 gen_helper_sve_stdd_be_zsu, }, 5689 { gen_helper_sve_stbd_zss, 5690 gen_helper_sve_sthd_be_zss, 5691 gen_helper_sve_stsd_be_zss, 5692 gen_helper_sve_stdd_be_zss, }, 5693 { gen_helper_sve_stbd_zd, 5694 gen_helper_sve_sthd_be_zd, 5695 gen_helper_sve_stsd_be_zd, 5696 gen_helper_sve_stdd_be_zd, } } }, 5697 { /* MTE Inactive */ 5698 { /* Little-endian */ 5699 { gen_helper_sve_stbd_zsu_mte, 5700 gen_helper_sve_sthd_le_zsu_mte, 5701 gen_helper_sve_stsd_le_zsu_mte, 5702 gen_helper_sve_stdd_le_zsu_mte, }, 5703 { gen_helper_sve_stbd_zss_mte, 5704 gen_helper_sve_sthd_le_zss_mte, 5705 gen_helper_sve_stsd_le_zss_mte, 5706 gen_helper_sve_stdd_le_zss_mte, }, 5707 { gen_helper_sve_stbd_zd_mte, 5708 gen_helper_sve_sthd_le_zd_mte, 5709 gen_helper_sve_stsd_le_zd_mte, 5710 gen_helper_sve_stdd_le_zd_mte, } }, 5711 { /* Big-endian */ 5712 { gen_helper_sve_stbd_zsu_mte, 5713 gen_helper_sve_sthd_be_zsu_mte, 5714 gen_helper_sve_stsd_be_zsu_mte, 5715 gen_helper_sve_stdd_be_zsu_mte, }, 5716 { gen_helper_sve_stbd_zss_mte, 5717 gen_helper_sve_sthd_be_zss_mte, 5718 gen_helper_sve_stsd_be_zss_mte, 5719 gen_helper_sve_stdd_be_zss_mte, }, 5720 { gen_helper_sve_stbd_zd_mte, 5721 gen_helper_sve_sthd_be_zd_mte, 5722 gen_helper_sve_stsd_be_zd_mte, 5723 gen_helper_sve_stdd_be_zd_mte, } } }, 5724 }; 5725 5726 static bool trans_ST1_zprz(DisasContext *s, arg_ST1_zprz *a) 5727 { 5728 gen_helper_gvec_mem_scatter *fn; 5729 bool be = s->be_data == MO_BE; 5730 bool mte = s->mte_active[0]; 5731 5732 if (a->esz < a->msz || (a->msz == 0 && a->scale)) { 5733 return false; 5734 } 5735 if (!dc_isar_feature(aa64_sve, s)) { 5736 return false; 5737 } 5738 s->is_nonstreaming = true; 5739 if (!sve_access_check(s)) { 5740 return true; 5741 } 5742 switch (a->esz) { 5743 case MO_32: 5744 fn = scatter_store_fn32[mte][be][a->xs][a->msz]; 5745 break; 5746 case MO_64: 5747 fn = scatter_store_fn64[mte][be][a->xs][a->msz]; 5748 break; 5749 default: 5750 g_assert_not_reached(); 5751 } 5752 do_mem_zpz(s, a->rd, a->pg, a->rm, a->scale * a->msz, 5753 cpu_reg_sp(s, a->rn), a->msz, true, fn); 5754 return true; 5755 } 5756 5757 static bool trans_ST1_zpiz(DisasContext *s, arg_ST1_zpiz *a) 5758 { 5759 gen_helper_gvec_mem_scatter *fn = NULL; 5760 bool be = s->be_data == MO_BE; 5761 bool mte = s->mte_active[0]; 5762 5763 if (a->esz < a->msz) { 5764 return false; 5765 } 5766 if (!dc_isar_feature(aa64_sve, s)) { 5767 return false; 5768 } 5769 s->is_nonstreaming = true; 5770 if (!sve_access_check(s)) { 5771 return true; 5772 } 5773 5774 switch (a->esz) { 5775 case MO_32: 5776 fn = scatter_store_fn32[mte][be][0][a->msz]; 5777 break; 5778 case MO_64: 5779 fn = scatter_store_fn64[mte][be][2][a->msz]; 5780 break; 5781 } 5782 assert(fn != NULL); 5783 5784 /* Treat ST1_zpiz (zn[x] + imm) the same way as ST1_zprz (rn + zm[x]) 5785 * by loading the immediate into the scalar parameter. 5786 */ 5787 do_mem_zpz(s, a->rd, a->pg, a->rn, 0, 5788 tcg_constant_i64(a->imm << a->msz), a->msz, true, fn); 5789 return true; 5790 } 5791 5792 static bool trans_STNT1_zprz(DisasContext *s, arg_ST1_zprz *a) 5793 { 5794 gen_helper_gvec_mem_scatter *fn; 5795 bool be = s->be_data == MO_BE; 5796 bool mte = s->mte_active[0]; 5797 5798 if (a->esz < a->msz) { 5799 return false; 5800 } 5801 if (!dc_isar_feature(aa64_sve2, s)) { 5802 return false; 5803 } 5804 s->is_nonstreaming = true; 5805 if (!sve_access_check(s)) { 5806 return true; 5807 } 5808 5809 switch (a->esz) { 5810 case MO_32: 5811 fn = scatter_store_fn32[mte][be][0][a->msz]; 5812 break; 5813 case MO_64: 5814 fn = scatter_store_fn64[mte][be][2][a->msz]; 5815 break; 5816 default: 5817 g_assert_not_reached(); 5818 } 5819 5820 do_mem_zpz(s, a->rd, a->pg, a->rn, 0, 5821 cpu_reg(s, a->rm), a->msz, true, fn); 5822 return true; 5823 } 5824 5825 /* 5826 * Prefetches 5827 */ 5828 5829 static bool trans_PRF(DisasContext *s, arg_PRF *a) 5830 { 5831 if (!dc_isar_feature(aa64_sve, s)) { 5832 return false; 5833 } 5834 /* Prefetch is a nop within QEMU. */ 5835 (void)sve_access_check(s); 5836 return true; 5837 } 5838 5839 static bool trans_PRF_rr(DisasContext *s, arg_PRF_rr *a) 5840 { 5841 if (a->rm == 31 || !dc_isar_feature(aa64_sve, s)) { 5842 return false; 5843 } 5844 /* Prefetch is a nop within QEMU. */ 5845 (void)sve_access_check(s); 5846 return true; 5847 } 5848 5849 static bool trans_PRF_ns(DisasContext *s, arg_PRF_ns *a) 5850 { 5851 if (!dc_isar_feature(aa64_sve, s)) { 5852 return false; 5853 } 5854 /* Prefetch is a nop within QEMU. */ 5855 s->is_nonstreaming = true; 5856 (void)sve_access_check(s); 5857 return true; 5858 } 5859 5860 /* 5861 * Move Prefix 5862 * 5863 * TODO: The implementation so far could handle predicated merging movprfx. 5864 * The helper functions as written take an extra source register to 5865 * use in the operation, but the result is only written when predication 5866 * succeeds. For unpredicated movprfx, we need to rearrange the helpers 5867 * to allow the final write back to the destination to be unconditional. 5868 * For predicated zeroing movprfx, we need to rearrange the helpers to 5869 * allow the final write back to zero inactives. 5870 * 5871 * In the meantime, just emit the moves. 5872 */ 5873 5874 TRANS_FEAT(MOVPRFX, aa64_sve, do_mov_z, a->rd, a->rn) 5875 TRANS_FEAT(MOVPRFX_m, aa64_sve, do_sel_z, a->rd, a->rn, a->rd, a->pg, a->esz) 5876 TRANS_FEAT(MOVPRFX_z, aa64_sve, do_movz_zpz, a->rd, a->rn, a->pg, a->esz, false) 5877 5878 /* 5879 * SVE2 Integer Multiply - Unpredicated 5880 */ 5881 5882 TRANS_FEAT(MUL_zzz, aa64_sve2, gen_gvec_fn_arg_zzz, tcg_gen_gvec_mul, a) 5883 5884 static gen_helper_gvec_3 * const smulh_zzz_fns[4] = { 5885 gen_helper_gvec_smulh_b, gen_helper_gvec_smulh_h, 5886 gen_helper_gvec_smulh_s, gen_helper_gvec_smulh_d, 5887 }; 5888 TRANS_FEAT(SMULH_zzz, aa64_sve2, gen_gvec_ool_arg_zzz, 5889 smulh_zzz_fns[a->esz], a, 0) 5890 5891 static gen_helper_gvec_3 * const umulh_zzz_fns[4] = { 5892 gen_helper_gvec_umulh_b, gen_helper_gvec_umulh_h, 5893 gen_helper_gvec_umulh_s, gen_helper_gvec_umulh_d, 5894 }; 5895 TRANS_FEAT(UMULH_zzz, aa64_sve2, gen_gvec_ool_arg_zzz, 5896 umulh_zzz_fns[a->esz], a, 0) 5897 5898 TRANS_FEAT(PMUL_zzz, aa64_sve2, gen_gvec_ool_arg_zzz, 5899 gen_helper_gvec_pmul_b, a, 0) 5900 5901 static gen_helper_gvec_3 * const sqdmulh_zzz_fns[4] = { 5902 gen_helper_sve2_sqdmulh_b, gen_helper_sve2_sqdmulh_h, 5903 gen_helper_sve2_sqdmulh_s, gen_helper_sve2_sqdmulh_d, 5904 }; 5905 TRANS_FEAT(SQDMULH_zzz, aa64_sve2, gen_gvec_ool_arg_zzz, 5906 sqdmulh_zzz_fns[a->esz], a, 0) 5907 5908 static gen_helper_gvec_3 * const sqrdmulh_zzz_fns[4] = { 5909 gen_helper_sve2_sqrdmulh_b, gen_helper_sve2_sqrdmulh_h, 5910 gen_helper_sve2_sqrdmulh_s, gen_helper_sve2_sqrdmulh_d, 5911 }; 5912 TRANS_FEAT(SQRDMULH_zzz, aa64_sve2, gen_gvec_ool_arg_zzz, 5913 sqrdmulh_zzz_fns[a->esz], a, 0) 5914 5915 /* 5916 * SVE2 Integer - Predicated 5917 */ 5918 5919 static gen_helper_gvec_4 * const sadlp_fns[4] = { 5920 NULL, gen_helper_sve2_sadalp_zpzz_h, 5921 gen_helper_sve2_sadalp_zpzz_s, gen_helper_sve2_sadalp_zpzz_d, 5922 }; 5923 TRANS_FEAT(SADALP_zpzz, aa64_sve2, gen_gvec_ool_arg_zpzz, 5924 sadlp_fns[a->esz], a, 0) 5925 5926 static gen_helper_gvec_4 * const uadlp_fns[4] = { 5927 NULL, gen_helper_sve2_uadalp_zpzz_h, 5928 gen_helper_sve2_uadalp_zpzz_s, gen_helper_sve2_uadalp_zpzz_d, 5929 }; 5930 TRANS_FEAT(UADALP_zpzz, aa64_sve2, gen_gvec_ool_arg_zpzz, 5931 uadlp_fns[a->esz], a, 0) 5932 5933 /* 5934 * SVE2 integer unary operations (predicated) 5935 */ 5936 5937 TRANS_FEAT(URECPE, aa64_sve2, gen_gvec_ool_arg_zpz, 5938 a->esz == 2 ? gen_helper_sve2_urecpe_s : NULL, a, 0) 5939 5940 TRANS_FEAT(URSQRTE, aa64_sve2, gen_gvec_ool_arg_zpz, 5941 a->esz == 2 ? gen_helper_sve2_ursqrte_s : NULL, a, 0) 5942 5943 static gen_helper_gvec_3 * const sqabs_fns[4] = { 5944 gen_helper_sve2_sqabs_b, gen_helper_sve2_sqabs_h, 5945 gen_helper_sve2_sqabs_s, gen_helper_sve2_sqabs_d, 5946 }; 5947 TRANS_FEAT(SQABS, aa64_sve2, gen_gvec_ool_arg_zpz, sqabs_fns[a->esz], a, 0) 5948 5949 static gen_helper_gvec_3 * const sqneg_fns[4] = { 5950 gen_helper_sve2_sqneg_b, gen_helper_sve2_sqneg_h, 5951 gen_helper_sve2_sqneg_s, gen_helper_sve2_sqneg_d, 5952 }; 5953 TRANS_FEAT(SQNEG, aa64_sve2, gen_gvec_ool_arg_zpz, sqneg_fns[a->esz], a, 0) 5954 5955 DO_ZPZZ(SQSHL, aa64_sve2, sve2_sqshl) 5956 DO_ZPZZ(SQRSHL, aa64_sve2, sve2_sqrshl) 5957 DO_ZPZZ(SRSHL, aa64_sve2, sve2_srshl) 5958 5959 DO_ZPZZ(UQSHL, aa64_sve2, sve2_uqshl) 5960 DO_ZPZZ(UQRSHL, aa64_sve2, sve2_uqrshl) 5961 DO_ZPZZ(URSHL, aa64_sve2, sve2_urshl) 5962 5963 DO_ZPZZ(SHADD, aa64_sve2, sve2_shadd) 5964 DO_ZPZZ(SRHADD, aa64_sve2, sve2_srhadd) 5965 DO_ZPZZ(SHSUB, aa64_sve2, sve2_shsub) 5966 5967 DO_ZPZZ(UHADD, aa64_sve2, sve2_uhadd) 5968 DO_ZPZZ(URHADD, aa64_sve2, sve2_urhadd) 5969 DO_ZPZZ(UHSUB, aa64_sve2, sve2_uhsub) 5970 5971 DO_ZPZZ(ADDP, aa64_sve2, sve2_addp) 5972 DO_ZPZZ(SMAXP, aa64_sve2, sve2_smaxp) 5973 DO_ZPZZ(UMAXP, aa64_sve2, sve2_umaxp) 5974 DO_ZPZZ(SMINP, aa64_sve2, sve2_sminp) 5975 DO_ZPZZ(UMINP, aa64_sve2, sve2_uminp) 5976 5977 DO_ZPZZ(SQADD_zpzz, aa64_sve2, sve2_sqadd) 5978 DO_ZPZZ(UQADD_zpzz, aa64_sve2, sve2_uqadd) 5979 DO_ZPZZ(SQSUB_zpzz, aa64_sve2, sve2_sqsub) 5980 DO_ZPZZ(UQSUB_zpzz, aa64_sve2, sve2_uqsub) 5981 DO_ZPZZ(SUQADD, aa64_sve2, sve2_suqadd) 5982 DO_ZPZZ(USQADD, aa64_sve2, sve2_usqadd) 5983 5984 /* 5985 * SVE2 Widening Integer Arithmetic 5986 */ 5987 5988 static gen_helper_gvec_3 * const saddl_fns[4] = { 5989 NULL, gen_helper_sve2_saddl_h, 5990 gen_helper_sve2_saddl_s, gen_helper_sve2_saddl_d, 5991 }; 5992 TRANS_FEAT(SADDLB, aa64_sve2, gen_gvec_ool_arg_zzz, 5993 saddl_fns[a->esz], a, 0) 5994 TRANS_FEAT(SADDLT, aa64_sve2, gen_gvec_ool_arg_zzz, 5995 saddl_fns[a->esz], a, 3) 5996 TRANS_FEAT(SADDLBT, aa64_sve2, gen_gvec_ool_arg_zzz, 5997 saddl_fns[a->esz], a, 2) 5998 5999 static gen_helper_gvec_3 * const ssubl_fns[4] = { 6000 NULL, gen_helper_sve2_ssubl_h, 6001 gen_helper_sve2_ssubl_s, gen_helper_sve2_ssubl_d, 6002 }; 6003 TRANS_FEAT(SSUBLB, aa64_sve2, gen_gvec_ool_arg_zzz, 6004 ssubl_fns[a->esz], a, 0) 6005 TRANS_FEAT(SSUBLT, aa64_sve2, gen_gvec_ool_arg_zzz, 6006 ssubl_fns[a->esz], a, 3) 6007 TRANS_FEAT(SSUBLBT, aa64_sve2, gen_gvec_ool_arg_zzz, 6008 ssubl_fns[a->esz], a, 2) 6009 TRANS_FEAT(SSUBLTB, aa64_sve2, gen_gvec_ool_arg_zzz, 6010 ssubl_fns[a->esz], a, 1) 6011 6012 static gen_helper_gvec_3 * const sabdl_fns[4] = { 6013 NULL, gen_helper_sve2_sabdl_h, 6014 gen_helper_sve2_sabdl_s, gen_helper_sve2_sabdl_d, 6015 }; 6016 TRANS_FEAT(SABDLB, aa64_sve2, gen_gvec_ool_arg_zzz, 6017 sabdl_fns[a->esz], a, 0) 6018 TRANS_FEAT(SABDLT, aa64_sve2, gen_gvec_ool_arg_zzz, 6019 sabdl_fns[a->esz], a, 3) 6020 6021 static gen_helper_gvec_3 * const uaddl_fns[4] = { 6022 NULL, gen_helper_sve2_uaddl_h, 6023 gen_helper_sve2_uaddl_s, gen_helper_sve2_uaddl_d, 6024 }; 6025 TRANS_FEAT(UADDLB, aa64_sve2, gen_gvec_ool_arg_zzz, 6026 uaddl_fns[a->esz], a, 0) 6027 TRANS_FEAT(UADDLT, aa64_sve2, gen_gvec_ool_arg_zzz, 6028 uaddl_fns[a->esz], a, 3) 6029 6030 static gen_helper_gvec_3 * const usubl_fns[4] = { 6031 NULL, gen_helper_sve2_usubl_h, 6032 gen_helper_sve2_usubl_s, gen_helper_sve2_usubl_d, 6033 }; 6034 TRANS_FEAT(USUBLB, aa64_sve2, gen_gvec_ool_arg_zzz, 6035 usubl_fns[a->esz], a, 0) 6036 TRANS_FEAT(USUBLT, aa64_sve2, gen_gvec_ool_arg_zzz, 6037 usubl_fns[a->esz], a, 3) 6038 6039 static gen_helper_gvec_3 * const uabdl_fns[4] = { 6040 NULL, gen_helper_sve2_uabdl_h, 6041 gen_helper_sve2_uabdl_s, gen_helper_sve2_uabdl_d, 6042 }; 6043 TRANS_FEAT(UABDLB, aa64_sve2, gen_gvec_ool_arg_zzz, 6044 uabdl_fns[a->esz], a, 0) 6045 TRANS_FEAT(UABDLT, aa64_sve2, gen_gvec_ool_arg_zzz, 6046 uabdl_fns[a->esz], a, 3) 6047 6048 static gen_helper_gvec_3 * const sqdmull_fns[4] = { 6049 NULL, gen_helper_sve2_sqdmull_zzz_h, 6050 gen_helper_sve2_sqdmull_zzz_s, gen_helper_sve2_sqdmull_zzz_d, 6051 }; 6052 TRANS_FEAT(SQDMULLB_zzz, aa64_sve2, gen_gvec_ool_arg_zzz, 6053 sqdmull_fns[a->esz], a, 0) 6054 TRANS_FEAT(SQDMULLT_zzz, aa64_sve2, gen_gvec_ool_arg_zzz, 6055 sqdmull_fns[a->esz], a, 3) 6056 6057 static gen_helper_gvec_3 * const smull_fns[4] = { 6058 NULL, gen_helper_sve2_smull_zzz_h, 6059 gen_helper_sve2_smull_zzz_s, gen_helper_sve2_smull_zzz_d, 6060 }; 6061 TRANS_FEAT(SMULLB_zzz, aa64_sve2, gen_gvec_ool_arg_zzz, 6062 smull_fns[a->esz], a, 0) 6063 TRANS_FEAT(SMULLT_zzz, aa64_sve2, gen_gvec_ool_arg_zzz, 6064 smull_fns[a->esz], a, 3) 6065 6066 static gen_helper_gvec_3 * const umull_fns[4] = { 6067 NULL, gen_helper_sve2_umull_zzz_h, 6068 gen_helper_sve2_umull_zzz_s, gen_helper_sve2_umull_zzz_d, 6069 }; 6070 TRANS_FEAT(UMULLB_zzz, aa64_sve2, gen_gvec_ool_arg_zzz, 6071 umull_fns[a->esz], a, 0) 6072 TRANS_FEAT(UMULLT_zzz, aa64_sve2, gen_gvec_ool_arg_zzz, 6073 umull_fns[a->esz], a, 3) 6074 6075 static gen_helper_gvec_3 * const eoril_fns[4] = { 6076 gen_helper_sve2_eoril_b, gen_helper_sve2_eoril_h, 6077 gen_helper_sve2_eoril_s, gen_helper_sve2_eoril_d, 6078 }; 6079 TRANS_FEAT(EORBT, aa64_sve2, gen_gvec_ool_arg_zzz, eoril_fns[a->esz], a, 2) 6080 TRANS_FEAT(EORTB, aa64_sve2, gen_gvec_ool_arg_zzz, eoril_fns[a->esz], a, 1) 6081 6082 static bool do_trans_pmull(DisasContext *s, arg_rrr_esz *a, bool sel) 6083 { 6084 static gen_helper_gvec_3 * const fns[4] = { 6085 gen_helper_gvec_pmull_q, gen_helper_sve2_pmull_h, 6086 NULL, gen_helper_sve2_pmull_d, 6087 }; 6088 6089 if (a->esz == 0) { 6090 if (!dc_isar_feature(aa64_sve2_pmull128, s)) { 6091 return false; 6092 } 6093 s->is_nonstreaming = true; 6094 } else if (!dc_isar_feature(aa64_sve, s)) { 6095 return false; 6096 } 6097 return gen_gvec_ool_arg_zzz(s, fns[a->esz], a, sel); 6098 } 6099 6100 TRANS_FEAT(PMULLB, aa64_sve2, do_trans_pmull, a, false) 6101 TRANS_FEAT(PMULLT, aa64_sve2, do_trans_pmull, a, true) 6102 6103 static gen_helper_gvec_3 * const saddw_fns[4] = { 6104 NULL, gen_helper_sve2_saddw_h, 6105 gen_helper_sve2_saddw_s, gen_helper_sve2_saddw_d, 6106 }; 6107 TRANS_FEAT(SADDWB, aa64_sve2, gen_gvec_ool_arg_zzz, saddw_fns[a->esz], a, 0) 6108 TRANS_FEAT(SADDWT, aa64_sve2, gen_gvec_ool_arg_zzz, saddw_fns[a->esz], a, 1) 6109 6110 static gen_helper_gvec_3 * const ssubw_fns[4] = { 6111 NULL, gen_helper_sve2_ssubw_h, 6112 gen_helper_sve2_ssubw_s, gen_helper_sve2_ssubw_d, 6113 }; 6114 TRANS_FEAT(SSUBWB, aa64_sve2, gen_gvec_ool_arg_zzz, ssubw_fns[a->esz], a, 0) 6115 TRANS_FEAT(SSUBWT, aa64_sve2, gen_gvec_ool_arg_zzz, ssubw_fns[a->esz], a, 1) 6116 6117 static gen_helper_gvec_3 * const uaddw_fns[4] = { 6118 NULL, gen_helper_sve2_uaddw_h, 6119 gen_helper_sve2_uaddw_s, gen_helper_sve2_uaddw_d, 6120 }; 6121 TRANS_FEAT(UADDWB, aa64_sve2, gen_gvec_ool_arg_zzz, uaddw_fns[a->esz], a, 0) 6122 TRANS_FEAT(UADDWT, aa64_sve2, gen_gvec_ool_arg_zzz, uaddw_fns[a->esz], a, 1) 6123 6124 static gen_helper_gvec_3 * const usubw_fns[4] = { 6125 NULL, gen_helper_sve2_usubw_h, 6126 gen_helper_sve2_usubw_s, gen_helper_sve2_usubw_d, 6127 }; 6128 TRANS_FEAT(USUBWB, aa64_sve2, gen_gvec_ool_arg_zzz, usubw_fns[a->esz], a, 0) 6129 TRANS_FEAT(USUBWT, aa64_sve2, gen_gvec_ool_arg_zzz, usubw_fns[a->esz], a, 1) 6130 6131 static void gen_sshll_vec(unsigned vece, TCGv_vec d, TCGv_vec n, int64_t imm) 6132 { 6133 int top = imm & 1; 6134 int shl = imm >> 1; 6135 int halfbits = 4 << vece; 6136 6137 if (top) { 6138 if (shl == halfbits) { 6139 TCGv_vec t = tcg_temp_new_vec_matching(d); 6140 tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(halfbits, halfbits)); 6141 tcg_gen_and_vec(vece, d, n, t); 6142 } else { 6143 tcg_gen_sari_vec(vece, d, n, halfbits); 6144 tcg_gen_shli_vec(vece, d, d, shl); 6145 } 6146 } else { 6147 tcg_gen_shli_vec(vece, d, n, halfbits); 6148 tcg_gen_sari_vec(vece, d, d, halfbits - shl); 6149 } 6150 } 6151 6152 static void gen_ushll_i64(unsigned vece, TCGv_i64 d, TCGv_i64 n, int imm) 6153 { 6154 int halfbits = 4 << vece; 6155 int top = imm & 1; 6156 int shl = (imm >> 1); 6157 int shift; 6158 uint64_t mask; 6159 6160 mask = MAKE_64BIT_MASK(0, halfbits); 6161 mask <<= shl; 6162 mask = dup_const(vece, mask); 6163 6164 shift = shl - top * halfbits; 6165 if (shift < 0) { 6166 tcg_gen_shri_i64(d, n, -shift); 6167 } else { 6168 tcg_gen_shli_i64(d, n, shift); 6169 } 6170 tcg_gen_andi_i64(d, d, mask); 6171 } 6172 6173 static void gen_ushll16_i64(TCGv_i64 d, TCGv_i64 n, int64_t imm) 6174 { 6175 gen_ushll_i64(MO_16, d, n, imm); 6176 } 6177 6178 static void gen_ushll32_i64(TCGv_i64 d, TCGv_i64 n, int64_t imm) 6179 { 6180 gen_ushll_i64(MO_32, d, n, imm); 6181 } 6182 6183 static void gen_ushll64_i64(TCGv_i64 d, TCGv_i64 n, int64_t imm) 6184 { 6185 gen_ushll_i64(MO_64, d, n, imm); 6186 } 6187 6188 static void gen_ushll_vec(unsigned vece, TCGv_vec d, TCGv_vec n, int64_t imm) 6189 { 6190 int halfbits = 4 << vece; 6191 int top = imm & 1; 6192 int shl = imm >> 1; 6193 6194 if (top) { 6195 if (shl == halfbits) { 6196 TCGv_vec t = tcg_temp_new_vec_matching(d); 6197 tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(halfbits, halfbits)); 6198 tcg_gen_and_vec(vece, d, n, t); 6199 } else { 6200 tcg_gen_shri_vec(vece, d, n, halfbits); 6201 tcg_gen_shli_vec(vece, d, d, shl); 6202 } 6203 } else { 6204 if (shl == 0) { 6205 TCGv_vec t = tcg_temp_new_vec_matching(d); 6206 tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(0, halfbits)); 6207 tcg_gen_and_vec(vece, d, n, t); 6208 } else { 6209 tcg_gen_shli_vec(vece, d, n, halfbits); 6210 tcg_gen_shri_vec(vece, d, d, halfbits - shl); 6211 } 6212 } 6213 } 6214 6215 static bool do_shll_tb(DisasContext *s, arg_rri_esz *a, 6216 const GVecGen2i ops[3], bool sel) 6217 { 6218 6219 if (a->esz < 0 || a->esz > 2) { 6220 return false; 6221 } 6222 if (sve_access_check(s)) { 6223 unsigned vsz = vec_full_reg_size(s); 6224 tcg_gen_gvec_2i(vec_full_reg_offset(s, a->rd), 6225 vec_full_reg_offset(s, a->rn), 6226 vsz, vsz, (a->imm << 1) | sel, 6227 &ops[a->esz]); 6228 } 6229 return true; 6230 } 6231 6232 static const TCGOpcode sshll_list[] = { 6233 INDEX_op_shli_vec, INDEX_op_sari_vec, 0 6234 }; 6235 static const GVecGen2i sshll_ops[3] = { 6236 { .fniv = gen_sshll_vec, 6237 .opt_opc = sshll_list, 6238 .fno = gen_helper_sve2_sshll_h, 6239 .vece = MO_16 }, 6240 { .fniv = gen_sshll_vec, 6241 .opt_opc = sshll_list, 6242 .fno = gen_helper_sve2_sshll_s, 6243 .vece = MO_32 }, 6244 { .fniv = gen_sshll_vec, 6245 .opt_opc = sshll_list, 6246 .fno = gen_helper_sve2_sshll_d, 6247 .vece = MO_64 } 6248 }; 6249 TRANS_FEAT(SSHLLB, aa64_sve2, do_shll_tb, a, sshll_ops, false) 6250 TRANS_FEAT(SSHLLT, aa64_sve2, do_shll_tb, a, sshll_ops, true) 6251 6252 static const TCGOpcode ushll_list[] = { 6253 INDEX_op_shli_vec, INDEX_op_shri_vec, 0 6254 }; 6255 static const GVecGen2i ushll_ops[3] = { 6256 { .fni8 = gen_ushll16_i64, 6257 .fniv = gen_ushll_vec, 6258 .opt_opc = ushll_list, 6259 .fno = gen_helper_sve2_ushll_h, 6260 .vece = MO_16 }, 6261 { .fni8 = gen_ushll32_i64, 6262 .fniv = gen_ushll_vec, 6263 .opt_opc = ushll_list, 6264 .fno = gen_helper_sve2_ushll_s, 6265 .vece = MO_32 }, 6266 { .fni8 = gen_ushll64_i64, 6267 .fniv = gen_ushll_vec, 6268 .opt_opc = ushll_list, 6269 .fno = gen_helper_sve2_ushll_d, 6270 .vece = MO_64 }, 6271 }; 6272 TRANS_FEAT(USHLLB, aa64_sve2, do_shll_tb, a, ushll_ops, false) 6273 TRANS_FEAT(USHLLT, aa64_sve2, do_shll_tb, a, ushll_ops, true) 6274 6275 static gen_helper_gvec_3 * const bext_fns[4] = { 6276 gen_helper_sve2_bext_b, gen_helper_sve2_bext_h, 6277 gen_helper_sve2_bext_s, gen_helper_sve2_bext_d, 6278 }; 6279 TRANS_FEAT_NONSTREAMING(BEXT, aa64_sve2_bitperm, gen_gvec_ool_arg_zzz, 6280 bext_fns[a->esz], a, 0) 6281 6282 static gen_helper_gvec_3 * const bdep_fns[4] = { 6283 gen_helper_sve2_bdep_b, gen_helper_sve2_bdep_h, 6284 gen_helper_sve2_bdep_s, gen_helper_sve2_bdep_d, 6285 }; 6286 TRANS_FEAT_NONSTREAMING(BDEP, aa64_sve2_bitperm, gen_gvec_ool_arg_zzz, 6287 bdep_fns[a->esz], a, 0) 6288 6289 static gen_helper_gvec_3 * const bgrp_fns[4] = { 6290 gen_helper_sve2_bgrp_b, gen_helper_sve2_bgrp_h, 6291 gen_helper_sve2_bgrp_s, gen_helper_sve2_bgrp_d, 6292 }; 6293 TRANS_FEAT_NONSTREAMING(BGRP, aa64_sve2_bitperm, gen_gvec_ool_arg_zzz, 6294 bgrp_fns[a->esz], a, 0) 6295 6296 static gen_helper_gvec_3 * const cadd_fns[4] = { 6297 gen_helper_sve2_cadd_b, gen_helper_sve2_cadd_h, 6298 gen_helper_sve2_cadd_s, gen_helper_sve2_cadd_d, 6299 }; 6300 TRANS_FEAT(CADD_rot90, aa64_sve2, gen_gvec_ool_arg_zzz, 6301 cadd_fns[a->esz], a, 0) 6302 TRANS_FEAT(CADD_rot270, aa64_sve2, gen_gvec_ool_arg_zzz, 6303 cadd_fns[a->esz], a, 1) 6304 6305 static gen_helper_gvec_3 * const sqcadd_fns[4] = { 6306 gen_helper_sve2_sqcadd_b, gen_helper_sve2_sqcadd_h, 6307 gen_helper_sve2_sqcadd_s, gen_helper_sve2_sqcadd_d, 6308 }; 6309 TRANS_FEAT(SQCADD_rot90, aa64_sve2, gen_gvec_ool_arg_zzz, 6310 sqcadd_fns[a->esz], a, 0) 6311 TRANS_FEAT(SQCADD_rot270, aa64_sve2, gen_gvec_ool_arg_zzz, 6312 sqcadd_fns[a->esz], a, 1) 6313 6314 static gen_helper_gvec_4 * const sabal_fns[4] = { 6315 NULL, gen_helper_sve2_sabal_h, 6316 gen_helper_sve2_sabal_s, gen_helper_sve2_sabal_d, 6317 }; 6318 TRANS_FEAT(SABALB, aa64_sve2, gen_gvec_ool_arg_zzzz, sabal_fns[a->esz], a, 0) 6319 TRANS_FEAT(SABALT, aa64_sve2, gen_gvec_ool_arg_zzzz, sabal_fns[a->esz], a, 1) 6320 6321 static gen_helper_gvec_4 * const uabal_fns[4] = { 6322 NULL, gen_helper_sve2_uabal_h, 6323 gen_helper_sve2_uabal_s, gen_helper_sve2_uabal_d, 6324 }; 6325 TRANS_FEAT(UABALB, aa64_sve2, gen_gvec_ool_arg_zzzz, uabal_fns[a->esz], a, 0) 6326 TRANS_FEAT(UABALT, aa64_sve2, gen_gvec_ool_arg_zzzz, uabal_fns[a->esz], a, 1) 6327 6328 static bool do_adcl(DisasContext *s, arg_rrrr_esz *a, bool sel) 6329 { 6330 static gen_helper_gvec_4 * const fns[2] = { 6331 gen_helper_sve2_adcl_s, 6332 gen_helper_sve2_adcl_d, 6333 }; 6334 /* 6335 * Note that in this case the ESZ field encodes both size and sign. 6336 * Split out 'subtract' into bit 1 of the data field for the helper. 6337 */ 6338 return gen_gvec_ool_arg_zzzz(s, fns[a->esz & 1], a, (a->esz & 2) | sel); 6339 } 6340 6341 TRANS_FEAT(ADCLB, aa64_sve2, do_adcl, a, false) 6342 TRANS_FEAT(ADCLT, aa64_sve2, do_adcl, a, true) 6343 6344 TRANS_FEAT(SSRA, aa64_sve2, gen_gvec_fn_arg_zzi, gen_gvec_ssra, a) 6345 TRANS_FEAT(USRA, aa64_sve2, gen_gvec_fn_arg_zzi, gen_gvec_usra, a) 6346 TRANS_FEAT(SRSRA, aa64_sve2, gen_gvec_fn_arg_zzi, gen_gvec_srsra, a) 6347 TRANS_FEAT(URSRA, aa64_sve2, gen_gvec_fn_arg_zzi, gen_gvec_ursra, a) 6348 TRANS_FEAT(SRI, aa64_sve2, gen_gvec_fn_arg_zzi, gen_gvec_sri, a) 6349 TRANS_FEAT(SLI, aa64_sve2, gen_gvec_fn_arg_zzi, gen_gvec_sli, a) 6350 6351 TRANS_FEAT(SABA, aa64_sve2, gen_gvec_fn_arg_zzz, gen_gvec_saba, a) 6352 TRANS_FEAT(UABA, aa64_sve2, gen_gvec_fn_arg_zzz, gen_gvec_uaba, a) 6353 6354 static bool do_narrow_extract(DisasContext *s, arg_rri_esz *a, 6355 const GVecGen2 ops[3]) 6356 { 6357 if (a->esz < 0 || a->esz > MO_32 || a->imm != 0) { 6358 return false; 6359 } 6360 if (sve_access_check(s)) { 6361 unsigned vsz = vec_full_reg_size(s); 6362 tcg_gen_gvec_2(vec_full_reg_offset(s, a->rd), 6363 vec_full_reg_offset(s, a->rn), 6364 vsz, vsz, &ops[a->esz]); 6365 } 6366 return true; 6367 } 6368 6369 static const TCGOpcode sqxtn_list[] = { 6370 INDEX_op_shli_vec, INDEX_op_smin_vec, INDEX_op_smax_vec, 0 6371 }; 6372 6373 static void gen_sqxtnb_vec(unsigned vece, TCGv_vec d, TCGv_vec n) 6374 { 6375 TCGv_vec t = tcg_temp_new_vec_matching(d); 6376 int halfbits = 4 << vece; 6377 int64_t mask = (1ull << halfbits) - 1; 6378 int64_t min = -1ull << (halfbits - 1); 6379 int64_t max = -min - 1; 6380 6381 tcg_gen_dupi_vec(vece, t, min); 6382 tcg_gen_smax_vec(vece, d, n, t); 6383 tcg_gen_dupi_vec(vece, t, max); 6384 tcg_gen_smin_vec(vece, d, d, t); 6385 tcg_gen_dupi_vec(vece, t, mask); 6386 tcg_gen_and_vec(vece, d, d, t); 6387 } 6388 6389 static const GVecGen2 sqxtnb_ops[3] = { 6390 { .fniv = gen_sqxtnb_vec, 6391 .opt_opc = sqxtn_list, 6392 .fno = gen_helper_sve2_sqxtnb_h, 6393 .vece = MO_16 }, 6394 { .fniv = gen_sqxtnb_vec, 6395 .opt_opc = sqxtn_list, 6396 .fno = gen_helper_sve2_sqxtnb_s, 6397 .vece = MO_32 }, 6398 { .fniv = gen_sqxtnb_vec, 6399 .opt_opc = sqxtn_list, 6400 .fno = gen_helper_sve2_sqxtnb_d, 6401 .vece = MO_64 }, 6402 }; 6403 TRANS_FEAT(SQXTNB, aa64_sve2, do_narrow_extract, a, sqxtnb_ops) 6404 6405 static void gen_sqxtnt_vec(unsigned vece, TCGv_vec d, TCGv_vec n) 6406 { 6407 TCGv_vec t = tcg_temp_new_vec_matching(d); 6408 int halfbits = 4 << vece; 6409 int64_t mask = (1ull << halfbits) - 1; 6410 int64_t min = -1ull << (halfbits - 1); 6411 int64_t max = -min - 1; 6412 6413 tcg_gen_dupi_vec(vece, t, min); 6414 tcg_gen_smax_vec(vece, n, n, t); 6415 tcg_gen_dupi_vec(vece, t, max); 6416 tcg_gen_smin_vec(vece, n, n, t); 6417 tcg_gen_shli_vec(vece, n, n, halfbits); 6418 tcg_gen_dupi_vec(vece, t, mask); 6419 tcg_gen_bitsel_vec(vece, d, t, d, n); 6420 } 6421 6422 static const GVecGen2 sqxtnt_ops[3] = { 6423 { .fniv = gen_sqxtnt_vec, 6424 .opt_opc = sqxtn_list, 6425 .load_dest = true, 6426 .fno = gen_helper_sve2_sqxtnt_h, 6427 .vece = MO_16 }, 6428 { .fniv = gen_sqxtnt_vec, 6429 .opt_opc = sqxtn_list, 6430 .load_dest = true, 6431 .fno = gen_helper_sve2_sqxtnt_s, 6432 .vece = MO_32 }, 6433 { .fniv = gen_sqxtnt_vec, 6434 .opt_opc = sqxtn_list, 6435 .load_dest = true, 6436 .fno = gen_helper_sve2_sqxtnt_d, 6437 .vece = MO_64 }, 6438 }; 6439 TRANS_FEAT(SQXTNT, aa64_sve2, do_narrow_extract, a, sqxtnt_ops) 6440 6441 static const TCGOpcode uqxtn_list[] = { 6442 INDEX_op_shli_vec, INDEX_op_umin_vec, 0 6443 }; 6444 6445 static void gen_uqxtnb_vec(unsigned vece, TCGv_vec d, TCGv_vec n) 6446 { 6447 TCGv_vec t = tcg_temp_new_vec_matching(d); 6448 int halfbits = 4 << vece; 6449 int64_t max = (1ull << halfbits) - 1; 6450 6451 tcg_gen_dupi_vec(vece, t, max); 6452 tcg_gen_umin_vec(vece, d, n, t); 6453 } 6454 6455 static const GVecGen2 uqxtnb_ops[3] = { 6456 { .fniv = gen_uqxtnb_vec, 6457 .opt_opc = uqxtn_list, 6458 .fno = gen_helper_sve2_uqxtnb_h, 6459 .vece = MO_16 }, 6460 { .fniv = gen_uqxtnb_vec, 6461 .opt_opc = uqxtn_list, 6462 .fno = gen_helper_sve2_uqxtnb_s, 6463 .vece = MO_32 }, 6464 { .fniv = gen_uqxtnb_vec, 6465 .opt_opc = uqxtn_list, 6466 .fno = gen_helper_sve2_uqxtnb_d, 6467 .vece = MO_64 }, 6468 }; 6469 TRANS_FEAT(UQXTNB, aa64_sve2, do_narrow_extract, a, uqxtnb_ops) 6470 6471 static void gen_uqxtnt_vec(unsigned vece, TCGv_vec d, TCGv_vec n) 6472 { 6473 TCGv_vec t = tcg_temp_new_vec_matching(d); 6474 int halfbits = 4 << vece; 6475 int64_t max = (1ull << halfbits) - 1; 6476 6477 tcg_gen_dupi_vec(vece, t, max); 6478 tcg_gen_umin_vec(vece, n, n, t); 6479 tcg_gen_shli_vec(vece, n, n, halfbits); 6480 tcg_gen_bitsel_vec(vece, d, t, d, n); 6481 } 6482 6483 static const GVecGen2 uqxtnt_ops[3] = { 6484 { .fniv = gen_uqxtnt_vec, 6485 .opt_opc = uqxtn_list, 6486 .load_dest = true, 6487 .fno = gen_helper_sve2_uqxtnt_h, 6488 .vece = MO_16 }, 6489 { .fniv = gen_uqxtnt_vec, 6490 .opt_opc = uqxtn_list, 6491 .load_dest = true, 6492 .fno = gen_helper_sve2_uqxtnt_s, 6493 .vece = MO_32 }, 6494 { .fniv = gen_uqxtnt_vec, 6495 .opt_opc = uqxtn_list, 6496 .load_dest = true, 6497 .fno = gen_helper_sve2_uqxtnt_d, 6498 .vece = MO_64 }, 6499 }; 6500 TRANS_FEAT(UQXTNT, aa64_sve2, do_narrow_extract, a, uqxtnt_ops) 6501 6502 static const TCGOpcode sqxtun_list[] = { 6503 INDEX_op_shli_vec, INDEX_op_umin_vec, INDEX_op_smax_vec, 0 6504 }; 6505 6506 static void gen_sqxtunb_vec(unsigned vece, TCGv_vec d, TCGv_vec n) 6507 { 6508 TCGv_vec t = tcg_temp_new_vec_matching(d); 6509 int halfbits = 4 << vece; 6510 int64_t max = (1ull << halfbits) - 1; 6511 6512 tcg_gen_dupi_vec(vece, t, 0); 6513 tcg_gen_smax_vec(vece, d, n, t); 6514 tcg_gen_dupi_vec(vece, t, max); 6515 tcg_gen_umin_vec(vece, d, d, t); 6516 } 6517 6518 static const GVecGen2 sqxtunb_ops[3] = { 6519 { .fniv = gen_sqxtunb_vec, 6520 .opt_opc = sqxtun_list, 6521 .fno = gen_helper_sve2_sqxtunb_h, 6522 .vece = MO_16 }, 6523 { .fniv = gen_sqxtunb_vec, 6524 .opt_opc = sqxtun_list, 6525 .fno = gen_helper_sve2_sqxtunb_s, 6526 .vece = MO_32 }, 6527 { .fniv = gen_sqxtunb_vec, 6528 .opt_opc = sqxtun_list, 6529 .fno = gen_helper_sve2_sqxtunb_d, 6530 .vece = MO_64 }, 6531 }; 6532 TRANS_FEAT(SQXTUNB, aa64_sve2, do_narrow_extract, a, sqxtunb_ops) 6533 6534 static void gen_sqxtunt_vec(unsigned vece, TCGv_vec d, TCGv_vec n) 6535 { 6536 TCGv_vec t = tcg_temp_new_vec_matching(d); 6537 int halfbits = 4 << vece; 6538 int64_t max = (1ull << halfbits) - 1; 6539 6540 tcg_gen_dupi_vec(vece, t, 0); 6541 tcg_gen_smax_vec(vece, n, n, t); 6542 tcg_gen_dupi_vec(vece, t, max); 6543 tcg_gen_umin_vec(vece, n, n, t); 6544 tcg_gen_shli_vec(vece, n, n, halfbits); 6545 tcg_gen_bitsel_vec(vece, d, t, d, n); 6546 } 6547 6548 static const GVecGen2 sqxtunt_ops[3] = { 6549 { .fniv = gen_sqxtunt_vec, 6550 .opt_opc = sqxtun_list, 6551 .load_dest = true, 6552 .fno = gen_helper_sve2_sqxtunt_h, 6553 .vece = MO_16 }, 6554 { .fniv = gen_sqxtunt_vec, 6555 .opt_opc = sqxtun_list, 6556 .load_dest = true, 6557 .fno = gen_helper_sve2_sqxtunt_s, 6558 .vece = MO_32 }, 6559 { .fniv = gen_sqxtunt_vec, 6560 .opt_opc = sqxtun_list, 6561 .load_dest = true, 6562 .fno = gen_helper_sve2_sqxtunt_d, 6563 .vece = MO_64 }, 6564 }; 6565 TRANS_FEAT(SQXTUNT, aa64_sve2, do_narrow_extract, a, sqxtunt_ops) 6566 6567 static bool do_shr_narrow(DisasContext *s, arg_rri_esz *a, 6568 const GVecGen2i ops[3]) 6569 { 6570 if (a->esz < 0 || a->esz > MO_32) { 6571 return false; 6572 } 6573 assert(a->imm > 0 && a->imm <= (8 << a->esz)); 6574 if (sve_access_check(s)) { 6575 unsigned vsz = vec_full_reg_size(s); 6576 tcg_gen_gvec_2i(vec_full_reg_offset(s, a->rd), 6577 vec_full_reg_offset(s, a->rn), 6578 vsz, vsz, a->imm, &ops[a->esz]); 6579 } 6580 return true; 6581 } 6582 6583 static void gen_shrnb_i64(unsigned vece, TCGv_i64 d, TCGv_i64 n, int shr) 6584 { 6585 int halfbits = 4 << vece; 6586 uint64_t mask = dup_const(vece, MAKE_64BIT_MASK(0, halfbits)); 6587 6588 tcg_gen_shri_i64(d, n, shr); 6589 tcg_gen_andi_i64(d, d, mask); 6590 } 6591 6592 static void gen_shrnb16_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr) 6593 { 6594 gen_shrnb_i64(MO_16, d, n, shr); 6595 } 6596 6597 static void gen_shrnb32_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr) 6598 { 6599 gen_shrnb_i64(MO_32, d, n, shr); 6600 } 6601 6602 static void gen_shrnb64_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr) 6603 { 6604 gen_shrnb_i64(MO_64, d, n, shr); 6605 } 6606 6607 static void gen_shrnb_vec(unsigned vece, TCGv_vec d, TCGv_vec n, int64_t shr) 6608 { 6609 TCGv_vec t = tcg_temp_new_vec_matching(d); 6610 int halfbits = 4 << vece; 6611 uint64_t mask = MAKE_64BIT_MASK(0, halfbits); 6612 6613 tcg_gen_shri_vec(vece, n, n, shr); 6614 tcg_gen_dupi_vec(vece, t, mask); 6615 tcg_gen_and_vec(vece, d, n, t); 6616 } 6617 6618 static const TCGOpcode shrnb_vec_list[] = { INDEX_op_shri_vec, 0 }; 6619 static const GVecGen2i shrnb_ops[3] = { 6620 { .fni8 = gen_shrnb16_i64, 6621 .fniv = gen_shrnb_vec, 6622 .opt_opc = shrnb_vec_list, 6623 .fno = gen_helper_sve2_shrnb_h, 6624 .vece = MO_16 }, 6625 { .fni8 = gen_shrnb32_i64, 6626 .fniv = gen_shrnb_vec, 6627 .opt_opc = shrnb_vec_list, 6628 .fno = gen_helper_sve2_shrnb_s, 6629 .vece = MO_32 }, 6630 { .fni8 = gen_shrnb64_i64, 6631 .fniv = gen_shrnb_vec, 6632 .opt_opc = shrnb_vec_list, 6633 .fno = gen_helper_sve2_shrnb_d, 6634 .vece = MO_64 }, 6635 }; 6636 TRANS_FEAT(SHRNB, aa64_sve2, do_shr_narrow, a, shrnb_ops) 6637 6638 static void gen_shrnt_i64(unsigned vece, TCGv_i64 d, TCGv_i64 n, int shr) 6639 { 6640 int halfbits = 4 << vece; 6641 uint64_t mask = dup_const(vece, MAKE_64BIT_MASK(0, halfbits)); 6642 6643 tcg_gen_shli_i64(n, n, halfbits - shr); 6644 tcg_gen_andi_i64(n, n, ~mask); 6645 tcg_gen_andi_i64(d, d, mask); 6646 tcg_gen_or_i64(d, d, n); 6647 } 6648 6649 static void gen_shrnt16_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr) 6650 { 6651 gen_shrnt_i64(MO_16, d, n, shr); 6652 } 6653 6654 static void gen_shrnt32_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr) 6655 { 6656 gen_shrnt_i64(MO_32, d, n, shr); 6657 } 6658 6659 static void gen_shrnt64_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr) 6660 { 6661 tcg_gen_shri_i64(n, n, shr); 6662 tcg_gen_deposit_i64(d, d, n, 32, 32); 6663 } 6664 6665 static void gen_shrnt_vec(unsigned vece, TCGv_vec d, TCGv_vec n, int64_t shr) 6666 { 6667 TCGv_vec t = tcg_temp_new_vec_matching(d); 6668 int halfbits = 4 << vece; 6669 uint64_t mask = MAKE_64BIT_MASK(0, halfbits); 6670 6671 tcg_gen_shli_vec(vece, n, n, halfbits - shr); 6672 tcg_gen_dupi_vec(vece, t, mask); 6673 tcg_gen_bitsel_vec(vece, d, t, d, n); 6674 } 6675 6676 static const TCGOpcode shrnt_vec_list[] = { INDEX_op_shli_vec, 0 }; 6677 static const GVecGen2i shrnt_ops[3] = { 6678 { .fni8 = gen_shrnt16_i64, 6679 .fniv = gen_shrnt_vec, 6680 .opt_opc = shrnt_vec_list, 6681 .load_dest = true, 6682 .fno = gen_helper_sve2_shrnt_h, 6683 .vece = MO_16 }, 6684 { .fni8 = gen_shrnt32_i64, 6685 .fniv = gen_shrnt_vec, 6686 .opt_opc = shrnt_vec_list, 6687 .load_dest = true, 6688 .fno = gen_helper_sve2_shrnt_s, 6689 .vece = MO_32 }, 6690 { .fni8 = gen_shrnt64_i64, 6691 .fniv = gen_shrnt_vec, 6692 .opt_opc = shrnt_vec_list, 6693 .load_dest = true, 6694 .fno = gen_helper_sve2_shrnt_d, 6695 .vece = MO_64 }, 6696 }; 6697 TRANS_FEAT(SHRNT, aa64_sve2, do_shr_narrow, a, shrnt_ops) 6698 6699 static const GVecGen2i rshrnb_ops[3] = { 6700 { .fno = gen_helper_sve2_rshrnb_h }, 6701 { .fno = gen_helper_sve2_rshrnb_s }, 6702 { .fno = gen_helper_sve2_rshrnb_d }, 6703 }; 6704 TRANS_FEAT(RSHRNB, aa64_sve2, do_shr_narrow, a, rshrnb_ops) 6705 6706 static const GVecGen2i rshrnt_ops[3] = { 6707 { .fno = gen_helper_sve2_rshrnt_h }, 6708 { .fno = gen_helper_sve2_rshrnt_s }, 6709 { .fno = gen_helper_sve2_rshrnt_d }, 6710 }; 6711 TRANS_FEAT(RSHRNT, aa64_sve2, do_shr_narrow, a, rshrnt_ops) 6712 6713 static void gen_sqshrunb_vec(unsigned vece, TCGv_vec d, 6714 TCGv_vec n, int64_t shr) 6715 { 6716 TCGv_vec t = tcg_temp_new_vec_matching(d); 6717 int halfbits = 4 << vece; 6718 6719 tcg_gen_sari_vec(vece, n, n, shr); 6720 tcg_gen_dupi_vec(vece, t, 0); 6721 tcg_gen_smax_vec(vece, n, n, t); 6722 tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(0, halfbits)); 6723 tcg_gen_umin_vec(vece, d, n, t); 6724 } 6725 6726 static const TCGOpcode sqshrunb_vec_list[] = { 6727 INDEX_op_sari_vec, INDEX_op_smax_vec, INDEX_op_umin_vec, 0 6728 }; 6729 static const GVecGen2i sqshrunb_ops[3] = { 6730 { .fniv = gen_sqshrunb_vec, 6731 .opt_opc = sqshrunb_vec_list, 6732 .fno = gen_helper_sve2_sqshrunb_h, 6733 .vece = MO_16 }, 6734 { .fniv = gen_sqshrunb_vec, 6735 .opt_opc = sqshrunb_vec_list, 6736 .fno = gen_helper_sve2_sqshrunb_s, 6737 .vece = MO_32 }, 6738 { .fniv = gen_sqshrunb_vec, 6739 .opt_opc = sqshrunb_vec_list, 6740 .fno = gen_helper_sve2_sqshrunb_d, 6741 .vece = MO_64 }, 6742 }; 6743 TRANS_FEAT(SQSHRUNB, aa64_sve2, do_shr_narrow, a, sqshrunb_ops) 6744 6745 static void gen_sqshrunt_vec(unsigned vece, TCGv_vec d, 6746 TCGv_vec n, int64_t shr) 6747 { 6748 TCGv_vec t = tcg_temp_new_vec_matching(d); 6749 int halfbits = 4 << vece; 6750 6751 tcg_gen_sari_vec(vece, n, n, shr); 6752 tcg_gen_dupi_vec(vece, t, 0); 6753 tcg_gen_smax_vec(vece, n, n, t); 6754 tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(0, halfbits)); 6755 tcg_gen_umin_vec(vece, n, n, t); 6756 tcg_gen_shli_vec(vece, n, n, halfbits); 6757 tcg_gen_bitsel_vec(vece, d, t, d, n); 6758 } 6759 6760 static const TCGOpcode sqshrunt_vec_list[] = { 6761 INDEX_op_shli_vec, INDEX_op_sari_vec, 6762 INDEX_op_smax_vec, INDEX_op_umin_vec, 0 6763 }; 6764 static const GVecGen2i sqshrunt_ops[3] = { 6765 { .fniv = gen_sqshrunt_vec, 6766 .opt_opc = sqshrunt_vec_list, 6767 .load_dest = true, 6768 .fno = gen_helper_sve2_sqshrunt_h, 6769 .vece = MO_16 }, 6770 { .fniv = gen_sqshrunt_vec, 6771 .opt_opc = sqshrunt_vec_list, 6772 .load_dest = true, 6773 .fno = gen_helper_sve2_sqshrunt_s, 6774 .vece = MO_32 }, 6775 { .fniv = gen_sqshrunt_vec, 6776 .opt_opc = sqshrunt_vec_list, 6777 .load_dest = true, 6778 .fno = gen_helper_sve2_sqshrunt_d, 6779 .vece = MO_64 }, 6780 }; 6781 TRANS_FEAT(SQSHRUNT, aa64_sve2, do_shr_narrow, a, sqshrunt_ops) 6782 6783 static const GVecGen2i sqrshrunb_ops[3] = { 6784 { .fno = gen_helper_sve2_sqrshrunb_h }, 6785 { .fno = gen_helper_sve2_sqrshrunb_s }, 6786 { .fno = gen_helper_sve2_sqrshrunb_d }, 6787 }; 6788 TRANS_FEAT(SQRSHRUNB, aa64_sve2, do_shr_narrow, a, sqrshrunb_ops) 6789 6790 static const GVecGen2i sqrshrunt_ops[3] = { 6791 { .fno = gen_helper_sve2_sqrshrunt_h }, 6792 { .fno = gen_helper_sve2_sqrshrunt_s }, 6793 { .fno = gen_helper_sve2_sqrshrunt_d }, 6794 }; 6795 TRANS_FEAT(SQRSHRUNT, aa64_sve2, do_shr_narrow, a, sqrshrunt_ops) 6796 6797 static void gen_sqshrnb_vec(unsigned vece, TCGv_vec d, 6798 TCGv_vec n, int64_t shr) 6799 { 6800 TCGv_vec t = tcg_temp_new_vec_matching(d); 6801 int halfbits = 4 << vece; 6802 int64_t max = MAKE_64BIT_MASK(0, halfbits - 1); 6803 int64_t min = -max - 1; 6804 6805 tcg_gen_sari_vec(vece, n, n, shr); 6806 tcg_gen_dupi_vec(vece, t, min); 6807 tcg_gen_smax_vec(vece, n, n, t); 6808 tcg_gen_dupi_vec(vece, t, max); 6809 tcg_gen_smin_vec(vece, n, n, t); 6810 tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(0, halfbits)); 6811 tcg_gen_and_vec(vece, d, n, t); 6812 } 6813 6814 static const TCGOpcode sqshrnb_vec_list[] = { 6815 INDEX_op_sari_vec, INDEX_op_smax_vec, INDEX_op_smin_vec, 0 6816 }; 6817 static const GVecGen2i sqshrnb_ops[3] = { 6818 { .fniv = gen_sqshrnb_vec, 6819 .opt_opc = sqshrnb_vec_list, 6820 .fno = gen_helper_sve2_sqshrnb_h, 6821 .vece = MO_16 }, 6822 { .fniv = gen_sqshrnb_vec, 6823 .opt_opc = sqshrnb_vec_list, 6824 .fno = gen_helper_sve2_sqshrnb_s, 6825 .vece = MO_32 }, 6826 { .fniv = gen_sqshrnb_vec, 6827 .opt_opc = sqshrnb_vec_list, 6828 .fno = gen_helper_sve2_sqshrnb_d, 6829 .vece = MO_64 }, 6830 }; 6831 TRANS_FEAT(SQSHRNB, aa64_sve2, do_shr_narrow, a, sqshrnb_ops) 6832 6833 static void gen_sqshrnt_vec(unsigned vece, TCGv_vec d, 6834 TCGv_vec n, int64_t shr) 6835 { 6836 TCGv_vec t = tcg_temp_new_vec_matching(d); 6837 int halfbits = 4 << vece; 6838 int64_t max = MAKE_64BIT_MASK(0, halfbits - 1); 6839 int64_t min = -max - 1; 6840 6841 tcg_gen_sari_vec(vece, n, n, shr); 6842 tcg_gen_dupi_vec(vece, t, min); 6843 tcg_gen_smax_vec(vece, n, n, t); 6844 tcg_gen_dupi_vec(vece, t, max); 6845 tcg_gen_smin_vec(vece, n, n, t); 6846 tcg_gen_shli_vec(vece, n, n, halfbits); 6847 tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(0, halfbits)); 6848 tcg_gen_bitsel_vec(vece, d, t, d, n); 6849 } 6850 6851 static const TCGOpcode sqshrnt_vec_list[] = { 6852 INDEX_op_shli_vec, INDEX_op_sari_vec, 6853 INDEX_op_smax_vec, INDEX_op_smin_vec, 0 6854 }; 6855 static const GVecGen2i sqshrnt_ops[3] = { 6856 { .fniv = gen_sqshrnt_vec, 6857 .opt_opc = sqshrnt_vec_list, 6858 .load_dest = true, 6859 .fno = gen_helper_sve2_sqshrnt_h, 6860 .vece = MO_16 }, 6861 { .fniv = gen_sqshrnt_vec, 6862 .opt_opc = sqshrnt_vec_list, 6863 .load_dest = true, 6864 .fno = gen_helper_sve2_sqshrnt_s, 6865 .vece = MO_32 }, 6866 { .fniv = gen_sqshrnt_vec, 6867 .opt_opc = sqshrnt_vec_list, 6868 .load_dest = true, 6869 .fno = gen_helper_sve2_sqshrnt_d, 6870 .vece = MO_64 }, 6871 }; 6872 TRANS_FEAT(SQSHRNT, aa64_sve2, do_shr_narrow, a, sqshrnt_ops) 6873 6874 static const GVecGen2i sqrshrnb_ops[3] = { 6875 { .fno = gen_helper_sve2_sqrshrnb_h }, 6876 { .fno = gen_helper_sve2_sqrshrnb_s }, 6877 { .fno = gen_helper_sve2_sqrshrnb_d }, 6878 }; 6879 TRANS_FEAT(SQRSHRNB, aa64_sve2, do_shr_narrow, a, sqrshrnb_ops) 6880 6881 static const GVecGen2i sqrshrnt_ops[3] = { 6882 { .fno = gen_helper_sve2_sqrshrnt_h }, 6883 { .fno = gen_helper_sve2_sqrshrnt_s }, 6884 { .fno = gen_helper_sve2_sqrshrnt_d }, 6885 }; 6886 TRANS_FEAT(SQRSHRNT, aa64_sve2, do_shr_narrow, a, sqrshrnt_ops) 6887 6888 static void gen_uqshrnb_vec(unsigned vece, TCGv_vec d, 6889 TCGv_vec n, int64_t shr) 6890 { 6891 TCGv_vec t = tcg_temp_new_vec_matching(d); 6892 int halfbits = 4 << vece; 6893 6894 tcg_gen_shri_vec(vece, n, n, shr); 6895 tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(0, halfbits)); 6896 tcg_gen_umin_vec(vece, d, n, t); 6897 } 6898 6899 static const TCGOpcode uqshrnb_vec_list[] = { 6900 INDEX_op_shri_vec, INDEX_op_umin_vec, 0 6901 }; 6902 static const GVecGen2i uqshrnb_ops[3] = { 6903 { .fniv = gen_uqshrnb_vec, 6904 .opt_opc = uqshrnb_vec_list, 6905 .fno = gen_helper_sve2_uqshrnb_h, 6906 .vece = MO_16 }, 6907 { .fniv = gen_uqshrnb_vec, 6908 .opt_opc = uqshrnb_vec_list, 6909 .fno = gen_helper_sve2_uqshrnb_s, 6910 .vece = MO_32 }, 6911 { .fniv = gen_uqshrnb_vec, 6912 .opt_opc = uqshrnb_vec_list, 6913 .fno = gen_helper_sve2_uqshrnb_d, 6914 .vece = MO_64 }, 6915 }; 6916 TRANS_FEAT(UQSHRNB, aa64_sve2, do_shr_narrow, a, uqshrnb_ops) 6917 6918 static void gen_uqshrnt_vec(unsigned vece, TCGv_vec d, 6919 TCGv_vec n, int64_t shr) 6920 { 6921 TCGv_vec t = tcg_temp_new_vec_matching(d); 6922 int halfbits = 4 << vece; 6923 6924 tcg_gen_shri_vec(vece, n, n, shr); 6925 tcg_gen_dupi_vec(vece, t, MAKE_64BIT_MASK(0, halfbits)); 6926 tcg_gen_umin_vec(vece, n, n, t); 6927 tcg_gen_shli_vec(vece, n, n, halfbits); 6928 tcg_gen_bitsel_vec(vece, d, t, d, n); 6929 } 6930 6931 static const TCGOpcode uqshrnt_vec_list[] = { 6932 INDEX_op_shli_vec, INDEX_op_shri_vec, INDEX_op_umin_vec, 0 6933 }; 6934 static const GVecGen2i uqshrnt_ops[3] = { 6935 { .fniv = gen_uqshrnt_vec, 6936 .opt_opc = uqshrnt_vec_list, 6937 .load_dest = true, 6938 .fno = gen_helper_sve2_uqshrnt_h, 6939 .vece = MO_16 }, 6940 { .fniv = gen_uqshrnt_vec, 6941 .opt_opc = uqshrnt_vec_list, 6942 .load_dest = true, 6943 .fno = gen_helper_sve2_uqshrnt_s, 6944 .vece = MO_32 }, 6945 { .fniv = gen_uqshrnt_vec, 6946 .opt_opc = uqshrnt_vec_list, 6947 .load_dest = true, 6948 .fno = gen_helper_sve2_uqshrnt_d, 6949 .vece = MO_64 }, 6950 }; 6951 TRANS_FEAT(UQSHRNT, aa64_sve2, do_shr_narrow, a, uqshrnt_ops) 6952 6953 static const GVecGen2i uqrshrnb_ops[3] = { 6954 { .fno = gen_helper_sve2_uqrshrnb_h }, 6955 { .fno = gen_helper_sve2_uqrshrnb_s }, 6956 { .fno = gen_helper_sve2_uqrshrnb_d }, 6957 }; 6958 TRANS_FEAT(UQRSHRNB, aa64_sve2, do_shr_narrow, a, uqrshrnb_ops) 6959 6960 static const GVecGen2i uqrshrnt_ops[3] = { 6961 { .fno = gen_helper_sve2_uqrshrnt_h }, 6962 { .fno = gen_helper_sve2_uqrshrnt_s }, 6963 { .fno = gen_helper_sve2_uqrshrnt_d }, 6964 }; 6965 TRANS_FEAT(UQRSHRNT, aa64_sve2, do_shr_narrow, a, uqrshrnt_ops) 6966 6967 #define DO_SVE2_ZZZ_NARROW(NAME, name) \ 6968 static gen_helper_gvec_3 * const name##_fns[4] = { \ 6969 NULL, gen_helper_sve2_##name##_h, \ 6970 gen_helper_sve2_##name##_s, gen_helper_sve2_##name##_d, \ 6971 }; \ 6972 TRANS_FEAT(NAME, aa64_sve2, gen_gvec_ool_arg_zzz, \ 6973 name##_fns[a->esz], a, 0) 6974 6975 DO_SVE2_ZZZ_NARROW(ADDHNB, addhnb) 6976 DO_SVE2_ZZZ_NARROW(ADDHNT, addhnt) 6977 DO_SVE2_ZZZ_NARROW(RADDHNB, raddhnb) 6978 DO_SVE2_ZZZ_NARROW(RADDHNT, raddhnt) 6979 6980 DO_SVE2_ZZZ_NARROW(SUBHNB, subhnb) 6981 DO_SVE2_ZZZ_NARROW(SUBHNT, subhnt) 6982 DO_SVE2_ZZZ_NARROW(RSUBHNB, rsubhnb) 6983 DO_SVE2_ZZZ_NARROW(RSUBHNT, rsubhnt) 6984 6985 static gen_helper_gvec_flags_4 * const match_fns[4] = { 6986 gen_helper_sve2_match_ppzz_b, gen_helper_sve2_match_ppzz_h, NULL, NULL 6987 }; 6988 TRANS_FEAT_NONSTREAMING(MATCH, aa64_sve2, do_ppzz_flags, a, match_fns[a->esz]) 6989 6990 static gen_helper_gvec_flags_4 * const nmatch_fns[4] = { 6991 gen_helper_sve2_nmatch_ppzz_b, gen_helper_sve2_nmatch_ppzz_h, NULL, NULL 6992 }; 6993 TRANS_FEAT_NONSTREAMING(NMATCH, aa64_sve2, do_ppzz_flags, a, nmatch_fns[a->esz]) 6994 6995 static gen_helper_gvec_4 * const histcnt_fns[4] = { 6996 NULL, NULL, gen_helper_sve2_histcnt_s, gen_helper_sve2_histcnt_d 6997 }; 6998 TRANS_FEAT_NONSTREAMING(HISTCNT, aa64_sve2, gen_gvec_ool_arg_zpzz, 6999 histcnt_fns[a->esz], a, 0) 7000 7001 TRANS_FEAT_NONSTREAMING(HISTSEG, aa64_sve2, gen_gvec_ool_arg_zzz, 7002 a->esz == 0 ? gen_helper_sve2_histseg : NULL, a, 0) 7003 7004 DO_ZPZZ_FP(FADDP, aa64_sve2, sve2_faddp_zpzz) 7005 DO_ZPZZ_FP(FMAXNMP, aa64_sve2, sve2_fmaxnmp_zpzz) 7006 DO_ZPZZ_FP(FMINNMP, aa64_sve2, sve2_fminnmp_zpzz) 7007 DO_ZPZZ_FP(FMAXP, aa64_sve2, sve2_fmaxp_zpzz) 7008 DO_ZPZZ_FP(FMINP, aa64_sve2, sve2_fminp_zpzz) 7009 7010 /* 7011 * SVE Integer Multiply-Add (unpredicated) 7012 */ 7013 7014 TRANS_FEAT_NONSTREAMING(FMMLA_s, aa64_sve_f32mm, gen_gvec_fpst_zzzz, 7015 gen_helper_fmmla_s, a->rd, a->rn, a->rm, a->ra, 7016 0, FPST_FPCR) 7017 TRANS_FEAT_NONSTREAMING(FMMLA_d, aa64_sve_f64mm, gen_gvec_fpst_zzzz, 7018 gen_helper_fmmla_d, a->rd, a->rn, a->rm, a->ra, 7019 0, FPST_FPCR) 7020 7021 static gen_helper_gvec_4 * const sqdmlal_zzzw_fns[] = { 7022 NULL, gen_helper_sve2_sqdmlal_zzzw_h, 7023 gen_helper_sve2_sqdmlal_zzzw_s, gen_helper_sve2_sqdmlal_zzzw_d, 7024 }; 7025 TRANS_FEAT(SQDMLALB_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz, 7026 sqdmlal_zzzw_fns[a->esz], a, 0) 7027 TRANS_FEAT(SQDMLALT_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz, 7028 sqdmlal_zzzw_fns[a->esz], a, 3) 7029 TRANS_FEAT(SQDMLALBT, aa64_sve2, gen_gvec_ool_arg_zzzz, 7030 sqdmlal_zzzw_fns[a->esz], a, 2) 7031 7032 static gen_helper_gvec_4 * const sqdmlsl_zzzw_fns[] = { 7033 NULL, gen_helper_sve2_sqdmlsl_zzzw_h, 7034 gen_helper_sve2_sqdmlsl_zzzw_s, gen_helper_sve2_sqdmlsl_zzzw_d, 7035 }; 7036 TRANS_FEAT(SQDMLSLB_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz, 7037 sqdmlsl_zzzw_fns[a->esz], a, 0) 7038 TRANS_FEAT(SQDMLSLT_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz, 7039 sqdmlsl_zzzw_fns[a->esz], a, 3) 7040 TRANS_FEAT(SQDMLSLBT, aa64_sve2, gen_gvec_ool_arg_zzzz, 7041 sqdmlsl_zzzw_fns[a->esz], a, 2) 7042 7043 static gen_helper_gvec_4 * const sqrdmlah_fns[] = { 7044 gen_helper_sve2_sqrdmlah_b, gen_helper_sve2_sqrdmlah_h, 7045 gen_helper_sve2_sqrdmlah_s, gen_helper_sve2_sqrdmlah_d, 7046 }; 7047 TRANS_FEAT(SQRDMLAH_zzzz, aa64_sve2, gen_gvec_ool_arg_zzzz, 7048 sqrdmlah_fns[a->esz], a, 0) 7049 7050 static gen_helper_gvec_4 * const sqrdmlsh_fns[] = { 7051 gen_helper_sve2_sqrdmlsh_b, gen_helper_sve2_sqrdmlsh_h, 7052 gen_helper_sve2_sqrdmlsh_s, gen_helper_sve2_sqrdmlsh_d, 7053 }; 7054 TRANS_FEAT(SQRDMLSH_zzzz, aa64_sve2, gen_gvec_ool_arg_zzzz, 7055 sqrdmlsh_fns[a->esz], a, 0) 7056 7057 static gen_helper_gvec_4 * const smlal_zzzw_fns[] = { 7058 NULL, gen_helper_sve2_smlal_zzzw_h, 7059 gen_helper_sve2_smlal_zzzw_s, gen_helper_sve2_smlal_zzzw_d, 7060 }; 7061 TRANS_FEAT(SMLALB_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz, 7062 smlal_zzzw_fns[a->esz], a, 0) 7063 TRANS_FEAT(SMLALT_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz, 7064 smlal_zzzw_fns[a->esz], a, 1) 7065 7066 static gen_helper_gvec_4 * const umlal_zzzw_fns[] = { 7067 NULL, gen_helper_sve2_umlal_zzzw_h, 7068 gen_helper_sve2_umlal_zzzw_s, gen_helper_sve2_umlal_zzzw_d, 7069 }; 7070 TRANS_FEAT(UMLALB_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz, 7071 umlal_zzzw_fns[a->esz], a, 0) 7072 TRANS_FEAT(UMLALT_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz, 7073 umlal_zzzw_fns[a->esz], a, 1) 7074 7075 static gen_helper_gvec_4 * const smlsl_zzzw_fns[] = { 7076 NULL, gen_helper_sve2_smlsl_zzzw_h, 7077 gen_helper_sve2_smlsl_zzzw_s, gen_helper_sve2_smlsl_zzzw_d, 7078 }; 7079 TRANS_FEAT(SMLSLB_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz, 7080 smlsl_zzzw_fns[a->esz], a, 0) 7081 TRANS_FEAT(SMLSLT_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz, 7082 smlsl_zzzw_fns[a->esz], a, 1) 7083 7084 static gen_helper_gvec_4 * const umlsl_zzzw_fns[] = { 7085 NULL, gen_helper_sve2_umlsl_zzzw_h, 7086 gen_helper_sve2_umlsl_zzzw_s, gen_helper_sve2_umlsl_zzzw_d, 7087 }; 7088 TRANS_FEAT(UMLSLB_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz, 7089 umlsl_zzzw_fns[a->esz], a, 0) 7090 TRANS_FEAT(UMLSLT_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz, 7091 umlsl_zzzw_fns[a->esz], a, 1) 7092 7093 static gen_helper_gvec_4 * const cmla_fns[] = { 7094 gen_helper_sve2_cmla_zzzz_b, gen_helper_sve2_cmla_zzzz_h, 7095 gen_helper_sve2_cmla_zzzz_s, gen_helper_sve2_cmla_zzzz_d, 7096 }; 7097 TRANS_FEAT(CMLA_zzzz, aa64_sve2, gen_gvec_ool_zzzz, 7098 cmla_fns[a->esz], a->rd, a->rn, a->rm, a->ra, a->rot) 7099 7100 static gen_helper_gvec_4 * const cdot_fns[] = { 7101 NULL, NULL, gen_helper_sve2_cdot_zzzz_s, gen_helper_sve2_cdot_zzzz_d 7102 }; 7103 TRANS_FEAT(CDOT_zzzz, aa64_sve2, gen_gvec_ool_zzzz, 7104 cdot_fns[a->esz], a->rd, a->rn, a->rm, a->ra, a->rot) 7105 7106 static gen_helper_gvec_4 * const sqrdcmlah_fns[] = { 7107 gen_helper_sve2_sqrdcmlah_zzzz_b, gen_helper_sve2_sqrdcmlah_zzzz_h, 7108 gen_helper_sve2_sqrdcmlah_zzzz_s, gen_helper_sve2_sqrdcmlah_zzzz_d, 7109 }; 7110 TRANS_FEAT(SQRDCMLAH_zzzz, aa64_sve2, gen_gvec_ool_zzzz, 7111 sqrdcmlah_fns[a->esz], a->rd, a->rn, a->rm, a->ra, a->rot) 7112 7113 TRANS_FEAT(USDOT_zzzz, aa64_sve_i8mm, gen_gvec_ool_arg_zzzz, 7114 a->esz == 2 ? gen_helper_gvec_usdot_b : NULL, a, 0) 7115 7116 TRANS_FEAT_NONSTREAMING(AESMC, aa64_sve2_aes, gen_gvec_ool_zz, 7117 gen_helper_crypto_aesmc, a->rd, a->rd, a->decrypt) 7118 7119 TRANS_FEAT_NONSTREAMING(AESE, aa64_sve2_aes, gen_gvec_ool_arg_zzz, 7120 gen_helper_crypto_aese, a, false) 7121 TRANS_FEAT_NONSTREAMING(AESD, aa64_sve2_aes, gen_gvec_ool_arg_zzz, 7122 gen_helper_crypto_aese, a, true) 7123 7124 TRANS_FEAT_NONSTREAMING(SM4E, aa64_sve2_sm4, gen_gvec_ool_arg_zzz, 7125 gen_helper_crypto_sm4e, a, 0) 7126 TRANS_FEAT_NONSTREAMING(SM4EKEY, aa64_sve2_sm4, gen_gvec_ool_arg_zzz, 7127 gen_helper_crypto_sm4ekey, a, 0) 7128 7129 TRANS_FEAT_NONSTREAMING(RAX1, aa64_sve2_sha3, gen_gvec_fn_arg_zzz, 7130 gen_gvec_rax1, a) 7131 7132 TRANS_FEAT(FCVTNT_sh, aa64_sve2, gen_gvec_fpst_arg_zpz, 7133 gen_helper_sve2_fcvtnt_sh, a, 0, FPST_FPCR) 7134 TRANS_FEAT(FCVTNT_ds, aa64_sve2, gen_gvec_fpst_arg_zpz, 7135 gen_helper_sve2_fcvtnt_ds, a, 0, FPST_FPCR) 7136 7137 TRANS_FEAT(BFCVTNT, aa64_sve_bf16, gen_gvec_fpst_arg_zpz, 7138 gen_helper_sve_bfcvtnt, a, 0, FPST_FPCR) 7139 7140 TRANS_FEAT(FCVTLT_hs, aa64_sve2, gen_gvec_fpst_arg_zpz, 7141 gen_helper_sve2_fcvtlt_hs, a, 0, FPST_FPCR) 7142 TRANS_FEAT(FCVTLT_sd, aa64_sve2, gen_gvec_fpst_arg_zpz, 7143 gen_helper_sve2_fcvtlt_sd, a, 0, FPST_FPCR) 7144 7145 TRANS_FEAT(FCVTX_ds, aa64_sve2, do_frint_mode, a, 7146 FPROUNDING_ODD, gen_helper_sve_fcvt_ds) 7147 TRANS_FEAT(FCVTXNT_ds, aa64_sve2, do_frint_mode, a, 7148 FPROUNDING_ODD, gen_helper_sve2_fcvtnt_ds) 7149 7150 static gen_helper_gvec_3_ptr * const flogb_fns[] = { 7151 NULL, gen_helper_flogb_h, 7152 gen_helper_flogb_s, gen_helper_flogb_d 7153 }; 7154 TRANS_FEAT(FLOGB, aa64_sve2, gen_gvec_fpst_arg_zpz, flogb_fns[a->esz], 7155 a, 0, a->esz == MO_16 ? FPST_FPCR_F16 : FPST_FPCR) 7156 7157 static bool do_FMLAL_zzzw(DisasContext *s, arg_rrrr_esz *a, bool sub, bool sel) 7158 { 7159 return gen_gvec_ptr_zzzz(s, gen_helper_sve2_fmlal_zzzw_s, 7160 a->rd, a->rn, a->rm, a->ra, 7161 (sel << 1) | sub, cpu_env); 7162 } 7163 7164 TRANS_FEAT(FMLALB_zzzw, aa64_sve2, do_FMLAL_zzzw, a, false, false) 7165 TRANS_FEAT(FMLALT_zzzw, aa64_sve2, do_FMLAL_zzzw, a, false, true) 7166 TRANS_FEAT(FMLSLB_zzzw, aa64_sve2, do_FMLAL_zzzw, a, true, false) 7167 TRANS_FEAT(FMLSLT_zzzw, aa64_sve2, do_FMLAL_zzzw, a, true, true) 7168 7169 static bool do_FMLAL_zzxw(DisasContext *s, arg_rrxr_esz *a, bool sub, bool sel) 7170 { 7171 return gen_gvec_ptr_zzzz(s, gen_helper_sve2_fmlal_zzxw_s, 7172 a->rd, a->rn, a->rm, a->ra, 7173 (a->index << 2) | (sel << 1) | sub, cpu_env); 7174 } 7175 7176 TRANS_FEAT(FMLALB_zzxw, aa64_sve2, do_FMLAL_zzxw, a, false, false) 7177 TRANS_FEAT(FMLALT_zzxw, aa64_sve2, do_FMLAL_zzxw, a, false, true) 7178 TRANS_FEAT(FMLSLB_zzxw, aa64_sve2, do_FMLAL_zzxw, a, true, false) 7179 TRANS_FEAT(FMLSLT_zzxw, aa64_sve2, do_FMLAL_zzxw, a, true, true) 7180 7181 TRANS_FEAT_NONSTREAMING(SMMLA, aa64_sve_i8mm, gen_gvec_ool_arg_zzzz, 7182 gen_helper_gvec_smmla_b, a, 0) 7183 TRANS_FEAT_NONSTREAMING(USMMLA, aa64_sve_i8mm, gen_gvec_ool_arg_zzzz, 7184 gen_helper_gvec_usmmla_b, a, 0) 7185 TRANS_FEAT_NONSTREAMING(UMMLA, aa64_sve_i8mm, gen_gvec_ool_arg_zzzz, 7186 gen_helper_gvec_ummla_b, a, 0) 7187 7188 TRANS_FEAT(BFDOT_zzzz, aa64_sve_bf16, gen_gvec_ool_arg_zzzz, 7189 gen_helper_gvec_bfdot, a, 0) 7190 TRANS_FEAT(BFDOT_zzxz, aa64_sve_bf16, gen_gvec_ool_arg_zzxz, 7191 gen_helper_gvec_bfdot_idx, a) 7192 7193 TRANS_FEAT_NONSTREAMING(BFMMLA, aa64_sve_bf16, gen_gvec_ool_arg_zzzz, 7194 gen_helper_gvec_bfmmla, a, 0) 7195 7196 static bool do_BFMLAL_zzzw(DisasContext *s, arg_rrrr_esz *a, bool sel) 7197 { 7198 return gen_gvec_fpst_zzzz(s, gen_helper_gvec_bfmlal, 7199 a->rd, a->rn, a->rm, a->ra, sel, FPST_FPCR); 7200 } 7201 7202 TRANS_FEAT(BFMLALB_zzzw, aa64_sve_bf16, do_BFMLAL_zzzw, a, false) 7203 TRANS_FEAT(BFMLALT_zzzw, aa64_sve_bf16, do_BFMLAL_zzzw, a, true) 7204 7205 static bool do_BFMLAL_zzxw(DisasContext *s, arg_rrxr_esz *a, bool sel) 7206 { 7207 return gen_gvec_fpst_zzzz(s, gen_helper_gvec_bfmlal_idx, 7208 a->rd, a->rn, a->rm, a->ra, 7209 (a->index << 1) | sel, FPST_FPCR); 7210 } 7211 7212 TRANS_FEAT(BFMLALB_zzxw, aa64_sve_bf16, do_BFMLAL_zzxw, a, false) 7213 TRANS_FEAT(BFMLALT_zzxw, aa64_sve_bf16, do_BFMLAL_zzxw, a, true) 7214 7215 static bool trans_PSEL(DisasContext *s, arg_psel *a) 7216 { 7217 int vl = vec_full_reg_size(s); 7218 int pl = pred_gvec_reg_size(s); 7219 int elements = vl >> a->esz; 7220 TCGv_i64 tmp, didx, dbit; 7221 TCGv_ptr ptr; 7222 7223 if (!dc_isar_feature(aa64_sme, s)) { 7224 return false; 7225 } 7226 if (!sve_access_check(s)) { 7227 return true; 7228 } 7229 7230 tmp = tcg_temp_new_i64(); 7231 dbit = tcg_temp_new_i64(); 7232 didx = tcg_temp_new_i64(); 7233 ptr = tcg_temp_new_ptr(); 7234 7235 /* Compute the predicate element. */ 7236 tcg_gen_addi_i64(tmp, cpu_reg(s, a->rv), a->imm); 7237 if (is_power_of_2(elements)) { 7238 tcg_gen_andi_i64(tmp, tmp, elements - 1); 7239 } else { 7240 tcg_gen_remu_i64(tmp, tmp, tcg_constant_i64(elements)); 7241 } 7242 7243 /* Extract the predicate byte and bit indices. */ 7244 tcg_gen_shli_i64(tmp, tmp, a->esz); 7245 tcg_gen_andi_i64(dbit, tmp, 7); 7246 tcg_gen_shri_i64(didx, tmp, 3); 7247 if (HOST_BIG_ENDIAN) { 7248 tcg_gen_xori_i64(didx, didx, 7); 7249 } 7250 7251 /* Load the predicate word. */ 7252 tcg_gen_trunc_i64_ptr(ptr, didx); 7253 tcg_gen_add_ptr(ptr, ptr, cpu_env); 7254 tcg_gen_ld8u_i64(tmp, ptr, pred_full_reg_offset(s, a->pm)); 7255 7256 /* Extract the predicate bit and replicate to MO_64. */ 7257 tcg_gen_shr_i64(tmp, tmp, dbit); 7258 tcg_gen_andi_i64(tmp, tmp, 1); 7259 tcg_gen_neg_i64(tmp, tmp); 7260 7261 /* Apply to either copy the source, or write zeros. */ 7262 tcg_gen_gvec_ands(MO_64, pred_full_reg_offset(s, a->pd), 7263 pred_full_reg_offset(s, a->pn), tmp, pl, pl); 7264 return true; 7265 } 7266 7267 static void gen_sclamp_i32(TCGv_i32 d, TCGv_i32 n, TCGv_i32 m, TCGv_i32 a) 7268 { 7269 tcg_gen_smax_i32(d, a, n); 7270 tcg_gen_smin_i32(d, d, m); 7271 } 7272 7273 static void gen_sclamp_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 a) 7274 { 7275 tcg_gen_smax_i64(d, a, n); 7276 tcg_gen_smin_i64(d, d, m); 7277 } 7278 7279 static void gen_sclamp_vec(unsigned vece, TCGv_vec d, TCGv_vec n, 7280 TCGv_vec m, TCGv_vec a) 7281 { 7282 tcg_gen_smax_vec(vece, d, a, n); 7283 tcg_gen_smin_vec(vece, d, d, m); 7284 } 7285 7286 static void gen_sclamp(unsigned vece, uint32_t d, uint32_t n, uint32_t m, 7287 uint32_t a, uint32_t oprsz, uint32_t maxsz) 7288 { 7289 static const TCGOpcode vecop[] = { 7290 INDEX_op_smin_vec, INDEX_op_smax_vec, 0 7291 }; 7292 static const GVecGen4 ops[4] = { 7293 { .fniv = gen_sclamp_vec, 7294 .fno = gen_helper_gvec_sclamp_b, 7295 .opt_opc = vecop, 7296 .vece = MO_8 }, 7297 { .fniv = gen_sclamp_vec, 7298 .fno = gen_helper_gvec_sclamp_h, 7299 .opt_opc = vecop, 7300 .vece = MO_16 }, 7301 { .fni4 = gen_sclamp_i32, 7302 .fniv = gen_sclamp_vec, 7303 .fno = gen_helper_gvec_sclamp_s, 7304 .opt_opc = vecop, 7305 .vece = MO_32 }, 7306 { .fni8 = gen_sclamp_i64, 7307 .fniv = gen_sclamp_vec, 7308 .fno = gen_helper_gvec_sclamp_d, 7309 .opt_opc = vecop, 7310 .vece = MO_64, 7311 .prefer_i64 = TCG_TARGET_REG_BITS == 64 } 7312 }; 7313 tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &ops[vece]); 7314 } 7315 7316 TRANS_FEAT(SCLAMP, aa64_sme, gen_gvec_fn_arg_zzzz, gen_sclamp, a) 7317 7318 static void gen_uclamp_i32(TCGv_i32 d, TCGv_i32 n, TCGv_i32 m, TCGv_i32 a) 7319 { 7320 tcg_gen_umax_i32(d, a, n); 7321 tcg_gen_umin_i32(d, d, m); 7322 } 7323 7324 static void gen_uclamp_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 a) 7325 { 7326 tcg_gen_umax_i64(d, a, n); 7327 tcg_gen_umin_i64(d, d, m); 7328 } 7329 7330 static void gen_uclamp_vec(unsigned vece, TCGv_vec d, TCGv_vec n, 7331 TCGv_vec m, TCGv_vec a) 7332 { 7333 tcg_gen_umax_vec(vece, d, a, n); 7334 tcg_gen_umin_vec(vece, d, d, m); 7335 } 7336 7337 static void gen_uclamp(unsigned vece, uint32_t d, uint32_t n, uint32_t m, 7338 uint32_t a, uint32_t oprsz, uint32_t maxsz) 7339 { 7340 static const TCGOpcode vecop[] = { 7341 INDEX_op_umin_vec, INDEX_op_umax_vec, 0 7342 }; 7343 static const GVecGen4 ops[4] = { 7344 { .fniv = gen_uclamp_vec, 7345 .fno = gen_helper_gvec_uclamp_b, 7346 .opt_opc = vecop, 7347 .vece = MO_8 }, 7348 { .fniv = gen_uclamp_vec, 7349 .fno = gen_helper_gvec_uclamp_h, 7350 .opt_opc = vecop, 7351 .vece = MO_16 }, 7352 { .fni4 = gen_uclamp_i32, 7353 .fniv = gen_uclamp_vec, 7354 .fno = gen_helper_gvec_uclamp_s, 7355 .opt_opc = vecop, 7356 .vece = MO_32 }, 7357 { .fni8 = gen_uclamp_i64, 7358 .fniv = gen_uclamp_vec, 7359 .fno = gen_helper_gvec_uclamp_d, 7360 .opt_opc = vecop, 7361 .vece = MO_64, 7362 .prefer_i64 = TCG_TARGET_REG_BITS == 64 } 7363 }; 7364 tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &ops[vece]); 7365 } 7366 7367 TRANS_FEAT(UCLAMP, aa64_sme, gen_gvec_fn_arg_zzzz, gen_uclamp, a) 7368