1 /* 2 * AArch64 translation 3 * 4 * Copyright (c) 2013 Alexander Graf <agraf@suse.de> 5 * 6 * This library is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Lesser General Public 8 * License as published by the Free Software Foundation; either 9 * version 2.1 of the License, or (at your option) any later version. 10 * 11 * This library is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * Lesser General Public License for more details. 15 * 16 * You should have received a copy of the GNU Lesser General Public 17 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 18 */ 19 #include "qemu/osdep.h" 20 21 #include "cpu.h" 22 #include "exec/exec-all.h" 23 #include "tcg/tcg-op.h" 24 #include "tcg/tcg-op-gvec.h" 25 #include "qemu/log.h" 26 #include "arm_ldst.h" 27 #include "translate.h" 28 #include "internals.h" 29 #include "qemu/host-utils.h" 30 #include "semihosting/semihost.h" 31 #include "exec/log.h" 32 #include "cpregs.h" 33 #include "translate-a64.h" 34 #include "qemu/atomic128.h" 35 36 static TCGv_i64 cpu_X[32]; 37 static TCGv_i64 cpu_pc; 38 39 /* Load/store exclusive handling */ 40 static TCGv_i64 cpu_exclusive_high; 41 42 static const char *regnames[] = { 43 "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", 44 "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", 45 "x16", "x17", "x18", "x19", "x20", "x21", "x22", "x23", 46 "x24", "x25", "x26", "x27", "x28", "x29", "lr", "sp" 47 }; 48 49 enum a64_shift_type { 50 A64_SHIFT_TYPE_LSL = 0, 51 A64_SHIFT_TYPE_LSR = 1, 52 A64_SHIFT_TYPE_ASR = 2, 53 A64_SHIFT_TYPE_ROR = 3 54 }; 55 56 /* 57 * Include the generated decoders. 58 */ 59 60 #include "decode-sme-fa64.c.inc" 61 #include "decode-a64.c.inc" 62 63 /* Table based decoder typedefs - used when the relevant bits for decode 64 * are too awkwardly scattered across the instruction (eg SIMD). 65 */ 66 typedef void AArch64DecodeFn(DisasContext *s, uint32_t insn); 67 68 typedef struct AArch64DecodeTable { 69 uint32_t pattern; 70 uint32_t mask; 71 AArch64DecodeFn *disas_fn; 72 } AArch64DecodeTable; 73 74 /* initialize TCG globals. */ 75 void a64_translate_init(void) 76 { 77 int i; 78 79 cpu_pc = tcg_global_mem_new_i64(cpu_env, 80 offsetof(CPUARMState, pc), 81 "pc"); 82 for (i = 0; i < 32; i++) { 83 cpu_X[i] = tcg_global_mem_new_i64(cpu_env, 84 offsetof(CPUARMState, xregs[i]), 85 regnames[i]); 86 } 87 88 cpu_exclusive_high = tcg_global_mem_new_i64(cpu_env, 89 offsetof(CPUARMState, exclusive_high), "exclusive_high"); 90 } 91 92 /* 93 * Return the core mmu_idx to use for A64 "unprivileged load/store" insns 94 */ 95 static int get_a64_user_mem_index(DisasContext *s) 96 { 97 /* 98 * If AccType_UNPRIV is not used, the insn uses AccType_NORMAL, 99 * which is the usual mmu_idx for this cpu state. 100 */ 101 ARMMMUIdx useridx = s->mmu_idx; 102 103 if (s->unpriv) { 104 /* 105 * We have pre-computed the condition for AccType_UNPRIV. 106 * Therefore we should never get here with a mmu_idx for 107 * which we do not know the corresponding user mmu_idx. 108 */ 109 switch (useridx) { 110 case ARMMMUIdx_E10_1: 111 case ARMMMUIdx_E10_1_PAN: 112 useridx = ARMMMUIdx_E10_0; 113 break; 114 case ARMMMUIdx_E20_2: 115 case ARMMMUIdx_E20_2_PAN: 116 useridx = ARMMMUIdx_E20_0; 117 break; 118 default: 119 g_assert_not_reached(); 120 } 121 } 122 return arm_to_core_mmu_idx(useridx); 123 } 124 125 static void set_btype_raw(int val) 126 { 127 tcg_gen_st_i32(tcg_constant_i32(val), cpu_env, 128 offsetof(CPUARMState, btype)); 129 } 130 131 static void set_btype(DisasContext *s, int val) 132 { 133 /* BTYPE is a 2-bit field, and 0 should be done with reset_btype. */ 134 tcg_debug_assert(val >= 1 && val <= 3); 135 set_btype_raw(val); 136 s->btype = -1; 137 } 138 139 static void reset_btype(DisasContext *s) 140 { 141 if (s->btype != 0) { 142 set_btype_raw(0); 143 s->btype = 0; 144 } 145 } 146 147 static void gen_pc_plus_diff(DisasContext *s, TCGv_i64 dest, target_long diff) 148 { 149 assert(s->pc_save != -1); 150 if (tb_cflags(s->base.tb) & CF_PCREL) { 151 tcg_gen_addi_i64(dest, cpu_pc, (s->pc_curr - s->pc_save) + diff); 152 } else { 153 tcg_gen_movi_i64(dest, s->pc_curr + diff); 154 } 155 } 156 157 void gen_a64_update_pc(DisasContext *s, target_long diff) 158 { 159 gen_pc_plus_diff(s, cpu_pc, diff); 160 s->pc_save = s->pc_curr + diff; 161 } 162 163 /* 164 * Handle Top Byte Ignore (TBI) bits. 165 * 166 * If address tagging is enabled via the TCR TBI bits: 167 * + for EL2 and EL3 there is only one TBI bit, and if it is set 168 * then the address is zero-extended, clearing bits [63:56] 169 * + for EL0 and EL1, TBI0 controls addresses with bit 55 == 0 170 * and TBI1 controls addressses with bit 55 == 1. 171 * If the appropriate TBI bit is set for the address then 172 * the address is sign-extended from bit 55 into bits [63:56] 173 * 174 * Here We have concatenated TBI{1,0} into tbi. 175 */ 176 static void gen_top_byte_ignore(DisasContext *s, TCGv_i64 dst, 177 TCGv_i64 src, int tbi) 178 { 179 if (tbi == 0) { 180 /* Load unmodified address */ 181 tcg_gen_mov_i64(dst, src); 182 } else if (!regime_has_2_ranges(s->mmu_idx)) { 183 /* Force tag byte to all zero */ 184 tcg_gen_extract_i64(dst, src, 0, 56); 185 } else { 186 /* Sign-extend from bit 55. */ 187 tcg_gen_sextract_i64(dst, src, 0, 56); 188 189 switch (tbi) { 190 case 1: 191 /* tbi0 but !tbi1: only use the extension if positive */ 192 tcg_gen_and_i64(dst, dst, src); 193 break; 194 case 2: 195 /* !tbi0 but tbi1: only use the extension if negative */ 196 tcg_gen_or_i64(dst, dst, src); 197 break; 198 case 3: 199 /* tbi0 and tbi1: always use the extension */ 200 break; 201 default: 202 g_assert_not_reached(); 203 } 204 } 205 } 206 207 static void gen_a64_set_pc(DisasContext *s, TCGv_i64 src) 208 { 209 /* 210 * If address tagging is enabled for instructions via the TCR TBI bits, 211 * then loading an address into the PC will clear out any tag. 212 */ 213 gen_top_byte_ignore(s, cpu_pc, src, s->tbii); 214 s->pc_save = -1; 215 } 216 217 /* 218 * Handle MTE and/or TBI. 219 * 220 * For TBI, ideally, we would do nothing. Proper behaviour on fault is 221 * for the tag to be present in the FAR_ELx register. But for user-only 222 * mode we do not have a TLB with which to implement this, so we must 223 * remove the top byte now. 224 * 225 * Always return a fresh temporary that we can increment independently 226 * of the write-back address. 227 */ 228 229 TCGv_i64 clean_data_tbi(DisasContext *s, TCGv_i64 addr) 230 { 231 TCGv_i64 clean = tcg_temp_new_i64(); 232 #ifdef CONFIG_USER_ONLY 233 gen_top_byte_ignore(s, clean, addr, s->tbid); 234 #else 235 tcg_gen_mov_i64(clean, addr); 236 #endif 237 return clean; 238 } 239 240 /* Insert a zero tag into src, with the result at dst. */ 241 static void gen_address_with_allocation_tag0(TCGv_i64 dst, TCGv_i64 src) 242 { 243 tcg_gen_andi_i64(dst, src, ~MAKE_64BIT_MASK(56, 4)); 244 } 245 246 static void gen_probe_access(DisasContext *s, TCGv_i64 ptr, 247 MMUAccessType acc, int log2_size) 248 { 249 gen_helper_probe_access(cpu_env, ptr, 250 tcg_constant_i32(acc), 251 tcg_constant_i32(get_mem_index(s)), 252 tcg_constant_i32(1 << log2_size)); 253 } 254 255 /* 256 * For MTE, check a single logical or atomic access. This probes a single 257 * address, the exact one specified. The size and alignment of the access 258 * is not relevant to MTE, per se, but watchpoints do require the size, 259 * and we want to recognize those before making any other changes to state. 260 */ 261 static TCGv_i64 gen_mte_check1_mmuidx(DisasContext *s, TCGv_i64 addr, 262 bool is_write, bool tag_checked, 263 int log2_size, bool is_unpriv, 264 int core_idx) 265 { 266 if (tag_checked && s->mte_active[is_unpriv]) { 267 TCGv_i64 ret; 268 int desc = 0; 269 270 desc = FIELD_DP32(desc, MTEDESC, MIDX, core_idx); 271 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid); 272 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma); 273 desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write); 274 desc = FIELD_DP32(desc, MTEDESC, SIZEM1, (1 << log2_size) - 1); 275 276 ret = tcg_temp_new_i64(); 277 gen_helper_mte_check(ret, cpu_env, tcg_constant_i32(desc), addr); 278 279 return ret; 280 } 281 return clean_data_tbi(s, addr); 282 } 283 284 TCGv_i64 gen_mte_check1(DisasContext *s, TCGv_i64 addr, bool is_write, 285 bool tag_checked, int log2_size) 286 { 287 return gen_mte_check1_mmuidx(s, addr, is_write, tag_checked, log2_size, 288 false, get_mem_index(s)); 289 } 290 291 /* 292 * For MTE, check multiple logical sequential accesses. 293 */ 294 TCGv_i64 gen_mte_checkN(DisasContext *s, TCGv_i64 addr, bool is_write, 295 bool tag_checked, int size) 296 { 297 if (tag_checked && s->mte_active[0]) { 298 TCGv_i64 ret; 299 int desc = 0; 300 301 desc = FIELD_DP32(desc, MTEDESC, MIDX, get_mem_index(s)); 302 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid); 303 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma); 304 desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write); 305 desc = FIELD_DP32(desc, MTEDESC, SIZEM1, size - 1); 306 307 ret = tcg_temp_new_i64(); 308 gen_helper_mte_check(ret, cpu_env, tcg_constant_i32(desc), addr); 309 310 return ret; 311 } 312 return clean_data_tbi(s, addr); 313 } 314 315 typedef struct DisasCompare64 { 316 TCGCond cond; 317 TCGv_i64 value; 318 } DisasCompare64; 319 320 static void a64_test_cc(DisasCompare64 *c64, int cc) 321 { 322 DisasCompare c32; 323 324 arm_test_cc(&c32, cc); 325 326 /* 327 * Sign-extend the 32-bit value so that the GE/LT comparisons work 328 * properly. The NE/EQ comparisons are also fine with this choice. 329 */ 330 c64->cond = c32.cond; 331 c64->value = tcg_temp_new_i64(); 332 tcg_gen_ext_i32_i64(c64->value, c32.value); 333 } 334 335 static void gen_rebuild_hflags(DisasContext *s) 336 { 337 gen_helper_rebuild_hflags_a64(cpu_env, tcg_constant_i32(s->current_el)); 338 } 339 340 static void gen_exception_internal(int excp) 341 { 342 assert(excp_is_internal(excp)); 343 gen_helper_exception_internal(cpu_env, tcg_constant_i32(excp)); 344 } 345 346 static void gen_exception_internal_insn(DisasContext *s, int excp) 347 { 348 gen_a64_update_pc(s, 0); 349 gen_exception_internal(excp); 350 s->base.is_jmp = DISAS_NORETURN; 351 } 352 353 static void gen_exception_bkpt_insn(DisasContext *s, uint32_t syndrome) 354 { 355 gen_a64_update_pc(s, 0); 356 gen_helper_exception_bkpt_insn(cpu_env, tcg_constant_i32(syndrome)); 357 s->base.is_jmp = DISAS_NORETURN; 358 } 359 360 static void gen_step_complete_exception(DisasContext *s) 361 { 362 /* We just completed step of an insn. Move from Active-not-pending 363 * to Active-pending, and then also take the swstep exception. 364 * This corresponds to making the (IMPDEF) choice to prioritize 365 * swstep exceptions over asynchronous exceptions taken to an exception 366 * level where debug is disabled. This choice has the advantage that 367 * we do not need to maintain internal state corresponding to the 368 * ISV/EX syndrome bits between completion of the step and generation 369 * of the exception, and our syndrome information is always correct. 370 */ 371 gen_ss_advance(s); 372 gen_swstep_exception(s, 1, s->is_ldex); 373 s->base.is_jmp = DISAS_NORETURN; 374 } 375 376 static inline bool use_goto_tb(DisasContext *s, uint64_t dest) 377 { 378 if (s->ss_active) { 379 return false; 380 } 381 return translator_use_goto_tb(&s->base, dest); 382 } 383 384 static void gen_goto_tb(DisasContext *s, int n, int64_t diff) 385 { 386 if (use_goto_tb(s, s->pc_curr + diff)) { 387 /* 388 * For pcrel, the pc must always be up-to-date on entry to 389 * the linked TB, so that it can use simple additions for all 390 * further adjustments. For !pcrel, the linked TB is compiled 391 * to know its full virtual address, so we can delay the 392 * update to pc to the unlinked path. A long chain of links 393 * can thus avoid many updates to the PC. 394 */ 395 if (tb_cflags(s->base.tb) & CF_PCREL) { 396 gen_a64_update_pc(s, diff); 397 tcg_gen_goto_tb(n); 398 } else { 399 tcg_gen_goto_tb(n); 400 gen_a64_update_pc(s, diff); 401 } 402 tcg_gen_exit_tb(s->base.tb, n); 403 s->base.is_jmp = DISAS_NORETURN; 404 } else { 405 gen_a64_update_pc(s, diff); 406 if (s->ss_active) { 407 gen_step_complete_exception(s); 408 } else { 409 tcg_gen_lookup_and_goto_ptr(); 410 s->base.is_jmp = DISAS_NORETURN; 411 } 412 } 413 } 414 415 /* 416 * Register access functions 417 * 418 * These functions are used for directly accessing a register in where 419 * changes to the final register value are likely to be made. If you 420 * need to use a register for temporary calculation (e.g. index type 421 * operations) use the read_* form. 422 * 423 * B1.2.1 Register mappings 424 * 425 * In instruction register encoding 31 can refer to ZR (zero register) or 426 * the SP (stack pointer) depending on context. In QEMU's case we map SP 427 * to cpu_X[31] and ZR accesses to a temporary which can be discarded. 428 * This is the point of the _sp forms. 429 */ 430 TCGv_i64 cpu_reg(DisasContext *s, int reg) 431 { 432 if (reg == 31) { 433 TCGv_i64 t = tcg_temp_new_i64(); 434 tcg_gen_movi_i64(t, 0); 435 return t; 436 } else { 437 return cpu_X[reg]; 438 } 439 } 440 441 /* register access for when 31 == SP */ 442 TCGv_i64 cpu_reg_sp(DisasContext *s, int reg) 443 { 444 return cpu_X[reg]; 445 } 446 447 /* read a cpu register in 32bit/64bit mode. Returns a TCGv_i64 448 * representing the register contents. This TCGv is an auto-freed 449 * temporary so it need not be explicitly freed, and may be modified. 450 */ 451 TCGv_i64 read_cpu_reg(DisasContext *s, int reg, int sf) 452 { 453 TCGv_i64 v = tcg_temp_new_i64(); 454 if (reg != 31) { 455 if (sf) { 456 tcg_gen_mov_i64(v, cpu_X[reg]); 457 } else { 458 tcg_gen_ext32u_i64(v, cpu_X[reg]); 459 } 460 } else { 461 tcg_gen_movi_i64(v, 0); 462 } 463 return v; 464 } 465 466 TCGv_i64 read_cpu_reg_sp(DisasContext *s, int reg, int sf) 467 { 468 TCGv_i64 v = tcg_temp_new_i64(); 469 if (sf) { 470 tcg_gen_mov_i64(v, cpu_X[reg]); 471 } else { 472 tcg_gen_ext32u_i64(v, cpu_X[reg]); 473 } 474 return v; 475 } 476 477 /* Return the offset into CPUARMState of a slice (from 478 * the least significant end) of FP register Qn (ie 479 * Dn, Sn, Hn or Bn). 480 * (Note that this is not the same mapping as for A32; see cpu.h) 481 */ 482 static inline int fp_reg_offset(DisasContext *s, int regno, MemOp size) 483 { 484 return vec_reg_offset(s, regno, 0, size); 485 } 486 487 /* Offset of the high half of the 128 bit vector Qn */ 488 static inline int fp_reg_hi_offset(DisasContext *s, int regno) 489 { 490 return vec_reg_offset(s, regno, 1, MO_64); 491 } 492 493 /* Convenience accessors for reading and writing single and double 494 * FP registers. Writing clears the upper parts of the associated 495 * 128 bit vector register, as required by the architecture. 496 * Note that unlike the GP register accessors, the values returned 497 * by the read functions must be manually freed. 498 */ 499 static TCGv_i64 read_fp_dreg(DisasContext *s, int reg) 500 { 501 TCGv_i64 v = tcg_temp_new_i64(); 502 503 tcg_gen_ld_i64(v, cpu_env, fp_reg_offset(s, reg, MO_64)); 504 return v; 505 } 506 507 static TCGv_i32 read_fp_sreg(DisasContext *s, int reg) 508 { 509 TCGv_i32 v = tcg_temp_new_i32(); 510 511 tcg_gen_ld_i32(v, cpu_env, fp_reg_offset(s, reg, MO_32)); 512 return v; 513 } 514 515 static TCGv_i32 read_fp_hreg(DisasContext *s, int reg) 516 { 517 TCGv_i32 v = tcg_temp_new_i32(); 518 519 tcg_gen_ld16u_i32(v, cpu_env, fp_reg_offset(s, reg, MO_16)); 520 return v; 521 } 522 523 /* Clear the bits above an N-bit vector, for N = (is_q ? 128 : 64). 524 * If SVE is not enabled, then there are only 128 bits in the vector. 525 */ 526 static void clear_vec_high(DisasContext *s, bool is_q, int rd) 527 { 528 unsigned ofs = fp_reg_offset(s, rd, MO_64); 529 unsigned vsz = vec_full_reg_size(s); 530 531 /* Nop move, with side effect of clearing the tail. */ 532 tcg_gen_gvec_mov(MO_64, ofs, ofs, is_q ? 16 : 8, vsz); 533 } 534 535 void write_fp_dreg(DisasContext *s, int reg, TCGv_i64 v) 536 { 537 unsigned ofs = fp_reg_offset(s, reg, MO_64); 538 539 tcg_gen_st_i64(v, cpu_env, ofs); 540 clear_vec_high(s, false, reg); 541 } 542 543 static void write_fp_sreg(DisasContext *s, int reg, TCGv_i32 v) 544 { 545 TCGv_i64 tmp = tcg_temp_new_i64(); 546 547 tcg_gen_extu_i32_i64(tmp, v); 548 write_fp_dreg(s, reg, tmp); 549 } 550 551 /* Expand a 2-operand AdvSIMD vector operation using an expander function. */ 552 static void gen_gvec_fn2(DisasContext *s, bool is_q, int rd, int rn, 553 GVecGen2Fn *gvec_fn, int vece) 554 { 555 gvec_fn(vece, vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn), 556 is_q ? 16 : 8, vec_full_reg_size(s)); 557 } 558 559 /* Expand a 2-operand + immediate AdvSIMD vector operation using 560 * an expander function. 561 */ 562 static void gen_gvec_fn2i(DisasContext *s, bool is_q, int rd, int rn, 563 int64_t imm, GVecGen2iFn *gvec_fn, int vece) 564 { 565 gvec_fn(vece, vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn), 566 imm, is_q ? 16 : 8, vec_full_reg_size(s)); 567 } 568 569 /* Expand a 3-operand AdvSIMD vector operation using an expander function. */ 570 static void gen_gvec_fn3(DisasContext *s, bool is_q, int rd, int rn, int rm, 571 GVecGen3Fn *gvec_fn, int vece) 572 { 573 gvec_fn(vece, vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn), 574 vec_full_reg_offset(s, rm), is_q ? 16 : 8, vec_full_reg_size(s)); 575 } 576 577 /* Expand a 4-operand AdvSIMD vector operation using an expander function. */ 578 static void gen_gvec_fn4(DisasContext *s, bool is_q, int rd, int rn, int rm, 579 int rx, GVecGen4Fn *gvec_fn, int vece) 580 { 581 gvec_fn(vece, vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn), 582 vec_full_reg_offset(s, rm), vec_full_reg_offset(s, rx), 583 is_q ? 16 : 8, vec_full_reg_size(s)); 584 } 585 586 /* Expand a 2-operand operation using an out-of-line helper. */ 587 static void gen_gvec_op2_ool(DisasContext *s, bool is_q, int rd, 588 int rn, int data, gen_helper_gvec_2 *fn) 589 { 590 tcg_gen_gvec_2_ool(vec_full_reg_offset(s, rd), 591 vec_full_reg_offset(s, rn), 592 is_q ? 16 : 8, vec_full_reg_size(s), data, fn); 593 } 594 595 /* Expand a 3-operand operation using an out-of-line helper. */ 596 static void gen_gvec_op3_ool(DisasContext *s, bool is_q, int rd, 597 int rn, int rm, int data, gen_helper_gvec_3 *fn) 598 { 599 tcg_gen_gvec_3_ool(vec_full_reg_offset(s, rd), 600 vec_full_reg_offset(s, rn), 601 vec_full_reg_offset(s, rm), 602 is_q ? 16 : 8, vec_full_reg_size(s), data, fn); 603 } 604 605 /* Expand a 3-operand + fpstatus pointer + simd data value operation using 606 * an out-of-line helper. 607 */ 608 static void gen_gvec_op3_fpst(DisasContext *s, bool is_q, int rd, int rn, 609 int rm, bool is_fp16, int data, 610 gen_helper_gvec_3_ptr *fn) 611 { 612 TCGv_ptr fpst = fpstatus_ptr(is_fp16 ? FPST_FPCR_F16 : FPST_FPCR); 613 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd), 614 vec_full_reg_offset(s, rn), 615 vec_full_reg_offset(s, rm), fpst, 616 is_q ? 16 : 8, vec_full_reg_size(s), data, fn); 617 } 618 619 /* Expand a 3-operand + qc + operation using an out-of-line helper. */ 620 static void gen_gvec_op3_qc(DisasContext *s, bool is_q, int rd, int rn, 621 int rm, gen_helper_gvec_3_ptr *fn) 622 { 623 TCGv_ptr qc_ptr = tcg_temp_new_ptr(); 624 625 tcg_gen_addi_ptr(qc_ptr, cpu_env, offsetof(CPUARMState, vfp.qc)); 626 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd), 627 vec_full_reg_offset(s, rn), 628 vec_full_reg_offset(s, rm), qc_ptr, 629 is_q ? 16 : 8, vec_full_reg_size(s), 0, fn); 630 } 631 632 /* Expand a 4-operand operation using an out-of-line helper. */ 633 static void gen_gvec_op4_ool(DisasContext *s, bool is_q, int rd, int rn, 634 int rm, int ra, int data, gen_helper_gvec_4 *fn) 635 { 636 tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd), 637 vec_full_reg_offset(s, rn), 638 vec_full_reg_offset(s, rm), 639 vec_full_reg_offset(s, ra), 640 is_q ? 16 : 8, vec_full_reg_size(s), data, fn); 641 } 642 643 /* 644 * Expand a 4-operand + fpstatus pointer + simd data value operation using 645 * an out-of-line helper. 646 */ 647 static void gen_gvec_op4_fpst(DisasContext *s, bool is_q, int rd, int rn, 648 int rm, int ra, bool is_fp16, int data, 649 gen_helper_gvec_4_ptr *fn) 650 { 651 TCGv_ptr fpst = fpstatus_ptr(is_fp16 ? FPST_FPCR_F16 : FPST_FPCR); 652 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, rd), 653 vec_full_reg_offset(s, rn), 654 vec_full_reg_offset(s, rm), 655 vec_full_reg_offset(s, ra), fpst, 656 is_q ? 16 : 8, vec_full_reg_size(s), data, fn); 657 } 658 659 /* Set ZF and NF based on a 64 bit result. This is alas fiddlier 660 * than the 32 bit equivalent. 661 */ 662 static inline void gen_set_NZ64(TCGv_i64 result) 663 { 664 tcg_gen_extr_i64_i32(cpu_ZF, cpu_NF, result); 665 tcg_gen_or_i32(cpu_ZF, cpu_ZF, cpu_NF); 666 } 667 668 /* Set NZCV as for a logical operation: NZ as per result, CV cleared. */ 669 static inline void gen_logic_CC(int sf, TCGv_i64 result) 670 { 671 if (sf) { 672 gen_set_NZ64(result); 673 } else { 674 tcg_gen_extrl_i64_i32(cpu_ZF, result); 675 tcg_gen_mov_i32(cpu_NF, cpu_ZF); 676 } 677 tcg_gen_movi_i32(cpu_CF, 0); 678 tcg_gen_movi_i32(cpu_VF, 0); 679 } 680 681 /* dest = T0 + T1; compute C, N, V and Z flags */ 682 static void gen_add64_CC(TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1) 683 { 684 TCGv_i64 result, flag, tmp; 685 result = tcg_temp_new_i64(); 686 flag = tcg_temp_new_i64(); 687 tmp = tcg_temp_new_i64(); 688 689 tcg_gen_movi_i64(tmp, 0); 690 tcg_gen_add2_i64(result, flag, t0, tmp, t1, tmp); 691 692 tcg_gen_extrl_i64_i32(cpu_CF, flag); 693 694 gen_set_NZ64(result); 695 696 tcg_gen_xor_i64(flag, result, t0); 697 tcg_gen_xor_i64(tmp, t0, t1); 698 tcg_gen_andc_i64(flag, flag, tmp); 699 tcg_gen_extrh_i64_i32(cpu_VF, flag); 700 701 tcg_gen_mov_i64(dest, result); 702 } 703 704 static void gen_add32_CC(TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1) 705 { 706 TCGv_i32 t0_32 = tcg_temp_new_i32(); 707 TCGv_i32 t1_32 = tcg_temp_new_i32(); 708 TCGv_i32 tmp = tcg_temp_new_i32(); 709 710 tcg_gen_movi_i32(tmp, 0); 711 tcg_gen_extrl_i64_i32(t0_32, t0); 712 tcg_gen_extrl_i64_i32(t1_32, t1); 713 tcg_gen_add2_i32(cpu_NF, cpu_CF, t0_32, tmp, t1_32, tmp); 714 tcg_gen_mov_i32(cpu_ZF, cpu_NF); 715 tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32); 716 tcg_gen_xor_i32(tmp, t0_32, t1_32); 717 tcg_gen_andc_i32(cpu_VF, cpu_VF, tmp); 718 tcg_gen_extu_i32_i64(dest, cpu_NF); 719 } 720 721 static void gen_add_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1) 722 { 723 if (sf) { 724 gen_add64_CC(dest, t0, t1); 725 } else { 726 gen_add32_CC(dest, t0, t1); 727 } 728 } 729 730 /* dest = T0 - T1; compute C, N, V and Z flags */ 731 static void gen_sub64_CC(TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1) 732 { 733 /* 64 bit arithmetic */ 734 TCGv_i64 result, flag, tmp; 735 736 result = tcg_temp_new_i64(); 737 flag = tcg_temp_new_i64(); 738 tcg_gen_sub_i64(result, t0, t1); 739 740 gen_set_NZ64(result); 741 742 tcg_gen_setcond_i64(TCG_COND_GEU, flag, t0, t1); 743 tcg_gen_extrl_i64_i32(cpu_CF, flag); 744 745 tcg_gen_xor_i64(flag, result, t0); 746 tmp = tcg_temp_new_i64(); 747 tcg_gen_xor_i64(tmp, t0, t1); 748 tcg_gen_and_i64(flag, flag, tmp); 749 tcg_gen_extrh_i64_i32(cpu_VF, flag); 750 tcg_gen_mov_i64(dest, result); 751 } 752 753 static void gen_sub32_CC(TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1) 754 { 755 /* 32 bit arithmetic */ 756 TCGv_i32 t0_32 = tcg_temp_new_i32(); 757 TCGv_i32 t1_32 = tcg_temp_new_i32(); 758 TCGv_i32 tmp; 759 760 tcg_gen_extrl_i64_i32(t0_32, t0); 761 tcg_gen_extrl_i64_i32(t1_32, t1); 762 tcg_gen_sub_i32(cpu_NF, t0_32, t1_32); 763 tcg_gen_mov_i32(cpu_ZF, cpu_NF); 764 tcg_gen_setcond_i32(TCG_COND_GEU, cpu_CF, t0_32, t1_32); 765 tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32); 766 tmp = tcg_temp_new_i32(); 767 tcg_gen_xor_i32(tmp, t0_32, t1_32); 768 tcg_gen_and_i32(cpu_VF, cpu_VF, tmp); 769 tcg_gen_extu_i32_i64(dest, cpu_NF); 770 } 771 772 static void gen_sub_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1) 773 { 774 if (sf) { 775 gen_sub64_CC(dest, t0, t1); 776 } else { 777 gen_sub32_CC(dest, t0, t1); 778 } 779 } 780 781 /* dest = T0 + T1 + CF; do not compute flags. */ 782 static void gen_adc(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1) 783 { 784 TCGv_i64 flag = tcg_temp_new_i64(); 785 tcg_gen_extu_i32_i64(flag, cpu_CF); 786 tcg_gen_add_i64(dest, t0, t1); 787 tcg_gen_add_i64(dest, dest, flag); 788 789 if (!sf) { 790 tcg_gen_ext32u_i64(dest, dest); 791 } 792 } 793 794 /* dest = T0 + T1 + CF; compute C, N, V and Z flags. */ 795 static void gen_adc_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1) 796 { 797 if (sf) { 798 TCGv_i64 result = tcg_temp_new_i64(); 799 TCGv_i64 cf_64 = tcg_temp_new_i64(); 800 TCGv_i64 vf_64 = tcg_temp_new_i64(); 801 TCGv_i64 tmp = tcg_temp_new_i64(); 802 TCGv_i64 zero = tcg_constant_i64(0); 803 804 tcg_gen_extu_i32_i64(cf_64, cpu_CF); 805 tcg_gen_add2_i64(result, cf_64, t0, zero, cf_64, zero); 806 tcg_gen_add2_i64(result, cf_64, result, cf_64, t1, zero); 807 tcg_gen_extrl_i64_i32(cpu_CF, cf_64); 808 gen_set_NZ64(result); 809 810 tcg_gen_xor_i64(vf_64, result, t0); 811 tcg_gen_xor_i64(tmp, t0, t1); 812 tcg_gen_andc_i64(vf_64, vf_64, tmp); 813 tcg_gen_extrh_i64_i32(cpu_VF, vf_64); 814 815 tcg_gen_mov_i64(dest, result); 816 } else { 817 TCGv_i32 t0_32 = tcg_temp_new_i32(); 818 TCGv_i32 t1_32 = tcg_temp_new_i32(); 819 TCGv_i32 tmp = tcg_temp_new_i32(); 820 TCGv_i32 zero = tcg_constant_i32(0); 821 822 tcg_gen_extrl_i64_i32(t0_32, t0); 823 tcg_gen_extrl_i64_i32(t1_32, t1); 824 tcg_gen_add2_i32(cpu_NF, cpu_CF, t0_32, zero, cpu_CF, zero); 825 tcg_gen_add2_i32(cpu_NF, cpu_CF, cpu_NF, cpu_CF, t1_32, zero); 826 827 tcg_gen_mov_i32(cpu_ZF, cpu_NF); 828 tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32); 829 tcg_gen_xor_i32(tmp, t0_32, t1_32); 830 tcg_gen_andc_i32(cpu_VF, cpu_VF, tmp); 831 tcg_gen_extu_i32_i64(dest, cpu_NF); 832 } 833 } 834 835 /* 836 * Load/Store generators 837 */ 838 839 /* 840 * Store from GPR register to memory. 841 */ 842 static void do_gpr_st_memidx(DisasContext *s, TCGv_i64 source, 843 TCGv_i64 tcg_addr, MemOp memop, int memidx, 844 bool iss_valid, 845 unsigned int iss_srt, 846 bool iss_sf, bool iss_ar) 847 { 848 memop = finalize_memop(s, memop); 849 tcg_gen_qemu_st_i64(source, tcg_addr, memidx, memop); 850 851 if (iss_valid) { 852 uint32_t syn; 853 854 syn = syn_data_abort_with_iss(0, 855 (memop & MO_SIZE), 856 false, 857 iss_srt, 858 iss_sf, 859 iss_ar, 860 0, 0, 0, 0, 0, false); 861 disas_set_insn_syndrome(s, syn); 862 } 863 } 864 865 static void do_gpr_st(DisasContext *s, TCGv_i64 source, 866 TCGv_i64 tcg_addr, MemOp memop, 867 bool iss_valid, 868 unsigned int iss_srt, 869 bool iss_sf, bool iss_ar) 870 { 871 do_gpr_st_memidx(s, source, tcg_addr, memop, get_mem_index(s), 872 iss_valid, iss_srt, iss_sf, iss_ar); 873 } 874 875 /* 876 * Load from memory to GPR register 877 */ 878 static void do_gpr_ld_memidx(DisasContext *s, TCGv_i64 dest, TCGv_i64 tcg_addr, 879 MemOp memop, bool extend, int memidx, 880 bool iss_valid, unsigned int iss_srt, 881 bool iss_sf, bool iss_ar) 882 { 883 memop = finalize_memop(s, memop); 884 tcg_gen_qemu_ld_i64(dest, tcg_addr, memidx, memop); 885 886 if (extend && (memop & MO_SIGN)) { 887 g_assert((memop & MO_SIZE) <= MO_32); 888 tcg_gen_ext32u_i64(dest, dest); 889 } 890 891 if (iss_valid) { 892 uint32_t syn; 893 894 syn = syn_data_abort_with_iss(0, 895 (memop & MO_SIZE), 896 (memop & MO_SIGN) != 0, 897 iss_srt, 898 iss_sf, 899 iss_ar, 900 0, 0, 0, 0, 0, false); 901 disas_set_insn_syndrome(s, syn); 902 } 903 } 904 905 static void do_gpr_ld(DisasContext *s, TCGv_i64 dest, TCGv_i64 tcg_addr, 906 MemOp memop, bool extend, 907 bool iss_valid, unsigned int iss_srt, 908 bool iss_sf, bool iss_ar) 909 { 910 do_gpr_ld_memidx(s, dest, tcg_addr, memop, extend, get_mem_index(s), 911 iss_valid, iss_srt, iss_sf, iss_ar); 912 } 913 914 /* 915 * Store from FP register to memory 916 */ 917 static void do_fp_st(DisasContext *s, int srcidx, TCGv_i64 tcg_addr, int size) 918 { 919 /* This writes the bottom N bits of a 128 bit wide vector to memory */ 920 TCGv_i64 tmplo = tcg_temp_new_i64(); 921 MemOp mop; 922 923 tcg_gen_ld_i64(tmplo, cpu_env, fp_reg_offset(s, srcidx, MO_64)); 924 925 if (size < 4) { 926 mop = finalize_memop(s, size); 927 tcg_gen_qemu_st_i64(tmplo, tcg_addr, get_mem_index(s), mop); 928 } else { 929 bool be = s->be_data == MO_BE; 930 TCGv_i64 tcg_hiaddr = tcg_temp_new_i64(); 931 TCGv_i64 tmphi = tcg_temp_new_i64(); 932 933 tcg_gen_ld_i64(tmphi, cpu_env, fp_reg_hi_offset(s, srcidx)); 934 935 mop = s->be_data | MO_UQ; 936 tcg_gen_qemu_st_i64(be ? tmphi : tmplo, tcg_addr, get_mem_index(s), 937 mop | (s->align_mem ? MO_ALIGN_16 : 0)); 938 tcg_gen_addi_i64(tcg_hiaddr, tcg_addr, 8); 939 tcg_gen_qemu_st_i64(be ? tmplo : tmphi, tcg_hiaddr, 940 get_mem_index(s), mop); 941 } 942 } 943 944 /* 945 * Load from memory to FP register 946 */ 947 static void do_fp_ld(DisasContext *s, int destidx, TCGv_i64 tcg_addr, int size) 948 { 949 /* This always zero-extends and writes to a full 128 bit wide vector */ 950 TCGv_i64 tmplo = tcg_temp_new_i64(); 951 TCGv_i64 tmphi = NULL; 952 MemOp mop; 953 954 if (size < 4) { 955 mop = finalize_memop(s, size); 956 tcg_gen_qemu_ld_i64(tmplo, tcg_addr, get_mem_index(s), mop); 957 } else { 958 bool be = s->be_data == MO_BE; 959 TCGv_i64 tcg_hiaddr; 960 961 tmphi = tcg_temp_new_i64(); 962 tcg_hiaddr = tcg_temp_new_i64(); 963 964 mop = s->be_data | MO_UQ; 965 tcg_gen_qemu_ld_i64(be ? tmphi : tmplo, tcg_addr, get_mem_index(s), 966 mop | (s->align_mem ? MO_ALIGN_16 : 0)); 967 tcg_gen_addi_i64(tcg_hiaddr, tcg_addr, 8); 968 tcg_gen_qemu_ld_i64(be ? tmplo : tmphi, tcg_hiaddr, 969 get_mem_index(s), mop); 970 } 971 972 tcg_gen_st_i64(tmplo, cpu_env, fp_reg_offset(s, destidx, MO_64)); 973 974 if (tmphi) { 975 tcg_gen_st_i64(tmphi, cpu_env, fp_reg_hi_offset(s, destidx)); 976 } 977 clear_vec_high(s, tmphi != NULL, destidx); 978 } 979 980 /* 981 * Vector load/store helpers. 982 * 983 * The principal difference between this and a FP load is that we don't 984 * zero extend as we are filling a partial chunk of the vector register. 985 * These functions don't support 128 bit loads/stores, which would be 986 * normal load/store operations. 987 * 988 * The _i32 versions are useful when operating on 32 bit quantities 989 * (eg for floating point single or using Neon helper functions). 990 */ 991 992 /* Get value of an element within a vector register */ 993 static void read_vec_element(DisasContext *s, TCGv_i64 tcg_dest, int srcidx, 994 int element, MemOp memop) 995 { 996 int vect_off = vec_reg_offset(s, srcidx, element, memop & MO_SIZE); 997 switch ((unsigned)memop) { 998 case MO_8: 999 tcg_gen_ld8u_i64(tcg_dest, cpu_env, vect_off); 1000 break; 1001 case MO_16: 1002 tcg_gen_ld16u_i64(tcg_dest, cpu_env, vect_off); 1003 break; 1004 case MO_32: 1005 tcg_gen_ld32u_i64(tcg_dest, cpu_env, vect_off); 1006 break; 1007 case MO_8|MO_SIGN: 1008 tcg_gen_ld8s_i64(tcg_dest, cpu_env, vect_off); 1009 break; 1010 case MO_16|MO_SIGN: 1011 tcg_gen_ld16s_i64(tcg_dest, cpu_env, vect_off); 1012 break; 1013 case MO_32|MO_SIGN: 1014 tcg_gen_ld32s_i64(tcg_dest, cpu_env, vect_off); 1015 break; 1016 case MO_64: 1017 case MO_64|MO_SIGN: 1018 tcg_gen_ld_i64(tcg_dest, cpu_env, vect_off); 1019 break; 1020 default: 1021 g_assert_not_reached(); 1022 } 1023 } 1024 1025 static void read_vec_element_i32(DisasContext *s, TCGv_i32 tcg_dest, int srcidx, 1026 int element, MemOp memop) 1027 { 1028 int vect_off = vec_reg_offset(s, srcidx, element, memop & MO_SIZE); 1029 switch (memop) { 1030 case MO_8: 1031 tcg_gen_ld8u_i32(tcg_dest, cpu_env, vect_off); 1032 break; 1033 case MO_16: 1034 tcg_gen_ld16u_i32(tcg_dest, cpu_env, vect_off); 1035 break; 1036 case MO_8|MO_SIGN: 1037 tcg_gen_ld8s_i32(tcg_dest, cpu_env, vect_off); 1038 break; 1039 case MO_16|MO_SIGN: 1040 tcg_gen_ld16s_i32(tcg_dest, cpu_env, vect_off); 1041 break; 1042 case MO_32: 1043 case MO_32|MO_SIGN: 1044 tcg_gen_ld_i32(tcg_dest, cpu_env, vect_off); 1045 break; 1046 default: 1047 g_assert_not_reached(); 1048 } 1049 } 1050 1051 /* Set value of an element within a vector register */ 1052 static void write_vec_element(DisasContext *s, TCGv_i64 tcg_src, int destidx, 1053 int element, MemOp memop) 1054 { 1055 int vect_off = vec_reg_offset(s, destidx, element, memop & MO_SIZE); 1056 switch (memop) { 1057 case MO_8: 1058 tcg_gen_st8_i64(tcg_src, cpu_env, vect_off); 1059 break; 1060 case MO_16: 1061 tcg_gen_st16_i64(tcg_src, cpu_env, vect_off); 1062 break; 1063 case MO_32: 1064 tcg_gen_st32_i64(tcg_src, cpu_env, vect_off); 1065 break; 1066 case MO_64: 1067 tcg_gen_st_i64(tcg_src, cpu_env, vect_off); 1068 break; 1069 default: 1070 g_assert_not_reached(); 1071 } 1072 } 1073 1074 static void write_vec_element_i32(DisasContext *s, TCGv_i32 tcg_src, 1075 int destidx, int element, MemOp memop) 1076 { 1077 int vect_off = vec_reg_offset(s, destidx, element, memop & MO_SIZE); 1078 switch (memop) { 1079 case MO_8: 1080 tcg_gen_st8_i32(tcg_src, cpu_env, vect_off); 1081 break; 1082 case MO_16: 1083 tcg_gen_st16_i32(tcg_src, cpu_env, vect_off); 1084 break; 1085 case MO_32: 1086 tcg_gen_st_i32(tcg_src, cpu_env, vect_off); 1087 break; 1088 default: 1089 g_assert_not_reached(); 1090 } 1091 } 1092 1093 /* Store from vector register to memory */ 1094 static void do_vec_st(DisasContext *s, int srcidx, int element, 1095 TCGv_i64 tcg_addr, MemOp mop) 1096 { 1097 TCGv_i64 tcg_tmp = tcg_temp_new_i64(); 1098 1099 read_vec_element(s, tcg_tmp, srcidx, element, mop & MO_SIZE); 1100 tcg_gen_qemu_st_i64(tcg_tmp, tcg_addr, get_mem_index(s), mop); 1101 } 1102 1103 /* Load from memory to vector register */ 1104 static void do_vec_ld(DisasContext *s, int destidx, int element, 1105 TCGv_i64 tcg_addr, MemOp mop) 1106 { 1107 TCGv_i64 tcg_tmp = tcg_temp_new_i64(); 1108 1109 tcg_gen_qemu_ld_i64(tcg_tmp, tcg_addr, get_mem_index(s), mop); 1110 write_vec_element(s, tcg_tmp, destidx, element, mop & MO_SIZE); 1111 } 1112 1113 /* Check that FP/Neon access is enabled. If it is, return 1114 * true. If not, emit code to generate an appropriate exception, 1115 * and return false; the caller should not emit any code for 1116 * the instruction. Note that this check must happen after all 1117 * unallocated-encoding checks (otherwise the syndrome information 1118 * for the resulting exception will be incorrect). 1119 */ 1120 static bool fp_access_check_only(DisasContext *s) 1121 { 1122 if (s->fp_excp_el) { 1123 assert(!s->fp_access_checked); 1124 s->fp_access_checked = true; 1125 1126 gen_exception_insn_el(s, 0, EXCP_UDEF, 1127 syn_fp_access_trap(1, 0xe, false, 0), 1128 s->fp_excp_el); 1129 return false; 1130 } 1131 s->fp_access_checked = true; 1132 return true; 1133 } 1134 1135 static bool fp_access_check(DisasContext *s) 1136 { 1137 if (!fp_access_check_only(s)) { 1138 return false; 1139 } 1140 if (s->sme_trap_nonstreaming && s->is_nonstreaming) { 1141 gen_exception_insn(s, 0, EXCP_UDEF, 1142 syn_smetrap(SME_ET_Streaming, false)); 1143 return false; 1144 } 1145 return true; 1146 } 1147 1148 /* 1149 * Check that SVE access is enabled. If it is, return true. 1150 * If not, emit code to generate an appropriate exception and return false. 1151 * This function corresponds to CheckSVEEnabled(). 1152 */ 1153 bool sve_access_check(DisasContext *s) 1154 { 1155 if (s->pstate_sm || !dc_isar_feature(aa64_sve, s)) { 1156 assert(dc_isar_feature(aa64_sme, s)); 1157 if (!sme_sm_enabled_check(s)) { 1158 goto fail_exit; 1159 } 1160 } else if (s->sve_excp_el) { 1161 gen_exception_insn_el(s, 0, EXCP_UDEF, 1162 syn_sve_access_trap(), s->sve_excp_el); 1163 goto fail_exit; 1164 } 1165 s->sve_access_checked = true; 1166 return fp_access_check(s); 1167 1168 fail_exit: 1169 /* Assert that we only raise one exception per instruction. */ 1170 assert(!s->sve_access_checked); 1171 s->sve_access_checked = true; 1172 return false; 1173 } 1174 1175 /* 1176 * Check that SME access is enabled, raise an exception if not. 1177 * Note that this function corresponds to CheckSMEAccess and is 1178 * only used directly for cpregs. 1179 */ 1180 static bool sme_access_check(DisasContext *s) 1181 { 1182 if (s->sme_excp_el) { 1183 gen_exception_insn_el(s, 0, EXCP_UDEF, 1184 syn_smetrap(SME_ET_AccessTrap, false), 1185 s->sme_excp_el); 1186 return false; 1187 } 1188 return true; 1189 } 1190 1191 /* This function corresponds to CheckSMEEnabled. */ 1192 bool sme_enabled_check(DisasContext *s) 1193 { 1194 /* 1195 * Note that unlike sve_excp_el, we have not constrained sme_excp_el 1196 * to be zero when fp_excp_el has priority. This is because we need 1197 * sme_excp_el by itself for cpregs access checks. 1198 */ 1199 if (!s->fp_excp_el || s->sme_excp_el < s->fp_excp_el) { 1200 s->fp_access_checked = true; 1201 return sme_access_check(s); 1202 } 1203 return fp_access_check_only(s); 1204 } 1205 1206 /* Common subroutine for CheckSMEAnd*Enabled. */ 1207 bool sme_enabled_check_with_svcr(DisasContext *s, unsigned req) 1208 { 1209 if (!sme_enabled_check(s)) { 1210 return false; 1211 } 1212 if (FIELD_EX64(req, SVCR, SM) && !s->pstate_sm) { 1213 gen_exception_insn(s, 0, EXCP_UDEF, 1214 syn_smetrap(SME_ET_NotStreaming, false)); 1215 return false; 1216 } 1217 if (FIELD_EX64(req, SVCR, ZA) && !s->pstate_za) { 1218 gen_exception_insn(s, 0, EXCP_UDEF, 1219 syn_smetrap(SME_ET_InactiveZA, false)); 1220 return false; 1221 } 1222 return true; 1223 } 1224 1225 /* 1226 * This utility function is for doing register extension with an 1227 * optional shift. You will likely want to pass a temporary for the 1228 * destination register. See DecodeRegExtend() in the ARM ARM. 1229 */ 1230 static void ext_and_shift_reg(TCGv_i64 tcg_out, TCGv_i64 tcg_in, 1231 int option, unsigned int shift) 1232 { 1233 int extsize = extract32(option, 0, 2); 1234 bool is_signed = extract32(option, 2, 1); 1235 1236 if (is_signed) { 1237 switch (extsize) { 1238 case 0: 1239 tcg_gen_ext8s_i64(tcg_out, tcg_in); 1240 break; 1241 case 1: 1242 tcg_gen_ext16s_i64(tcg_out, tcg_in); 1243 break; 1244 case 2: 1245 tcg_gen_ext32s_i64(tcg_out, tcg_in); 1246 break; 1247 case 3: 1248 tcg_gen_mov_i64(tcg_out, tcg_in); 1249 break; 1250 } 1251 } else { 1252 switch (extsize) { 1253 case 0: 1254 tcg_gen_ext8u_i64(tcg_out, tcg_in); 1255 break; 1256 case 1: 1257 tcg_gen_ext16u_i64(tcg_out, tcg_in); 1258 break; 1259 case 2: 1260 tcg_gen_ext32u_i64(tcg_out, tcg_in); 1261 break; 1262 case 3: 1263 tcg_gen_mov_i64(tcg_out, tcg_in); 1264 break; 1265 } 1266 } 1267 1268 if (shift) { 1269 tcg_gen_shli_i64(tcg_out, tcg_out, shift); 1270 } 1271 } 1272 1273 static inline void gen_check_sp_alignment(DisasContext *s) 1274 { 1275 /* The AArch64 architecture mandates that (if enabled via PSTATE 1276 * or SCTLR bits) there is a check that SP is 16-aligned on every 1277 * SP-relative load or store (with an exception generated if it is not). 1278 * In line with general QEMU practice regarding misaligned accesses, 1279 * we omit these checks for the sake of guest program performance. 1280 * This function is provided as a hook so we can more easily add these 1281 * checks in future (possibly as a "favour catching guest program bugs 1282 * over speed" user selectable option). 1283 */ 1284 } 1285 1286 /* 1287 * This provides a simple table based table lookup decoder. It is 1288 * intended to be used when the relevant bits for decode are too 1289 * awkwardly placed and switch/if based logic would be confusing and 1290 * deeply nested. Since it's a linear search through the table, tables 1291 * should be kept small. 1292 * 1293 * It returns the first handler where insn & mask == pattern, or 1294 * NULL if there is no match. 1295 * The table is terminated by an empty mask (i.e. 0) 1296 */ 1297 static inline AArch64DecodeFn *lookup_disas_fn(const AArch64DecodeTable *table, 1298 uint32_t insn) 1299 { 1300 const AArch64DecodeTable *tptr = table; 1301 1302 while (tptr->mask) { 1303 if ((insn & tptr->mask) == tptr->pattern) { 1304 return tptr->disas_fn; 1305 } 1306 tptr++; 1307 } 1308 return NULL; 1309 } 1310 1311 /* 1312 * The instruction disassembly implemented here matches 1313 * the instruction encoding classifications in chapter C4 1314 * of the ARM Architecture Reference Manual (DDI0487B_a); 1315 * classification names and decode diagrams here should generally 1316 * match up with those in the manual. 1317 */ 1318 1319 static bool trans_B(DisasContext *s, arg_i *a) 1320 { 1321 reset_btype(s); 1322 gen_goto_tb(s, 0, a->imm); 1323 return true; 1324 } 1325 1326 static bool trans_BL(DisasContext *s, arg_i *a) 1327 { 1328 gen_pc_plus_diff(s, cpu_reg(s, 30), curr_insn_len(s)); 1329 reset_btype(s); 1330 gen_goto_tb(s, 0, a->imm); 1331 return true; 1332 } 1333 1334 1335 static bool trans_CBZ(DisasContext *s, arg_cbz *a) 1336 { 1337 DisasLabel match; 1338 TCGv_i64 tcg_cmp; 1339 1340 tcg_cmp = read_cpu_reg(s, a->rt, a->sf); 1341 reset_btype(s); 1342 1343 match = gen_disas_label(s); 1344 tcg_gen_brcondi_i64(a->nz ? TCG_COND_NE : TCG_COND_EQ, 1345 tcg_cmp, 0, match.label); 1346 gen_goto_tb(s, 0, 4); 1347 set_disas_label(s, match); 1348 gen_goto_tb(s, 1, a->imm); 1349 return true; 1350 } 1351 1352 static bool trans_TBZ(DisasContext *s, arg_tbz *a) 1353 { 1354 DisasLabel match; 1355 TCGv_i64 tcg_cmp; 1356 1357 tcg_cmp = tcg_temp_new_i64(); 1358 tcg_gen_andi_i64(tcg_cmp, cpu_reg(s, a->rt), 1ULL << a->bitpos); 1359 1360 reset_btype(s); 1361 1362 match = gen_disas_label(s); 1363 tcg_gen_brcondi_i64(a->nz ? TCG_COND_NE : TCG_COND_EQ, 1364 tcg_cmp, 0, match.label); 1365 gen_goto_tb(s, 0, 4); 1366 set_disas_label(s, match); 1367 gen_goto_tb(s, 1, a->imm); 1368 return true; 1369 } 1370 1371 static bool trans_B_cond(DisasContext *s, arg_B_cond *a) 1372 { 1373 reset_btype(s); 1374 if (a->cond < 0x0e) { 1375 /* genuinely conditional branches */ 1376 DisasLabel match = gen_disas_label(s); 1377 arm_gen_test_cc(a->cond, match.label); 1378 gen_goto_tb(s, 0, 4); 1379 set_disas_label(s, match); 1380 gen_goto_tb(s, 1, a->imm); 1381 } else { 1382 /* 0xe and 0xf are both "always" conditions */ 1383 gen_goto_tb(s, 0, a->imm); 1384 } 1385 return true; 1386 } 1387 1388 static void set_btype_for_br(DisasContext *s, int rn) 1389 { 1390 if (dc_isar_feature(aa64_bti, s)) { 1391 /* BR to {x16,x17} or !guard -> 1, else 3. */ 1392 set_btype(s, rn == 16 || rn == 17 || !s->guarded_page ? 1 : 3); 1393 } 1394 } 1395 1396 static void set_btype_for_blr(DisasContext *s) 1397 { 1398 if (dc_isar_feature(aa64_bti, s)) { 1399 /* BLR sets BTYPE to 2, regardless of source guarded page. */ 1400 set_btype(s, 2); 1401 } 1402 } 1403 1404 static bool trans_BR(DisasContext *s, arg_r *a) 1405 { 1406 gen_a64_set_pc(s, cpu_reg(s, a->rn)); 1407 set_btype_for_br(s, a->rn); 1408 s->base.is_jmp = DISAS_JUMP; 1409 return true; 1410 } 1411 1412 static bool trans_BLR(DisasContext *s, arg_r *a) 1413 { 1414 TCGv_i64 dst = cpu_reg(s, a->rn); 1415 TCGv_i64 lr = cpu_reg(s, 30); 1416 if (dst == lr) { 1417 TCGv_i64 tmp = tcg_temp_new_i64(); 1418 tcg_gen_mov_i64(tmp, dst); 1419 dst = tmp; 1420 } 1421 gen_pc_plus_diff(s, lr, curr_insn_len(s)); 1422 gen_a64_set_pc(s, dst); 1423 set_btype_for_blr(s); 1424 s->base.is_jmp = DISAS_JUMP; 1425 return true; 1426 } 1427 1428 static bool trans_RET(DisasContext *s, arg_r *a) 1429 { 1430 gen_a64_set_pc(s, cpu_reg(s, a->rn)); 1431 s->base.is_jmp = DISAS_JUMP; 1432 return true; 1433 } 1434 1435 static TCGv_i64 auth_branch_target(DisasContext *s, TCGv_i64 dst, 1436 TCGv_i64 modifier, bool use_key_a) 1437 { 1438 TCGv_i64 truedst; 1439 /* 1440 * Return the branch target for a BRAA/RETA/etc, which is either 1441 * just the destination dst, or that value with the pauth check 1442 * done and the code removed from the high bits. 1443 */ 1444 if (!s->pauth_active) { 1445 return dst; 1446 } 1447 1448 truedst = tcg_temp_new_i64(); 1449 if (use_key_a) { 1450 gen_helper_autia(truedst, cpu_env, dst, modifier); 1451 } else { 1452 gen_helper_autib(truedst, cpu_env, dst, modifier); 1453 } 1454 return truedst; 1455 } 1456 1457 static bool trans_BRAZ(DisasContext *s, arg_braz *a) 1458 { 1459 TCGv_i64 dst; 1460 1461 if (!dc_isar_feature(aa64_pauth, s)) { 1462 return false; 1463 } 1464 1465 dst = auth_branch_target(s, cpu_reg(s, a->rn), tcg_constant_i64(0), !a->m); 1466 gen_a64_set_pc(s, dst); 1467 set_btype_for_br(s, a->rn); 1468 s->base.is_jmp = DISAS_JUMP; 1469 return true; 1470 } 1471 1472 static bool trans_BLRAZ(DisasContext *s, arg_braz *a) 1473 { 1474 TCGv_i64 dst, lr; 1475 1476 if (!dc_isar_feature(aa64_pauth, s)) { 1477 return false; 1478 } 1479 1480 dst = auth_branch_target(s, cpu_reg(s, a->rn), tcg_constant_i64(0), !a->m); 1481 lr = cpu_reg(s, 30); 1482 if (dst == lr) { 1483 TCGv_i64 tmp = tcg_temp_new_i64(); 1484 tcg_gen_mov_i64(tmp, dst); 1485 dst = tmp; 1486 } 1487 gen_pc_plus_diff(s, lr, curr_insn_len(s)); 1488 gen_a64_set_pc(s, dst); 1489 set_btype_for_blr(s); 1490 s->base.is_jmp = DISAS_JUMP; 1491 return true; 1492 } 1493 1494 static bool trans_RETA(DisasContext *s, arg_reta *a) 1495 { 1496 TCGv_i64 dst; 1497 1498 dst = auth_branch_target(s, cpu_reg(s, 30), cpu_X[31], !a->m); 1499 gen_a64_set_pc(s, dst); 1500 s->base.is_jmp = DISAS_JUMP; 1501 return true; 1502 } 1503 1504 static bool trans_BRA(DisasContext *s, arg_bra *a) 1505 { 1506 TCGv_i64 dst; 1507 1508 if (!dc_isar_feature(aa64_pauth, s)) { 1509 return false; 1510 } 1511 dst = auth_branch_target(s, cpu_reg(s,a->rn), cpu_reg_sp(s, a->rm), !a->m); 1512 gen_a64_set_pc(s, dst); 1513 set_btype_for_br(s, a->rn); 1514 s->base.is_jmp = DISAS_JUMP; 1515 return true; 1516 } 1517 1518 static bool trans_BLRA(DisasContext *s, arg_bra *a) 1519 { 1520 TCGv_i64 dst, lr; 1521 1522 if (!dc_isar_feature(aa64_pauth, s)) { 1523 return false; 1524 } 1525 dst = auth_branch_target(s, cpu_reg(s, a->rn), cpu_reg_sp(s, a->rm), !a->m); 1526 lr = cpu_reg(s, 30); 1527 if (dst == lr) { 1528 TCGv_i64 tmp = tcg_temp_new_i64(); 1529 tcg_gen_mov_i64(tmp, dst); 1530 dst = tmp; 1531 } 1532 gen_pc_plus_diff(s, lr, curr_insn_len(s)); 1533 gen_a64_set_pc(s, dst); 1534 set_btype_for_blr(s); 1535 s->base.is_jmp = DISAS_JUMP; 1536 return true; 1537 } 1538 1539 static bool trans_ERET(DisasContext *s, arg_ERET *a) 1540 { 1541 TCGv_i64 dst; 1542 1543 if (s->current_el == 0) { 1544 return false; 1545 } 1546 if (s->fgt_eret) { 1547 gen_exception_insn_el(s, 0, EXCP_UDEF, 0, 2); 1548 return true; 1549 } 1550 dst = tcg_temp_new_i64(); 1551 tcg_gen_ld_i64(dst, cpu_env, 1552 offsetof(CPUARMState, elr_el[s->current_el])); 1553 1554 translator_io_start(&s->base); 1555 1556 gen_helper_exception_return(cpu_env, dst); 1557 /* Must exit loop to check un-masked IRQs */ 1558 s->base.is_jmp = DISAS_EXIT; 1559 return true; 1560 } 1561 1562 static bool trans_ERETA(DisasContext *s, arg_reta *a) 1563 { 1564 TCGv_i64 dst; 1565 1566 if (!dc_isar_feature(aa64_pauth, s)) { 1567 return false; 1568 } 1569 if (s->current_el == 0) { 1570 return false; 1571 } 1572 /* The FGT trap takes precedence over an auth trap. */ 1573 if (s->fgt_eret) { 1574 gen_exception_insn_el(s, 0, EXCP_UDEF, a->m ? 3 : 2, 2); 1575 return true; 1576 } 1577 dst = tcg_temp_new_i64(); 1578 tcg_gen_ld_i64(dst, cpu_env, 1579 offsetof(CPUARMState, elr_el[s->current_el])); 1580 1581 dst = auth_branch_target(s, dst, cpu_X[31], !a->m); 1582 1583 translator_io_start(&s->base); 1584 1585 gen_helper_exception_return(cpu_env, dst); 1586 /* Must exit loop to check un-masked IRQs */ 1587 s->base.is_jmp = DISAS_EXIT; 1588 return true; 1589 } 1590 1591 /* HINT instruction group, including various allocated HINTs */ 1592 static void handle_hint(DisasContext *s, uint32_t insn, 1593 unsigned int op1, unsigned int op2, unsigned int crm) 1594 { 1595 unsigned int selector = crm << 3 | op2; 1596 1597 if (op1 != 3) { 1598 unallocated_encoding(s); 1599 return; 1600 } 1601 1602 switch (selector) { 1603 case 0b00000: /* NOP */ 1604 break; 1605 case 0b00011: /* WFI */ 1606 s->base.is_jmp = DISAS_WFI; 1607 break; 1608 case 0b00001: /* YIELD */ 1609 /* When running in MTTCG we don't generate jumps to the yield and 1610 * WFE helpers as it won't affect the scheduling of other vCPUs. 1611 * If we wanted to more completely model WFE/SEV so we don't busy 1612 * spin unnecessarily we would need to do something more involved. 1613 */ 1614 if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) { 1615 s->base.is_jmp = DISAS_YIELD; 1616 } 1617 break; 1618 case 0b00010: /* WFE */ 1619 if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) { 1620 s->base.is_jmp = DISAS_WFE; 1621 } 1622 break; 1623 case 0b00100: /* SEV */ 1624 case 0b00101: /* SEVL */ 1625 case 0b00110: /* DGH */ 1626 /* we treat all as NOP at least for now */ 1627 break; 1628 case 0b00111: /* XPACLRI */ 1629 if (s->pauth_active) { 1630 gen_helper_xpaci(cpu_X[30], cpu_env, cpu_X[30]); 1631 } 1632 break; 1633 case 0b01000: /* PACIA1716 */ 1634 if (s->pauth_active) { 1635 gen_helper_pacia(cpu_X[17], cpu_env, cpu_X[17], cpu_X[16]); 1636 } 1637 break; 1638 case 0b01010: /* PACIB1716 */ 1639 if (s->pauth_active) { 1640 gen_helper_pacib(cpu_X[17], cpu_env, cpu_X[17], cpu_X[16]); 1641 } 1642 break; 1643 case 0b01100: /* AUTIA1716 */ 1644 if (s->pauth_active) { 1645 gen_helper_autia(cpu_X[17], cpu_env, cpu_X[17], cpu_X[16]); 1646 } 1647 break; 1648 case 0b01110: /* AUTIB1716 */ 1649 if (s->pauth_active) { 1650 gen_helper_autib(cpu_X[17], cpu_env, cpu_X[17], cpu_X[16]); 1651 } 1652 break; 1653 case 0b10000: /* ESB */ 1654 /* Without RAS, we must implement this as NOP. */ 1655 if (dc_isar_feature(aa64_ras, s)) { 1656 /* 1657 * QEMU does not have a source of physical SErrors, 1658 * so we are only concerned with virtual SErrors. 1659 * The pseudocode in the ARM for this case is 1660 * if PSTATE.EL IN {EL0, EL1} && EL2Enabled() then 1661 * AArch64.vESBOperation(); 1662 * Most of the condition can be evaluated at translation time. 1663 * Test for EL2 present, and defer test for SEL2 to runtime. 1664 */ 1665 if (s->current_el <= 1 && arm_dc_feature(s, ARM_FEATURE_EL2)) { 1666 gen_helper_vesb(cpu_env); 1667 } 1668 } 1669 break; 1670 case 0b11000: /* PACIAZ */ 1671 if (s->pauth_active) { 1672 gen_helper_pacia(cpu_X[30], cpu_env, cpu_X[30], 1673 tcg_constant_i64(0)); 1674 } 1675 break; 1676 case 0b11001: /* PACIASP */ 1677 if (s->pauth_active) { 1678 gen_helper_pacia(cpu_X[30], cpu_env, cpu_X[30], cpu_X[31]); 1679 } 1680 break; 1681 case 0b11010: /* PACIBZ */ 1682 if (s->pauth_active) { 1683 gen_helper_pacib(cpu_X[30], cpu_env, cpu_X[30], 1684 tcg_constant_i64(0)); 1685 } 1686 break; 1687 case 0b11011: /* PACIBSP */ 1688 if (s->pauth_active) { 1689 gen_helper_pacib(cpu_X[30], cpu_env, cpu_X[30], cpu_X[31]); 1690 } 1691 break; 1692 case 0b11100: /* AUTIAZ */ 1693 if (s->pauth_active) { 1694 gen_helper_autia(cpu_X[30], cpu_env, cpu_X[30], 1695 tcg_constant_i64(0)); 1696 } 1697 break; 1698 case 0b11101: /* AUTIASP */ 1699 if (s->pauth_active) { 1700 gen_helper_autia(cpu_X[30], cpu_env, cpu_X[30], cpu_X[31]); 1701 } 1702 break; 1703 case 0b11110: /* AUTIBZ */ 1704 if (s->pauth_active) { 1705 gen_helper_autib(cpu_X[30], cpu_env, cpu_X[30], 1706 tcg_constant_i64(0)); 1707 } 1708 break; 1709 case 0b11111: /* AUTIBSP */ 1710 if (s->pauth_active) { 1711 gen_helper_autib(cpu_X[30], cpu_env, cpu_X[30], cpu_X[31]); 1712 } 1713 break; 1714 default: 1715 /* default specified as NOP equivalent */ 1716 break; 1717 } 1718 } 1719 1720 static void gen_clrex(DisasContext *s, uint32_t insn) 1721 { 1722 tcg_gen_movi_i64(cpu_exclusive_addr, -1); 1723 } 1724 1725 /* CLREX, DSB, DMB, ISB */ 1726 static void handle_sync(DisasContext *s, uint32_t insn, 1727 unsigned int op1, unsigned int op2, unsigned int crm) 1728 { 1729 TCGBar bar; 1730 1731 if (op1 != 3) { 1732 unallocated_encoding(s); 1733 return; 1734 } 1735 1736 switch (op2) { 1737 case 2: /* CLREX */ 1738 gen_clrex(s, insn); 1739 return; 1740 case 4: /* DSB */ 1741 case 5: /* DMB */ 1742 switch (crm & 3) { 1743 case 1: /* MBReqTypes_Reads */ 1744 bar = TCG_BAR_SC | TCG_MO_LD_LD | TCG_MO_LD_ST; 1745 break; 1746 case 2: /* MBReqTypes_Writes */ 1747 bar = TCG_BAR_SC | TCG_MO_ST_ST; 1748 break; 1749 default: /* MBReqTypes_All */ 1750 bar = TCG_BAR_SC | TCG_MO_ALL; 1751 break; 1752 } 1753 tcg_gen_mb(bar); 1754 return; 1755 case 6: /* ISB */ 1756 /* We need to break the TB after this insn to execute 1757 * a self-modified code correctly and also to take 1758 * any pending interrupts immediately. 1759 */ 1760 reset_btype(s); 1761 gen_goto_tb(s, 0, 4); 1762 return; 1763 1764 case 7: /* SB */ 1765 if (crm != 0 || !dc_isar_feature(aa64_sb, s)) { 1766 goto do_unallocated; 1767 } 1768 /* 1769 * TODO: There is no speculation barrier opcode for TCG; 1770 * MB and end the TB instead. 1771 */ 1772 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_SC); 1773 gen_goto_tb(s, 0, 4); 1774 return; 1775 1776 default: 1777 do_unallocated: 1778 unallocated_encoding(s); 1779 return; 1780 } 1781 } 1782 1783 static void gen_xaflag(void) 1784 { 1785 TCGv_i32 z = tcg_temp_new_i32(); 1786 1787 tcg_gen_setcondi_i32(TCG_COND_EQ, z, cpu_ZF, 0); 1788 1789 /* 1790 * (!C & !Z) << 31 1791 * (!(C | Z)) << 31 1792 * ~((C | Z) << 31) 1793 * ~-(C | Z) 1794 * (C | Z) - 1 1795 */ 1796 tcg_gen_or_i32(cpu_NF, cpu_CF, z); 1797 tcg_gen_subi_i32(cpu_NF, cpu_NF, 1); 1798 1799 /* !(Z & C) */ 1800 tcg_gen_and_i32(cpu_ZF, z, cpu_CF); 1801 tcg_gen_xori_i32(cpu_ZF, cpu_ZF, 1); 1802 1803 /* (!C & Z) << 31 -> -(Z & ~C) */ 1804 tcg_gen_andc_i32(cpu_VF, z, cpu_CF); 1805 tcg_gen_neg_i32(cpu_VF, cpu_VF); 1806 1807 /* C | Z */ 1808 tcg_gen_or_i32(cpu_CF, cpu_CF, z); 1809 } 1810 1811 static void gen_axflag(void) 1812 { 1813 tcg_gen_sari_i32(cpu_VF, cpu_VF, 31); /* V ? -1 : 0 */ 1814 tcg_gen_andc_i32(cpu_CF, cpu_CF, cpu_VF); /* C & !V */ 1815 1816 /* !(Z | V) -> !(!ZF | V) -> ZF & !V -> ZF & ~VF */ 1817 tcg_gen_andc_i32(cpu_ZF, cpu_ZF, cpu_VF); 1818 1819 tcg_gen_movi_i32(cpu_NF, 0); 1820 tcg_gen_movi_i32(cpu_VF, 0); 1821 } 1822 1823 /* MSR (immediate) - move immediate to processor state field */ 1824 static void handle_msr_i(DisasContext *s, uint32_t insn, 1825 unsigned int op1, unsigned int op2, unsigned int crm) 1826 { 1827 int op = op1 << 3 | op2; 1828 1829 /* End the TB by default, chaining is ok. */ 1830 s->base.is_jmp = DISAS_TOO_MANY; 1831 1832 switch (op) { 1833 case 0x00: /* CFINV */ 1834 if (crm != 0 || !dc_isar_feature(aa64_condm_4, s)) { 1835 goto do_unallocated; 1836 } 1837 tcg_gen_xori_i32(cpu_CF, cpu_CF, 1); 1838 s->base.is_jmp = DISAS_NEXT; 1839 break; 1840 1841 case 0x01: /* XAFlag */ 1842 if (crm != 0 || !dc_isar_feature(aa64_condm_5, s)) { 1843 goto do_unallocated; 1844 } 1845 gen_xaflag(); 1846 s->base.is_jmp = DISAS_NEXT; 1847 break; 1848 1849 case 0x02: /* AXFlag */ 1850 if (crm != 0 || !dc_isar_feature(aa64_condm_5, s)) { 1851 goto do_unallocated; 1852 } 1853 gen_axflag(); 1854 s->base.is_jmp = DISAS_NEXT; 1855 break; 1856 1857 case 0x03: /* UAO */ 1858 if (!dc_isar_feature(aa64_uao, s) || s->current_el == 0) { 1859 goto do_unallocated; 1860 } 1861 if (crm & 1) { 1862 set_pstate_bits(PSTATE_UAO); 1863 } else { 1864 clear_pstate_bits(PSTATE_UAO); 1865 } 1866 gen_rebuild_hflags(s); 1867 break; 1868 1869 case 0x04: /* PAN */ 1870 if (!dc_isar_feature(aa64_pan, s) || s->current_el == 0) { 1871 goto do_unallocated; 1872 } 1873 if (crm & 1) { 1874 set_pstate_bits(PSTATE_PAN); 1875 } else { 1876 clear_pstate_bits(PSTATE_PAN); 1877 } 1878 gen_rebuild_hflags(s); 1879 break; 1880 1881 case 0x05: /* SPSel */ 1882 if (s->current_el == 0) { 1883 goto do_unallocated; 1884 } 1885 gen_helper_msr_i_spsel(cpu_env, tcg_constant_i32(crm & PSTATE_SP)); 1886 break; 1887 1888 case 0x19: /* SSBS */ 1889 if (!dc_isar_feature(aa64_ssbs, s)) { 1890 goto do_unallocated; 1891 } 1892 if (crm & 1) { 1893 set_pstate_bits(PSTATE_SSBS); 1894 } else { 1895 clear_pstate_bits(PSTATE_SSBS); 1896 } 1897 /* Don't need to rebuild hflags since SSBS is a nop */ 1898 break; 1899 1900 case 0x1a: /* DIT */ 1901 if (!dc_isar_feature(aa64_dit, s)) { 1902 goto do_unallocated; 1903 } 1904 if (crm & 1) { 1905 set_pstate_bits(PSTATE_DIT); 1906 } else { 1907 clear_pstate_bits(PSTATE_DIT); 1908 } 1909 /* There's no need to rebuild hflags because DIT is a nop */ 1910 break; 1911 1912 case 0x1e: /* DAIFSet */ 1913 gen_helper_msr_i_daifset(cpu_env, tcg_constant_i32(crm)); 1914 break; 1915 1916 case 0x1f: /* DAIFClear */ 1917 gen_helper_msr_i_daifclear(cpu_env, tcg_constant_i32(crm)); 1918 /* For DAIFClear, exit the cpu loop to re-evaluate pending IRQs. */ 1919 s->base.is_jmp = DISAS_UPDATE_EXIT; 1920 break; 1921 1922 case 0x1c: /* TCO */ 1923 if (dc_isar_feature(aa64_mte, s)) { 1924 /* Full MTE is enabled -- set the TCO bit as directed. */ 1925 if (crm & 1) { 1926 set_pstate_bits(PSTATE_TCO); 1927 } else { 1928 clear_pstate_bits(PSTATE_TCO); 1929 } 1930 gen_rebuild_hflags(s); 1931 /* Many factors, including TCO, go into MTE_ACTIVE. */ 1932 s->base.is_jmp = DISAS_UPDATE_NOCHAIN; 1933 } else if (dc_isar_feature(aa64_mte_insn_reg, s)) { 1934 /* Only "instructions accessible at EL0" -- PSTATE.TCO is WI. */ 1935 s->base.is_jmp = DISAS_NEXT; 1936 } else { 1937 goto do_unallocated; 1938 } 1939 break; 1940 1941 case 0x1b: /* SVCR* */ 1942 if (!dc_isar_feature(aa64_sme, s) || crm < 2 || crm > 7) { 1943 goto do_unallocated; 1944 } 1945 if (sme_access_check(s)) { 1946 int old = s->pstate_sm | (s->pstate_za << 1); 1947 int new = (crm & 1) * 3; 1948 int msk = (crm >> 1) & 3; 1949 1950 if ((old ^ new) & msk) { 1951 /* At least one bit changes. */ 1952 gen_helper_set_svcr(cpu_env, tcg_constant_i32(new), 1953 tcg_constant_i32(msk)); 1954 } else { 1955 s->base.is_jmp = DISAS_NEXT; 1956 } 1957 } 1958 break; 1959 1960 default: 1961 do_unallocated: 1962 unallocated_encoding(s); 1963 return; 1964 } 1965 } 1966 1967 static void gen_get_nzcv(TCGv_i64 tcg_rt) 1968 { 1969 TCGv_i32 tmp = tcg_temp_new_i32(); 1970 TCGv_i32 nzcv = tcg_temp_new_i32(); 1971 1972 /* build bit 31, N */ 1973 tcg_gen_andi_i32(nzcv, cpu_NF, (1U << 31)); 1974 /* build bit 30, Z */ 1975 tcg_gen_setcondi_i32(TCG_COND_EQ, tmp, cpu_ZF, 0); 1976 tcg_gen_deposit_i32(nzcv, nzcv, tmp, 30, 1); 1977 /* build bit 29, C */ 1978 tcg_gen_deposit_i32(nzcv, nzcv, cpu_CF, 29, 1); 1979 /* build bit 28, V */ 1980 tcg_gen_shri_i32(tmp, cpu_VF, 31); 1981 tcg_gen_deposit_i32(nzcv, nzcv, tmp, 28, 1); 1982 /* generate result */ 1983 tcg_gen_extu_i32_i64(tcg_rt, nzcv); 1984 } 1985 1986 static void gen_set_nzcv(TCGv_i64 tcg_rt) 1987 { 1988 TCGv_i32 nzcv = tcg_temp_new_i32(); 1989 1990 /* take NZCV from R[t] */ 1991 tcg_gen_extrl_i64_i32(nzcv, tcg_rt); 1992 1993 /* bit 31, N */ 1994 tcg_gen_andi_i32(cpu_NF, nzcv, (1U << 31)); 1995 /* bit 30, Z */ 1996 tcg_gen_andi_i32(cpu_ZF, nzcv, (1 << 30)); 1997 tcg_gen_setcondi_i32(TCG_COND_EQ, cpu_ZF, cpu_ZF, 0); 1998 /* bit 29, C */ 1999 tcg_gen_andi_i32(cpu_CF, nzcv, (1 << 29)); 2000 tcg_gen_shri_i32(cpu_CF, cpu_CF, 29); 2001 /* bit 28, V */ 2002 tcg_gen_andi_i32(cpu_VF, nzcv, (1 << 28)); 2003 tcg_gen_shli_i32(cpu_VF, cpu_VF, 3); 2004 } 2005 2006 static void gen_sysreg_undef(DisasContext *s, bool isread, 2007 uint8_t op0, uint8_t op1, uint8_t op2, 2008 uint8_t crn, uint8_t crm, uint8_t rt) 2009 { 2010 /* 2011 * Generate code to emit an UNDEF with correct syndrome 2012 * information for a failed system register access. 2013 * This is EC_UNCATEGORIZED (ie a standard UNDEF) in most cases, 2014 * but if FEAT_IDST is implemented then read accesses to registers 2015 * in the feature ID space are reported with the EC_SYSTEMREGISTERTRAP 2016 * syndrome. 2017 */ 2018 uint32_t syndrome; 2019 2020 if (isread && dc_isar_feature(aa64_ids, s) && 2021 arm_cpreg_encoding_in_idspace(op0, op1, op2, crn, crm)) { 2022 syndrome = syn_aa64_sysregtrap(op0, op1, op2, crn, crm, rt, isread); 2023 } else { 2024 syndrome = syn_uncategorized(); 2025 } 2026 gen_exception_insn(s, 0, EXCP_UDEF, syndrome); 2027 } 2028 2029 /* MRS - move from system register 2030 * MSR (register) - move to system register 2031 * SYS 2032 * SYSL 2033 * These are all essentially the same insn in 'read' and 'write' 2034 * versions, with varying op0 fields. 2035 */ 2036 static void handle_sys(DisasContext *s, uint32_t insn, bool isread, 2037 unsigned int op0, unsigned int op1, unsigned int op2, 2038 unsigned int crn, unsigned int crm, unsigned int rt) 2039 { 2040 uint32_t key = ENCODE_AA64_CP_REG(CP_REG_ARM64_SYSREG_CP, 2041 crn, crm, op0, op1, op2); 2042 const ARMCPRegInfo *ri = get_arm_cp_reginfo(s->cp_regs, key); 2043 bool need_exit_tb = false; 2044 TCGv_ptr tcg_ri = NULL; 2045 TCGv_i64 tcg_rt; 2046 2047 if (!ri) { 2048 /* Unknown register; this might be a guest error or a QEMU 2049 * unimplemented feature. 2050 */ 2051 qemu_log_mask(LOG_UNIMP, "%s access to unsupported AArch64 " 2052 "system register op0:%d op1:%d crn:%d crm:%d op2:%d\n", 2053 isread ? "read" : "write", op0, op1, crn, crm, op2); 2054 gen_sysreg_undef(s, isread, op0, op1, op2, crn, crm, rt); 2055 return; 2056 } 2057 2058 /* Check access permissions */ 2059 if (!cp_access_ok(s->current_el, ri, isread)) { 2060 gen_sysreg_undef(s, isread, op0, op1, op2, crn, crm, rt); 2061 return; 2062 } 2063 2064 if (ri->accessfn || (ri->fgt && s->fgt_active)) { 2065 /* Emit code to perform further access permissions checks at 2066 * runtime; this may result in an exception. 2067 */ 2068 uint32_t syndrome; 2069 2070 syndrome = syn_aa64_sysregtrap(op0, op1, op2, crn, crm, rt, isread); 2071 gen_a64_update_pc(s, 0); 2072 tcg_ri = tcg_temp_new_ptr(); 2073 gen_helper_access_check_cp_reg(tcg_ri, cpu_env, 2074 tcg_constant_i32(key), 2075 tcg_constant_i32(syndrome), 2076 tcg_constant_i32(isread)); 2077 } else if (ri->type & ARM_CP_RAISES_EXC) { 2078 /* 2079 * The readfn or writefn might raise an exception; 2080 * synchronize the CPU state in case it does. 2081 */ 2082 gen_a64_update_pc(s, 0); 2083 } 2084 2085 /* Handle special cases first */ 2086 switch (ri->type & ARM_CP_SPECIAL_MASK) { 2087 case 0: 2088 break; 2089 case ARM_CP_NOP: 2090 return; 2091 case ARM_CP_NZCV: 2092 tcg_rt = cpu_reg(s, rt); 2093 if (isread) { 2094 gen_get_nzcv(tcg_rt); 2095 } else { 2096 gen_set_nzcv(tcg_rt); 2097 } 2098 return; 2099 case ARM_CP_CURRENTEL: 2100 /* Reads as current EL value from pstate, which is 2101 * guaranteed to be constant by the tb flags. 2102 */ 2103 tcg_rt = cpu_reg(s, rt); 2104 tcg_gen_movi_i64(tcg_rt, s->current_el << 2); 2105 return; 2106 case ARM_CP_DC_ZVA: 2107 /* Writes clear the aligned block of memory which rt points into. */ 2108 if (s->mte_active[0]) { 2109 int desc = 0; 2110 2111 desc = FIELD_DP32(desc, MTEDESC, MIDX, get_mem_index(s)); 2112 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid); 2113 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma); 2114 2115 tcg_rt = tcg_temp_new_i64(); 2116 gen_helper_mte_check_zva(tcg_rt, cpu_env, 2117 tcg_constant_i32(desc), cpu_reg(s, rt)); 2118 } else { 2119 tcg_rt = clean_data_tbi(s, cpu_reg(s, rt)); 2120 } 2121 gen_helper_dc_zva(cpu_env, tcg_rt); 2122 return; 2123 case ARM_CP_DC_GVA: 2124 { 2125 TCGv_i64 clean_addr, tag; 2126 2127 /* 2128 * DC_GVA, like DC_ZVA, requires that we supply the original 2129 * pointer for an invalid page. Probe that address first. 2130 */ 2131 tcg_rt = cpu_reg(s, rt); 2132 clean_addr = clean_data_tbi(s, tcg_rt); 2133 gen_probe_access(s, clean_addr, MMU_DATA_STORE, MO_8); 2134 2135 if (s->ata) { 2136 /* Extract the tag from the register to match STZGM. */ 2137 tag = tcg_temp_new_i64(); 2138 tcg_gen_shri_i64(tag, tcg_rt, 56); 2139 gen_helper_stzgm_tags(cpu_env, clean_addr, tag); 2140 } 2141 } 2142 return; 2143 case ARM_CP_DC_GZVA: 2144 { 2145 TCGv_i64 clean_addr, tag; 2146 2147 /* For DC_GZVA, we can rely on DC_ZVA for the proper fault. */ 2148 tcg_rt = cpu_reg(s, rt); 2149 clean_addr = clean_data_tbi(s, tcg_rt); 2150 gen_helper_dc_zva(cpu_env, clean_addr); 2151 2152 if (s->ata) { 2153 /* Extract the tag from the register to match STZGM. */ 2154 tag = tcg_temp_new_i64(); 2155 tcg_gen_shri_i64(tag, tcg_rt, 56); 2156 gen_helper_stzgm_tags(cpu_env, clean_addr, tag); 2157 } 2158 } 2159 return; 2160 default: 2161 g_assert_not_reached(); 2162 } 2163 if ((ri->type & ARM_CP_FPU) && !fp_access_check_only(s)) { 2164 return; 2165 } else if ((ri->type & ARM_CP_SVE) && !sve_access_check(s)) { 2166 return; 2167 } else if ((ri->type & ARM_CP_SME) && !sme_access_check(s)) { 2168 return; 2169 } 2170 2171 if (ri->type & ARM_CP_IO) { 2172 /* I/O operations must end the TB here (whether read or write) */ 2173 need_exit_tb = translator_io_start(&s->base); 2174 } 2175 2176 tcg_rt = cpu_reg(s, rt); 2177 2178 if (isread) { 2179 if (ri->type & ARM_CP_CONST) { 2180 tcg_gen_movi_i64(tcg_rt, ri->resetvalue); 2181 } else if (ri->readfn) { 2182 if (!tcg_ri) { 2183 tcg_ri = gen_lookup_cp_reg(key); 2184 } 2185 gen_helper_get_cp_reg64(tcg_rt, cpu_env, tcg_ri); 2186 } else { 2187 tcg_gen_ld_i64(tcg_rt, cpu_env, ri->fieldoffset); 2188 } 2189 } else { 2190 if (ri->type & ARM_CP_CONST) { 2191 /* If not forbidden by access permissions, treat as WI */ 2192 return; 2193 } else if (ri->writefn) { 2194 if (!tcg_ri) { 2195 tcg_ri = gen_lookup_cp_reg(key); 2196 } 2197 gen_helper_set_cp_reg64(cpu_env, tcg_ri, tcg_rt); 2198 } else { 2199 tcg_gen_st_i64(tcg_rt, cpu_env, ri->fieldoffset); 2200 } 2201 } 2202 2203 if (!isread && !(ri->type & ARM_CP_SUPPRESS_TB_END)) { 2204 /* 2205 * A write to any coprocessor regiser that ends a TB 2206 * must rebuild the hflags for the next TB. 2207 */ 2208 gen_rebuild_hflags(s); 2209 /* 2210 * We default to ending the TB on a coprocessor register write, 2211 * but allow this to be suppressed by the register definition 2212 * (usually only necessary to work around guest bugs). 2213 */ 2214 need_exit_tb = true; 2215 } 2216 if (need_exit_tb) { 2217 s->base.is_jmp = DISAS_UPDATE_EXIT; 2218 } 2219 } 2220 2221 /* System 2222 * 31 22 21 20 19 18 16 15 12 11 8 7 5 4 0 2223 * +---------------------+---+-----+-----+-------+-------+-----+------+ 2224 * | 1 1 0 1 0 1 0 1 0 0 | L | op0 | op1 | CRn | CRm | op2 | Rt | 2225 * +---------------------+---+-----+-----+-------+-------+-----+------+ 2226 */ 2227 static void disas_system(DisasContext *s, uint32_t insn) 2228 { 2229 unsigned int l, op0, op1, crn, crm, op2, rt; 2230 l = extract32(insn, 21, 1); 2231 op0 = extract32(insn, 19, 2); 2232 op1 = extract32(insn, 16, 3); 2233 crn = extract32(insn, 12, 4); 2234 crm = extract32(insn, 8, 4); 2235 op2 = extract32(insn, 5, 3); 2236 rt = extract32(insn, 0, 5); 2237 2238 if (op0 == 0) { 2239 if (l || rt != 31) { 2240 unallocated_encoding(s); 2241 return; 2242 } 2243 switch (crn) { 2244 case 2: /* HINT (including allocated hints like NOP, YIELD, etc) */ 2245 handle_hint(s, insn, op1, op2, crm); 2246 break; 2247 case 3: /* CLREX, DSB, DMB, ISB */ 2248 handle_sync(s, insn, op1, op2, crm); 2249 break; 2250 case 4: /* MSR (immediate) */ 2251 handle_msr_i(s, insn, op1, op2, crm); 2252 break; 2253 default: 2254 unallocated_encoding(s); 2255 break; 2256 } 2257 return; 2258 } 2259 handle_sys(s, insn, l, op0, op1, op2, crn, crm, rt); 2260 } 2261 2262 /* Exception generation 2263 * 2264 * 31 24 23 21 20 5 4 2 1 0 2265 * +-----------------+-----+------------------------+-----+----+ 2266 * | 1 1 0 1 0 1 0 0 | opc | imm16 | op2 | LL | 2267 * +-----------------------+------------------------+----------+ 2268 */ 2269 static void disas_exc(DisasContext *s, uint32_t insn) 2270 { 2271 int opc = extract32(insn, 21, 3); 2272 int op2_ll = extract32(insn, 0, 5); 2273 int imm16 = extract32(insn, 5, 16); 2274 uint32_t syndrome; 2275 2276 switch (opc) { 2277 case 0: 2278 /* For SVC, HVC and SMC we advance the single-step state 2279 * machine before taking the exception. This is architecturally 2280 * mandated, to ensure that single-stepping a system call 2281 * instruction works properly. 2282 */ 2283 switch (op2_ll) { 2284 case 1: /* SVC */ 2285 syndrome = syn_aa64_svc(imm16); 2286 if (s->fgt_svc) { 2287 gen_exception_insn_el(s, 0, EXCP_UDEF, syndrome, 2); 2288 break; 2289 } 2290 gen_ss_advance(s); 2291 gen_exception_insn(s, 4, EXCP_SWI, syndrome); 2292 break; 2293 case 2: /* HVC */ 2294 if (s->current_el == 0) { 2295 unallocated_encoding(s); 2296 break; 2297 } 2298 /* The pre HVC helper handles cases when HVC gets trapped 2299 * as an undefined insn by runtime configuration. 2300 */ 2301 gen_a64_update_pc(s, 0); 2302 gen_helper_pre_hvc(cpu_env); 2303 gen_ss_advance(s); 2304 gen_exception_insn_el(s, 4, EXCP_HVC, syn_aa64_hvc(imm16), 2); 2305 break; 2306 case 3: /* SMC */ 2307 if (s->current_el == 0) { 2308 unallocated_encoding(s); 2309 break; 2310 } 2311 gen_a64_update_pc(s, 0); 2312 gen_helper_pre_smc(cpu_env, tcg_constant_i32(syn_aa64_smc(imm16))); 2313 gen_ss_advance(s); 2314 gen_exception_insn_el(s, 4, EXCP_SMC, syn_aa64_smc(imm16), 3); 2315 break; 2316 default: 2317 unallocated_encoding(s); 2318 break; 2319 } 2320 break; 2321 case 1: 2322 if (op2_ll != 0) { 2323 unallocated_encoding(s); 2324 break; 2325 } 2326 /* BRK */ 2327 gen_exception_bkpt_insn(s, syn_aa64_bkpt(imm16)); 2328 break; 2329 case 2: 2330 if (op2_ll != 0) { 2331 unallocated_encoding(s); 2332 break; 2333 } 2334 /* HLT. This has two purposes. 2335 * Architecturally, it is an external halting debug instruction. 2336 * Since QEMU doesn't implement external debug, we treat this as 2337 * it is required for halting debug disabled: it will UNDEF. 2338 * Secondly, "HLT 0xf000" is the A64 semihosting syscall instruction. 2339 */ 2340 if (semihosting_enabled(s->current_el == 0) && imm16 == 0xf000) { 2341 gen_exception_internal_insn(s, EXCP_SEMIHOST); 2342 } else { 2343 unallocated_encoding(s); 2344 } 2345 break; 2346 case 5: 2347 if (op2_ll < 1 || op2_ll > 3) { 2348 unallocated_encoding(s); 2349 break; 2350 } 2351 /* DCPS1, DCPS2, DCPS3 */ 2352 unallocated_encoding(s); 2353 break; 2354 default: 2355 unallocated_encoding(s); 2356 break; 2357 } 2358 } 2359 2360 /* Branches, exception generating and system instructions */ 2361 static void disas_b_exc_sys(DisasContext *s, uint32_t insn) 2362 { 2363 switch (extract32(insn, 25, 7)) { 2364 case 0x6a: /* Exception generation / System */ 2365 if (insn & (1 << 24)) { 2366 if (extract32(insn, 22, 2) == 0) { 2367 disas_system(s, insn); 2368 } else { 2369 unallocated_encoding(s); 2370 } 2371 } else { 2372 disas_exc(s, insn); 2373 } 2374 break; 2375 default: 2376 unallocated_encoding(s); 2377 break; 2378 } 2379 } 2380 2381 /* 2382 * Load/Store exclusive instructions are implemented by remembering 2383 * the value/address loaded, and seeing if these are the same 2384 * when the store is performed. This is not actually the architecturally 2385 * mandated semantics, but it works for typical guest code sequences 2386 * and avoids having to monitor regular stores. 2387 * 2388 * The store exclusive uses the atomic cmpxchg primitives to avoid 2389 * races in multi-threaded linux-user and when MTTCG softmmu is 2390 * enabled. 2391 */ 2392 static void gen_load_exclusive(DisasContext *s, int rt, int rt2, 2393 TCGv_i64 addr, int size, bool is_pair) 2394 { 2395 int idx = get_mem_index(s); 2396 MemOp memop = s->be_data; 2397 2398 g_assert(size <= 3); 2399 if (is_pair) { 2400 g_assert(size >= 2); 2401 if (size == 2) { 2402 /* The pair must be single-copy atomic for the doubleword. */ 2403 memop |= MO_64 | MO_ALIGN; 2404 tcg_gen_qemu_ld_i64(cpu_exclusive_val, addr, idx, memop); 2405 if (s->be_data == MO_LE) { 2406 tcg_gen_extract_i64(cpu_reg(s, rt), cpu_exclusive_val, 0, 32); 2407 tcg_gen_extract_i64(cpu_reg(s, rt2), cpu_exclusive_val, 32, 32); 2408 } else { 2409 tcg_gen_extract_i64(cpu_reg(s, rt), cpu_exclusive_val, 32, 32); 2410 tcg_gen_extract_i64(cpu_reg(s, rt2), cpu_exclusive_val, 0, 32); 2411 } 2412 } else { 2413 /* The pair must be single-copy atomic for *each* doubleword, not 2414 the entire quadword, however it must be quadword aligned. */ 2415 memop |= MO_64; 2416 tcg_gen_qemu_ld_i64(cpu_exclusive_val, addr, idx, 2417 memop | MO_ALIGN_16); 2418 2419 TCGv_i64 addr2 = tcg_temp_new_i64(); 2420 tcg_gen_addi_i64(addr2, addr, 8); 2421 tcg_gen_qemu_ld_i64(cpu_exclusive_high, addr2, idx, memop); 2422 2423 tcg_gen_mov_i64(cpu_reg(s, rt), cpu_exclusive_val); 2424 tcg_gen_mov_i64(cpu_reg(s, rt2), cpu_exclusive_high); 2425 } 2426 } else { 2427 memop |= size | MO_ALIGN; 2428 tcg_gen_qemu_ld_i64(cpu_exclusive_val, addr, idx, memop); 2429 tcg_gen_mov_i64(cpu_reg(s, rt), cpu_exclusive_val); 2430 } 2431 tcg_gen_mov_i64(cpu_exclusive_addr, addr); 2432 } 2433 2434 static void gen_store_exclusive(DisasContext *s, int rd, int rt, int rt2, 2435 TCGv_i64 addr, int size, int is_pair) 2436 { 2437 /* if (env->exclusive_addr == addr && env->exclusive_val == [addr] 2438 * && (!is_pair || env->exclusive_high == [addr + datasize])) { 2439 * [addr] = {Rt}; 2440 * if (is_pair) { 2441 * [addr + datasize] = {Rt2}; 2442 * } 2443 * {Rd} = 0; 2444 * } else { 2445 * {Rd} = 1; 2446 * } 2447 * env->exclusive_addr = -1; 2448 */ 2449 TCGLabel *fail_label = gen_new_label(); 2450 TCGLabel *done_label = gen_new_label(); 2451 TCGv_i64 tmp; 2452 2453 tcg_gen_brcond_i64(TCG_COND_NE, addr, cpu_exclusive_addr, fail_label); 2454 2455 tmp = tcg_temp_new_i64(); 2456 if (is_pair) { 2457 if (size == 2) { 2458 if (s->be_data == MO_LE) { 2459 tcg_gen_concat32_i64(tmp, cpu_reg(s, rt), cpu_reg(s, rt2)); 2460 } else { 2461 tcg_gen_concat32_i64(tmp, cpu_reg(s, rt2), cpu_reg(s, rt)); 2462 } 2463 tcg_gen_atomic_cmpxchg_i64(tmp, cpu_exclusive_addr, 2464 cpu_exclusive_val, tmp, 2465 get_mem_index(s), 2466 MO_64 | MO_ALIGN | s->be_data); 2467 tcg_gen_setcond_i64(TCG_COND_NE, tmp, tmp, cpu_exclusive_val); 2468 } else { 2469 TCGv_i128 t16 = tcg_temp_new_i128(); 2470 TCGv_i128 c16 = tcg_temp_new_i128(); 2471 TCGv_i64 a, b; 2472 2473 if (s->be_data == MO_LE) { 2474 tcg_gen_concat_i64_i128(t16, cpu_reg(s, rt), cpu_reg(s, rt2)); 2475 tcg_gen_concat_i64_i128(c16, cpu_exclusive_val, 2476 cpu_exclusive_high); 2477 } else { 2478 tcg_gen_concat_i64_i128(t16, cpu_reg(s, rt2), cpu_reg(s, rt)); 2479 tcg_gen_concat_i64_i128(c16, cpu_exclusive_high, 2480 cpu_exclusive_val); 2481 } 2482 2483 tcg_gen_atomic_cmpxchg_i128(t16, cpu_exclusive_addr, c16, t16, 2484 get_mem_index(s), 2485 MO_128 | MO_ALIGN | s->be_data); 2486 2487 a = tcg_temp_new_i64(); 2488 b = tcg_temp_new_i64(); 2489 if (s->be_data == MO_LE) { 2490 tcg_gen_extr_i128_i64(a, b, t16); 2491 } else { 2492 tcg_gen_extr_i128_i64(b, a, t16); 2493 } 2494 2495 tcg_gen_xor_i64(a, a, cpu_exclusive_val); 2496 tcg_gen_xor_i64(b, b, cpu_exclusive_high); 2497 tcg_gen_or_i64(tmp, a, b); 2498 2499 tcg_gen_setcondi_i64(TCG_COND_NE, tmp, tmp, 0); 2500 } 2501 } else { 2502 tcg_gen_atomic_cmpxchg_i64(tmp, cpu_exclusive_addr, cpu_exclusive_val, 2503 cpu_reg(s, rt), get_mem_index(s), 2504 size | MO_ALIGN | s->be_data); 2505 tcg_gen_setcond_i64(TCG_COND_NE, tmp, tmp, cpu_exclusive_val); 2506 } 2507 tcg_gen_mov_i64(cpu_reg(s, rd), tmp); 2508 tcg_gen_br(done_label); 2509 2510 gen_set_label(fail_label); 2511 tcg_gen_movi_i64(cpu_reg(s, rd), 1); 2512 gen_set_label(done_label); 2513 tcg_gen_movi_i64(cpu_exclusive_addr, -1); 2514 } 2515 2516 static void gen_compare_and_swap(DisasContext *s, int rs, int rt, 2517 int rn, int size) 2518 { 2519 TCGv_i64 tcg_rs = cpu_reg(s, rs); 2520 TCGv_i64 tcg_rt = cpu_reg(s, rt); 2521 int memidx = get_mem_index(s); 2522 TCGv_i64 clean_addr; 2523 2524 if (rn == 31) { 2525 gen_check_sp_alignment(s); 2526 } 2527 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn), true, rn != 31, size); 2528 tcg_gen_atomic_cmpxchg_i64(tcg_rs, clean_addr, tcg_rs, tcg_rt, memidx, 2529 size | MO_ALIGN | s->be_data); 2530 } 2531 2532 static void gen_compare_and_swap_pair(DisasContext *s, int rs, int rt, 2533 int rn, int size) 2534 { 2535 TCGv_i64 s1 = cpu_reg(s, rs); 2536 TCGv_i64 s2 = cpu_reg(s, rs + 1); 2537 TCGv_i64 t1 = cpu_reg(s, rt); 2538 TCGv_i64 t2 = cpu_reg(s, rt + 1); 2539 TCGv_i64 clean_addr; 2540 int memidx = get_mem_index(s); 2541 2542 if (rn == 31) { 2543 gen_check_sp_alignment(s); 2544 } 2545 2546 /* This is a single atomic access, despite the "pair". */ 2547 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn), true, rn != 31, size + 1); 2548 2549 if (size == 2) { 2550 TCGv_i64 cmp = tcg_temp_new_i64(); 2551 TCGv_i64 val = tcg_temp_new_i64(); 2552 2553 if (s->be_data == MO_LE) { 2554 tcg_gen_concat32_i64(val, t1, t2); 2555 tcg_gen_concat32_i64(cmp, s1, s2); 2556 } else { 2557 tcg_gen_concat32_i64(val, t2, t1); 2558 tcg_gen_concat32_i64(cmp, s2, s1); 2559 } 2560 2561 tcg_gen_atomic_cmpxchg_i64(cmp, clean_addr, cmp, val, memidx, 2562 MO_64 | MO_ALIGN | s->be_data); 2563 2564 if (s->be_data == MO_LE) { 2565 tcg_gen_extr32_i64(s1, s2, cmp); 2566 } else { 2567 tcg_gen_extr32_i64(s2, s1, cmp); 2568 } 2569 } else { 2570 TCGv_i128 cmp = tcg_temp_new_i128(); 2571 TCGv_i128 val = tcg_temp_new_i128(); 2572 2573 if (s->be_data == MO_LE) { 2574 tcg_gen_concat_i64_i128(val, t1, t2); 2575 tcg_gen_concat_i64_i128(cmp, s1, s2); 2576 } else { 2577 tcg_gen_concat_i64_i128(val, t2, t1); 2578 tcg_gen_concat_i64_i128(cmp, s2, s1); 2579 } 2580 2581 tcg_gen_atomic_cmpxchg_i128(cmp, clean_addr, cmp, val, memidx, 2582 MO_128 | MO_ALIGN | s->be_data); 2583 2584 if (s->be_data == MO_LE) { 2585 tcg_gen_extr_i128_i64(s1, s2, cmp); 2586 } else { 2587 tcg_gen_extr_i128_i64(s2, s1, cmp); 2588 } 2589 } 2590 } 2591 2592 /* Update the Sixty-Four bit (SF) registersize. This logic is derived 2593 * from the ARMv8 specs for LDR (Shared decode for all encodings). 2594 */ 2595 static bool disas_ldst_compute_iss_sf(int size, bool is_signed, int opc) 2596 { 2597 int opc0 = extract32(opc, 0, 1); 2598 int regsize; 2599 2600 if (is_signed) { 2601 regsize = opc0 ? 32 : 64; 2602 } else { 2603 regsize = size == 3 ? 64 : 32; 2604 } 2605 return regsize == 64; 2606 } 2607 2608 /* Load/store exclusive 2609 * 2610 * 31 30 29 24 23 22 21 20 16 15 14 10 9 5 4 0 2611 * +-----+-------------+----+---+----+------+----+-------+------+------+ 2612 * | sz | 0 0 1 0 0 0 | o2 | L | o1 | Rs | o0 | Rt2 | Rn | Rt | 2613 * +-----+-------------+----+---+----+------+----+-------+------+------+ 2614 * 2615 * sz: 00 -> 8 bit, 01 -> 16 bit, 10 -> 32 bit, 11 -> 64 bit 2616 * L: 0 -> store, 1 -> load 2617 * o2: 0 -> exclusive, 1 -> not 2618 * o1: 0 -> single register, 1 -> register pair 2619 * o0: 1 -> load-acquire/store-release, 0 -> not 2620 */ 2621 static void disas_ldst_excl(DisasContext *s, uint32_t insn) 2622 { 2623 int rt = extract32(insn, 0, 5); 2624 int rn = extract32(insn, 5, 5); 2625 int rt2 = extract32(insn, 10, 5); 2626 int rs = extract32(insn, 16, 5); 2627 int is_lasr = extract32(insn, 15, 1); 2628 int o2_L_o1_o0 = extract32(insn, 21, 3) * 2 | is_lasr; 2629 int size = extract32(insn, 30, 2); 2630 TCGv_i64 clean_addr; 2631 2632 switch (o2_L_o1_o0) { 2633 case 0x0: /* STXR */ 2634 case 0x1: /* STLXR */ 2635 if (rn == 31) { 2636 gen_check_sp_alignment(s); 2637 } 2638 if (is_lasr) { 2639 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_STRL); 2640 } 2641 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn), 2642 true, rn != 31, size); 2643 gen_store_exclusive(s, rs, rt, rt2, clean_addr, size, false); 2644 return; 2645 2646 case 0x4: /* LDXR */ 2647 case 0x5: /* LDAXR */ 2648 if (rn == 31) { 2649 gen_check_sp_alignment(s); 2650 } 2651 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn), 2652 false, rn != 31, size); 2653 s->is_ldex = true; 2654 gen_load_exclusive(s, rt, rt2, clean_addr, size, false); 2655 if (is_lasr) { 2656 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ); 2657 } 2658 return; 2659 2660 case 0x8: /* STLLR */ 2661 if (!dc_isar_feature(aa64_lor, s)) { 2662 break; 2663 } 2664 /* StoreLORelease is the same as Store-Release for QEMU. */ 2665 /* fall through */ 2666 case 0x9: /* STLR */ 2667 /* Generate ISS for non-exclusive accesses including LASR. */ 2668 if (rn == 31) { 2669 gen_check_sp_alignment(s); 2670 } 2671 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_STRL); 2672 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn), 2673 true, rn != 31, size); 2674 /* TODO: ARMv8.4-LSE SCTLR.nAA */ 2675 do_gpr_st(s, cpu_reg(s, rt), clean_addr, size | MO_ALIGN, true, rt, 2676 disas_ldst_compute_iss_sf(size, false, 0), is_lasr); 2677 return; 2678 2679 case 0xc: /* LDLAR */ 2680 if (!dc_isar_feature(aa64_lor, s)) { 2681 break; 2682 } 2683 /* LoadLOAcquire is the same as Load-Acquire for QEMU. */ 2684 /* fall through */ 2685 case 0xd: /* LDAR */ 2686 /* Generate ISS for non-exclusive accesses including LASR. */ 2687 if (rn == 31) { 2688 gen_check_sp_alignment(s); 2689 } 2690 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn), 2691 false, rn != 31, size); 2692 /* TODO: ARMv8.4-LSE SCTLR.nAA */ 2693 do_gpr_ld(s, cpu_reg(s, rt), clean_addr, size | MO_ALIGN, false, true, 2694 rt, disas_ldst_compute_iss_sf(size, false, 0), is_lasr); 2695 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ); 2696 return; 2697 2698 case 0x2: case 0x3: /* CASP / STXP */ 2699 if (size & 2) { /* STXP / STLXP */ 2700 if (rn == 31) { 2701 gen_check_sp_alignment(s); 2702 } 2703 if (is_lasr) { 2704 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_STRL); 2705 } 2706 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn), 2707 true, rn != 31, size); 2708 gen_store_exclusive(s, rs, rt, rt2, clean_addr, size, true); 2709 return; 2710 } 2711 if (rt2 == 31 2712 && ((rt | rs) & 1) == 0 2713 && dc_isar_feature(aa64_atomics, s)) { 2714 /* CASP / CASPL */ 2715 gen_compare_and_swap_pair(s, rs, rt, rn, size | 2); 2716 return; 2717 } 2718 break; 2719 2720 case 0x6: case 0x7: /* CASPA / LDXP */ 2721 if (size & 2) { /* LDXP / LDAXP */ 2722 if (rn == 31) { 2723 gen_check_sp_alignment(s); 2724 } 2725 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn), 2726 false, rn != 31, size); 2727 s->is_ldex = true; 2728 gen_load_exclusive(s, rt, rt2, clean_addr, size, true); 2729 if (is_lasr) { 2730 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ); 2731 } 2732 return; 2733 } 2734 if (rt2 == 31 2735 && ((rt | rs) & 1) == 0 2736 && dc_isar_feature(aa64_atomics, s)) { 2737 /* CASPA / CASPAL */ 2738 gen_compare_and_swap_pair(s, rs, rt, rn, size | 2); 2739 return; 2740 } 2741 break; 2742 2743 case 0xa: /* CAS */ 2744 case 0xb: /* CASL */ 2745 case 0xe: /* CASA */ 2746 case 0xf: /* CASAL */ 2747 if (rt2 == 31 && dc_isar_feature(aa64_atomics, s)) { 2748 gen_compare_and_swap(s, rs, rt, rn, size); 2749 return; 2750 } 2751 break; 2752 } 2753 unallocated_encoding(s); 2754 } 2755 2756 /* 2757 * Load register (literal) 2758 * 2759 * 31 30 29 27 26 25 24 23 5 4 0 2760 * +-----+-------+---+-----+-------------------+-------+ 2761 * | opc | 0 1 1 | V | 0 0 | imm19 | Rt | 2762 * +-----+-------+---+-----+-------------------+-------+ 2763 * 2764 * V: 1 -> vector (simd/fp) 2765 * opc (non-vector): 00 -> 32 bit, 01 -> 64 bit, 2766 * 10-> 32 bit signed, 11 -> prefetch 2767 * opc (vector): 00 -> 32 bit, 01 -> 64 bit, 10 -> 128 bit (11 unallocated) 2768 */ 2769 static void disas_ld_lit(DisasContext *s, uint32_t insn) 2770 { 2771 int rt = extract32(insn, 0, 5); 2772 int64_t imm = sextract32(insn, 5, 19) << 2; 2773 bool is_vector = extract32(insn, 26, 1); 2774 int opc = extract32(insn, 30, 2); 2775 bool is_signed = false; 2776 int size = 2; 2777 TCGv_i64 tcg_rt, clean_addr; 2778 2779 if (is_vector) { 2780 if (opc == 3) { 2781 unallocated_encoding(s); 2782 return; 2783 } 2784 size = 2 + opc; 2785 if (!fp_access_check(s)) { 2786 return; 2787 } 2788 } else { 2789 if (opc == 3) { 2790 /* PRFM (literal) : prefetch */ 2791 return; 2792 } 2793 size = 2 + extract32(opc, 0, 1); 2794 is_signed = extract32(opc, 1, 1); 2795 } 2796 2797 tcg_rt = cpu_reg(s, rt); 2798 2799 clean_addr = tcg_temp_new_i64(); 2800 gen_pc_plus_diff(s, clean_addr, imm); 2801 if (is_vector) { 2802 do_fp_ld(s, rt, clean_addr, size); 2803 } else { 2804 /* Only unsigned 32bit loads target 32bit registers. */ 2805 bool iss_sf = opc != 0; 2806 2807 do_gpr_ld(s, tcg_rt, clean_addr, size + is_signed * MO_SIGN, 2808 false, true, rt, iss_sf, false); 2809 } 2810 } 2811 2812 /* 2813 * LDNP (Load Pair - non-temporal hint) 2814 * LDP (Load Pair - non vector) 2815 * LDPSW (Load Pair Signed Word - non vector) 2816 * STNP (Store Pair - non-temporal hint) 2817 * STP (Store Pair - non vector) 2818 * LDNP (Load Pair of SIMD&FP - non-temporal hint) 2819 * LDP (Load Pair of SIMD&FP) 2820 * STNP (Store Pair of SIMD&FP - non-temporal hint) 2821 * STP (Store Pair of SIMD&FP) 2822 * 2823 * 31 30 29 27 26 25 24 23 22 21 15 14 10 9 5 4 0 2824 * +-----+-------+---+---+-------+---+-----------------------------+ 2825 * | opc | 1 0 1 | V | 0 | index | L | imm7 | Rt2 | Rn | Rt | 2826 * +-----+-------+---+---+-------+---+-------+-------+------+------+ 2827 * 2828 * opc: LDP/STP/LDNP/STNP 00 -> 32 bit, 10 -> 64 bit 2829 * LDPSW/STGP 01 2830 * LDP/STP/LDNP/STNP (SIMD) 00 -> 32 bit, 01 -> 64 bit, 10 -> 128 bit 2831 * V: 0 -> GPR, 1 -> Vector 2832 * idx: 00 -> signed offset with non-temporal hint, 01 -> post-index, 2833 * 10 -> signed offset, 11 -> pre-index 2834 * L: 0 -> Store 1 -> Load 2835 * 2836 * Rt, Rt2 = GPR or SIMD registers to be stored 2837 * Rn = general purpose register containing address 2838 * imm7 = signed offset (multiple of 4 or 8 depending on size) 2839 */ 2840 static void disas_ldst_pair(DisasContext *s, uint32_t insn) 2841 { 2842 int rt = extract32(insn, 0, 5); 2843 int rn = extract32(insn, 5, 5); 2844 int rt2 = extract32(insn, 10, 5); 2845 uint64_t offset = sextract64(insn, 15, 7); 2846 int index = extract32(insn, 23, 2); 2847 bool is_vector = extract32(insn, 26, 1); 2848 bool is_load = extract32(insn, 22, 1); 2849 int opc = extract32(insn, 30, 2); 2850 2851 bool is_signed = false; 2852 bool postindex = false; 2853 bool wback = false; 2854 bool set_tag = false; 2855 2856 TCGv_i64 clean_addr, dirty_addr; 2857 2858 int size; 2859 2860 if (opc == 3) { 2861 unallocated_encoding(s); 2862 return; 2863 } 2864 2865 if (is_vector) { 2866 size = 2 + opc; 2867 } else if (opc == 1 && !is_load) { 2868 /* STGP */ 2869 if (!dc_isar_feature(aa64_mte_insn_reg, s) || index == 0) { 2870 unallocated_encoding(s); 2871 return; 2872 } 2873 size = 3; 2874 set_tag = true; 2875 } else { 2876 size = 2 + extract32(opc, 1, 1); 2877 is_signed = extract32(opc, 0, 1); 2878 if (!is_load && is_signed) { 2879 unallocated_encoding(s); 2880 return; 2881 } 2882 } 2883 2884 switch (index) { 2885 case 1: /* post-index */ 2886 postindex = true; 2887 wback = true; 2888 break; 2889 case 0: 2890 /* signed offset with "non-temporal" hint. Since we don't emulate 2891 * caches we don't care about hints to the cache system about 2892 * data access patterns, and handle this identically to plain 2893 * signed offset. 2894 */ 2895 if (is_signed) { 2896 /* There is no non-temporal-hint version of LDPSW */ 2897 unallocated_encoding(s); 2898 return; 2899 } 2900 postindex = false; 2901 break; 2902 case 2: /* signed offset, rn not updated */ 2903 postindex = false; 2904 break; 2905 case 3: /* pre-index */ 2906 postindex = false; 2907 wback = true; 2908 break; 2909 } 2910 2911 if (is_vector && !fp_access_check(s)) { 2912 return; 2913 } 2914 2915 offset <<= (set_tag ? LOG2_TAG_GRANULE : size); 2916 2917 if (rn == 31) { 2918 gen_check_sp_alignment(s); 2919 } 2920 2921 dirty_addr = read_cpu_reg_sp(s, rn, 1); 2922 if (!postindex) { 2923 tcg_gen_addi_i64(dirty_addr, dirty_addr, offset); 2924 } 2925 2926 if (set_tag) { 2927 if (!s->ata) { 2928 /* 2929 * TODO: We could rely on the stores below, at least for 2930 * system mode, if we arrange to add MO_ALIGN_16. 2931 */ 2932 gen_helper_stg_stub(cpu_env, dirty_addr); 2933 } else if (tb_cflags(s->base.tb) & CF_PARALLEL) { 2934 gen_helper_stg_parallel(cpu_env, dirty_addr, dirty_addr); 2935 } else { 2936 gen_helper_stg(cpu_env, dirty_addr, dirty_addr); 2937 } 2938 } 2939 2940 clean_addr = gen_mte_checkN(s, dirty_addr, !is_load, 2941 (wback || rn != 31) && !set_tag, 2 << size); 2942 2943 if (is_vector) { 2944 if (is_load) { 2945 do_fp_ld(s, rt, clean_addr, size); 2946 } else { 2947 do_fp_st(s, rt, clean_addr, size); 2948 } 2949 tcg_gen_addi_i64(clean_addr, clean_addr, 1 << size); 2950 if (is_load) { 2951 do_fp_ld(s, rt2, clean_addr, size); 2952 } else { 2953 do_fp_st(s, rt2, clean_addr, size); 2954 } 2955 } else { 2956 TCGv_i64 tcg_rt = cpu_reg(s, rt); 2957 TCGv_i64 tcg_rt2 = cpu_reg(s, rt2); 2958 2959 if (is_load) { 2960 TCGv_i64 tmp = tcg_temp_new_i64(); 2961 2962 /* Do not modify tcg_rt before recognizing any exception 2963 * from the second load. 2964 */ 2965 do_gpr_ld(s, tmp, clean_addr, size + is_signed * MO_SIGN, 2966 false, false, 0, false, false); 2967 tcg_gen_addi_i64(clean_addr, clean_addr, 1 << size); 2968 do_gpr_ld(s, tcg_rt2, clean_addr, size + is_signed * MO_SIGN, 2969 false, false, 0, false, false); 2970 2971 tcg_gen_mov_i64(tcg_rt, tmp); 2972 } else { 2973 do_gpr_st(s, tcg_rt, clean_addr, size, 2974 false, 0, false, false); 2975 tcg_gen_addi_i64(clean_addr, clean_addr, 1 << size); 2976 do_gpr_st(s, tcg_rt2, clean_addr, size, 2977 false, 0, false, false); 2978 } 2979 } 2980 2981 if (wback) { 2982 if (postindex) { 2983 tcg_gen_addi_i64(dirty_addr, dirty_addr, offset); 2984 } 2985 tcg_gen_mov_i64(cpu_reg_sp(s, rn), dirty_addr); 2986 } 2987 } 2988 2989 /* 2990 * Load/store (immediate post-indexed) 2991 * Load/store (immediate pre-indexed) 2992 * Load/store (unscaled immediate) 2993 * 2994 * 31 30 29 27 26 25 24 23 22 21 20 12 11 10 9 5 4 0 2995 * +----+-------+---+-----+-----+---+--------+-----+------+------+ 2996 * |size| 1 1 1 | V | 0 0 | opc | 0 | imm9 | idx | Rn | Rt | 2997 * +----+-------+---+-----+-----+---+--------+-----+------+------+ 2998 * 2999 * idx = 01 -> post-indexed, 11 pre-indexed, 00 unscaled imm. (no writeback) 3000 10 -> unprivileged 3001 * V = 0 -> non-vector 3002 * size: 00 -> 8 bit, 01 -> 16 bit, 10 -> 32 bit, 11 -> 64bit 3003 * opc: 00 -> store, 01 -> loadu, 10 -> loads 64, 11 -> loads 32 3004 */ 3005 static void disas_ldst_reg_imm9(DisasContext *s, uint32_t insn, 3006 int opc, 3007 int size, 3008 int rt, 3009 bool is_vector) 3010 { 3011 int rn = extract32(insn, 5, 5); 3012 int imm9 = sextract32(insn, 12, 9); 3013 int idx = extract32(insn, 10, 2); 3014 bool is_signed = false; 3015 bool is_store = false; 3016 bool is_extended = false; 3017 bool is_unpriv = (idx == 2); 3018 bool iss_valid; 3019 bool post_index; 3020 bool writeback; 3021 int memidx; 3022 3023 TCGv_i64 clean_addr, dirty_addr; 3024 3025 if (is_vector) { 3026 size |= (opc & 2) << 1; 3027 if (size > 4 || is_unpriv) { 3028 unallocated_encoding(s); 3029 return; 3030 } 3031 is_store = ((opc & 1) == 0); 3032 if (!fp_access_check(s)) { 3033 return; 3034 } 3035 } else { 3036 if (size == 3 && opc == 2) { 3037 /* PRFM - prefetch */ 3038 if (idx != 0) { 3039 unallocated_encoding(s); 3040 return; 3041 } 3042 return; 3043 } 3044 if (opc == 3 && size > 1) { 3045 unallocated_encoding(s); 3046 return; 3047 } 3048 is_store = (opc == 0); 3049 is_signed = extract32(opc, 1, 1); 3050 is_extended = (size < 3) && extract32(opc, 0, 1); 3051 } 3052 3053 switch (idx) { 3054 case 0: 3055 case 2: 3056 post_index = false; 3057 writeback = false; 3058 break; 3059 case 1: 3060 post_index = true; 3061 writeback = true; 3062 break; 3063 case 3: 3064 post_index = false; 3065 writeback = true; 3066 break; 3067 default: 3068 g_assert_not_reached(); 3069 } 3070 3071 iss_valid = !is_vector && !writeback; 3072 3073 if (rn == 31) { 3074 gen_check_sp_alignment(s); 3075 } 3076 3077 dirty_addr = read_cpu_reg_sp(s, rn, 1); 3078 if (!post_index) { 3079 tcg_gen_addi_i64(dirty_addr, dirty_addr, imm9); 3080 } 3081 3082 memidx = is_unpriv ? get_a64_user_mem_index(s) : get_mem_index(s); 3083 clean_addr = gen_mte_check1_mmuidx(s, dirty_addr, is_store, 3084 writeback || rn != 31, 3085 size, is_unpriv, memidx); 3086 3087 if (is_vector) { 3088 if (is_store) { 3089 do_fp_st(s, rt, clean_addr, size); 3090 } else { 3091 do_fp_ld(s, rt, clean_addr, size); 3092 } 3093 } else { 3094 TCGv_i64 tcg_rt = cpu_reg(s, rt); 3095 bool iss_sf = disas_ldst_compute_iss_sf(size, is_signed, opc); 3096 3097 if (is_store) { 3098 do_gpr_st_memidx(s, tcg_rt, clean_addr, size, memidx, 3099 iss_valid, rt, iss_sf, false); 3100 } else { 3101 do_gpr_ld_memidx(s, tcg_rt, clean_addr, size + is_signed * MO_SIGN, 3102 is_extended, memidx, 3103 iss_valid, rt, iss_sf, false); 3104 } 3105 } 3106 3107 if (writeback) { 3108 TCGv_i64 tcg_rn = cpu_reg_sp(s, rn); 3109 if (post_index) { 3110 tcg_gen_addi_i64(dirty_addr, dirty_addr, imm9); 3111 } 3112 tcg_gen_mov_i64(tcg_rn, dirty_addr); 3113 } 3114 } 3115 3116 /* 3117 * Load/store (register offset) 3118 * 3119 * 31 30 29 27 26 25 24 23 22 21 20 16 15 13 12 11 10 9 5 4 0 3120 * +----+-------+---+-----+-----+---+------+-----+--+-----+----+----+ 3121 * |size| 1 1 1 | V | 0 0 | opc | 1 | Rm | opt | S| 1 0 | Rn | Rt | 3122 * +----+-------+---+-----+-----+---+------+-----+--+-----+----+----+ 3123 * 3124 * For non-vector: 3125 * size: 00-> byte, 01 -> 16 bit, 10 -> 32bit, 11 -> 64bit 3126 * opc: 00 -> store, 01 -> loadu, 10 -> loads 64, 11 -> loads 32 3127 * For vector: 3128 * size is opc<1>:size<1:0> so 100 -> 128 bit; 110 and 111 unallocated 3129 * opc<0>: 0 -> store, 1 -> load 3130 * V: 1 -> vector/simd 3131 * opt: extend encoding (see DecodeRegExtend) 3132 * S: if S=1 then scale (essentially index by sizeof(size)) 3133 * Rt: register to transfer into/out of 3134 * Rn: address register or SP for base 3135 * Rm: offset register or ZR for offset 3136 */ 3137 static void disas_ldst_reg_roffset(DisasContext *s, uint32_t insn, 3138 int opc, 3139 int size, 3140 int rt, 3141 bool is_vector) 3142 { 3143 int rn = extract32(insn, 5, 5); 3144 int shift = extract32(insn, 12, 1); 3145 int rm = extract32(insn, 16, 5); 3146 int opt = extract32(insn, 13, 3); 3147 bool is_signed = false; 3148 bool is_store = false; 3149 bool is_extended = false; 3150 3151 TCGv_i64 tcg_rm, clean_addr, dirty_addr; 3152 3153 if (extract32(opt, 1, 1) == 0) { 3154 unallocated_encoding(s); 3155 return; 3156 } 3157 3158 if (is_vector) { 3159 size |= (opc & 2) << 1; 3160 if (size > 4) { 3161 unallocated_encoding(s); 3162 return; 3163 } 3164 is_store = !extract32(opc, 0, 1); 3165 if (!fp_access_check(s)) { 3166 return; 3167 } 3168 } else { 3169 if (size == 3 && opc == 2) { 3170 /* PRFM - prefetch */ 3171 return; 3172 } 3173 if (opc == 3 && size > 1) { 3174 unallocated_encoding(s); 3175 return; 3176 } 3177 is_store = (opc == 0); 3178 is_signed = extract32(opc, 1, 1); 3179 is_extended = (size < 3) && extract32(opc, 0, 1); 3180 } 3181 3182 if (rn == 31) { 3183 gen_check_sp_alignment(s); 3184 } 3185 dirty_addr = read_cpu_reg_sp(s, rn, 1); 3186 3187 tcg_rm = read_cpu_reg(s, rm, 1); 3188 ext_and_shift_reg(tcg_rm, tcg_rm, opt, shift ? size : 0); 3189 3190 tcg_gen_add_i64(dirty_addr, dirty_addr, tcg_rm); 3191 clean_addr = gen_mte_check1(s, dirty_addr, is_store, true, size); 3192 3193 if (is_vector) { 3194 if (is_store) { 3195 do_fp_st(s, rt, clean_addr, size); 3196 } else { 3197 do_fp_ld(s, rt, clean_addr, size); 3198 } 3199 } else { 3200 TCGv_i64 tcg_rt = cpu_reg(s, rt); 3201 bool iss_sf = disas_ldst_compute_iss_sf(size, is_signed, opc); 3202 if (is_store) { 3203 do_gpr_st(s, tcg_rt, clean_addr, size, 3204 true, rt, iss_sf, false); 3205 } else { 3206 do_gpr_ld(s, tcg_rt, clean_addr, size + is_signed * MO_SIGN, 3207 is_extended, true, rt, iss_sf, false); 3208 } 3209 } 3210 } 3211 3212 /* 3213 * Load/store (unsigned immediate) 3214 * 3215 * 31 30 29 27 26 25 24 23 22 21 10 9 5 3216 * +----+-------+---+-----+-----+------------+-------+------+ 3217 * |size| 1 1 1 | V | 0 1 | opc | imm12 | Rn | Rt | 3218 * +----+-------+---+-----+-----+------------+-------+------+ 3219 * 3220 * For non-vector: 3221 * size: 00-> byte, 01 -> 16 bit, 10 -> 32bit, 11 -> 64bit 3222 * opc: 00 -> store, 01 -> loadu, 10 -> loads 64, 11 -> loads 32 3223 * For vector: 3224 * size is opc<1>:size<1:0> so 100 -> 128 bit; 110 and 111 unallocated 3225 * opc<0>: 0 -> store, 1 -> load 3226 * Rn: base address register (inc SP) 3227 * Rt: target register 3228 */ 3229 static void disas_ldst_reg_unsigned_imm(DisasContext *s, uint32_t insn, 3230 int opc, 3231 int size, 3232 int rt, 3233 bool is_vector) 3234 { 3235 int rn = extract32(insn, 5, 5); 3236 unsigned int imm12 = extract32(insn, 10, 12); 3237 unsigned int offset; 3238 3239 TCGv_i64 clean_addr, dirty_addr; 3240 3241 bool is_store; 3242 bool is_signed = false; 3243 bool is_extended = false; 3244 3245 if (is_vector) { 3246 size |= (opc & 2) << 1; 3247 if (size > 4) { 3248 unallocated_encoding(s); 3249 return; 3250 } 3251 is_store = !extract32(opc, 0, 1); 3252 if (!fp_access_check(s)) { 3253 return; 3254 } 3255 } else { 3256 if (size == 3 && opc == 2) { 3257 /* PRFM - prefetch */ 3258 return; 3259 } 3260 if (opc == 3 && size > 1) { 3261 unallocated_encoding(s); 3262 return; 3263 } 3264 is_store = (opc == 0); 3265 is_signed = extract32(opc, 1, 1); 3266 is_extended = (size < 3) && extract32(opc, 0, 1); 3267 } 3268 3269 if (rn == 31) { 3270 gen_check_sp_alignment(s); 3271 } 3272 dirty_addr = read_cpu_reg_sp(s, rn, 1); 3273 offset = imm12 << size; 3274 tcg_gen_addi_i64(dirty_addr, dirty_addr, offset); 3275 clean_addr = gen_mte_check1(s, dirty_addr, is_store, rn != 31, size); 3276 3277 if (is_vector) { 3278 if (is_store) { 3279 do_fp_st(s, rt, clean_addr, size); 3280 } else { 3281 do_fp_ld(s, rt, clean_addr, size); 3282 } 3283 } else { 3284 TCGv_i64 tcg_rt = cpu_reg(s, rt); 3285 bool iss_sf = disas_ldst_compute_iss_sf(size, is_signed, opc); 3286 if (is_store) { 3287 do_gpr_st(s, tcg_rt, clean_addr, size, 3288 true, rt, iss_sf, false); 3289 } else { 3290 do_gpr_ld(s, tcg_rt, clean_addr, size + is_signed * MO_SIGN, 3291 is_extended, true, rt, iss_sf, false); 3292 } 3293 } 3294 } 3295 3296 /* Atomic memory operations 3297 * 3298 * 31 30 27 26 24 22 21 16 15 12 10 5 0 3299 * +------+-------+---+-----+-----+---+----+----+-----+-----+----+-----+ 3300 * | size | 1 1 1 | V | 0 0 | A R | 1 | Rs | o3 | opc | 0 0 | Rn | Rt | 3301 * +------+-------+---+-----+-----+--------+----+-----+-----+----+-----+ 3302 * 3303 * Rt: the result register 3304 * Rn: base address or SP 3305 * Rs: the source register for the operation 3306 * V: vector flag (always 0 as of v8.3) 3307 * A: acquire flag 3308 * R: release flag 3309 */ 3310 static void disas_ldst_atomic(DisasContext *s, uint32_t insn, 3311 int size, int rt, bool is_vector) 3312 { 3313 int rs = extract32(insn, 16, 5); 3314 int rn = extract32(insn, 5, 5); 3315 int o3_opc = extract32(insn, 12, 4); 3316 bool r = extract32(insn, 22, 1); 3317 bool a = extract32(insn, 23, 1); 3318 TCGv_i64 tcg_rs, tcg_rt, clean_addr; 3319 AtomicThreeOpFn *fn = NULL; 3320 MemOp mop = s->be_data | size | MO_ALIGN; 3321 3322 if (is_vector || !dc_isar_feature(aa64_atomics, s)) { 3323 unallocated_encoding(s); 3324 return; 3325 } 3326 switch (o3_opc) { 3327 case 000: /* LDADD */ 3328 fn = tcg_gen_atomic_fetch_add_i64; 3329 break; 3330 case 001: /* LDCLR */ 3331 fn = tcg_gen_atomic_fetch_and_i64; 3332 break; 3333 case 002: /* LDEOR */ 3334 fn = tcg_gen_atomic_fetch_xor_i64; 3335 break; 3336 case 003: /* LDSET */ 3337 fn = tcg_gen_atomic_fetch_or_i64; 3338 break; 3339 case 004: /* LDSMAX */ 3340 fn = tcg_gen_atomic_fetch_smax_i64; 3341 mop |= MO_SIGN; 3342 break; 3343 case 005: /* LDSMIN */ 3344 fn = tcg_gen_atomic_fetch_smin_i64; 3345 mop |= MO_SIGN; 3346 break; 3347 case 006: /* LDUMAX */ 3348 fn = tcg_gen_atomic_fetch_umax_i64; 3349 break; 3350 case 007: /* LDUMIN */ 3351 fn = tcg_gen_atomic_fetch_umin_i64; 3352 break; 3353 case 010: /* SWP */ 3354 fn = tcg_gen_atomic_xchg_i64; 3355 break; 3356 case 014: /* LDAPR, LDAPRH, LDAPRB */ 3357 if (!dc_isar_feature(aa64_rcpc_8_3, s) || 3358 rs != 31 || a != 1 || r != 0) { 3359 unallocated_encoding(s); 3360 return; 3361 } 3362 break; 3363 default: 3364 unallocated_encoding(s); 3365 return; 3366 } 3367 3368 if (rn == 31) { 3369 gen_check_sp_alignment(s); 3370 } 3371 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn), false, rn != 31, size); 3372 3373 if (o3_opc == 014) { 3374 /* 3375 * LDAPR* are a special case because they are a simple load, not a 3376 * fetch-and-do-something op. 3377 * The architectural consistency requirements here are weaker than 3378 * full load-acquire (we only need "load-acquire processor consistent"), 3379 * but we choose to implement them as full LDAQ. 3380 */ 3381 do_gpr_ld(s, cpu_reg(s, rt), clean_addr, size, false, 3382 true, rt, disas_ldst_compute_iss_sf(size, false, 0), true); 3383 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ); 3384 return; 3385 } 3386 3387 tcg_rs = read_cpu_reg(s, rs, true); 3388 tcg_rt = cpu_reg(s, rt); 3389 3390 if (o3_opc == 1) { /* LDCLR */ 3391 tcg_gen_not_i64(tcg_rs, tcg_rs); 3392 } 3393 3394 /* The tcg atomic primitives are all full barriers. Therefore we 3395 * can ignore the Acquire and Release bits of this instruction. 3396 */ 3397 fn(tcg_rt, clean_addr, tcg_rs, get_mem_index(s), mop); 3398 3399 if ((mop & MO_SIGN) && size != MO_64) { 3400 tcg_gen_ext32u_i64(tcg_rt, tcg_rt); 3401 } 3402 } 3403 3404 /* 3405 * PAC memory operations 3406 * 3407 * 31 30 27 26 24 22 21 12 11 10 5 0 3408 * +------+-------+---+-----+-----+---+--------+---+---+----+-----+ 3409 * | size | 1 1 1 | V | 0 0 | M S | 1 | imm9 | W | 1 | Rn | Rt | 3410 * +------+-------+---+-----+-----+---+--------+---+---+----+-----+ 3411 * 3412 * Rt: the result register 3413 * Rn: base address or SP 3414 * V: vector flag (always 0 as of v8.3) 3415 * M: clear for key DA, set for key DB 3416 * W: pre-indexing flag 3417 * S: sign for imm9. 3418 */ 3419 static void disas_ldst_pac(DisasContext *s, uint32_t insn, 3420 int size, int rt, bool is_vector) 3421 { 3422 int rn = extract32(insn, 5, 5); 3423 bool is_wback = extract32(insn, 11, 1); 3424 bool use_key_a = !extract32(insn, 23, 1); 3425 int offset; 3426 TCGv_i64 clean_addr, dirty_addr, tcg_rt; 3427 3428 if (size != 3 || is_vector || !dc_isar_feature(aa64_pauth, s)) { 3429 unallocated_encoding(s); 3430 return; 3431 } 3432 3433 if (rn == 31) { 3434 gen_check_sp_alignment(s); 3435 } 3436 dirty_addr = read_cpu_reg_sp(s, rn, 1); 3437 3438 if (s->pauth_active) { 3439 if (use_key_a) { 3440 gen_helper_autda(dirty_addr, cpu_env, dirty_addr, 3441 tcg_constant_i64(0)); 3442 } else { 3443 gen_helper_autdb(dirty_addr, cpu_env, dirty_addr, 3444 tcg_constant_i64(0)); 3445 } 3446 } 3447 3448 /* Form the 10-bit signed, scaled offset. */ 3449 offset = (extract32(insn, 22, 1) << 9) | extract32(insn, 12, 9); 3450 offset = sextract32(offset << size, 0, 10 + size); 3451 tcg_gen_addi_i64(dirty_addr, dirty_addr, offset); 3452 3453 /* Note that "clean" and "dirty" here refer to TBI not PAC. */ 3454 clean_addr = gen_mte_check1(s, dirty_addr, false, 3455 is_wback || rn != 31, size); 3456 3457 tcg_rt = cpu_reg(s, rt); 3458 do_gpr_ld(s, tcg_rt, clean_addr, size, 3459 /* extend */ false, /* iss_valid */ !is_wback, 3460 /* iss_srt */ rt, /* iss_sf */ true, /* iss_ar */ false); 3461 3462 if (is_wback) { 3463 tcg_gen_mov_i64(cpu_reg_sp(s, rn), dirty_addr); 3464 } 3465 } 3466 3467 /* 3468 * LDAPR/STLR (unscaled immediate) 3469 * 3470 * 31 30 24 22 21 12 10 5 0 3471 * +------+-------------+-----+---+--------+-----+----+-----+ 3472 * | size | 0 1 1 0 0 1 | opc | 0 | imm9 | 0 0 | Rn | Rt | 3473 * +------+-------------+-----+---+--------+-----+----+-----+ 3474 * 3475 * Rt: source or destination register 3476 * Rn: base register 3477 * imm9: unscaled immediate offset 3478 * opc: 00: STLUR*, 01/10/11: various LDAPUR* 3479 * size: size of load/store 3480 */ 3481 static void disas_ldst_ldapr_stlr(DisasContext *s, uint32_t insn) 3482 { 3483 int rt = extract32(insn, 0, 5); 3484 int rn = extract32(insn, 5, 5); 3485 int offset = sextract32(insn, 12, 9); 3486 int opc = extract32(insn, 22, 2); 3487 int size = extract32(insn, 30, 2); 3488 TCGv_i64 clean_addr, dirty_addr; 3489 bool is_store = false; 3490 bool extend = false; 3491 bool iss_sf; 3492 MemOp mop; 3493 3494 if (!dc_isar_feature(aa64_rcpc_8_4, s)) { 3495 unallocated_encoding(s); 3496 return; 3497 } 3498 3499 /* TODO: ARMv8.4-LSE SCTLR.nAA */ 3500 mop = size | MO_ALIGN; 3501 3502 switch (opc) { 3503 case 0: /* STLURB */ 3504 is_store = true; 3505 break; 3506 case 1: /* LDAPUR* */ 3507 break; 3508 case 2: /* LDAPURS* 64-bit variant */ 3509 if (size == 3) { 3510 unallocated_encoding(s); 3511 return; 3512 } 3513 mop |= MO_SIGN; 3514 break; 3515 case 3: /* LDAPURS* 32-bit variant */ 3516 if (size > 1) { 3517 unallocated_encoding(s); 3518 return; 3519 } 3520 mop |= MO_SIGN; 3521 extend = true; /* zero-extend 32->64 after signed load */ 3522 break; 3523 default: 3524 g_assert_not_reached(); 3525 } 3526 3527 iss_sf = disas_ldst_compute_iss_sf(size, (mop & MO_SIGN) != 0, opc); 3528 3529 if (rn == 31) { 3530 gen_check_sp_alignment(s); 3531 } 3532 3533 dirty_addr = read_cpu_reg_sp(s, rn, 1); 3534 tcg_gen_addi_i64(dirty_addr, dirty_addr, offset); 3535 clean_addr = clean_data_tbi(s, dirty_addr); 3536 3537 if (is_store) { 3538 /* Store-Release semantics */ 3539 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_STRL); 3540 do_gpr_st(s, cpu_reg(s, rt), clean_addr, mop, true, rt, iss_sf, true); 3541 } else { 3542 /* 3543 * Load-AcquirePC semantics; we implement as the slightly more 3544 * restrictive Load-Acquire. 3545 */ 3546 do_gpr_ld(s, cpu_reg(s, rt), clean_addr, mop, 3547 extend, true, rt, iss_sf, true); 3548 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ); 3549 } 3550 } 3551 3552 /* Load/store register (all forms) */ 3553 static void disas_ldst_reg(DisasContext *s, uint32_t insn) 3554 { 3555 int rt = extract32(insn, 0, 5); 3556 int opc = extract32(insn, 22, 2); 3557 bool is_vector = extract32(insn, 26, 1); 3558 int size = extract32(insn, 30, 2); 3559 3560 switch (extract32(insn, 24, 2)) { 3561 case 0: 3562 if (extract32(insn, 21, 1) == 0) { 3563 /* Load/store register (unscaled immediate) 3564 * Load/store immediate pre/post-indexed 3565 * Load/store register unprivileged 3566 */ 3567 disas_ldst_reg_imm9(s, insn, opc, size, rt, is_vector); 3568 return; 3569 } 3570 switch (extract32(insn, 10, 2)) { 3571 case 0: 3572 disas_ldst_atomic(s, insn, size, rt, is_vector); 3573 return; 3574 case 2: 3575 disas_ldst_reg_roffset(s, insn, opc, size, rt, is_vector); 3576 return; 3577 default: 3578 disas_ldst_pac(s, insn, size, rt, is_vector); 3579 return; 3580 } 3581 break; 3582 case 1: 3583 disas_ldst_reg_unsigned_imm(s, insn, opc, size, rt, is_vector); 3584 return; 3585 } 3586 unallocated_encoding(s); 3587 } 3588 3589 /* AdvSIMD load/store multiple structures 3590 * 3591 * 31 30 29 23 22 21 16 15 12 11 10 9 5 4 0 3592 * +---+---+---------------+---+-------------+--------+------+------+------+ 3593 * | 0 | Q | 0 0 1 1 0 0 0 | L | 0 0 0 0 0 0 | opcode | size | Rn | Rt | 3594 * +---+---+---------------+---+-------------+--------+------+------+------+ 3595 * 3596 * AdvSIMD load/store multiple structures (post-indexed) 3597 * 3598 * 31 30 29 23 22 21 20 16 15 12 11 10 9 5 4 0 3599 * +---+---+---------------+---+---+---------+--------+------+------+------+ 3600 * | 0 | Q | 0 0 1 1 0 0 1 | L | 0 | Rm | opcode | size | Rn | Rt | 3601 * +---+---+---------------+---+---+---------+--------+------+------+------+ 3602 * 3603 * Rt: first (or only) SIMD&FP register to be transferred 3604 * Rn: base address or SP 3605 * Rm (post-index only): post-index register (when !31) or size dependent #imm 3606 */ 3607 static void disas_ldst_multiple_struct(DisasContext *s, uint32_t insn) 3608 { 3609 int rt = extract32(insn, 0, 5); 3610 int rn = extract32(insn, 5, 5); 3611 int rm = extract32(insn, 16, 5); 3612 int size = extract32(insn, 10, 2); 3613 int opcode = extract32(insn, 12, 4); 3614 bool is_store = !extract32(insn, 22, 1); 3615 bool is_postidx = extract32(insn, 23, 1); 3616 bool is_q = extract32(insn, 30, 1); 3617 TCGv_i64 clean_addr, tcg_rn, tcg_ebytes; 3618 MemOp endian, align, mop; 3619 3620 int total; /* total bytes */ 3621 int elements; /* elements per vector */ 3622 int rpt; /* num iterations */ 3623 int selem; /* structure elements */ 3624 int r; 3625 3626 if (extract32(insn, 31, 1) || extract32(insn, 21, 1)) { 3627 unallocated_encoding(s); 3628 return; 3629 } 3630 3631 if (!is_postidx && rm != 0) { 3632 unallocated_encoding(s); 3633 return; 3634 } 3635 3636 /* From the shared decode logic */ 3637 switch (opcode) { 3638 case 0x0: 3639 rpt = 1; 3640 selem = 4; 3641 break; 3642 case 0x2: 3643 rpt = 4; 3644 selem = 1; 3645 break; 3646 case 0x4: 3647 rpt = 1; 3648 selem = 3; 3649 break; 3650 case 0x6: 3651 rpt = 3; 3652 selem = 1; 3653 break; 3654 case 0x7: 3655 rpt = 1; 3656 selem = 1; 3657 break; 3658 case 0x8: 3659 rpt = 1; 3660 selem = 2; 3661 break; 3662 case 0xa: 3663 rpt = 2; 3664 selem = 1; 3665 break; 3666 default: 3667 unallocated_encoding(s); 3668 return; 3669 } 3670 3671 if (size == 3 && !is_q && selem != 1) { 3672 /* reserved */ 3673 unallocated_encoding(s); 3674 return; 3675 } 3676 3677 if (!fp_access_check(s)) { 3678 return; 3679 } 3680 3681 if (rn == 31) { 3682 gen_check_sp_alignment(s); 3683 } 3684 3685 /* For our purposes, bytes are always little-endian. */ 3686 endian = s->be_data; 3687 if (size == 0) { 3688 endian = MO_LE; 3689 } 3690 3691 total = rpt * selem * (is_q ? 16 : 8); 3692 tcg_rn = cpu_reg_sp(s, rn); 3693 3694 /* 3695 * Issue the MTE check vs the logical repeat count, before we 3696 * promote consecutive little-endian elements below. 3697 */ 3698 clean_addr = gen_mte_checkN(s, tcg_rn, is_store, is_postidx || rn != 31, 3699 total); 3700 3701 /* 3702 * Consecutive little-endian elements from a single register 3703 * can be promoted to a larger little-endian operation. 3704 */ 3705 align = MO_ALIGN; 3706 if (selem == 1 && endian == MO_LE) { 3707 align = pow2_align(size); 3708 size = 3; 3709 } 3710 if (!s->align_mem) { 3711 align = 0; 3712 } 3713 mop = endian | size | align; 3714 3715 elements = (is_q ? 16 : 8) >> size; 3716 tcg_ebytes = tcg_constant_i64(1 << size); 3717 for (r = 0; r < rpt; r++) { 3718 int e; 3719 for (e = 0; e < elements; e++) { 3720 int xs; 3721 for (xs = 0; xs < selem; xs++) { 3722 int tt = (rt + r + xs) % 32; 3723 if (is_store) { 3724 do_vec_st(s, tt, e, clean_addr, mop); 3725 } else { 3726 do_vec_ld(s, tt, e, clean_addr, mop); 3727 } 3728 tcg_gen_add_i64(clean_addr, clean_addr, tcg_ebytes); 3729 } 3730 } 3731 } 3732 3733 if (!is_store) { 3734 /* For non-quad operations, setting a slice of the low 3735 * 64 bits of the register clears the high 64 bits (in 3736 * the ARM ARM pseudocode this is implicit in the fact 3737 * that 'rval' is a 64 bit wide variable). 3738 * For quad operations, we might still need to zero the 3739 * high bits of SVE. 3740 */ 3741 for (r = 0; r < rpt * selem; r++) { 3742 int tt = (rt + r) % 32; 3743 clear_vec_high(s, is_q, tt); 3744 } 3745 } 3746 3747 if (is_postidx) { 3748 if (rm == 31) { 3749 tcg_gen_addi_i64(tcg_rn, tcg_rn, total); 3750 } else { 3751 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, rm)); 3752 } 3753 } 3754 } 3755 3756 /* AdvSIMD load/store single structure 3757 * 3758 * 31 30 29 23 22 21 20 16 15 13 12 11 10 9 5 4 0 3759 * +---+---+---------------+-----+-----------+-----+---+------+------+------+ 3760 * | 0 | Q | 0 0 1 1 0 1 0 | L R | 0 0 0 0 0 | opc | S | size | Rn | Rt | 3761 * +---+---+---------------+-----+-----------+-----+---+------+------+------+ 3762 * 3763 * AdvSIMD load/store single structure (post-indexed) 3764 * 3765 * 31 30 29 23 22 21 20 16 15 13 12 11 10 9 5 4 0 3766 * +---+---+---------------+-----+-----------+-----+---+------+------+------+ 3767 * | 0 | Q | 0 0 1 1 0 1 1 | L R | Rm | opc | S | size | Rn | Rt | 3768 * +---+---+---------------+-----+-----------+-----+---+------+------+------+ 3769 * 3770 * Rt: first (or only) SIMD&FP register to be transferred 3771 * Rn: base address or SP 3772 * Rm (post-index only): post-index register (when !31) or size dependent #imm 3773 * index = encoded in Q:S:size dependent on size 3774 * 3775 * lane_size = encoded in R, opc 3776 * transfer width = encoded in opc, S, size 3777 */ 3778 static void disas_ldst_single_struct(DisasContext *s, uint32_t insn) 3779 { 3780 int rt = extract32(insn, 0, 5); 3781 int rn = extract32(insn, 5, 5); 3782 int rm = extract32(insn, 16, 5); 3783 int size = extract32(insn, 10, 2); 3784 int S = extract32(insn, 12, 1); 3785 int opc = extract32(insn, 13, 3); 3786 int R = extract32(insn, 21, 1); 3787 int is_load = extract32(insn, 22, 1); 3788 int is_postidx = extract32(insn, 23, 1); 3789 int is_q = extract32(insn, 30, 1); 3790 3791 int scale = extract32(opc, 1, 2); 3792 int selem = (extract32(opc, 0, 1) << 1 | R) + 1; 3793 bool replicate = false; 3794 int index = is_q << 3 | S << 2 | size; 3795 int xs, total; 3796 TCGv_i64 clean_addr, tcg_rn, tcg_ebytes; 3797 MemOp mop; 3798 3799 if (extract32(insn, 31, 1)) { 3800 unallocated_encoding(s); 3801 return; 3802 } 3803 if (!is_postidx && rm != 0) { 3804 unallocated_encoding(s); 3805 return; 3806 } 3807 3808 switch (scale) { 3809 case 3: 3810 if (!is_load || S) { 3811 unallocated_encoding(s); 3812 return; 3813 } 3814 scale = size; 3815 replicate = true; 3816 break; 3817 case 0: 3818 break; 3819 case 1: 3820 if (extract32(size, 0, 1)) { 3821 unallocated_encoding(s); 3822 return; 3823 } 3824 index >>= 1; 3825 break; 3826 case 2: 3827 if (extract32(size, 1, 1)) { 3828 unallocated_encoding(s); 3829 return; 3830 } 3831 if (!extract32(size, 0, 1)) { 3832 index >>= 2; 3833 } else { 3834 if (S) { 3835 unallocated_encoding(s); 3836 return; 3837 } 3838 index >>= 3; 3839 scale = 3; 3840 } 3841 break; 3842 default: 3843 g_assert_not_reached(); 3844 } 3845 3846 if (!fp_access_check(s)) { 3847 return; 3848 } 3849 3850 if (rn == 31) { 3851 gen_check_sp_alignment(s); 3852 } 3853 3854 total = selem << scale; 3855 tcg_rn = cpu_reg_sp(s, rn); 3856 3857 clean_addr = gen_mte_checkN(s, tcg_rn, !is_load, is_postidx || rn != 31, 3858 total); 3859 mop = finalize_memop(s, scale); 3860 3861 tcg_ebytes = tcg_constant_i64(1 << scale); 3862 for (xs = 0; xs < selem; xs++) { 3863 if (replicate) { 3864 /* Load and replicate to all elements */ 3865 TCGv_i64 tcg_tmp = tcg_temp_new_i64(); 3866 3867 tcg_gen_qemu_ld_i64(tcg_tmp, clean_addr, get_mem_index(s), mop); 3868 tcg_gen_gvec_dup_i64(scale, vec_full_reg_offset(s, rt), 3869 (is_q + 1) * 8, vec_full_reg_size(s), 3870 tcg_tmp); 3871 } else { 3872 /* Load/store one element per register */ 3873 if (is_load) { 3874 do_vec_ld(s, rt, index, clean_addr, mop); 3875 } else { 3876 do_vec_st(s, rt, index, clean_addr, mop); 3877 } 3878 } 3879 tcg_gen_add_i64(clean_addr, clean_addr, tcg_ebytes); 3880 rt = (rt + 1) % 32; 3881 } 3882 3883 if (is_postidx) { 3884 if (rm == 31) { 3885 tcg_gen_addi_i64(tcg_rn, tcg_rn, total); 3886 } else { 3887 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, rm)); 3888 } 3889 } 3890 } 3891 3892 /* 3893 * Load/Store memory tags 3894 * 3895 * 31 30 29 24 22 21 12 10 5 0 3896 * +-----+-------------+-----+---+------+-----+------+------+ 3897 * | 1 1 | 0 1 1 0 0 1 | op1 | 1 | imm9 | op2 | Rn | Rt | 3898 * +-----+-------------+-----+---+------+-----+------+------+ 3899 */ 3900 static void disas_ldst_tag(DisasContext *s, uint32_t insn) 3901 { 3902 int rt = extract32(insn, 0, 5); 3903 int rn = extract32(insn, 5, 5); 3904 uint64_t offset = sextract64(insn, 12, 9) << LOG2_TAG_GRANULE; 3905 int op2 = extract32(insn, 10, 2); 3906 int op1 = extract32(insn, 22, 2); 3907 bool is_load = false, is_pair = false, is_zero = false, is_mult = false; 3908 int index = 0; 3909 TCGv_i64 addr, clean_addr, tcg_rt; 3910 3911 /* We checked insn bits [29:24,21] in the caller. */ 3912 if (extract32(insn, 30, 2) != 3) { 3913 goto do_unallocated; 3914 } 3915 3916 /* 3917 * @index is a tri-state variable which has 3 states: 3918 * < 0 : post-index, writeback 3919 * = 0 : signed offset 3920 * > 0 : pre-index, writeback 3921 */ 3922 switch (op1) { 3923 case 0: 3924 if (op2 != 0) { 3925 /* STG */ 3926 index = op2 - 2; 3927 } else { 3928 /* STZGM */ 3929 if (s->current_el == 0 || offset != 0) { 3930 goto do_unallocated; 3931 } 3932 is_mult = is_zero = true; 3933 } 3934 break; 3935 case 1: 3936 if (op2 != 0) { 3937 /* STZG */ 3938 is_zero = true; 3939 index = op2 - 2; 3940 } else { 3941 /* LDG */ 3942 is_load = true; 3943 } 3944 break; 3945 case 2: 3946 if (op2 != 0) { 3947 /* ST2G */ 3948 is_pair = true; 3949 index = op2 - 2; 3950 } else { 3951 /* STGM */ 3952 if (s->current_el == 0 || offset != 0) { 3953 goto do_unallocated; 3954 } 3955 is_mult = true; 3956 } 3957 break; 3958 case 3: 3959 if (op2 != 0) { 3960 /* STZ2G */ 3961 is_pair = is_zero = true; 3962 index = op2 - 2; 3963 } else { 3964 /* LDGM */ 3965 if (s->current_el == 0 || offset != 0) { 3966 goto do_unallocated; 3967 } 3968 is_mult = is_load = true; 3969 } 3970 break; 3971 3972 default: 3973 do_unallocated: 3974 unallocated_encoding(s); 3975 return; 3976 } 3977 3978 if (is_mult 3979 ? !dc_isar_feature(aa64_mte, s) 3980 : !dc_isar_feature(aa64_mte_insn_reg, s)) { 3981 goto do_unallocated; 3982 } 3983 3984 if (rn == 31) { 3985 gen_check_sp_alignment(s); 3986 } 3987 3988 addr = read_cpu_reg_sp(s, rn, true); 3989 if (index >= 0) { 3990 /* pre-index or signed offset */ 3991 tcg_gen_addi_i64(addr, addr, offset); 3992 } 3993 3994 if (is_mult) { 3995 tcg_rt = cpu_reg(s, rt); 3996 3997 if (is_zero) { 3998 int size = 4 << s->dcz_blocksize; 3999 4000 if (s->ata) { 4001 gen_helper_stzgm_tags(cpu_env, addr, tcg_rt); 4002 } 4003 /* 4004 * The non-tags portion of STZGM is mostly like DC_ZVA, 4005 * except the alignment happens before the access. 4006 */ 4007 clean_addr = clean_data_tbi(s, addr); 4008 tcg_gen_andi_i64(clean_addr, clean_addr, -size); 4009 gen_helper_dc_zva(cpu_env, clean_addr); 4010 } else if (s->ata) { 4011 if (is_load) { 4012 gen_helper_ldgm(tcg_rt, cpu_env, addr); 4013 } else { 4014 gen_helper_stgm(cpu_env, addr, tcg_rt); 4015 } 4016 } else { 4017 MMUAccessType acc = is_load ? MMU_DATA_LOAD : MMU_DATA_STORE; 4018 int size = 4 << GMID_EL1_BS; 4019 4020 clean_addr = clean_data_tbi(s, addr); 4021 tcg_gen_andi_i64(clean_addr, clean_addr, -size); 4022 gen_probe_access(s, clean_addr, acc, size); 4023 4024 if (is_load) { 4025 /* The result tags are zeros. */ 4026 tcg_gen_movi_i64(tcg_rt, 0); 4027 } 4028 } 4029 return; 4030 } 4031 4032 if (is_load) { 4033 tcg_gen_andi_i64(addr, addr, -TAG_GRANULE); 4034 tcg_rt = cpu_reg(s, rt); 4035 if (s->ata) { 4036 gen_helper_ldg(tcg_rt, cpu_env, addr, tcg_rt); 4037 } else { 4038 clean_addr = clean_data_tbi(s, addr); 4039 gen_probe_access(s, clean_addr, MMU_DATA_LOAD, MO_8); 4040 gen_address_with_allocation_tag0(tcg_rt, addr); 4041 } 4042 } else { 4043 tcg_rt = cpu_reg_sp(s, rt); 4044 if (!s->ata) { 4045 /* 4046 * For STG and ST2G, we need to check alignment and probe memory. 4047 * TODO: For STZG and STZ2G, we could rely on the stores below, 4048 * at least for system mode; user-only won't enforce alignment. 4049 */ 4050 if (is_pair) { 4051 gen_helper_st2g_stub(cpu_env, addr); 4052 } else { 4053 gen_helper_stg_stub(cpu_env, addr); 4054 } 4055 } else if (tb_cflags(s->base.tb) & CF_PARALLEL) { 4056 if (is_pair) { 4057 gen_helper_st2g_parallel(cpu_env, addr, tcg_rt); 4058 } else { 4059 gen_helper_stg_parallel(cpu_env, addr, tcg_rt); 4060 } 4061 } else { 4062 if (is_pair) { 4063 gen_helper_st2g(cpu_env, addr, tcg_rt); 4064 } else { 4065 gen_helper_stg(cpu_env, addr, tcg_rt); 4066 } 4067 } 4068 } 4069 4070 if (is_zero) { 4071 TCGv_i64 clean_addr = clean_data_tbi(s, addr); 4072 TCGv_i64 tcg_zero = tcg_constant_i64(0); 4073 int mem_index = get_mem_index(s); 4074 int i, n = (1 + is_pair) << LOG2_TAG_GRANULE; 4075 4076 tcg_gen_qemu_st_i64(tcg_zero, clean_addr, mem_index, 4077 MO_UQ | MO_ALIGN_16); 4078 for (i = 8; i < n; i += 8) { 4079 tcg_gen_addi_i64(clean_addr, clean_addr, 8); 4080 tcg_gen_qemu_st_i64(tcg_zero, clean_addr, mem_index, MO_UQ); 4081 } 4082 } 4083 4084 if (index != 0) { 4085 /* pre-index or post-index */ 4086 if (index < 0) { 4087 /* post-index */ 4088 tcg_gen_addi_i64(addr, addr, offset); 4089 } 4090 tcg_gen_mov_i64(cpu_reg_sp(s, rn), addr); 4091 } 4092 } 4093 4094 /* Loads and stores */ 4095 static void disas_ldst(DisasContext *s, uint32_t insn) 4096 { 4097 switch (extract32(insn, 24, 6)) { 4098 case 0x08: /* Load/store exclusive */ 4099 disas_ldst_excl(s, insn); 4100 break; 4101 case 0x18: case 0x1c: /* Load register (literal) */ 4102 disas_ld_lit(s, insn); 4103 break; 4104 case 0x28: case 0x29: 4105 case 0x2c: case 0x2d: /* Load/store pair (all forms) */ 4106 disas_ldst_pair(s, insn); 4107 break; 4108 case 0x38: case 0x39: 4109 case 0x3c: case 0x3d: /* Load/store register (all forms) */ 4110 disas_ldst_reg(s, insn); 4111 break; 4112 case 0x0c: /* AdvSIMD load/store multiple structures */ 4113 disas_ldst_multiple_struct(s, insn); 4114 break; 4115 case 0x0d: /* AdvSIMD load/store single structure */ 4116 disas_ldst_single_struct(s, insn); 4117 break; 4118 case 0x19: 4119 if (extract32(insn, 21, 1) != 0) { 4120 disas_ldst_tag(s, insn); 4121 } else if (extract32(insn, 10, 2) == 0) { 4122 disas_ldst_ldapr_stlr(s, insn); 4123 } else { 4124 unallocated_encoding(s); 4125 } 4126 break; 4127 default: 4128 unallocated_encoding(s); 4129 break; 4130 } 4131 } 4132 4133 typedef void ArithTwoOp(TCGv_i64, TCGv_i64, TCGv_i64); 4134 4135 static bool gen_rri(DisasContext *s, arg_rri_sf *a, 4136 bool rd_sp, bool rn_sp, ArithTwoOp *fn) 4137 { 4138 TCGv_i64 tcg_rn = rn_sp ? cpu_reg_sp(s, a->rn) : cpu_reg(s, a->rn); 4139 TCGv_i64 tcg_rd = rd_sp ? cpu_reg_sp(s, a->rd) : cpu_reg(s, a->rd); 4140 TCGv_i64 tcg_imm = tcg_constant_i64(a->imm); 4141 4142 fn(tcg_rd, tcg_rn, tcg_imm); 4143 if (!a->sf) { 4144 tcg_gen_ext32u_i64(tcg_rd, tcg_rd); 4145 } 4146 return true; 4147 } 4148 4149 /* 4150 * PC-rel. addressing 4151 */ 4152 4153 static bool trans_ADR(DisasContext *s, arg_ri *a) 4154 { 4155 gen_pc_plus_diff(s, cpu_reg(s, a->rd), a->imm); 4156 return true; 4157 } 4158 4159 static bool trans_ADRP(DisasContext *s, arg_ri *a) 4160 { 4161 int64_t offset = (int64_t)a->imm << 12; 4162 4163 /* The page offset is ok for CF_PCREL. */ 4164 offset -= s->pc_curr & 0xfff; 4165 gen_pc_plus_diff(s, cpu_reg(s, a->rd), offset); 4166 return true; 4167 } 4168 4169 /* 4170 * Add/subtract (immediate) 4171 */ 4172 TRANS(ADD_i, gen_rri, a, 1, 1, tcg_gen_add_i64) 4173 TRANS(SUB_i, gen_rri, a, 1, 1, tcg_gen_sub_i64) 4174 TRANS(ADDS_i, gen_rri, a, 0, 1, a->sf ? gen_add64_CC : gen_add32_CC) 4175 TRANS(SUBS_i, gen_rri, a, 0, 1, a->sf ? gen_sub64_CC : gen_sub32_CC) 4176 4177 /* 4178 * Add/subtract (immediate, with tags) 4179 */ 4180 4181 static bool gen_add_sub_imm_with_tags(DisasContext *s, arg_rri_tag *a, 4182 bool sub_op) 4183 { 4184 TCGv_i64 tcg_rn, tcg_rd; 4185 int imm; 4186 4187 imm = a->uimm6 << LOG2_TAG_GRANULE; 4188 if (sub_op) { 4189 imm = -imm; 4190 } 4191 4192 tcg_rn = cpu_reg_sp(s, a->rn); 4193 tcg_rd = cpu_reg_sp(s, a->rd); 4194 4195 if (s->ata) { 4196 gen_helper_addsubg(tcg_rd, cpu_env, tcg_rn, 4197 tcg_constant_i32(imm), 4198 tcg_constant_i32(a->uimm4)); 4199 } else { 4200 tcg_gen_addi_i64(tcg_rd, tcg_rn, imm); 4201 gen_address_with_allocation_tag0(tcg_rd, tcg_rd); 4202 } 4203 return true; 4204 } 4205 4206 TRANS_FEAT(ADDG_i, aa64_mte_insn_reg, gen_add_sub_imm_with_tags, a, false) 4207 TRANS_FEAT(SUBG_i, aa64_mte_insn_reg, gen_add_sub_imm_with_tags, a, true) 4208 4209 /* The input should be a value in the bottom e bits (with higher 4210 * bits zero); returns that value replicated into every element 4211 * of size e in a 64 bit integer. 4212 */ 4213 static uint64_t bitfield_replicate(uint64_t mask, unsigned int e) 4214 { 4215 assert(e != 0); 4216 while (e < 64) { 4217 mask |= mask << e; 4218 e *= 2; 4219 } 4220 return mask; 4221 } 4222 4223 /* 4224 * Logical (immediate) 4225 */ 4226 4227 /* 4228 * Simplified variant of pseudocode DecodeBitMasks() for the case where we 4229 * only require the wmask. Returns false if the imms/immr/immn are a reserved 4230 * value (ie should cause a guest UNDEF exception), and true if they are 4231 * valid, in which case the decoded bit pattern is written to result. 4232 */ 4233 bool logic_imm_decode_wmask(uint64_t *result, unsigned int immn, 4234 unsigned int imms, unsigned int immr) 4235 { 4236 uint64_t mask; 4237 unsigned e, levels, s, r; 4238 int len; 4239 4240 assert(immn < 2 && imms < 64 && immr < 64); 4241 4242 /* The bit patterns we create here are 64 bit patterns which 4243 * are vectors of identical elements of size e = 2, 4, 8, 16, 32 or 4244 * 64 bits each. Each element contains the same value: a run 4245 * of between 1 and e-1 non-zero bits, rotated within the 4246 * element by between 0 and e-1 bits. 4247 * 4248 * The element size and run length are encoded into immn (1 bit) 4249 * and imms (6 bits) as follows: 4250 * 64 bit elements: immn = 1, imms = <length of run - 1> 4251 * 32 bit elements: immn = 0, imms = 0 : <length of run - 1> 4252 * 16 bit elements: immn = 0, imms = 10 : <length of run - 1> 4253 * 8 bit elements: immn = 0, imms = 110 : <length of run - 1> 4254 * 4 bit elements: immn = 0, imms = 1110 : <length of run - 1> 4255 * 2 bit elements: immn = 0, imms = 11110 : <length of run - 1> 4256 * Notice that immn = 0, imms = 11111x is the only combination 4257 * not covered by one of the above options; this is reserved. 4258 * Further, <length of run - 1> all-ones is a reserved pattern. 4259 * 4260 * In all cases the rotation is by immr % e (and immr is 6 bits). 4261 */ 4262 4263 /* First determine the element size */ 4264 len = 31 - clz32((immn << 6) | (~imms & 0x3f)); 4265 if (len < 1) { 4266 /* This is the immn == 0, imms == 0x11111x case */ 4267 return false; 4268 } 4269 e = 1 << len; 4270 4271 levels = e - 1; 4272 s = imms & levels; 4273 r = immr & levels; 4274 4275 if (s == levels) { 4276 /* <length of run - 1> mustn't be all-ones. */ 4277 return false; 4278 } 4279 4280 /* Create the value of one element: s+1 set bits rotated 4281 * by r within the element (which is e bits wide)... 4282 */ 4283 mask = MAKE_64BIT_MASK(0, s + 1); 4284 if (r) { 4285 mask = (mask >> r) | (mask << (e - r)); 4286 mask &= MAKE_64BIT_MASK(0, e); 4287 } 4288 /* ...then replicate the element over the whole 64 bit value */ 4289 mask = bitfield_replicate(mask, e); 4290 *result = mask; 4291 return true; 4292 } 4293 4294 static bool gen_rri_log(DisasContext *s, arg_rri_log *a, bool set_cc, 4295 void (*fn)(TCGv_i64, TCGv_i64, int64_t)) 4296 { 4297 TCGv_i64 tcg_rd, tcg_rn; 4298 uint64_t imm; 4299 4300 /* Some immediate field values are reserved. */ 4301 if (!logic_imm_decode_wmask(&imm, extract32(a->dbm, 12, 1), 4302 extract32(a->dbm, 0, 6), 4303 extract32(a->dbm, 6, 6))) { 4304 return false; 4305 } 4306 if (!a->sf) { 4307 imm &= 0xffffffffull; 4308 } 4309 4310 tcg_rd = set_cc ? cpu_reg(s, a->rd) : cpu_reg_sp(s, a->rd); 4311 tcg_rn = cpu_reg(s, a->rn); 4312 4313 fn(tcg_rd, tcg_rn, imm); 4314 if (set_cc) { 4315 gen_logic_CC(a->sf, tcg_rd); 4316 } 4317 if (!a->sf) { 4318 tcg_gen_ext32u_i64(tcg_rd, tcg_rd); 4319 } 4320 return true; 4321 } 4322 4323 TRANS(AND_i, gen_rri_log, a, false, tcg_gen_andi_i64) 4324 TRANS(ORR_i, gen_rri_log, a, false, tcg_gen_ori_i64) 4325 TRANS(EOR_i, gen_rri_log, a, false, tcg_gen_xori_i64) 4326 TRANS(ANDS_i, gen_rri_log, a, true, tcg_gen_andi_i64) 4327 4328 /* 4329 * Move wide (immediate) 4330 */ 4331 4332 static bool trans_MOVZ(DisasContext *s, arg_movw *a) 4333 { 4334 int pos = a->hw << 4; 4335 tcg_gen_movi_i64(cpu_reg(s, a->rd), (uint64_t)a->imm << pos); 4336 return true; 4337 } 4338 4339 static bool trans_MOVN(DisasContext *s, arg_movw *a) 4340 { 4341 int pos = a->hw << 4; 4342 uint64_t imm = a->imm; 4343 4344 imm = ~(imm << pos); 4345 if (!a->sf) { 4346 imm = (uint32_t)imm; 4347 } 4348 tcg_gen_movi_i64(cpu_reg(s, a->rd), imm); 4349 return true; 4350 } 4351 4352 static bool trans_MOVK(DisasContext *s, arg_movw *a) 4353 { 4354 int pos = a->hw << 4; 4355 TCGv_i64 tcg_rd, tcg_im; 4356 4357 tcg_rd = cpu_reg(s, a->rd); 4358 tcg_im = tcg_constant_i64(a->imm); 4359 tcg_gen_deposit_i64(tcg_rd, tcg_rd, tcg_im, pos, 16); 4360 if (!a->sf) { 4361 tcg_gen_ext32u_i64(tcg_rd, tcg_rd); 4362 } 4363 return true; 4364 } 4365 4366 /* 4367 * Bitfield 4368 */ 4369 4370 static bool trans_SBFM(DisasContext *s, arg_SBFM *a) 4371 { 4372 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); 4373 TCGv_i64 tcg_tmp = read_cpu_reg(s, a->rn, 1); 4374 unsigned int bitsize = a->sf ? 64 : 32; 4375 unsigned int ri = a->immr; 4376 unsigned int si = a->imms; 4377 unsigned int pos, len; 4378 4379 if (si >= ri) { 4380 /* Wd<s-r:0> = Wn<s:r> */ 4381 len = (si - ri) + 1; 4382 tcg_gen_sextract_i64(tcg_rd, tcg_tmp, ri, len); 4383 if (!a->sf) { 4384 tcg_gen_ext32u_i64(tcg_rd, tcg_rd); 4385 } 4386 } else { 4387 /* Wd<32+s-r,32-r> = Wn<s:0> */ 4388 len = si + 1; 4389 pos = (bitsize - ri) & (bitsize - 1); 4390 4391 if (len < ri) { 4392 /* 4393 * Sign extend the destination field from len to fill the 4394 * balance of the word. Let the deposit below insert all 4395 * of those sign bits. 4396 */ 4397 tcg_gen_sextract_i64(tcg_tmp, tcg_tmp, 0, len); 4398 len = ri; 4399 } 4400 4401 /* 4402 * We start with zero, and we haven't modified any bits outside 4403 * bitsize, therefore no final zero-extension is unneeded for !sf. 4404 */ 4405 tcg_gen_deposit_z_i64(tcg_rd, tcg_tmp, pos, len); 4406 } 4407 return true; 4408 } 4409 4410 static bool trans_UBFM(DisasContext *s, arg_UBFM *a) 4411 { 4412 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); 4413 TCGv_i64 tcg_tmp = read_cpu_reg(s, a->rn, 1); 4414 unsigned int bitsize = a->sf ? 64 : 32; 4415 unsigned int ri = a->immr; 4416 unsigned int si = a->imms; 4417 unsigned int pos, len; 4418 4419 tcg_rd = cpu_reg(s, a->rd); 4420 tcg_tmp = read_cpu_reg(s, a->rn, 1); 4421 4422 if (si >= ri) { 4423 /* Wd<s-r:0> = Wn<s:r> */ 4424 len = (si - ri) + 1; 4425 tcg_gen_extract_i64(tcg_rd, tcg_tmp, ri, len); 4426 } else { 4427 /* Wd<32+s-r,32-r> = Wn<s:0> */ 4428 len = si + 1; 4429 pos = (bitsize - ri) & (bitsize - 1); 4430 tcg_gen_deposit_z_i64(tcg_rd, tcg_tmp, pos, len); 4431 } 4432 return true; 4433 } 4434 4435 static bool trans_BFM(DisasContext *s, arg_BFM *a) 4436 { 4437 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); 4438 TCGv_i64 tcg_tmp = read_cpu_reg(s, a->rn, 1); 4439 unsigned int bitsize = a->sf ? 64 : 32; 4440 unsigned int ri = a->immr; 4441 unsigned int si = a->imms; 4442 unsigned int pos, len; 4443 4444 tcg_rd = cpu_reg(s, a->rd); 4445 tcg_tmp = read_cpu_reg(s, a->rn, 1); 4446 4447 if (si >= ri) { 4448 /* Wd<s-r:0> = Wn<s:r> */ 4449 tcg_gen_shri_i64(tcg_tmp, tcg_tmp, ri); 4450 len = (si - ri) + 1; 4451 pos = 0; 4452 } else { 4453 /* Wd<32+s-r,32-r> = Wn<s:0> */ 4454 len = si + 1; 4455 pos = (bitsize - ri) & (bitsize - 1); 4456 } 4457 4458 tcg_gen_deposit_i64(tcg_rd, tcg_rd, tcg_tmp, pos, len); 4459 if (!a->sf) { 4460 tcg_gen_ext32u_i64(tcg_rd, tcg_rd); 4461 } 4462 return true; 4463 } 4464 4465 static bool trans_EXTR(DisasContext *s, arg_extract *a) 4466 { 4467 TCGv_i64 tcg_rd, tcg_rm, tcg_rn; 4468 4469 tcg_rd = cpu_reg(s, a->rd); 4470 4471 if (unlikely(a->imm == 0)) { 4472 /* 4473 * tcg shl_i32/shl_i64 is undefined for 32/64 bit shifts, 4474 * so an extract from bit 0 is a special case. 4475 */ 4476 if (a->sf) { 4477 tcg_gen_mov_i64(tcg_rd, cpu_reg(s, a->rm)); 4478 } else { 4479 tcg_gen_ext32u_i64(tcg_rd, cpu_reg(s, a->rm)); 4480 } 4481 } else { 4482 tcg_rm = cpu_reg(s, a->rm); 4483 tcg_rn = cpu_reg(s, a->rn); 4484 4485 if (a->sf) { 4486 /* Specialization to ROR happens in EXTRACT2. */ 4487 tcg_gen_extract2_i64(tcg_rd, tcg_rm, tcg_rn, a->imm); 4488 } else { 4489 TCGv_i32 t0 = tcg_temp_new_i32(); 4490 4491 tcg_gen_extrl_i64_i32(t0, tcg_rm); 4492 if (a->rm == a->rn) { 4493 tcg_gen_rotri_i32(t0, t0, a->imm); 4494 } else { 4495 TCGv_i32 t1 = tcg_temp_new_i32(); 4496 tcg_gen_extrl_i64_i32(t1, tcg_rn); 4497 tcg_gen_extract2_i32(t0, t0, t1, a->imm); 4498 } 4499 tcg_gen_extu_i32_i64(tcg_rd, t0); 4500 } 4501 } 4502 return true; 4503 } 4504 4505 /* Shift a TCGv src by TCGv shift_amount, put result in dst. 4506 * Note that it is the caller's responsibility to ensure that the 4507 * shift amount is in range (ie 0..31 or 0..63) and provide the ARM 4508 * mandated semantics for out of range shifts. 4509 */ 4510 static void shift_reg(TCGv_i64 dst, TCGv_i64 src, int sf, 4511 enum a64_shift_type shift_type, TCGv_i64 shift_amount) 4512 { 4513 switch (shift_type) { 4514 case A64_SHIFT_TYPE_LSL: 4515 tcg_gen_shl_i64(dst, src, shift_amount); 4516 break; 4517 case A64_SHIFT_TYPE_LSR: 4518 tcg_gen_shr_i64(dst, src, shift_amount); 4519 break; 4520 case A64_SHIFT_TYPE_ASR: 4521 if (!sf) { 4522 tcg_gen_ext32s_i64(dst, src); 4523 } 4524 tcg_gen_sar_i64(dst, sf ? src : dst, shift_amount); 4525 break; 4526 case A64_SHIFT_TYPE_ROR: 4527 if (sf) { 4528 tcg_gen_rotr_i64(dst, src, shift_amount); 4529 } else { 4530 TCGv_i32 t0, t1; 4531 t0 = tcg_temp_new_i32(); 4532 t1 = tcg_temp_new_i32(); 4533 tcg_gen_extrl_i64_i32(t0, src); 4534 tcg_gen_extrl_i64_i32(t1, shift_amount); 4535 tcg_gen_rotr_i32(t0, t0, t1); 4536 tcg_gen_extu_i32_i64(dst, t0); 4537 } 4538 break; 4539 default: 4540 assert(FALSE); /* all shift types should be handled */ 4541 break; 4542 } 4543 4544 if (!sf) { /* zero extend final result */ 4545 tcg_gen_ext32u_i64(dst, dst); 4546 } 4547 } 4548 4549 /* Shift a TCGv src by immediate, put result in dst. 4550 * The shift amount must be in range (this should always be true as the 4551 * relevant instructions will UNDEF on bad shift immediates). 4552 */ 4553 static void shift_reg_imm(TCGv_i64 dst, TCGv_i64 src, int sf, 4554 enum a64_shift_type shift_type, unsigned int shift_i) 4555 { 4556 assert(shift_i < (sf ? 64 : 32)); 4557 4558 if (shift_i == 0) { 4559 tcg_gen_mov_i64(dst, src); 4560 } else { 4561 shift_reg(dst, src, sf, shift_type, tcg_constant_i64(shift_i)); 4562 } 4563 } 4564 4565 /* Logical (shifted register) 4566 * 31 30 29 28 24 23 22 21 20 16 15 10 9 5 4 0 4567 * +----+-----+-----------+-------+---+------+--------+------+------+ 4568 * | sf | opc | 0 1 0 1 0 | shift | N | Rm | imm6 | Rn | Rd | 4569 * +----+-----+-----------+-------+---+------+--------+------+------+ 4570 */ 4571 static void disas_logic_reg(DisasContext *s, uint32_t insn) 4572 { 4573 TCGv_i64 tcg_rd, tcg_rn, tcg_rm; 4574 unsigned int sf, opc, shift_type, invert, rm, shift_amount, rn, rd; 4575 4576 sf = extract32(insn, 31, 1); 4577 opc = extract32(insn, 29, 2); 4578 shift_type = extract32(insn, 22, 2); 4579 invert = extract32(insn, 21, 1); 4580 rm = extract32(insn, 16, 5); 4581 shift_amount = extract32(insn, 10, 6); 4582 rn = extract32(insn, 5, 5); 4583 rd = extract32(insn, 0, 5); 4584 4585 if (!sf && (shift_amount & (1 << 5))) { 4586 unallocated_encoding(s); 4587 return; 4588 } 4589 4590 tcg_rd = cpu_reg(s, rd); 4591 4592 if (opc == 1 && shift_amount == 0 && shift_type == 0 && rn == 31) { 4593 /* Unshifted ORR and ORN with WZR/XZR is the standard encoding for 4594 * register-register MOV and MVN, so it is worth special casing. 4595 */ 4596 tcg_rm = cpu_reg(s, rm); 4597 if (invert) { 4598 tcg_gen_not_i64(tcg_rd, tcg_rm); 4599 if (!sf) { 4600 tcg_gen_ext32u_i64(tcg_rd, tcg_rd); 4601 } 4602 } else { 4603 if (sf) { 4604 tcg_gen_mov_i64(tcg_rd, tcg_rm); 4605 } else { 4606 tcg_gen_ext32u_i64(tcg_rd, tcg_rm); 4607 } 4608 } 4609 return; 4610 } 4611 4612 tcg_rm = read_cpu_reg(s, rm, sf); 4613 4614 if (shift_amount) { 4615 shift_reg_imm(tcg_rm, tcg_rm, sf, shift_type, shift_amount); 4616 } 4617 4618 tcg_rn = cpu_reg(s, rn); 4619 4620 switch (opc | (invert << 2)) { 4621 case 0: /* AND */ 4622 case 3: /* ANDS */ 4623 tcg_gen_and_i64(tcg_rd, tcg_rn, tcg_rm); 4624 break; 4625 case 1: /* ORR */ 4626 tcg_gen_or_i64(tcg_rd, tcg_rn, tcg_rm); 4627 break; 4628 case 2: /* EOR */ 4629 tcg_gen_xor_i64(tcg_rd, tcg_rn, tcg_rm); 4630 break; 4631 case 4: /* BIC */ 4632 case 7: /* BICS */ 4633 tcg_gen_andc_i64(tcg_rd, tcg_rn, tcg_rm); 4634 break; 4635 case 5: /* ORN */ 4636 tcg_gen_orc_i64(tcg_rd, tcg_rn, tcg_rm); 4637 break; 4638 case 6: /* EON */ 4639 tcg_gen_eqv_i64(tcg_rd, tcg_rn, tcg_rm); 4640 break; 4641 default: 4642 assert(FALSE); 4643 break; 4644 } 4645 4646 if (!sf) { 4647 tcg_gen_ext32u_i64(tcg_rd, tcg_rd); 4648 } 4649 4650 if (opc == 3) { 4651 gen_logic_CC(sf, tcg_rd); 4652 } 4653 } 4654 4655 /* 4656 * Add/subtract (extended register) 4657 * 4658 * 31|30|29|28 24|23 22|21|20 16|15 13|12 10|9 5|4 0| 4659 * +--+--+--+-----------+-----+--+-------+------+------+----+----+ 4660 * |sf|op| S| 0 1 0 1 1 | opt | 1| Rm |option| imm3 | Rn | Rd | 4661 * +--+--+--+-----------+-----+--+-------+------+------+----+----+ 4662 * 4663 * sf: 0 -> 32bit, 1 -> 64bit 4664 * op: 0 -> add , 1 -> sub 4665 * S: 1 -> set flags 4666 * opt: 00 4667 * option: extension type (see DecodeRegExtend) 4668 * imm3: optional shift to Rm 4669 * 4670 * Rd = Rn + LSL(extend(Rm), amount) 4671 */ 4672 static void disas_add_sub_ext_reg(DisasContext *s, uint32_t insn) 4673 { 4674 int rd = extract32(insn, 0, 5); 4675 int rn = extract32(insn, 5, 5); 4676 int imm3 = extract32(insn, 10, 3); 4677 int option = extract32(insn, 13, 3); 4678 int rm = extract32(insn, 16, 5); 4679 int opt = extract32(insn, 22, 2); 4680 bool setflags = extract32(insn, 29, 1); 4681 bool sub_op = extract32(insn, 30, 1); 4682 bool sf = extract32(insn, 31, 1); 4683 4684 TCGv_i64 tcg_rm, tcg_rn; /* temps */ 4685 TCGv_i64 tcg_rd; 4686 TCGv_i64 tcg_result; 4687 4688 if (imm3 > 4 || opt != 0) { 4689 unallocated_encoding(s); 4690 return; 4691 } 4692 4693 /* non-flag setting ops may use SP */ 4694 if (!setflags) { 4695 tcg_rd = cpu_reg_sp(s, rd); 4696 } else { 4697 tcg_rd = cpu_reg(s, rd); 4698 } 4699 tcg_rn = read_cpu_reg_sp(s, rn, sf); 4700 4701 tcg_rm = read_cpu_reg(s, rm, sf); 4702 ext_and_shift_reg(tcg_rm, tcg_rm, option, imm3); 4703 4704 tcg_result = tcg_temp_new_i64(); 4705 4706 if (!setflags) { 4707 if (sub_op) { 4708 tcg_gen_sub_i64(tcg_result, tcg_rn, tcg_rm); 4709 } else { 4710 tcg_gen_add_i64(tcg_result, tcg_rn, tcg_rm); 4711 } 4712 } else { 4713 if (sub_op) { 4714 gen_sub_CC(sf, tcg_result, tcg_rn, tcg_rm); 4715 } else { 4716 gen_add_CC(sf, tcg_result, tcg_rn, tcg_rm); 4717 } 4718 } 4719 4720 if (sf) { 4721 tcg_gen_mov_i64(tcg_rd, tcg_result); 4722 } else { 4723 tcg_gen_ext32u_i64(tcg_rd, tcg_result); 4724 } 4725 } 4726 4727 /* 4728 * Add/subtract (shifted register) 4729 * 4730 * 31 30 29 28 24 23 22 21 20 16 15 10 9 5 4 0 4731 * +--+--+--+-----------+-----+--+-------+---------+------+------+ 4732 * |sf|op| S| 0 1 0 1 1 |shift| 0| Rm | imm6 | Rn | Rd | 4733 * +--+--+--+-----------+-----+--+-------+---------+------+------+ 4734 * 4735 * sf: 0 -> 32bit, 1 -> 64bit 4736 * op: 0 -> add , 1 -> sub 4737 * S: 1 -> set flags 4738 * shift: 00 -> LSL, 01 -> LSR, 10 -> ASR, 11 -> RESERVED 4739 * imm6: Shift amount to apply to Rm before the add/sub 4740 */ 4741 static void disas_add_sub_reg(DisasContext *s, uint32_t insn) 4742 { 4743 int rd = extract32(insn, 0, 5); 4744 int rn = extract32(insn, 5, 5); 4745 int imm6 = extract32(insn, 10, 6); 4746 int rm = extract32(insn, 16, 5); 4747 int shift_type = extract32(insn, 22, 2); 4748 bool setflags = extract32(insn, 29, 1); 4749 bool sub_op = extract32(insn, 30, 1); 4750 bool sf = extract32(insn, 31, 1); 4751 4752 TCGv_i64 tcg_rd = cpu_reg(s, rd); 4753 TCGv_i64 tcg_rn, tcg_rm; 4754 TCGv_i64 tcg_result; 4755 4756 if ((shift_type == 3) || (!sf && (imm6 > 31))) { 4757 unallocated_encoding(s); 4758 return; 4759 } 4760 4761 tcg_rn = read_cpu_reg(s, rn, sf); 4762 tcg_rm = read_cpu_reg(s, rm, sf); 4763 4764 shift_reg_imm(tcg_rm, tcg_rm, sf, shift_type, imm6); 4765 4766 tcg_result = tcg_temp_new_i64(); 4767 4768 if (!setflags) { 4769 if (sub_op) { 4770 tcg_gen_sub_i64(tcg_result, tcg_rn, tcg_rm); 4771 } else { 4772 tcg_gen_add_i64(tcg_result, tcg_rn, tcg_rm); 4773 } 4774 } else { 4775 if (sub_op) { 4776 gen_sub_CC(sf, tcg_result, tcg_rn, tcg_rm); 4777 } else { 4778 gen_add_CC(sf, tcg_result, tcg_rn, tcg_rm); 4779 } 4780 } 4781 4782 if (sf) { 4783 tcg_gen_mov_i64(tcg_rd, tcg_result); 4784 } else { 4785 tcg_gen_ext32u_i64(tcg_rd, tcg_result); 4786 } 4787 } 4788 4789 /* Data-processing (3 source) 4790 * 4791 * 31 30 29 28 24 23 21 20 16 15 14 10 9 5 4 0 4792 * +--+------+-----------+------+------+----+------+------+------+ 4793 * |sf| op54 | 1 1 0 1 1 | op31 | Rm | o0 | Ra | Rn | Rd | 4794 * +--+------+-----------+------+------+----+------+------+------+ 4795 */ 4796 static void disas_data_proc_3src(DisasContext *s, uint32_t insn) 4797 { 4798 int rd = extract32(insn, 0, 5); 4799 int rn = extract32(insn, 5, 5); 4800 int ra = extract32(insn, 10, 5); 4801 int rm = extract32(insn, 16, 5); 4802 int op_id = (extract32(insn, 29, 3) << 4) | 4803 (extract32(insn, 21, 3) << 1) | 4804 extract32(insn, 15, 1); 4805 bool sf = extract32(insn, 31, 1); 4806 bool is_sub = extract32(op_id, 0, 1); 4807 bool is_high = extract32(op_id, 2, 1); 4808 bool is_signed = false; 4809 TCGv_i64 tcg_op1; 4810 TCGv_i64 tcg_op2; 4811 TCGv_i64 tcg_tmp; 4812 4813 /* Note that op_id is sf:op54:op31:o0 so it includes the 32/64 size flag */ 4814 switch (op_id) { 4815 case 0x42: /* SMADDL */ 4816 case 0x43: /* SMSUBL */ 4817 case 0x44: /* SMULH */ 4818 is_signed = true; 4819 break; 4820 case 0x0: /* MADD (32bit) */ 4821 case 0x1: /* MSUB (32bit) */ 4822 case 0x40: /* MADD (64bit) */ 4823 case 0x41: /* MSUB (64bit) */ 4824 case 0x4a: /* UMADDL */ 4825 case 0x4b: /* UMSUBL */ 4826 case 0x4c: /* UMULH */ 4827 break; 4828 default: 4829 unallocated_encoding(s); 4830 return; 4831 } 4832 4833 if (is_high) { 4834 TCGv_i64 low_bits = tcg_temp_new_i64(); /* low bits discarded */ 4835 TCGv_i64 tcg_rd = cpu_reg(s, rd); 4836 TCGv_i64 tcg_rn = cpu_reg(s, rn); 4837 TCGv_i64 tcg_rm = cpu_reg(s, rm); 4838 4839 if (is_signed) { 4840 tcg_gen_muls2_i64(low_bits, tcg_rd, tcg_rn, tcg_rm); 4841 } else { 4842 tcg_gen_mulu2_i64(low_bits, tcg_rd, tcg_rn, tcg_rm); 4843 } 4844 return; 4845 } 4846 4847 tcg_op1 = tcg_temp_new_i64(); 4848 tcg_op2 = tcg_temp_new_i64(); 4849 tcg_tmp = tcg_temp_new_i64(); 4850 4851 if (op_id < 0x42) { 4852 tcg_gen_mov_i64(tcg_op1, cpu_reg(s, rn)); 4853 tcg_gen_mov_i64(tcg_op2, cpu_reg(s, rm)); 4854 } else { 4855 if (is_signed) { 4856 tcg_gen_ext32s_i64(tcg_op1, cpu_reg(s, rn)); 4857 tcg_gen_ext32s_i64(tcg_op2, cpu_reg(s, rm)); 4858 } else { 4859 tcg_gen_ext32u_i64(tcg_op1, cpu_reg(s, rn)); 4860 tcg_gen_ext32u_i64(tcg_op2, cpu_reg(s, rm)); 4861 } 4862 } 4863 4864 if (ra == 31 && !is_sub) { 4865 /* Special-case MADD with rA == XZR; it is the standard MUL alias */ 4866 tcg_gen_mul_i64(cpu_reg(s, rd), tcg_op1, tcg_op2); 4867 } else { 4868 tcg_gen_mul_i64(tcg_tmp, tcg_op1, tcg_op2); 4869 if (is_sub) { 4870 tcg_gen_sub_i64(cpu_reg(s, rd), cpu_reg(s, ra), tcg_tmp); 4871 } else { 4872 tcg_gen_add_i64(cpu_reg(s, rd), cpu_reg(s, ra), tcg_tmp); 4873 } 4874 } 4875 4876 if (!sf) { 4877 tcg_gen_ext32u_i64(cpu_reg(s, rd), cpu_reg(s, rd)); 4878 } 4879 } 4880 4881 /* Add/subtract (with carry) 4882 * 31 30 29 28 27 26 25 24 23 22 21 20 16 15 10 9 5 4 0 4883 * +--+--+--+------------------------+------+-------------+------+-----+ 4884 * |sf|op| S| 1 1 0 1 0 0 0 0 | rm | 0 0 0 0 0 0 | Rn | Rd | 4885 * +--+--+--+------------------------+------+-------------+------+-----+ 4886 */ 4887 4888 static void disas_adc_sbc(DisasContext *s, uint32_t insn) 4889 { 4890 unsigned int sf, op, setflags, rm, rn, rd; 4891 TCGv_i64 tcg_y, tcg_rn, tcg_rd; 4892 4893 sf = extract32(insn, 31, 1); 4894 op = extract32(insn, 30, 1); 4895 setflags = extract32(insn, 29, 1); 4896 rm = extract32(insn, 16, 5); 4897 rn = extract32(insn, 5, 5); 4898 rd = extract32(insn, 0, 5); 4899 4900 tcg_rd = cpu_reg(s, rd); 4901 tcg_rn = cpu_reg(s, rn); 4902 4903 if (op) { 4904 tcg_y = tcg_temp_new_i64(); 4905 tcg_gen_not_i64(tcg_y, cpu_reg(s, rm)); 4906 } else { 4907 tcg_y = cpu_reg(s, rm); 4908 } 4909 4910 if (setflags) { 4911 gen_adc_CC(sf, tcg_rd, tcg_rn, tcg_y); 4912 } else { 4913 gen_adc(sf, tcg_rd, tcg_rn, tcg_y); 4914 } 4915 } 4916 4917 /* 4918 * Rotate right into flags 4919 * 31 30 29 21 15 10 5 4 0 4920 * +--+--+--+-----------------+--------+-----------+------+--+------+ 4921 * |sf|op| S| 1 1 0 1 0 0 0 0 | imm6 | 0 0 0 0 1 | Rn |o2| mask | 4922 * +--+--+--+-----------------+--------+-----------+------+--+------+ 4923 */ 4924 static void disas_rotate_right_into_flags(DisasContext *s, uint32_t insn) 4925 { 4926 int mask = extract32(insn, 0, 4); 4927 int o2 = extract32(insn, 4, 1); 4928 int rn = extract32(insn, 5, 5); 4929 int imm6 = extract32(insn, 15, 6); 4930 int sf_op_s = extract32(insn, 29, 3); 4931 TCGv_i64 tcg_rn; 4932 TCGv_i32 nzcv; 4933 4934 if (sf_op_s != 5 || o2 != 0 || !dc_isar_feature(aa64_condm_4, s)) { 4935 unallocated_encoding(s); 4936 return; 4937 } 4938 4939 tcg_rn = read_cpu_reg(s, rn, 1); 4940 tcg_gen_rotri_i64(tcg_rn, tcg_rn, imm6); 4941 4942 nzcv = tcg_temp_new_i32(); 4943 tcg_gen_extrl_i64_i32(nzcv, tcg_rn); 4944 4945 if (mask & 8) { /* N */ 4946 tcg_gen_shli_i32(cpu_NF, nzcv, 31 - 3); 4947 } 4948 if (mask & 4) { /* Z */ 4949 tcg_gen_not_i32(cpu_ZF, nzcv); 4950 tcg_gen_andi_i32(cpu_ZF, cpu_ZF, 4); 4951 } 4952 if (mask & 2) { /* C */ 4953 tcg_gen_extract_i32(cpu_CF, nzcv, 1, 1); 4954 } 4955 if (mask & 1) { /* V */ 4956 tcg_gen_shli_i32(cpu_VF, nzcv, 31 - 0); 4957 } 4958 } 4959 4960 /* 4961 * Evaluate into flags 4962 * 31 30 29 21 15 14 10 5 4 0 4963 * +--+--+--+-----------------+---------+----+---------+------+--+------+ 4964 * |sf|op| S| 1 1 0 1 0 0 0 0 | opcode2 | sz | 0 0 1 0 | Rn |o3| mask | 4965 * +--+--+--+-----------------+---------+----+---------+------+--+------+ 4966 */ 4967 static void disas_evaluate_into_flags(DisasContext *s, uint32_t insn) 4968 { 4969 int o3_mask = extract32(insn, 0, 5); 4970 int rn = extract32(insn, 5, 5); 4971 int o2 = extract32(insn, 15, 6); 4972 int sz = extract32(insn, 14, 1); 4973 int sf_op_s = extract32(insn, 29, 3); 4974 TCGv_i32 tmp; 4975 int shift; 4976 4977 if (sf_op_s != 1 || o2 != 0 || o3_mask != 0xd || 4978 !dc_isar_feature(aa64_condm_4, s)) { 4979 unallocated_encoding(s); 4980 return; 4981 } 4982 shift = sz ? 16 : 24; /* SETF16 or SETF8 */ 4983 4984 tmp = tcg_temp_new_i32(); 4985 tcg_gen_extrl_i64_i32(tmp, cpu_reg(s, rn)); 4986 tcg_gen_shli_i32(cpu_NF, tmp, shift); 4987 tcg_gen_shli_i32(cpu_VF, tmp, shift - 1); 4988 tcg_gen_mov_i32(cpu_ZF, cpu_NF); 4989 tcg_gen_xor_i32(cpu_VF, cpu_VF, cpu_NF); 4990 } 4991 4992 /* Conditional compare (immediate / register) 4993 * 31 30 29 28 27 26 25 24 23 22 21 20 16 15 12 11 10 9 5 4 3 0 4994 * +--+--+--+------------------------+--------+------+----+--+------+--+-----+ 4995 * |sf|op| S| 1 1 0 1 0 0 1 0 |imm5/rm | cond |i/r |o2| Rn |o3|nzcv | 4996 * +--+--+--+------------------------+--------+------+----+--+------+--+-----+ 4997 * [1] y [0] [0] 4998 */ 4999 static void disas_cc(DisasContext *s, uint32_t insn) 5000 { 5001 unsigned int sf, op, y, cond, rn, nzcv, is_imm; 5002 TCGv_i32 tcg_t0, tcg_t1, tcg_t2; 5003 TCGv_i64 tcg_tmp, tcg_y, tcg_rn; 5004 DisasCompare c; 5005 5006 if (!extract32(insn, 29, 1)) { 5007 unallocated_encoding(s); 5008 return; 5009 } 5010 if (insn & (1 << 10 | 1 << 4)) { 5011 unallocated_encoding(s); 5012 return; 5013 } 5014 sf = extract32(insn, 31, 1); 5015 op = extract32(insn, 30, 1); 5016 is_imm = extract32(insn, 11, 1); 5017 y = extract32(insn, 16, 5); /* y = rm (reg) or imm5 (imm) */ 5018 cond = extract32(insn, 12, 4); 5019 rn = extract32(insn, 5, 5); 5020 nzcv = extract32(insn, 0, 4); 5021 5022 /* Set T0 = !COND. */ 5023 tcg_t0 = tcg_temp_new_i32(); 5024 arm_test_cc(&c, cond); 5025 tcg_gen_setcondi_i32(tcg_invert_cond(c.cond), tcg_t0, c.value, 0); 5026 5027 /* Load the arguments for the new comparison. */ 5028 if (is_imm) { 5029 tcg_y = tcg_temp_new_i64(); 5030 tcg_gen_movi_i64(tcg_y, y); 5031 } else { 5032 tcg_y = cpu_reg(s, y); 5033 } 5034 tcg_rn = cpu_reg(s, rn); 5035 5036 /* Set the flags for the new comparison. */ 5037 tcg_tmp = tcg_temp_new_i64(); 5038 if (op) { 5039 gen_sub_CC(sf, tcg_tmp, tcg_rn, tcg_y); 5040 } else { 5041 gen_add_CC(sf, tcg_tmp, tcg_rn, tcg_y); 5042 } 5043 5044 /* If COND was false, force the flags to #nzcv. Compute two masks 5045 * to help with this: T1 = (COND ? 0 : -1), T2 = (COND ? -1 : 0). 5046 * For tcg hosts that support ANDC, we can make do with just T1. 5047 * In either case, allow the tcg optimizer to delete any unused mask. 5048 */ 5049 tcg_t1 = tcg_temp_new_i32(); 5050 tcg_t2 = tcg_temp_new_i32(); 5051 tcg_gen_neg_i32(tcg_t1, tcg_t0); 5052 tcg_gen_subi_i32(tcg_t2, tcg_t0, 1); 5053 5054 if (nzcv & 8) { /* N */ 5055 tcg_gen_or_i32(cpu_NF, cpu_NF, tcg_t1); 5056 } else { 5057 if (TCG_TARGET_HAS_andc_i32) { 5058 tcg_gen_andc_i32(cpu_NF, cpu_NF, tcg_t1); 5059 } else { 5060 tcg_gen_and_i32(cpu_NF, cpu_NF, tcg_t2); 5061 } 5062 } 5063 if (nzcv & 4) { /* Z */ 5064 if (TCG_TARGET_HAS_andc_i32) { 5065 tcg_gen_andc_i32(cpu_ZF, cpu_ZF, tcg_t1); 5066 } else { 5067 tcg_gen_and_i32(cpu_ZF, cpu_ZF, tcg_t2); 5068 } 5069 } else { 5070 tcg_gen_or_i32(cpu_ZF, cpu_ZF, tcg_t0); 5071 } 5072 if (nzcv & 2) { /* C */ 5073 tcg_gen_or_i32(cpu_CF, cpu_CF, tcg_t0); 5074 } else { 5075 if (TCG_TARGET_HAS_andc_i32) { 5076 tcg_gen_andc_i32(cpu_CF, cpu_CF, tcg_t1); 5077 } else { 5078 tcg_gen_and_i32(cpu_CF, cpu_CF, tcg_t2); 5079 } 5080 } 5081 if (nzcv & 1) { /* V */ 5082 tcg_gen_or_i32(cpu_VF, cpu_VF, tcg_t1); 5083 } else { 5084 if (TCG_TARGET_HAS_andc_i32) { 5085 tcg_gen_andc_i32(cpu_VF, cpu_VF, tcg_t1); 5086 } else { 5087 tcg_gen_and_i32(cpu_VF, cpu_VF, tcg_t2); 5088 } 5089 } 5090 } 5091 5092 /* Conditional select 5093 * 31 30 29 28 21 20 16 15 12 11 10 9 5 4 0 5094 * +----+----+---+-----------------+------+------+-----+------+------+ 5095 * | sf | op | S | 1 1 0 1 0 1 0 0 | Rm | cond | op2 | Rn | Rd | 5096 * +----+----+---+-----------------+------+------+-----+------+------+ 5097 */ 5098 static void disas_cond_select(DisasContext *s, uint32_t insn) 5099 { 5100 unsigned int sf, else_inv, rm, cond, else_inc, rn, rd; 5101 TCGv_i64 tcg_rd, zero; 5102 DisasCompare64 c; 5103 5104 if (extract32(insn, 29, 1) || extract32(insn, 11, 1)) { 5105 /* S == 1 or op2<1> == 1 */ 5106 unallocated_encoding(s); 5107 return; 5108 } 5109 sf = extract32(insn, 31, 1); 5110 else_inv = extract32(insn, 30, 1); 5111 rm = extract32(insn, 16, 5); 5112 cond = extract32(insn, 12, 4); 5113 else_inc = extract32(insn, 10, 1); 5114 rn = extract32(insn, 5, 5); 5115 rd = extract32(insn, 0, 5); 5116 5117 tcg_rd = cpu_reg(s, rd); 5118 5119 a64_test_cc(&c, cond); 5120 zero = tcg_constant_i64(0); 5121 5122 if (rn == 31 && rm == 31 && (else_inc ^ else_inv)) { 5123 /* CSET & CSETM. */ 5124 tcg_gen_setcond_i64(tcg_invert_cond(c.cond), tcg_rd, c.value, zero); 5125 if (else_inv) { 5126 tcg_gen_neg_i64(tcg_rd, tcg_rd); 5127 } 5128 } else { 5129 TCGv_i64 t_true = cpu_reg(s, rn); 5130 TCGv_i64 t_false = read_cpu_reg(s, rm, 1); 5131 if (else_inv && else_inc) { 5132 tcg_gen_neg_i64(t_false, t_false); 5133 } else if (else_inv) { 5134 tcg_gen_not_i64(t_false, t_false); 5135 } else if (else_inc) { 5136 tcg_gen_addi_i64(t_false, t_false, 1); 5137 } 5138 tcg_gen_movcond_i64(c.cond, tcg_rd, c.value, zero, t_true, t_false); 5139 } 5140 5141 if (!sf) { 5142 tcg_gen_ext32u_i64(tcg_rd, tcg_rd); 5143 } 5144 } 5145 5146 static void handle_clz(DisasContext *s, unsigned int sf, 5147 unsigned int rn, unsigned int rd) 5148 { 5149 TCGv_i64 tcg_rd, tcg_rn; 5150 tcg_rd = cpu_reg(s, rd); 5151 tcg_rn = cpu_reg(s, rn); 5152 5153 if (sf) { 5154 tcg_gen_clzi_i64(tcg_rd, tcg_rn, 64); 5155 } else { 5156 TCGv_i32 tcg_tmp32 = tcg_temp_new_i32(); 5157 tcg_gen_extrl_i64_i32(tcg_tmp32, tcg_rn); 5158 tcg_gen_clzi_i32(tcg_tmp32, tcg_tmp32, 32); 5159 tcg_gen_extu_i32_i64(tcg_rd, tcg_tmp32); 5160 } 5161 } 5162 5163 static void handle_cls(DisasContext *s, unsigned int sf, 5164 unsigned int rn, unsigned int rd) 5165 { 5166 TCGv_i64 tcg_rd, tcg_rn; 5167 tcg_rd = cpu_reg(s, rd); 5168 tcg_rn = cpu_reg(s, rn); 5169 5170 if (sf) { 5171 tcg_gen_clrsb_i64(tcg_rd, tcg_rn); 5172 } else { 5173 TCGv_i32 tcg_tmp32 = tcg_temp_new_i32(); 5174 tcg_gen_extrl_i64_i32(tcg_tmp32, tcg_rn); 5175 tcg_gen_clrsb_i32(tcg_tmp32, tcg_tmp32); 5176 tcg_gen_extu_i32_i64(tcg_rd, tcg_tmp32); 5177 } 5178 } 5179 5180 static void handle_rbit(DisasContext *s, unsigned int sf, 5181 unsigned int rn, unsigned int rd) 5182 { 5183 TCGv_i64 tcg_rd, tcg_rn; 5184 tcg_rd = cpu_reg(s, rd); 5185 tcg_rn = cpu_reg(s, rn); 5186 5187 if (sf) { 5188 gen_helper_rbit64(tcg_rd, tcg_rn); 5189 } else { 5190 TCGv_i32 tcg_tmp32 = tcg_temp_new_i32(); 5191 tcg_gen_extrl_i64_i32(tcg_tmp32, tcg_rn); 5192 gen_helper_rbit(tcg_tmp32, tcg_tmp32); 5193 tcg_gen_extu_i32_i64(tcg_rd, tcg_tmp32); 5194 } 5195 } 5196 5197 /* REV with sf==1, opcode==3 ("REV64") */ 5198 static void handle_rev64(DisasContext *s, unsigned int sf, 5199 unsigned int rn, unsigned int rd) 5200 { 5201 if (!sf) { 5202 unallocated_encoding(s); 5203 return; 5204 } 5205 tcg_gen_bswap64_i64(cpu_reg(s, rd), cpu_reg(s, rn)); 5206 } 5207 5208 /* REV with sf==0, opcode==2 5209 * REV32 (sf==1, opcode==2) 5210 */ 5211 static void handle_rev32(DisasContext *s, unsigned int sf, 5212 unsigned int rn, unsigned int rd) 5213 { 5214 TCGv_i64 tcg_rd = cpu_reg(s, rd); 5215 TCGv_i64 tcg_rn = cpu_reg(s, rn); 5216 5217 if (sf) { 5218 tcg_gen_bswap64_i64(tcg_rd, tcg_rn); 5219 tcg_gen_rotri_i64(tcg_rd, tcg_rd, 32); 5220 } else { 5221 tcg_gen_bswap32_i64(tcg_rd, tcg_rn, TCG_BSWAP_OZ); 5222 } 5223 } 5224 5225 /* REV16 (opcode==1) */ 5226 static void handle_rev16(DisasContext *s, unsigned int sf, 5227 unsigned int rn, unsigned int rd) 5228 { 5229 TCGv_i64 tcg_rd = cpu_reg(s, rd); 5230 TCGv_i64 tcg_tmp = tcg_temp_new_i64(); 5231 TCGv_i64 tcg_rn = read_cpu_reg(s, rn, sf); 5232 TCGv_i64 mask = tcg_constant_i64(sf ? 0x00ff00ff00ff00ffull : 0x00ff00ff); 5233 5234 tcg_gen_shri_i64(tcg_tmp, tcg_rn, 8); 5235 tcg_gen_and_i64(tcg_rd, tcg_rn, mask); 5236 tcg_gen_and_i64(tcg_tmp, tcg_tmp, mask); 5237 tcg_gen_shli_i64(tcg_rd, tcg_rd, 8); 5238 tcg_gen_or_i64(tcg_rd, tcg_rd, tcg_tmp); 5239 } 5240 5241 /* Data-processing (1 source) 5242 * 31 30 29 28 21 20 16 15 10 9 5 4 0 5243 * +----+---+---+-----------------+---------+--------+------+------+ 5244 * | sf | 1 | S | 1 1 0 1 0 1 1 0 | opcode2 | opcode | Rn | Rd | 5245 * +----+---+---+-----------------+---------+--------+------+------+ 5246 */ 5247 static void disas_data_proc_1src(DisasContext *s, uint32_t insn) 5248 { 5249 unsigned int sf, opcode, opcode2, rn, rd; 5250 TCGv_i64 tcg_rd; 5251 5252 if (extract32(insn, 29, 1)) { 5253 unallocated_encoding(s); 5254 return; 5255 } 5256 5257 sf = extract32(insn, 31, 1); 5258 opcode = extract32(insn, 10, 6); 5259 opcode2 = extract32(insn, 16, 5); 5260 rn = extract32(insn, 5, 5); 5261 rd = extract32(insn, 0, 5); 5262 5263 #define MAP(SF, O2, O1) ((SF) | (O1 << 1) | (O2 << 7)) 5264 5265 switch (MAP(sf, opcode2, opcode)) { 5266 case MAP(0, 0x00, 0x00): /* RBIT */ 5267 case MAP(1, 0x00, 0x00): 5268 handle_rbit(s, sf, rn, rd); 5269 break; 5270 case MAP(0, 0x00, 0x01): /* REV16 */ 5271 case MAP(1, 0x00, 0x01): 5272 handle_rev16(s, sf, rn, rd); 5273 break; 5274 case MAP(0, 0x00, 0x02): /* REV/REV32 */ 5275 case MAP(1, 0x00, 0x02): 5276 handle_rev32(s, sf, rn, rd); 5277 break; 5278 case MAP(1, 0x00, 0x03): /* REV64 */ 5279 handle_rev64(s, sf, rn, rd); 5280 break; 5281 case MAP(0, 0x00, 0x04): /* CLZ */ 5282 case MAP(1, 0x00, 0x04): 5283 handle_clz(s, sf, rn, rd); 5284 break; 5285 case MAP(0, 0x00, 0x05): /* CLS */ 5286 case MAP(1, 0x00, 0x05): 5287 handle_cls(s, sf, rn, rd); 5288 break; 5289 case MAP(1, 0x01, 0x00): /* PACIA */ 5290 if (s->pauth_active) { 5291 tcg_rd = cpu_reg(s, rd); 5292 gen_helper_pacia(tcg_rd, cpu_env, tcg_rd, cpu_reg_sp(s, rn)); 5293 } else if (!dc_isar_feature(aa64_pauth, s)) { 5294 goto do_unallocated; 5295 } 5296 break; 5297 case MAP(1, 0x01, 0x01): /* PACIB */ 5298 if (s->pauth_active) { 5299 tcg_rd = cpu_reg(s, rd); 5300 gen_helper_pacib(tcg_rd, cpu_env, tcg_rd, cpu_reg_sp(s, rn)); 5301 } else if (!dc_isar_feature(aa64_pauth, s)) { 5302 goto do_unallocated; 5303 } 5304 break; 5305 case MAP(1, 0x01, 0x02): /* PACDA */ 5306 if (s->pauth_active) { 5307 tcg_rd = cpu_reg(s, rd); 5308 gen_helper_pacda(tcg_rd, cpu_env, tcg_rd, cpu_reg_sp(s, rn)); 5309 } else if (!dc_isar_feature(aa64_pauth, s)) { 5310 goto do_unallocated; 5311 } 5312 break; 5313 case MAP(1, 0x01, 0x03): /* PACDB */ 5314 if (s->pauth_active) { 5315 tcg_rd = cpu_reg(s, rd); 5316 gen_helper_pacdb(tcg_rd, cpu_env, tcg_rd, cpu_reg_sp(s, rn)); 5317 } else if (!dc_isar_feature(aa64_pauth, s)) { 5318 goto do_unallocated; 5319 } 5320 break; 5321 case MAP(1, 0x01, 0x04): /* AUTIA */ 5322 if (s->pauth_active) { 5323 tcg_rd = cpu_reg(s, rd); 5324 gen_helper_autia(tcg_rd, cpu_env, tcg_rd, cpu_reg_sp(s, rn)); 5325 } else if (!dc_isar_feature(aa64_pauth, s)) { 5326 goto do_unallocated; 5327 } 5328 break; 5329 case MAP(1, 0x01, 0x05): /* AUTIB */ 5330 if (s->pauth_active) { 5331 tcg_rd = cpu_reg(s, rd); 5332 gen_helper_autib(tcg_rd, cpu_env, tcg_rd, cpu_reg_sp(s, rn)); 5333 } else if (!dc_isar_feature(aa64_pauth, s)) { 5334 goto do_unallocated; 5335 } 5336 break; 5337 case MAP(1, 0x01, 0x06): /* AUTDA */ 5338 if (s->pauth_active) { 5339 tcg_rd = cpu_reg(s, rd); 5340 gen_helper_autda(tcg_rd, cpu_env, tcg_rd, cpu_reg_sp(s, rn)); 5341 } else if (!dc_isar_feature(aa64_pauth, s)) { 5342 goto do_unallocated; 5343 } 5344 break; 5345 case MAP(1, 0x01, 0x07): /* AUTDB */ 5346 if (s->pauth_active) { 5347 tcg_rd = cpu_reg(s, rd); 5348 gen_helper_autdb(tcg_rd, cpu_env, tcg_rd, cpu_reg_sp(s, rn)); 5349 } else if (!dc_isar_feature(aa64_pauth, s)) { 5350 goto do_unallocated; 5351 } 5352 break; 5353 case MAP(1, 0x01, 0x08): /* PACIZA */ 5354 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) { 5355 goto do_unallocated; 5356 } else if (s->pauth_active) { 5357 tcg_rd = cpu_reg(s, rd); 5358 gen_helper_pacia(tcg_rd, cpu_env, tcg_rd, tcg_constant_i64(0)); 5359 } 5360 break; 5361 case MAP(1, 0x01, 0x09): /* PACIZB */ 5362 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) { 5363 goto do_unallocated; 5364 } else if (s->pauth_active) { 5365 tcg_rd = cpu_reg(s, rd); 5366 gen_helper_pacib(tcg_rd, cpu_env, tcg_rd, tcg_constant_i64(0)); 5367 } 5368 break; 5369 case MAP(1, 0x01, 0x0a): /* PACDZA */ 5370 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) { 5371 goto do_unallocated; 5372 } else if (s->pauth_active) { 5373 tcg_rd = cpu_reg(s, rd); 5374 gen_helper_pacda(tcg_rd, cpu_env, tcg_rd, tcg_constant_i64(0)); 5375 } 5376 break; 5377 case MAP(1, 0x01, 0x0b): /* PACDZB */ 5378 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) { 5379 goto do_unallocated; 5380 } else if (s->pauth_active) { 5381 tcg_rd = cpu_reg(s, rd); 5382 gen_helper_pacdb(tcg_rd, cpu_env, tcg_rd, tcg_constant_i64(0)); 5383 } 5384 break; 5385 case MAP(1, 0x01, 0x0c): /* AUTIZA */ 5386 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) { 5387 goto do_unallocated; 5388 } else if (s->pauth_active) { 5389 tcg_rd = cpu_reg(s, rd); 5390 gen_helper_autia(tcg_rd, cpu_env, tcg_rd, tcg_constant_i64(0)); 5391 } 5392 break; 5393 case MAP(1, 0x01, 0x0d): /* AUTIZB */ 5394 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) { 5395 goto do_unallocated; 5396 } else if (s->pauth_active) { 5397 tcg_rd = cpu_reg(s, rd); 5398 gen_helper_autib(tcg_rd, cpu_env, tcg_rd, tcg_constant_i64(0)); 5399 } 5400 break; 5401 case MAP(1, 0x01, 0x0e): /* AUTDZA */ 5402 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) { 5403 goto do_unallocated; 5404 } else if (s->pauth_active) { 5405 tcg_rd = cpu_reg(s, rd); 5406 gen_helper_autda(tcg_rd, cpu_env, tcg_rd, tcg_constant_i64(0)); 5407 } 5408 break; 5409 case MAP(1, 0x01, 0x0f): /* AUTDZB */ 5410 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) { 5411 goto do_unallocated; 5412 } else if (s->pauth_active) { 5413 tcg_rd = cpu_reg(s, rd); 5414 gen_helper_autdb(tcg_rd, cpu_env, tcg_rd, tcg_constant_i64(0)); 5415 } 5416 break; 5417 case MAP(1, 0x01, 0x10): /* XPACI */ 5418 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) { 5419 goto do_unallocated; 5420 } else if (s->pauth_active) { 5421 tcg_rd = cpu_reg(s, rd); 5422 gen_helper_xpaci(tcg_rd, cpu_env, tcg_rd); 5423 } 5424 break; 5425 case MAP(1, 0x01, 0x11): /* XPACD */ 5426 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) { 5427 goto do_unallocated; 5428 } else if (s->pauth_active) { 5429 tcg_rd = cpu_reg(s, rd); 5430 gen_helper_xpacd(tcg_rd, cpu_env, tcg_rd); 5431 } 5432 break; 5433 default: 5434 do_unallocated: 5435 unallocated_encoding(s); 5436 break; 5437 } 5438 5439 #undef MAP 5440 } 5441 5442 static void handle_div(DisasContext *s, bool is_signed, unsigned int sf, 5443 unsigned int rm, unsigned int rn, unsigned int rd) 5444 { 5445 TCGv_i64 tcg_n, tcg_m, tcg_rd; 5446 tcg_rd = cpu_reg(s, rd); 5447 5448 if (!sf && is_signed) { 5449 tcg_n = tcg_temp_new_i64(); 5450 tcg_m = tcg_temp_new_i64(); 5451 tcg_gen_ext32s_i64(tcg_n, cpu_reg(s, rn)); 5452 tcg_gen_ext32s_i64(tcg_m, cpu_reg(s, rm)); 5453 } else { 5454 tcg_n = read_cpu_reg(s, rn, sf); 5455 tcg_m = read_cpu_reg(s, rm, sf); 5456 } 5457 5458 if (is_signed) { 5459 gen_helper_sdiv64(tcg_rd, tcg_n, tcg_m); 5460 } else { 5461 gen_helper_udiv64(tcg_rd, tcg_n, tcg_m); 5462 } 5463 5464 if (!sf) { /* zero extend final result */ 5465 tcg_gen_ext32u_i64(tcg_rd, tcg_rd); 5466 } 5467 } 5468 5469 /* LSLV, LSRV, ASRV, RORV */ 5470 static void handle_shift_reg(DisasContext *s, 5471 enum a64_shift_type shift_type, unsigned int sf, 5472 unsigned int rm, unsigned int rn, unsigned int rd) 5473 { 5474 TCGv_i64 tcg_shift = tcg_temp_new_i64(); 5475 TCGv_i64 tcg_rd = cpu_reg(s, rd); 5476 TCGv_i64 tcg_rn = read_cpu_reg(s, rn, sf); 5477 5478 tcg_gen_andi_i64(tcg_shift, cpu_reg(s, rm), sf ? 63 : 31); 5479 shift_reg(tcg_rd, tcg_rn, sf, shift_type, tcg_shift); 5480 } 5481 5482 /* CRC32[BHWX], CRC32C[BHWX] */ 5483 static void handle_crc32(DisasContext *s, 5484 unsigned int sf, unsigned int sz, bool crc32c, 5485 unsigned int rm, unsigned int rn, unsigned int rd) 5486 { 5487 TCGv_i64 tcg_acc, tcg_val; 5488 TCGv_i32 tcg_bytes; 5489 5490 if (!dc_isar_feature(aa64_crc32, s) 5491 || (sf == 1 && sz != 3) 5492 || (sf == 0 && sz == 3)) { 5493 unallocated_encoding(s); 5494 return; 5495 } 5496 5497 if (sz == 3) { 5498 tcg_val = cpu_reg(s, rm); 5499 } else { 5500 uint64_t mask; 5501 switch (sz) { 5502 case 0: 5503 mask = 0xFF; 5504 break; 5505 case 1: 5506 mask = 0xFFFF; 5507 break; 5508 case 2: 5509 mask = 0xFFFFFFFF; 5510 break; 5511 default: 5512 g_assert_not_reached(); 5513 } 5514 tcg_val = tcg_temp_new_i64(); 5515 tcg_gen_andi_i64(tcg_val, cpu_reg(s, rm), mask); 5516 } 5517 5518 tcg_acc = cpu_reg(s, rn); 5519 tcg_bytes = tcg_constant_i32(1 << sz); 5520 5521 if (crc32c) { 5522 gen_helper_crc32c_64(cpu_reg(s, rd), tcg_acc, tcg_val, tcg_bytes); 5523 } else { 5524 gen_helper_crc32_64(cpu_reg(s, rd), tcg_acc, tcg_val, tcg_bytes); 5525 } 5526 } 5527 5528 /* Data-processing (2 source) 5529 * 31 30 29 28 21 20 16 15 10 9 5 4 0 5530 * +----+---+---+-----------------+------+--------+------+------+ 5531 * | sf | 0 | S | 1 1 0 1 0 1 1 0 | Rm | opcode | Rn | Rd | 5532 * +----+---+---+-----------------+------+--------+------+------+ 5533 */ 5534 static void disas_data_proc_2src(DisasContext *s, uint32_t insn) 5535 { 5536 unsigned int sf, rm, opcode, rn, rd, setflag; 5537 sf = extract32(insn, 31, 1); 5538 setflag = extract32(insn, 29, 1); 5539 rm = extract32(insn, 16, 5); 5540 opcode = extract32(insn, 10, 6); 5541 rn = extract32(insn, 5, 5); 5542 rd = extract32(insn, 0, 5); 5543 5544 if (setflag && opcode != 0) { 5545 unallocated_encoding(s); 5546 return; 5547 } 5548 5549 switch (opcode) { 5550 case 0: /* SUBP(S) */ 5551 if (sf == 0 || !dc_isar_feature(aa64_mte_insn_reg, s)) { 5552 goto do_unallocated; 5553 } else { 5554 TCGv_i64 tcg_n, tcg_m, tcg_d; 5555 5556 tcg_n = read_cpu_reg_sp(s, rn, true); 5557 tcg_m = read_cpu_reg_sp(s, rm, true); 5558 tcg_gen_sextract_i64(tcg_n, tcg_n, 0, 56); 5559 tcg_gen_sextract_i64(tcg_m, tcg_m, 0, 56); 5560 tcg_d = cpu_reg(s, rd); 5561 5562 if (setflag) { 5563 gen_sub_CC(true, tcg_d, tcg_n, tcg_m); 5564 } else { 5565 tcg_gen_sub_i64(tcg_d, tcg_n, tcg_m); 5566 } 5567 } 5568 break; 5569 case 2: /* UDIV */ 5570 handle_div(s, false, sf, rm, rn, rd); 5571 break; 5572 case 3: /* SDIV */ 5573 handle_div(s, true, sf, rm, rn, rd); 5574 break; 5575 case 4: /* IRG */ 5576 if (sf == 0 || !dc_isar_feature(aa64_mte_insn_reg, s)) { 5577 goto do_unallocated; 5578 } 5579 if (s->ata) { 5580 gen_helper_irg(cpu_reg_sp(s, rd), cpu_env, 5581 cpu_reg_sp(s, rn), cpu_reg(s, rm)); 5582 } else { 5583 gen_address_with_allocation_tag0(cpu_reg_sp(s, rd), 5584 cpu_reg_sp(s, rn)); 5585 } 5586 break; 5587 case 5: /* GMI */ 5588 if (sf == 0 || !dc_isar_feature(aa64_mte_insn_reg, s)) { 5589 goto do_unallocated; 5590 } else { 5591 TCGv_i64 t = tcg_temp_new_i64(); 5592 5593 tcg_gen_extract_i64(t, cpu_reg_sp(s, rn), 56, 4); 5594 tcg_gen_shl_i64(t, tcg_constant_i64(1), t); 5595 tcg_gen_or_i64(cpu_reg(s, rd), cpu_reg(s, rm), t); 5596 } 5597 break; 5598 case 8: /* LSLV */ 5599 handle_shift_reg(s, A64_SHIFT_TYPE_LSL, sf, rm, rn, rd); 5600 break; 5601 case 9: /* LSRV */ 5602 handle_shift_reg(s, A64_SHIFT_TYPE_LSR, sf, rm, rn, rd); 5603 break; 5604 case 10: /* ASRV */ 5605 handle_shift_reg(s, A64_SHIFT_TYPE_ASR, sf, rm, rn, rd); 5606 break; 5607 case 11: /* RORV */ 5608 handle_shift_reg(s, A64_SHIFT_TYPE_ROR, sf, rm, rn, rd); 5609 break; 5610 case 12: /* PACGA */ 5611 if (sf == 0 || !dc_isar_feature(aa64_pauth, s)) { 5612 goto do_unallocated; 5613 } 5614 gen_helper_pacga(cpu_reg(s, rd), cpu_env, 5615 cpu_reg(s, rn), cpu_reg_sp(s, rm)); 5616 break; 5617 case 16: 5618 case 17: 5619 case 18: 5620 case 19: 5621 case 20: 5622 case 21: 5623 case 22: 5624 case 23: /* CRC32 */ 5625 { 5626 int sz = extract32(opcode, 0, 2); 5627 bool crc32c = extract32(opcode, 2, 1); 5628 handle_crc32(s, sf, sz, crc32c, rm, rn, rd); 5629 break; 5630 } 5631 default: 5632 do_unallocated: 5633 unallocated_encoding(s); 5634 break; 5635 } 5636 } 5637 5638 /* 5639 * Data processing - register 5640 * 31 30 29 28 25 21 20 16 10 0 5641 * +--+---+--+---+-------+-----+-------+-------+---------+ 5642 * | |op0| |op1| 1 0 1 | op2 | | op3 | | 5643 * +--+---+--+---+-------+-----+-------+-------+---------+ 5644 */ 5645 static void disas_data_proc_reg(DisasContext *s, uint32_t insn) 5646 { 5647 int op0 = extract32(insn, 30, 1); 5648 int op1 = extract32(insn, 28, 1); 5649 int op2 = extract32(insn, 21, 4); 5650 int op3 = extract32(insn, 10, 6); 5651 5652 if (!op1) { 5653 if (op2 & 8) { 5654 if (op2 & 1) { 5655 /* Add/sub (extended register) */ 5656 disas_add_sub_ext_reg(s, insn); 5657 } else { 5658 /* Add/sub (shifted register) */ 5659 disas_add_sub_reg(s, insn); 5660 } 5661 } else { 5662 /* Logical (shifted register) */ 5663 disas_logic_reg(s, insn); 5664 } 5665 return; 5666 } 5667 5668 switch (op2) { 5669 case 0x0: 5670 switch (op3) { 5671 case 0x00: /* Add/subtract (with carry) */ 5672 disas_adc_sbc(s, insn); 5673 break; 5674 5675 case 0x01: /* Rotate right into flags */ 5676 case 0x21: 5677 disas_rotate_right_into_flags(s, insn); 5678 break; 5679 5680 case 0x02: /* Evaluate into flags */ 5681 case 0x12: 5682 case 0x22: 5683 case 0x32: 5684 disas_evaluate_into_flags(s, insn); 5685 break; 5686 5687 default: 5688 goto do_unallocated; 5689 } 5690 break; 5691 5692 case 0x2: /* Conditional compare */ 5693 disas_cc(s, insn); /* both imm and reg forms */ 5694 break; 5695 5696 case 0x4: /* Conditional select */ 5697 disas_cond_select(s, insn); 5698 break; 5699 5700 case 0x6: /* Data-processing */ 5701 if (op0) { /* (1 source) */ 5702 disas_data_proc_1src(s, insn); 5703 } else { /* (2 source) */ 5704 disas_data_proc_2src(s, insn); 5705 } 5706 break; 5707 case 0x8 ... 0xf: /* (3 source) */ 5708 disas_data_proc_3src(s, insn); 5709 break; 5710 5711 default: 5712 do_unallocated: 5713 unallocated_encoding(s); 5714 break; 5715 } 5716 } 5717 5718 static void handle_fp_compare(DisasContext *s, int size, 5719 unsigned int rn, unsigned int rm, 5720 bool cmp_with_zero, bool signal_all_nans) 5721 { 5722 TCGv_i64 tcg_flags = tcg_temp_new_i64(); 5723 TCGv_ptr fpst = fpstatus_ptr(size == MO_16 ? FPST_FPCR_F16 : FPST_FPCR); 5724 5725 if (size == MO_64) { 5726 TCGv_i64 tcg_vn, tcg_vm; 5727 5728 tcg_vn = read_fp_dreg(s, rn); 5729 if (cmp_with_zero) { 5730 tcg_vm = tcg_constant_i64(0); 5731 } else { 5732 tcg_vm = read_fp_dreg(s, rm); 5733 } 5734 if (signal_all_nans) { 5735 gen_helper_vfp_cmped_a64(tcg_flags, tcg_vn, tcg_vm, fpst); 5736 } else { 5737 gen_helper_vfp_cmpd_a64(tcg_flags, tcg_vn, tcg_vm, fpst); 5738 } 5739 } else { 5740 TCGv_i32 tcg_vn = tcg_temp_new_i32(); 5741 TCGv_i32 tcg_vm = tcg_temp_new_i32(); 5742 5743 read_vec_element_i32(s, tcg_vn, rn, 0, size); 5744 if (cmp_with_zero) { 5745 tcg_gen_movi_i32(tcg_vm, 0); 5746 } else { 5747 read_vec_element_i32(s, tcg_vm, rm, 0, size); 5748 } 5749 5750 switch (size) { 5751 case MO_32: 5752 if (signal_all_nans) { 5753 gen_helper_vfp_cmpes_a64(tcg_flags, tcg_vn, tcg_vm, fpst); 5754 } else { 5755 gen_helper_vfp_cmps_a64(tcg_flags, tcg_vn, tcg_vm, fpst); 5756 } 5757 break; 5758 case MO_16: 5759 if (signal_all_nans) { 5760 gen_helper_vfp_cmpeh_a64(tcg_flags, tcg_vn, tcg_vm, fpst); 5761 } else { 5762 gen_helper_vfp_cmph_a64(tcg_flags, tcg_vn, tcg_vm, fpst); 5763 } 5764 break; 5765 default: 5766 g_assert_not_reached(); 5767 } 5768 } 5769 5770 gen_set_nzcv(tcg_flags); 5771 } 5772 5773 /* Floating point compare 5774 * 31 30 29 28 24 23 22 21 20 16 15 14 13 10 9 5 4 0 5775 * +---+---+---+-----------+------+---+------+-----+---------+------+-------+ 5776 * | M | 0 | S | 1 1 1 1 0 | type | 1 | Rm | op | 1 0 0 0 | Rn | op2 | 5777 * +---+---+---+-----------+------+---+------+-----+---------+------+-------+ 5778 */ 5779 static void disas_fp_compare(DisasContext *s, uint32_t insn) 5780 { 5781 unsigned int mos, type, rm, op, rn, opc, op2r; 5782 int size; 5783 5784 mos = extract32(insn, 29, 3); 5785 type = extract32(insn, 22, 2); 5786 rm = extract32(insn, 16, 5); 5787 op = extract32(insn, 14, 2); 5788 rn = extract32(insn, 5, 5); 5789 opc = extract32(insn, 3, 2); 5790 op2r = extract32(insn, 0, 3); 5791 5792 if (mos || op || op2r) { 5793 unallocated_encoding(s); 5794 return; 5795 } 5796 5797 switch (type) { 5798 case 0: 5799 size = MO_32; 5800 break; 5801 case 1: 5802 size = MO_64; 5803 break; 5804 case 3: 5805 size = MO_16; 5806 if (dc_isar_feature(aa64_fp16, s)) { 5807 break; 5808 } 5809 /* fallthru */ 5810 default: 5811 unallocated_encoding(s); 5812 return; 5813 } 5814 5815 if (!fp_access_check(s)) { 5816 return; 5817 } 5818 5819 handle_fp_compare(s, size, rn, rm, opc & 1, opc & 2); 5820 } 5821 5822 /* Floating point conditional compare 5823 * 31 30 29 28 24 23 22 21 20 16 15 12 11 10 9 5 4 3 0 5824 * +---+---+---+-----------+------+---+------+------+-----+------+----+------+ 5825 * | M | 0 | S | 1 1 1 1 0 | type | 1 | Rm | cond | 0 1 | Rn | op | nzcv | 5826 * +---+---+---+-----------+------+---+------+------+-----+------+----+------+ 5827 */ 5828 static void disas_fp_ccomp(DisasContext *s, uint32_t insn) 5829 { 5830 unsigned int mos, type, rm, cond, rn, op, nzcv; 5831 TCGLabel *label_continue = NULL; 5832 int size; 5833 5834 mos = extract32(insn, 29, 3); 5835 type = extract32(insn, 22, 2); 5836 rm = extract32(insn, 16, 5); 5837 cond = extract32(insn, 12, 4); 5838 rn = extract32(insn, 5, 5); 5839 op = extract32(insn, 4, 1); 5840 nzcv = extract32(insn, 0, 4); 5841 5842 if (mos) { 5843 unallocated_encoding(s); 5844 return; 5845 } 5846 5847 switch (type) { 5848 case 0: 5849 size = MO_32; 5850 break; 5851 case 1: 5852 size = MO_64; 5853 break; 5854 case 3: 5855 size = MO_16; 5856 if (dc_isar_feature(aa64_fp16, s)) { 5857 break; 5858 } 5859 /* fallthru */ 5860 default: 5861 unallocated_encoding(s); 5862 return; 5863 } 5864 5865 if (!fp_access_check(s)) { 5866 return; 5867 } 5868 5869 if (cond < 0x0e) { /* not always */ 5870 TCGLabel *label_match = gen_new_label(); 5871 label_continue = gen_new_label(); 5872 arm_gen_test_cc(cond, label_match); 5873 /* nomatch: */ 5874 gen_set_nzcv(tcg_constant_i64(nzcv << 28)); 5875 tcg_gen_br(label_continue); 5876 gen_set_label(label_match); 5877 } 5878 5879 handle_fp_compare(s, size, rn, rm, false, op); 5880 5881 if (cond < 0x0e) { 5882 gen_set_label(label_continue); 5883 } 5884 } 5885 5886 /* Floating point conditional select 5887 * 31 30 29 28 24 23 22 21 20 16 15 12 11 10 9 5 4 0 5888 * +---+---+---+-----------+------+---+------+------+-----+------+------+ 5889 * | M | 0 | S | 1 1 1 1 0 | type | 1 | Rm | cond | 1 1 | Rn | Rd | 5890 * +---+---+---+-----------+------+---+------+------+-----+------+------+ 5891 */ 5892 static void disas_fp_csel(DisasContext *s, uint32_t insn) 5893 { 5894 unsigned int mos, type, rm, cond, rn, rd; 5895 TCGv_i64 t_true, t_false; 5896 DisasCompare64 c; 5897 MemOp sz; 5898 5899 mos = extract32(insn, 29, 3); 5900 type = extract32(insn, 22, 2); 5901 rm = extract32(insn, 16, 5); 5902 cond = extract32(insn, 12, 4); 5903 rn = extract32(insn, 5, 5); 5904 rd = extract32(insn, 0, 5); 5905 5906 if (mos) { 5907 unallocated_encoding(s); 5908 return; 5909 } 5910 5911 switch (type) { 5912 case 0: 5913 sz = MO_32; 5914 break; 5915 case 1: 5916 sz = MO_64; 5917 break; 5918 case 3: 5919 sz = MO_16; 5920 if (dc_isar_feature(aa64_fp16, s)) { 5921 break; 5922 } 5923 /* fallthru */ 5924 default: 5925 unallocated_encoding(s); 5926 return; 5927 } 5928 5929 if (!fp_access_check(s)) { 5930 return; 5931 } 5932 5933 /* Zero extend sreg & hreg inputs to 64 bits now. */ 5934 t_true = tcg_temp_new_i64(); 5935 t_false = tcg_temp_new_i64(); 5936 read_vec_element(s, t_true, rn, 0, sz); 5937 read_vec_element(s, t_false, rm, 0, sz); 5938 5939 a64_test_cc(&c, cond); 5940 tcg_gen_movcond_i64(c.cond, t_true, c.value, tcg_constant_i64(0), 5941 t_true, t_false); 5942 5943 /* Note that sregs & hregs write back zeros to the high bits, 5944 and we've already done the zero-extension. */ 5945 write_fp_dreg(s, rd, t_true); 5946 } 5947 5948 /* Floating-point data-processing (1 source) - half precision */ 5949 static void handle_fp_1src_half(DisasContext *s, int opcode, int rd, int rn) 5950 { 5951 TCGv_ptr fpst = NULL; 5952 TCGv_i32 tcg_op = read_fp_hreg(s, rn); 5953 TCGv_i32 tcg_res = tcg_temp_new_i32(); 5954 5955 switch (opcode) { 5956 case 0x0: /* FMOV */ 5957 tcg_gen_mov_i32(tcg_res, tcg_op); 5958 break; 5959 case 0x1: /* FABS */ 5960 tcg_gen_andi_i32(tcg_res, tcg_op, 0x7fff); 5961 break; 5962 case 0x2: /* FNEG */ 5963 tcg_gen_xori_i32(tcg_res, tcg_op, 0x8000); 5964 break; 5965 case 0x3: /* FSQRT */ 5966 fpst = fpstatus_ptr(FPST_FPCR_F16); 5967 gen_helper_sqrt_f16(tcg_res, tcg_op, fpst); 5968 break; 5969 case 0x8: /* FRINTN */ 5970 case 0x9: /* FRINTP */ 5971 case 0xa: /* FRINTM */ 5972 case 0xb: /* FRINTZ */ 5973 case 0xc: /* FRINTA */ 5974 { 5975 TCGv_i32 tcg_rmode; 5976 5977 fpst = fpstatus_ptr(FPST_FPCR_F16); 5978 tcg_rmode = gen_set_rmode(opcode & 7, fpst); 5979 gen_helper_advsimd_rinth(tcg_res, tcg_op, fpst); 5980 gen_restore_rmode(tcg_rmode, fpst); 5981 break; 5982 } 5983 case 0xe: /* FRINTX */ 5984 fpst = fpstatus_ptr(FPST_FPCR_F16); 5985 gen_helper_advsimd_rinth_exact(tcg_res, tcg_op, fpst); 5986 break; 5987 case 0xf: /* FRINTI */ 5988 fpst = fpstatus_ptr(FPST_FPCR_F16); 5989 gen_helper_advsimd_rinth(tcg_res, tcg_op, fpst); 5990 break; 5991 default: 5992 g_assert_not_reached(); 5993 } 5994 5995 write_fp_sreg(s, rd, tcg_res); 5996 } 5997 5998 /* Floating-point data-processing (1 source) - single precision */ 5999 static void handle_fp_1src_single(DisasContext *s, int opcode, int rd, int rn) 6000 { 6001 void (*gen_fpst)(TCGv_i32, TCGv_i32, TCGv_ptr); 6002 TCGv_i32 tcg_op, tcg_res; 6003 TCGv_ptr fpst; 6004 int rmode = -1; 6005 6006 tcg_op = read_fp_sreg(s, rn); 6007 tcg_res = tcg_temp_new_i32(); 6008 6009 switch (opcode) { 6010 case 0x0: /* FMOV */ 6011 tcg_gen_mov_i32(tcg_res, tcg_op); 6012 goto done; 6013 case 0x1: /* FABS */ 6014 gen_helper_vfp_abss(tcg_res, tcg_op); 6015 goto done; 6016 case 0x2: /* FNEG */ 6017 gen_helper_vfp_negs(tcg_res, tcg_op); 6018 goto done; 6019 case 0x3: /* FSQRT */ 6020 gen_helper_vfp_sqrts(tcg_res, tcg_op, cpu_env); 6021 goto done; 6022 case 0x6: /* BFCVT */ 6023 gen_fpst = gen_helper_bfcvt; 6024 break; 6025 case 0x8: /* FRINTN */ 6026 case 0x9: /* FRINTP */ 6027 case 0xa: /* FRINTM */ 6028 case 0xb: /* FRINTZ */ 6029 case 0xc: /* FRINTA */ 6030 rmode = opcode & 7; 6031 gen_fpst = gen_helper_rints; 6032 break; 6033 case 0xe: /* FRINTX */ 6034 gen_fpst = gen_helper_rints_exact; 6035 break; 6036 case 0xf: /* FRINTI */ 6037 gen_fpst = gen_helper_rints; 6038 break; 6039 case 0x10: /* FRINT32Z */ 6040 rmode = FPROUNDING_ZERO; 6041 gen_fpst = gen_helper_frint32_s; 6042 break; 6043 case 0x11: /* FRINT32X */ 6044 gen_fpst = gen_helper_frint32_s; 6045 break; 6046 case 0x12: /* FRINT64Z */ 6047 rmode = FPROUNDING_ZERO; 6048 gen_fpst = gen_helper_frint64_s; 6049 break; 6050 case 0x13: /* FRINT64X */ 6051 gen_fpst = gen_helper_frint64_s; 6052 break; 6053 default: 6054 g_assert_not_reached(); 6055 } 6056 6057 fpst = fpstatus_ptr(FPST_FPCR); 6058 if (rmode >= 0) { 6059 TCGv_i32 tcg_rmode = gen_set_rmode(rmode, fpst); 6060 gen_fpst(tcg_res, tcg_op, fpst); 6061 gen_restore_rmode(tcg_rmode, fpst); 6062 } else { 6063 gen_fpst(tcg_res, tcg_op, fpst); 6064 } 6065 6066 done: 6067 write_fp_sreg(s, rd, tcg_res); 6068 } 6069 6070 /* Floating-point data-processing (1 source) - double precision */ 6071 static void handle_fp_1src_double(DisasContext *s, int opcode, int rd, int rn) 6072 { 6073 void (*gen_fpst)(TCGv_i64, TCGv_i64, TCGv_ptr); 6074 TCGv_i64 tcg_op, tcg_res; 6075 TCGv_ptr fpst; 6076 int rmode = -1; 6077 6078 switch (opcode) { 6079 case 0x0: /* FMOV */ 6080 gen_gvec_fn2(s, false, rd, rn, tcg_gen_gvec_mov, 0); 6081 return; 6082 } 6083 6084 tcg_op = read_fp_dreg(s, rn); 6085 tcg_res = tcg_temp_new_i64(); 6086 6087 switch (opcode) { 6088 case 0x1: /* FABS */ 6089 gen_helper_vfp_absd(tcg_res, tcg_op); 6090 goto done; 6091 case 0x2: /* FNEG */ 6092 gen_helper_vfp_negd(tcg_res, tcg_op); 6093 goto done; 6094 case 0x3: /* FSQRT */ 6095 gen_helper_vfp_sqrtd(tcg_res, tcg_op, cpu_env); 6096 goto done; 6097 case 0x8: /* FRINTN */ 6098 case 0x9: /* FRINTP */ 6099 case 0xa: /* FRINTM */ 6100 case 0xb: /* FRINTZ */ 6101 case 0xc: /* FRINTA */ 6102 rmode = opcode & 7; 6103 gen_fpst = gen_helper_rintd; 6104 break; 6105 case 0xe: /* FRINTX */ 6106 gen_fpst = gen_helper_rintd_exact; 6107 break; 6108 case 0xf: /* FRINTI */ 6109 gen_fpst = gen_helper_rintd; 6110 break; 6111 case 0x10: /* FRINT32Z */ 6112 rmode = FPROUNDING_ZERO; 6113 gen_fpst = gen_helper_frint32_d; 6114 break; 6115 case 0x11: /* FRINT32X */ 6116 gen_fpst = gen_helper_frint32_d; 6117 break; 6118 case 0x12: /* FRINT64Z */ 6119 rmode = FPROUNDING_ZERO; 6120 gen_fpst = gen_helper_frint64_d; 6121 break; 6122 case 0x13: /* FRINT64X */ 6123 gen_fpst = gen_helper_frint64_d; 6124 break; 6125 default: 6126 g_assert_not_reached(); 6127 } 6128 6129 fpst = fpstatus_ptr(FPST_FPCR); 6130 if (rmode >= 0) { 6131 TCGv_i32 tcg_rmode = gen_set_rmode(rmode, fpst); 6132 gen_fpst(tcg_res, tcg_op, fpst); 6133 gen_restore_rmode(tcg_rmode, fpst); 6134 } else { 6135 gen_fpst(tcg_res, tcg_op, fpst); 6136 } 6137 6138 done: 6139 write_fp_dreg(s, rd, tcg_res); 6140 } 6141 6142 static void handle_fp_fcvt(DisasContext *s, int opcode, 6143 int rd, int rn, int dtype, int ntype) 6144 { 6145 switch (ntype) { 6146 case 0x0: 6147 { 6148 TCGv_i32 tcg_rn = read_fp_sreg(s, rn); 6149 if (dtype == 1) { 6150 /* Single to double */ 6151 TCGv_i64 tcg_rd = tcg_temp_new_i64(); 6152 gen_helper_vfp_fcvtds(tcg_rd, tcg_rn, cpu_env); 6153 write_fp_dreg(s, rd, tcg_rd); 6154 } else { 6155 /* Single to half */ 6156 TCGv_i32 tcg_rd = tcg_temp_new_i32(); 6157 TCGv_i32 ahp = get_ahp_flag(); 6158 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR); 6159 6160 gen_helper_vfp_fcvt_f32_to_f16(tcg_rd, tcg_rn, fpst, ahp); 6161 /* write_fp_sreg is OK here because top half of tcg_rd is zero */ 6162 write_fp_sreg(s, rd, tcg_rd); 6163 } 6164 break; 6165 } 6166 case 0x1: 6167 { 6168 TCGv_i64 tcg_rn = read_fp_dreg(s, rn); 6169 TCGv_i32 tcg_rd = tcg_temp_new_i32(); 6170 if (dtype == 0) { 6171 /* Double to single */ 6172 gen_helper_vfp_fcvtsd(tcg_rd, tcg_rn, cpu_env); 6173 } else { 6174 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR); 6175 TCGv_i32 ahp = get_ahp_flag(); 6176 /* Double to half */ 6177 gen_helper_vfp_fcvt_f64_to_f16(tcg_rd, tcg_rn, fpst, ahp); 6178 /* write_fp_sreg is OK here because top half of tcg_rd is zero */ 6179 } 6180 write_fp_sreg(s, rd, tcg_rd); 6181 break; 6182 } 6183 case 0x3: 6184 { 6185 TCGv_i32 tcg_rn = read_fp_sreg(s, rn); 6186 TCGv_ptr tcg_fpst = fpstatus_ptr(FPST_FPCR); 6187 TCGv_i32 tcg_ahp = get_ahp_flag(); 6188 tcg_gen_ext16u_i32(tcg_rn, tcg_rn); 6189 if (dtype == 0) { 6190 /* Half to single */ 6191 TCGv_i32 tcg_rd = tcg_temp_new_i32(); 6192 gen_helper_vfp_fcvt_f16_to_f32(tcg_rd, tcg_rn, tcg_fpst, tcg_ahp); 6193 write_fp_sreg(s, rd, tcg_rd); 6194 } else { 6195 /* Half to double */ 6196 TCGv_i64 tcg_rd = tcg_temp_new_i64(); 6197 gen_helper_vfp_fcvt_f16_to_f64(tcg_rd, tcg_rn, tcg_fpst, tcg_ahp); 6198 write_fp_dreg(s, rd, tcg_rd); 6199 } 6200 break; 6201 } 6202 default: 6203 g_assert_not_reached(); 6204 } 6205 } 6206 6207 /* Floating point data-processing (1 source) 6208 * 31 30 29 28 24 23 22 21 20 15 14 10 9 5 4 0 6209 * +---+---+---+-----------+------+---+--------+-----------+------+------+ 6210 * | M | 0 | S | 1 1 1 1 0 | type | 1 | opcode | 1 0 0 0 0 | Rn | Rd | 6211 * +---+---+---+-----------+------+---+--------+-----------+------+------+ 6212 */ 6213 static void disas_fp_1src(DisasContext *s, uint32_t insn) 6214 { 6215 int mos = extract32(insn, 29, 3); 6216 int type = extract32(insn, 22, 2); 6217 int opcode = extract32(insn, 15, 6); 6218 int rn = extract32(insn, 5, 5); 6219 int rd = extract32(insn, 0, 5); 6220 6221 if (mos) { 6222 goto do_unallocated; 6223 } 6224 6225 switch (opcode) { 6226 case 0x4: case 0x5: case 0x7: 6227 { 6228 /* FCVT between half, single and double precision */ 6229 int dtype = extract32(opcode, 0, 2); 6230 if (type == 2 || dtype == type) { 6231 goto do_unallocated; 6232 } 6233 if (!fp_access_check(s)) { 6234 return; 6235 } 6236 6237 handle_fp_fcvt(s, opcode, rd, rn, dtype, type); 6238 break; 6239 } 6240 6241 case 0x10 ... 0x13: /* FRINT{32,64}{X,Z} */ 6242 if (type > 1 || !dc_isar_feature(aa64_frint, s)) { 6243 goto do_unallocated; 6244 } 6245 /* fall through */ 6246 case 0x0 ... 0x3: 6247 case 0x8 ... 0xc: 6248 case 0xe ... 0xf: 6249 /* 32-to-32 and 64-to-64 ops */ 6250 switch (type) { 6251 case 0: 6252 if (!fp_access_check(s)) { 6253 return; 6254 } 6255 handle_fp_1src_single(s, opcode, rd, rn); 6256 break; 6257 case 1: 6258 if (!fp_access_check(s)) { 6259 return; 6260 } 6261 handle_fp_1src_double(s, opcode, rd, rn); 6262 break; 6263 case 3: 6264 if (!dc_isar_feature(aa64_fp16, s)) { 6265 goto do_unallocated; 6266 } 6267 6268 if (!fp_access_check(s)) { 6269 return; 6270 } 6271 handle_fp_1src_half(s, opcode, rd, rn); 6272 break; 6273 default: 6274 goto do_unallocated; 6275 } 6276 break; 6277 6278 case 0x6: 6279 switch (type) { 6280 case 1: /* BFCVT */ 6281 if (!dc_isar_feature(aa64_bf16, s)) { 6282 goto do_unallocated; 6283 } 6284 if (!fp_access_check(s)) { 6285 return; 6286 } 6287 handle_fp_1src_single(s, opcode, rd, rn); 6288 break; 6289 default: 6290 goto do_unallocated; 6291 } 6292 break; 6293 6294 default: 6295 do_unallocated: 6296 unallocated_encoding(s); 6297 break; 6298 } 6299 } 6300 6301 /* Floating-point data-processing (2 source) - single precision */ 6302 static void handle_fp_2src_single(DisasContext *s, int opcode, 6303 int rd, int rn, int rm) 6304 { 6305 TCGv_i32 tcg_op1; 6306 TCGv_i32 tcg_op2; 6307 TCGv_i32 tcg_res; 6308 TCGv_ptr fpst; 6309 6310 tcg_res = tcg_temp_new_i32(); 6311 fpst = fpstatus_ptr(FPST_FPCR); 6312 tcg_op1 = read_fp_sreg(s, rn); 6313 tcg_op2 = read_fp_sreg(s, rm); 6314 6315 switch (opcode) { 6316 case 0x0: /* FMUL */ 6317 gen_helper_vfp_muls(tcg_res, tcg_op1, tcg_op2, fpst); 6318 break; 6319 case 0x1: /* FDIV */ 6320 gen_helper_vfp_divs(tcg_res, tcg_op1, tcg_op2, fpst); 6321 break; 6322 case 0x2: /* FADD */ 6323 gen_helper_vfp_adds(tcg_res, tcg_op1, tcg_op2, fpst); 6324 break; 6325 case 0x3: /* FSUB */ 6326 gen_helper_vfp_subs(tcg_res, tcg_op1, tcg_op2, fpst); 6327 break; 6328 case 0x4: /* FMAX */ 6329 gen_helper_vfp_maxs(tcg_res, tcg_op1, tcg_op2, fpst); 6330 break; 6331 case 0x5: /* FMIN */ 6332 gen_helper_vfp_mins(tcg_res, tcg_op1, tcg_op2, fpst); 6333 break; 6334 case 0x6: /* FMAXNM */ 6335 gen_helper_vfp_maxnums(tcg_res, tcg_op1, tcg_op2, fpst); 6336 break; 6337 case 0x7: /* FMINNM */ 6338 gen_helper_vfp_minnums(tcg_res, tcg_op1, tcg_op2, fpst); 6339 break; 6340 case 0x8: /* FNMUL */ 6341 gen_helper_vfp_muls(tcg_res, tcg_op1, tcg_op2, fpst); 6342 gen_helper_vfp_negs(tcg_res, tcg_res); 6343 break; 6344 } 6345 6346 write_fp_sreg(s, rd, tcg_res); 6347 } 6348 6349 /* Floating-point data-processing (2 source) - double precision */ 6350 static void handle_fp_2src_double(DisasContext *s, int opcode, 6351 int rd, int rn, int rm) 6352 { 6353 TCGv_i64 tcg_op1; 6354 TCGv_i64 tcg_op2; 6355 TCGv_i64 tcg_res; 6356 TCGv_ptr fpst; 6357 6358 tcg_res = tcg_temp_new_i64(); 6359 fpst = fpstatus_ptr(FPST_FPCR); 6360 tcg_op1 = read_fp_dreg(s, rn); 6361 tcg_op2 = read_fp_dreg(s, rm); 6362 6363 switch (opcode) { 6364 case 0x0: /* FMUL */ 6365 gen_helper_vfp_muld(tcg_res, tcg_op1, tcg_op2, fpst); 6366 break; 6367 case 0x1: /* FDIV */ 6368 gen_helper_vfp_divd(tcg_res, tcg_op1, tcg_op2, fpst); 6369 break; 6370 case 0x2: /* FADD */ 6371 gen_helper_vfp_addd(tcg_res, tcg_op1, tcg_op2, fpst); 6372 break; 6373 case 0x3: /* FSUB */ 6374 gen_helper_vfp_subd(tcg_res, tcg_op1, tcg_op2, fpst); 6375 break; 6376 case 0x4: /* FMAX */ 6377 gen_helper_vfp_maxd(tcg_res, tcg_op1, tcg_op2, fpst); 6378 break; 6379 case 0x5: /* FMIN */ 6380 gen_helper_vfp_mind(tcg_res, tcg_op1, tcg_op2, fpst); 6381 break; 6382 case 0x6: /* FMAXNM */ 6383 gen_helper_vfp_maxnumd(tcg_res, tcg_op1, tcg_op2, fpst); 6384 break; 6385 case 0x7: /* FMINNM */ 6386 gen_helper_vfp_minnumd(tcg_res, tcg_op1, tcg_op2, fpst); 6387 break; 6388 case 0x8: /* FNMUL */ 6389 gen_helper_vfp_muld(tcg_res, tcg_op1, tcg_op2, fpst); 6390 gen_helper_vfp_negd(tcg_res, tcg_res); 6391 break; 6392 } 6393 6394 write_fp_dreg(s, rd, tcg_res); 6395 } 6396 6397 /* Floating-point data-processing (2 source) - half precision */ 6398 static void handle_fp_2src_half(DisasContext *s, int opcode, 6399 int rd, int rn, int rm) 6400 { 6401 TCGv_i32 tcg_op1; 6402 TCGv_i32 tcg_op2; 6403 TCGv_i32 tcg_res; 6404 TCGv_ptr fpst; 6405 6406 tcg_res = tcg_temp_new_i32(); 6407 fpst = fpstatus_ptr(FPST_FPCR_F16); 6408 tcg_op1 = read_fp_hreg(s, rn); 6409 tcg_op2 = read_fp_hreg(s, rm); 6410 6411 switch (opcode) { 6412 case 0x0: /* FMUL */ 6413 gen_helper_advsimd_mulh(tcg_res, tcg_op1, tcg_op2, fpst); 6414 break; 6415 case 0x1: /* FDIV */ 6416 gen_helper_advsimd_divh(tcg_res, tcg_op1, tcg_op2, fpst); 6417 break; 6418 case 0x2: /* FADD */ 6419 gen_helper_advsimd_addh(tcg_res, tcg_op1, tcg_op2, fpst); 6420 break; 6421 case 0x3: /* FSUB */ 6422 gen_helper_advsimd_subh(tcg_res, tcg_op1, tcg_op2, fpst); 6423 break; 6424 case 0x4: /* FMAX */ 6425 gen_helper_advsimd_maxh(tcg_res, tcg_op1, tcg_op2, fpst); 6426 break; 6427 case 0x5: /* FMIN */ 6428 gen_helper_advsimd_minh(tcg_res, tcg_op1, tcg_op2, fpst); 6429 break; 6430 case 0x6: /* FMAXNM */ 6431 gen_helper_advsimd_maxnumh(tcg_res, tcg_op1, tcg_op2, fpst); 6432 break; 6433 case 0x7: /* FMINNM */ 6434 gen_helper_advsimd_minnumh(tcg_res, tcg_op1, tcg_op2, fpst); 6435 break; 6436 case 0x8: /* FNMUL */ 6437 gen_helper_advsimd_mulh(tcg_res, tcg_op1, tcg_op2, fpst); 6438 tcg_gen_xori_i32(tcg_res, tcg_res, 0x8000); 6439 break; 6440 default: 6441 g_assert_not_reached(); 6442 } 6443 6444 write_fp_sreg(s, rd, tcg_res); 6445 } 6446 6447 /* Floating point data-processing (2 source) 6448 * 31 30 29 28 24 23 22 21 20 16 15 12 11 10 9 5 4 0 6449 * +---+---+---+-----------+------+---+------+--------+-----+------+------+ 6450 * | M | 0 | S | 1 1 1 1 0 | type | 1 | Rm | opcode | 1 0 | Rn | Rd | 6451 * +---+---+---+-----------+------+---+------+--------+-----+------+------+ 6452 */ 6453 static void disas_fp_2src(DisasContext *s, uint32_t insn) 6454 { 6455 int mos = extract32(insn, 29, 3); 6456 int type = extract32(insn, 22, 2); 6457 int rd = extract32(insn, 0, 5); 6458 int rn = extract32(insn, 5, 5); 6459 int rm = extract32(insn, 16, 5); 6460 int opcode = extract32(insn, 12, 4); 6461 6462 if (opcode > 8 || mos) { 6463 unallocated_encoding(s); 6464 return; 6465 } 6466 6467 switch (type) { 6468 case 0: 6469 if (!fp_access_check(s)) { 6470 return; 6471 } 6472 handle_fp_2src_single(s, opcode, rd, rn, rm); 6473 break; 6474 case 1: 6475 if (!fp_access_check(s)) { 6476 return; 6477 } 6478 handle_fp_2src_double(s, opcode, rd, rn, rm); 6479 break; 6480 case 3: 6481 if (!dc_isar_feature(aa64_fp16, s)) { 6482 unallocated_encoding(s); 6483 return; 6484 } 6485 if (!fp_access_check(s)) { 6486 return; 6487 } 6488 handle_fp_2src_half(s, opcode, rd, rn, rm); 6489 break; 6490 default: 6491 unallocated_encoding(s); 6492 } 6493 } 6494 6495 /* Floating-point data-processing (3 source) - single precision */ 6496 static void handle_fp_3src_single(DisasContext *s, bool o0, bool o1, 6497 int rd, int rn, int rm, int ra) 6498 { 6499 TCGv_i32 tcg_op1, tcg_op2, tcg_op3; 6500 TCGv_i32 tcg_res = tcg_temp_new_i32(); 6501 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR); 6502 6503 tcg_op1 = read_fp_sreg(s, rn); 6504 tcg_op2 = read_fp_sreg(s, rm); 6505 tcg_op3 = read_fp_sreg(s, ra); 6506 6507 /* These are fused multiply-add, and must be done as one 6508 * floating point operation with no rounding between the 6509 * multiplication and addition steps. 6510 * NB that doing the negations here as separate steps is 6511 * correct : an input NaN should come out with its sign bit 6512 * flipped if it is a negated-input. 6513 */ 6514 if (o1 == true) { 6515 gen_helper_vfp_negs(tcg_op3, tcg_op3); 6516 } 6517 6518 if (o0 != o1) { 6519 gen_helper_vfp_negs(tcg_op1, tcg_op1); 6520 } 6521 6522 gen_helper_vfp_muladds(tcg_res, tcg_op1, tcg_op2, tcg_op3, fpst); 6523 6524 write_fp_sreg(s, rd, tcg_res); 6525 } 6526 6527 /* Floating-point data-processing (3 source) - double precision */ 6528 static void handle_fp_3src_double(DisasContext *s, bool o0, bool o1, 6529 int rd, int rn, int rm, int ra) 6530 { 6531 TCGv_i64 tcg_op1, tcg_op2, tcg_op3; 6532 TCGv_i64 tcg_res = tcg_temp_new_i64(); 6533 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR); 6534 6535 tcg_op1 = read_fp_dreg(s, rn); 6536 tcg_op2 = read_fp_dreg(s, rm); 6537 tcg_op3 = read_fp_dreg(s, ra); 6538 6539 /* These are fused multiply-add, and must be done as one 6540 * floating point operation with no rounding between the 6541 * multiplication and addition steps. 6542 * NB that doing the negations here as separate steps is 6543 * correct : an input NaN should come out with its sign bit 6544 * flipped if it is a negated-input. 6545 */ 6546 if (o1 == true) { 6547 gen_helper_vfp_negd(tcg_op3, tcg_op3); 6548 } 6549 6550 if (o0 != o1) { 6551 gen_helper_vfp_negd(tcg_op1, tcg_op1); 6552 } 6553 6554 gen_helper_vfp_muladdd(tcg_res, tcg_op1, tcg_op2, tcg_op3, fpst); 6555 6556 write_fp_dreg(s, rd, tcg_res); 6557 } 6558 6559 /* Floating-point data-processing (3 source) - half precision */ 6560 static void handle_fp_3src_half(DisasContext *s, bool o0, bool o1, 6561 int rd, int rn, int rm, int ra) 6562 { 6563 TCGv_i32 tcg_op1, tcg_op2, tcg_op3; 6564 TCGv_i32 tcg_res = tcg_temp_new_i32(); 6565 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR_F16); 6566 6567 tcg_op1 = read_fp_hreg(s, rn); 6568 tcg_op2 = read_fp_hreg(s, rm); 6569 tcg_op3 = read_fp_hreg(s, ra); 6570 6571 /* These are fused multiply-add, and must be done as one 6572 * floating point operation with no rounding between the 6573 * multiplication and addition steps. 6574 * NB that doing the negations here as separate steps is 6575 * correct : an input NaN should come out with its sign bit 6576 * flipped if it is a negated-input. 6577 */ 6578 if (o1 == true) { 6579 tcg_gen_xori_i32(tcg_op3, tcg_op3, 0x8000); 6580 } 6581 6582 if (o0 != o1) { 6583 tcg_gen_xori_i32(tcg_op1, tcg_op1, 0x8000); 6584 } 6585 6586 gen_helper_advsimd_muladdh(tcg_res, tcg_op1, tcg_op2, tcg_op3, fpst); 6587 6588 write_fp_sreg(s, rd, tcg_res); 6589 } 6590 6591 /* Floating point data-processing (3 source) 6592 * 31 30 29 28 24 23 22 21 20 16 15 14 10 9 5 4 0 6593 * +---+---+---+-----------+------+----+------+----+------+------+------+ 6594 * | M | 0 | S | 1 1 1 1 1 | type | o1 | Rm | o0 | Ra | Rn | Rd | 6595 * +---+---+---+-----------+------+----+------+----+------+------+------+ 6596 */ 6597 static void disas_fp_3src(DisasContext *s, uint32_t insn) 6598 { 6599 int mos = extract32(insn, 29, 3); 6600 int type = extract32(insn, 22, 2); 6601 int rd = extract32(insn, 0, 5); 6602 int rn = extract32(insn, 5, 5); 6603 int ra = extract32(insn, 10, 5); 6604 int rm = extract32(insn, 16, 5); 6605 bool o0 = extract32(insn, 15, 1); 6606 bool o1 = extract32(insn, 21, 1); 6607 6608 if (mos) { 6609 unallocated_encoding(s); 6610 return; 6611 } 6612 6613 switch (type) { 6614 case 0: 6615 if (!fp_access_check(s)) { 6616 return; 6617 } 6618 handle_fp_3src_single(s, o0, o1, rd, rn, rm, ra); 6619 break; 6620 case 1: 6621 if (!fp_access_check(s)) { 6622 return; 6623 } 6624 handle_fp_3src_double(s, o0, o1, rd, rn, rm, ra); 6625 break; 6626 case 3: 6627 if (!dc_isar_feature(aa64_fp16, s)) { 6628 unallocated_encoding(s); 6629 return; 6630 } 6631 if (!fp_access_check(s)) { 6632 return; 6633 } 6634 handle_fp_3src_half(s, o0, o1, rd, rn, rm, ra); 6635 break; 6636 default: 6637 unallocated_encoding(s); 6638 } 6639 } 6640 6641 /* Floating point immediate 6642 * 31 30 29 28 24 23 22 21 20 13 12 10 9 5 4 0 6643 * +---+---+---+-----------+------+---+------------+-------+------+------+ 6644 * | M | 0 | S | 1 1 1 1 0 | type | 1 | imm8 | 1 0 0 | imm5 | Rd | 6645 * +---+---+---+-----------+------+---+------------+-------+------+------+ 6646 */ 6647 static void disas_fp_imm(DisasContext *s, uint32_t insn) 6648 { 6649 int rd = extract32(insn, 0, 5); 6650 int imm5 = extract32(insn, 5, 5); 6651 int imm8 = extract32(insn, 13, 8); 6652 int type = extract32(insn, 22, 2); 6653 int mos = extract32(insn, 29, 3); 6654 uint64_t imm; 6655 MemOp sz; 6656 6657 if (mos || imm5) { 6658 unallocated_encoding(s); 6659 return; 6660 } 6661 6662 switch (type) { 6663 case 0: 6664 sz = MO_32; 6665 break; 6666 case 1: 6667 sz = MO_64; 6668 break; 6669 case 3: 6670 sz = MO_16; 6671 if (dc_isar_feature(aa64_fp16, s)) { 6672 break; 6673 } 6674 /* fallthru */ 6675 default: 6676 unallocated_encoding(s); 6677 return; 6678 } 6679 6680 if (!fp_access_check(s)) { 6681 return; 6682 } 6683 6684 imm = vfp_expand_imm(sz, imm8); 6685 write_fp_dreg(s, rd, tcg_constant_i64(imm)); 6686 } 6687 6688 /* Handle floating point <=> fixed point conversions. Note that we can 6689 * also deal with fp <=> integer conversions as a special case (scale == 64) 6690 * OPTME: consider handling that special case specially or at least skipping 6691 * the call to scalbn in the helpers for zero shifts. 6692 */ 6693 static void handle_fpfpcvt(DisasContext *s, int rd, int rn, int opcode, 6694 bool itof, int rmode, int scale, int sf, int type) 6695 { 6696 bool is_signed = !(opcode & 1); 6697 TCGv_ptr tcg_fpstatus; 6698 TCGv_i32 tcg_shift, tcg_single; 6699 TCGv_i64 tcg_double; 6700 6701 tcg_fpstatus = fpstatus_ptr(type == 3 ? FPST_FPCR_F16 : FPST_FPCR); 6702 6703 tcg_shift = tcg_constant_i32(64 - scale); 6704 6705 if (itof) { 6706 TCGv_i64 tcg_int = cpu_reg(s, rn); 6707 if (!sf) { 6708 TCGv_i64 tcg_extend = tcg_temp_new_i64(); 6709 6710 if (is_signed) { 6711 tcg_gen_ext32s_i64(tcg_extend, tcg_int); 6712 } else { 6713 tcg_gen_ext32u_i64(tcg_extend, tcg_int); 6714 } 6715 6716 tcg_int = tcg_extend; 6717 } 6718 6719 switch (type) { 6720 case 1: /* float64 */ 6721 tcg_double = tcg_temp_new_i64(); 6722 if (is_signed) { 6723 gen_helper_vfp_sqtod(tcg_double, tcg_int, 6724 tcg_shift, tcg_fpstatus); 6725 } else { 6726 gen_helper_vfp_uqtod(tcg_double, tcg_int, 6727 tcg_shift, tcg_fpstatus); 6728 } 6729 write_fp_dreg(s, rd, tcg_double); 6730 break; 6731 6732 case 0: /* float32 */ 6733 tcg_single = tcg_temp_new_i32(); 6734 if (is_signed) { 6735 gen_helper_vfp_sqtos(tcg_single, tcg_int, 6736 tcg_shift, tcg_fpstatus); 6737 } else { 6738 gen_helper_vfp_uqtos(tcg_single, tcg_int, 6739 tcg_shift, tcg_fpstatus); 6740 } 6741 write_fp_sreg(s, rd, tcg_single); 6742 break; 6743 6744 case 3: /* float16 */ 6745 tcg_single = tcg_temp_new_i32(); 6746 if (is_signed) { 6747 gen_helper_vfp_sqtoh(tcg_single, tcg_int, 6748 tcg_shift, tcg_fpstatus); 6749 } else { 6750 gen_helper_vfp_uqtoh(tcg_single, tcg_int, 6751 tcg_shift, tcg_fpstatus); 6752 } 6753 write_fp_sreg(s, rd, tcg_single); 6754 break; 6755 6756 default: 6757 g_assert_not_reached(); 6758 } 6759 } else { 6760 TCGv_i64 tcg_int = cpu_reg(s, rd); 6761 TCGv_i32 tcg_rmode; 6762 6763 if (extract32(opcode, 2, 1)) { 6764 /* There are too many rounding modes to all fit into rmode, 6765 * so FCVTA[US] is a special case. 6766 */ 6767 rmode = FPROUNDING_TIEAWAY; 6768 } 6769 6770 tcg_rmode = gen_set_rmode(rmode, tcg_fpstatus); 6771 6772 switch (type) { 6773 case 1: /* float64 */ 6774 tcg_double = read_fp_dreg(s, rn); 6775 if (is_signed) { 6776 if (!sf) { 6777 gen_helper_vfp_tosld(tcg_int, tcg_double, 6778 tcg_shift, tcg_fpstatus); 6779 } else { 6780 gen_helper_vfp_tosqd(tcg_int, tcg_double, 6781 tcg_shift, tcg_fpstatus); 6782 } 6783 } else { 6784 if (!sf) { 6785 gen_helper_vfp_tould(tcg_int, tcg_double, 6786 tcg_shift, tcg_fpstatus); 6787 } else { 6788 gen_helper_vfp_touqd(tcg_int, tcg_double, 6789 tcg_shift, tcg_fpstatus); 6790 } 6791 } 6792 if (!sf) { 6793 tcg_gen_ext32u_i64(tcg_int, tcg_int); 6794 } 6795 break; 6796 6797 case 0: /* float32 */ 6798 tcg_single = read_fp_sreg(s, rn); 6799 if (sf) { 6800 if (is_signed) { 6801 gen_helper_vfp_tosqs(tcg_int, tcg_single, 6802 tcg_shift, tcg_fpstatus); 6803 } else { 6804 gen_helper_vfp_touqs(tcg_int, tcg_single, 6805 tcg_shift, tcg_fpstatus); 6806 } 6807 } else { 6808 TCGv_i32 tcg_dest = tcg_temp_new_i32(); 6809 if (is_signed) { 6810 gen_helper_vfp_tosls(tcg_dest, tcg_single, 6811 tcg_shift, tcg_fpstatus); 6812 } else { 6813 gen_helper_vfp_touls(tcg_dest, tcg_single, 6814 tcg_shift, tcg_fpstatus); 6815 } 6816 tcg_gen_extu_i32_i64(tcg_int, tcg_dest); 6817 } 6818 break; 6819 6820 case 3: /* float16 */ 6821 tcg_single = read_fp_sreg(s, rn); 6822 if (sf) { 6823 if (is_signed) { 6824 gen_helper_vfp_tosqh(tcg_int, tcg_single, 6825 tcg_shift, tcg_fpstatus); 6826 } else { 6827 gen_helper_vfp_touqh(tcg_int, tcg_single, 6828 tcg_shift, tcg_fpstatus); 6829 } 6830 } else { 6831 TCGv_i32 tcg_dest = tcg_temp_new_i32(); 6832 if (is_signed) { 6833 gen_helper_vfp_toslh(tcg_dest, tcg_single, 6834 tcg_shift, tcg_fpstatus); 6835 } else { 6836 gen_helper_vfp_toulh(tcg_dest, tcg_single, 6837 tcg_shift, tcg_fpstatus); 6838 } 6839 tcg_gen_extu_i32_i64(tcg_int, tcg_dest); 6840 } 6841 break; 6842 6843 default: 6844 g_assert_not_reached(); 6845 } 6846 6847 gen_restore_rmode(tcg_rmode, tcg_fpstatus); 6848 } 6849 } 6850 6851 /* Floating point <-> fixed point conversions 6852 * 31 30 29 28 24 23 22 21 20 19 18 16 15 10 9 5 4 0 6853 * +----+---+---+-----------+------+---+-------+--------+-------+------+------+ 6854 * | sf | 0 | S | 1 1 1 1 0 | type | 0 | rmode | opcode | scale | Rn | Rd | 6855 * +----+---+---+-----------+------+---+-------+--------+-------+------+------+ 6856 */ 6857 static void disas_fp_fixed_conv(DisasContext *s, uint32_t insn) 6858 { 6859 int rd = extract32(insn, 0, 5); 6860 int rn = extract32(insn, 5, 5); 6861 int scale = extract32(insn, 10, 6); 6862 int opcode = extract32(insn, 16, 3); 6863 int rmode = extract32(insn, 19, 2); 6864 int type = extract32(insn, 22, 2); 6865 bool sbit = extract32(insn, 29, 1); 6866 bool sf = extract32(insn, 31, 1); 6867 bool itof; 6868 6869 if (sbit || (!sf && scale < 32)) { 6870 unallocated_encoding(s); 6871 return; 6872 } 6873 6874 switch (type) { 6875 case 0: /* float32 */ 6876 case 1: /* float64 */ 6877 break; 6878 case 3: /* float16 */ 6879 if (dc_isar_feature(aa64_fp16, s)) { 6880 break; 6881 } 6882 /* fallthru */ 6883 default: 6884 unallocated_encoding(s); 6885 return; 6886 } 6887 6888 switch ((rmode << 3) | opcode) { 6889 case 0x2: /* SCVTF */ 6890 case 0x3: /* UCVTF */ 6891 itof = true; 6892 break; 6893 case 0x18: /* FCVTZS */ 6894 case 0x19: /* FCVTZU */ 6895 itof = false; 6896 break; 6897 default: 6898 unallocated_encoding(s); 6899 return; 6900 } 6901 6902 if (!fp_access_check(s)) { 6903 return; 6904 } 6905 6906 handle_fpfpcvt(s, rd, rn, opcode, itof, FPROUNDING_ZERO, scale, sf, type); 6907 } 6908 6909 static void handle_fmov(DisasContext *s, int rd, int rn, int type, bool itof) 6910 { 6911 /* FMOV: gpr to or from float, double, or top half of quad fp reg, 6912 * without conversion. 6913 */ 6914 6915 if (itof) { 6916 TCGv_i64 tcg_rn = cpu_reg(s, rn); 6917 TCGv_i64 tmp; 6918 6919 switch (type) { 6920 case 0: 6921 /* 32 bit */ 6922 tmp = tcg_temp_new_i64(); 6923 tcg_gen_ext32u_i64(tmp, tcg_rn); 6924 write_fp_dreg(s, rd, tmp); 6925 break; 6926 case 1: 6927 /* 64 bit */ 6928 write_fp_dreg(s, rd, tcg_rn); 6929 break; 6930 case 2: 6931 /* 64 bit to top half. */ 6932 tcg_gen_st_i64(tcg_rn, cpu_env, fp_reg_hi_offset(s, rd)); 6933 clear_vec_high(s, true, rd); 6934 break; 6935 case 3: 6936 /* 16 bit */ 6937 tmp = tcg_temp_new_i64(); 6938 tcg_gen_ext16u_i64(tmp, tcg_rn); 6939 write_fp_dreg(s, rd, tmp); 6940 break; 6941 default: 6942 g_assert_not_reached(); 6943 } 6944 } else { 6945 TCGv_i64 tcg_rd = cpu_reg(s, rd); 6946 6947 switch (type) { 6948 case 0: 6949 /* 32 bit */ 6950 tcg_gen_ld32u_i64(tcg_rd, cpu_env, fp_reg_offset(s, rn, MO_32)); 6951 break; 6952 case 1: 6953 /* 64 bit */ 6954 tcg_gen_ld_i64(tcg_rd, cpu_env, fp_reg_offset(s, rn, MO_64)); 6955 break; 6956 case 2: 6957 /* 64 bits from top half */ 6958 tcg_gen_ld_i64(tcg_rd, cpu_env, fp_reg_hi_offset(s, rn)); 6959 break; 6960 case 3: 6961 /* 16 bit */ 6962 tcg_gen_ld16u_i64(tcg_rd, cpu_env, fp_reg_offset(s, rn, MO_16)); 6963 break; 6964 default: 6965 g_assert_not_reached(); 6966 } 6967 } 6968 } 6969 6970 static void handle_fjcvtzs(DisasContext *s, int rd, int rn) 6971 { 6972 TCGv_i64 t = read_fp_dreg(s, rn); 6973 TCGv_ptr fpstatus = fpstatus_ptr(FPST_FPCR); 6974 6975 gen_helper_fjcvtzs(t, t, fpstatus); 6976 6977 tcg_gen_ext32u_i64(cpu_reg(s, rd), t); 6978 tcg_gen_extrh_i64_i32(cpu_ZF, t); 6979 tcg_gen_movi_i32(cpu_CF, 0); 6980 tcg_gen_movi_i32(cpu_NF, 0); 6981 tcg_gen_movi_i32(cpu_VF, 0); 6982 } 6983 6984 /* Floating point <-> integer conversions 6985 * 31 30 29 28 24 23 22 21 20 19 18 16 15 10 9 5 4 0 6986 * +----+---+---+-----------+------+---+-------+-----+-------------+----+----+ 6987 * | sf | 0 | S | 1 1 1 1 0 | type | 1 | rmode | opc | 0 0 0 0 0 0 | Rn | Rd | 6988 * +----+---+---+-----------+------+---+-------+-----+-------------+----+----+ 6989 */ 6990 static void disas_fp_int_conv(DisasContext *s, uint32_t insn) 6991 { 6992 int rd = extract32(insn, 0, 5); 6993 int rn = extract32(insn, 5, 5); 6994 int opcode = extract32(insn, 16, 3); 6995 int rmode = extract32(insn, 19, 2); 6996 int type = extract32(insn, 22, 2); 6997 bool sbit = extract32(insn, 29, 1); 6998 bool sf = extract32(insn, 31, 1); 6999 bool itof = false; 7000 7001 if (sbit) { 7002 goto do_unallocated; 7003 } 7004 7005 switch (opcode) { 7006 case 2: /* SCVTF */ 7007 case 3: /* UCVTF */ 7008 itof = true; 7009 /* fallthru */ 7010 case 4: /* FCVTAS */ 7011 case 5: /* FCVTAU */ 7012 if (rmode != 0) { 7013 goto do_unallocated; 7014 } 7015 /* fallthru */ 7016 case 0: /* FCVT[NPMZ]S */ 7017 case 1: /* FCVT[NPMZ]U */ 7018 switch (type) { 7019 case 0: /* float32 */ 7020 case 1: /* float64 */ 7021 break; 7022 case 3: /* float16 */ 7023 if (!dc_isar_feature(aa64_fp16, s)) { 7024 goto do_unallocated; 7025 } 7026 break; 7027 default: 7028 goto do_unallocated; 7029 } 7030 if (!fp_access_check(s)) { 7031 return; 7032 } 7033 handle_fpfpcvt(s, rd, rn, opcode, itof, rmode, 64, sf, type); 7034 break; 7035 7036 default: 7037 switch (sf << 7 | type << 5 | rmode << 3 | opcode) { 7038 case 0b01100110: /* FMOV half <-> 32-bit int */ 7039 case 0b01100111: 7040 case 0b11100110: /* FMOV half <-> 64-bit int */ 7041 case 0b11100111: 7042 if (!dc_isar_feature(aa64_fp16, s)) { 7043 goto do_unallocated; 7044 } 7045 /* fallthru */ 7046 case 0b00000110: /* FMOV 32-bit */ 7047 case 0b00000111: 7048 case 0b10100110: /* FMOV 64-bit */ 7049 case 0b10100111: 7050 case 0b11001110: /* FMOV top half of 128-bit */ 7051 case 0b11001111: 7052 if (!fp_access_check(s)) { 7053 return; 7054 } 7055 itof = opcode & 1; 7056 handle_fmov(s, rd, rn, type, itof); 7057 break; 7058 7059 case 0b00111110: /* FJCVTZS */ 7060 if (!dc_isar_feature(aa64_jscvt, s)) { 7061 goto do_unallocated; 7062 } else if (fp_access_check(s)) { 7063 handle_fjcvtzs(s, rd, rn); 7064 } 7065 break; 7066 7067 default: 7068 do_unallocated: 7069 unallocated_encoding(s); 7070 return; 7071 } 7072 break; 7073 } 7074 } 7075 7076 /* FP-specific subcases of table C3-6 (SIMD and FP data processing) 7077 * 31 30 29 28 25 24 0 7078 * +---+---+---+---------+-----------------------------+ 7079 * | | 0 | | 1 1 1 1 | | 7080 * +---+---+---+---------+-----------------------------+ 7081 */ 7082 static void disas_data_proc_fp(DisasContext *s, uint32_t insn) 7083 { 7084 if (extract32(insn, 24, 1)) { 7085 /* Floating point data-processing (3 source) */ 7086 disas_fp_3src(s, insn); 7087 } else if (extract32(insn, 21, 1) == 0) { 7088 /* Floating point to fixed point conversions */ 7089 disas_fp_fixed_conv(s, insn); 7090 } else { 7091 switch (extract32(insn, 10, 2)) { 7092 case 1: 7093 /* Floating point conditional compare */ 7094 disas_fp_ccomp(s, insn); 7095 break; 7096 case 2: 7097 /* Floating point data-processing (2 source) */ 7098 disas_fp_2src(s, insn); 7099 break; 7100 case 3: 7101 /* Floating point conditional select */ 7102 disas_fp_csel(s, insn); 7103 break; 7104 case 0: 7105 switch (ctz32(extract32(insn, 12, 4))) { 7106 case 0: /* [15:12] == xxx1 */ 7107 /* Floating point immediate */ 7108 disas_fp_imm(s, insn); 7109 break; 7110 case 1: /* [15:12] == xx10 */ 7111 /* Floating point compare */ 7112 disas_fp_compare(s, insn); 7113 break; 7114 case 2: /* [15:12] == x100 */ 7115 /* Floating point data-processing (1 source) */ 7116 disas_fp_1src(s, insn); 7117 break; 7118 case 3: /* [15:12] == 1000 */ 7119 unallocated_encoding(s); 7120 break; 7121 default: /* [15:12] == 0000 */ 7122 /* Floating point <-> integer conversions */ 7123 disas_fp_int_conv(s, insn); 7124 break; 7125 } 7126 break; 7127 } 7128 } 7129 } 7130 7131 static void do_ext64(DisasContext *s, TCGv_i64 tcg_left, TCGv_i64 tcg_right, 7132 int pos) 7133 { 7134 /* Extract 64 bits from the middle of two concatenated 64 bit 7135 * vector register slices left:right. The extracted bits start 7136 * at 'pos' bits into the right (least significant) side. 7137 * We return the result in tcg_right, and guarantee not to 7138 * trash tcg_left. 7139 */ 7140 TCGv_i64 tcg_tmp = tcg_temp_new_i64(); 7141 assert(pos > 0 && pos < 64); 7142 7143 tcg_gen_shri_i64(tcg_right, tcg_right, pos); 7144 tcg_gen_shli_i64(tcg_tmp, tcg_left, 64 - pos); 7145 tcg_gen_or_i64(tcg_right, tcg_right, tcg_tmp); 7146 } 7147 7148 /* EXT 7149 * 31 30 29 24 23 22 21 20 16 15 14 11 10 9 5 4 0 7150 * +---+---+-------------+-----+---+------+---+------+---+------+------+ 7151 * | 0 | Q | 1 0 1 1 1 0 | op2 | 0 | Rm | 0 | imm4 | 0 | Rn | Rd | 7152 * +---+---+-------------+-----+---+------+---+------+---+------+------+ 7153 */ 7154 static void disas_simd_ext(DisasContext *s, uint32_t insn) 7155 { 7156 int is_q = extract32(insn, 30, 1); 7157 int op2 = extract32(insn, 22, 2); 7158 int imm4 = extract32(insn, 11, 4); 7159 int rm = extract32(insn, 16, 5); 7160 int rn = extract32(insn, 5, 5); 7161 int rd = extract32(insn, 0, 5); 7162 int pos = imm4 << 3; 7163 TCGv_i64 tcg_resl, tcg_resh; 7164 7165 if (op2 != 0 || (!is_q && extract32(imm4, 3, 1))) { 7166 unallocated_encoding(s); 7167 return; 7168 } 7169 7170 if (!fp_access_check(s)) { 7171 return; 7172 } 7173 7174 tcg_resh = tcg_temp_new_i64(); 7175 tcg_resl = tcg_temp_new_i64(); 7176 7177 /* Vd gets bits starting at pos bits into Vm:Vn. This is 7178 * either extracting 128 bits from a 128:128 concatenation, or 7179 * extracting 64 bits from a 64:64 concatenation. 7180 */ 7181 if (!is_q) { 7182 read_vec_element(s, tcg_resl, rn, 0, MO_64); 7183 if (pos != 0) { 7184 read_vec_element(s, tcg_resh, rm, 0, MO_64); 7185 do_ext64(s, tcg_resh, tcg_resl, pos); 7186 } 7187 } else { 7188 TCGv_i64 tcg_hh; 7189 typedef struct { 7190 int reg; 7191 int elt; 7192 } EltPosns; 7193 EltPosns eltposns[] = { {rn, 0}, {rn, 1}, {rm, 0}, {rm, 1} }; 7194 EltPosns *elt = eltposns; 7195 7196 if (pos >= 64) { 7197 elt++; 7198 pos -= 64; 7199 } 7200 7201 read_vec_element(s, tcg_resl, elt->reg, elt->elt, MO_64); 7202 elt++; 7203 read_vec_element(s, tcg_resh, elt->reg, elt->elt, MO_64); 7204 elt++; 7205 if (pos != 0) { 7206 do_ext64(s, tcg_resh, tcg_resl, pos); 7207 tcg_hh = tcg_temp_new_i64(); 7208 read_vec_element(s, tcg_hh, elt->reg, elt->elt, MO_64); 7209 do_ext64(s, tcg_hh, tcg_resh, pos); 7210 } 7211 } 7212 7213 write_vec_element(s, tcg_resl, rd, 0, MO_64); 7214 if (is_q) { 7215 write_vec_element(s, tcg_resh, rd, 1, MO_64); 7216 } 7217 clear_vec_high(s, is_q, rd); 7218 } 7219 7220 /* TBL/TBX 7221 * 31 30 29 24 23 22 21 20 16 15 14 13 12 11 10 9 5 4 0 7222 * +---+---+-------------+-----+---+------+---+-----+----+-----+------+------+ 7223 * | 0 | Q | 0 0 1 1 1 0 | op2 | 0 | Rm | 0 | len | op | 0 0 | Rn | Rd | 7224 * +---+---+-------------+-----+---+------+---+-----+----+-----+------+------+ 7225 */ 7226 static void disas_simd_tb(DisasContext *s, uint32_t insn) 7227 { 7228 int op2 = extract32(insn, 22, 2); 7229 int is_q = extract32(insn, 30, 1); 7230 int rm = extract32(insn, 16, 5); 7231 int rn = extract32(insn, 5, 5); 7232 int rd = extract32(insn, 0, 5); 7233 int is_tbx = extract32(insn, 12, 1); 7234 int len = (extract32(insn, 13, 2) + 1) * 16; 7235 7236 if (op2 != 0) { 7237 unallocated_encoding(s); 7238 return; 7239 } 7240 7241 if (!fp_access_check(s)) { 7242 return; 7243 } 7244 7245 tcg_gen_gvec_2_ptr(vec_full_reg_offset(s, rd), 7246 vec_full_reg_offset(s, rm), cpu_env, 7247 is_q ? 16 : 8, vec_full_reg_size(s), 7248 (len << 6) | (is_tbx << 5) | rn, 7249 gen_helper_simd_tblx); 7250 } 7251 7252 /* ZIP/UZP/TRN 7253 * 31 30 29 24 23 22 21 20 16 15 14 12 11 10 9 5 4 0 7254 * +---+---+-------------+------+---+------+---+------------------+------+ 7255 * | 0 | Q | 0 0 1 1 1 0 | size | 0 | Rm | 0 | opc | 1 0 | Rn | Rd | 7256 * +---+---+-------------+------+---+------+---+------------------+------+ 7257 */ 7258 static void disas_simd_zip_trn(DisasContext *s, uint32_t insn) 7259 { 7260 int rd = extract32(insn, 0, 5); 7261 int rn = extract32(insn, 5, 5); 7262 int rm = extract32(insn, 16, 5); 7263 int size = extract32(insn, 22, 2); 7264 /* opc field bits [1:0] indicate ZIP/UZP/TRN; 7265 * bit 2 indicates 1 vs 2 variant of the insn. 7266 */ 7267 int opcode = extract32(insn, 12, 2); 7268 bool part = extract32(insn, 14, 1); 7269 bool is_q = extract32(insn, 30, 1); 7270 int esize = 8 << size; 7271 int i; 7272 int datasize = is_q ? 128 : 64; 7273 int elements = datasize / esize; 7274 TCGv_i64 tcg_res[2], tcg_ele; 7275 7276 if (opcode == 0 || (size == 3 && !is_q)) { 7277 unallocated_encoding(s); 7278 return; 7279 } 7280 7281 if (!fp_access_check(s)) { 7282 return; 7283 } 7284 7285 tcg_res[0] = tcg_temp_new_i64(); 7286 tcg_res[1] = is_q ? tcg_temp_new_i64() : NULL; 7287 tcg_ele = tcg_temp_new_i64(); 7288 7289 for (i = 0; i < elements; i++) { 7290 int o, w; 7291 7292 switch (opcode) { 7293 case 1: /* UZP1/2 */ 7294 { 7295 int midpoint = elements / 2; 7296 if (i < midpoint) { 7297 read_vec_element(s, tcg_ele, rn, 2 * i + part, size); 7298 } else { 7299 read_vec_element(s, tcg_ele, rm, 7300 2 * (i - midpoint) + part, size); 7301 } 7302 break; 7303 } 7304 case 2: /* TRN1/2 */ 7305 if (i & 1) { 7306 read_vec_element(s, tcg_ele, rm, (i & ~1) + part, size); 7307 } else { 7308 read_vec_element(s, tcg_ele, rn, (i & ~1) + part, size); 7309 } 7310 break; 7311 case 3: /* ZIP1/2 */ 7312 { 7313 int base = part * elements / 2; 7314 if (i & 1) { 7315 read_vec_element(s, tcg_ele, rm, base + (i >> 1), size); 7316 } else { 7317 read_vec_element(s, tcg_ele, rn, base + (i >> 1), size); 7318 } 7319 break; 7320 } 7321 default: 7322 g_assert_not_reached(); 7323 } 7324 7325 w = (i * esize) / 64; 7326 o = (i * esize) % 64; 7327 if (o == 0) { 7328 tcg_gen_mov_i64(tcg_res[w], tcg_ele); 7329 } else { 7330 tcg_gen_shli_i64(tcg_ele, tcg_ele, o); 7331 tcg_gen_or_i64(tcg_res[w], tcg_res[w], tcg_ele); 7332 } 7333 } 7334 7335 for (i = 0; i <= is_q; ++i) { 7336 write_vec_element(s, tcg_res[i], rd, i, MO_64); 7337 } 7338 clear_vec_high(s, is_q, rd); 7339 } 7340 7341 /* 7342 * do_reduction_op helper 7343 * 7344 * This mirrors the Reduce() pseudocode in the ARM ARM. It is 7345 * important for correct NaN propagation that we do these 7346 * operations in exactly the order specified by the pseudocode. 7347 * 7348 * This is a recursive function, TCG temps should be freed by the 7349 * calling function once it is done with the values. 7350 */ 7351 static TCGv_i32 do_reduction_op(DisasContext *s, int fpopcode, int rn, 7352 int esize, int size, int vmap, TCGv_ptr fpst) 7353 { 7354 if (esize == size) { 7355 int element; 7356 MemOp msize = esize == 16 ? MO_16 : MO_32; 7357 TCGv_i32 tcg_elem; 7358 7359 /* We should have one register left here */ 7360 assert(ctpop8(vmap) == 1); 7361 element = ctz32(vmap); 7362 assert(element < 8); 7363 7364 tcg_elem = tcg_temp_new_i32(); 7365 read_vec_element_i32(s, tcg_elem, rn, element, msize); 7366 return tcg_elem; 7367 } else { 7368 int bits = size / 2; 7369 int shift = ctpop8(vmap) / 2; 7370 int vmap_lo = (vmap >> shift) & vmap; 7371 int vmap_hi = (vmap & ~vmap_lo); 7372 TCGv_i32 tcg_hi, tcg_lo, tcg_res; 7373 7374 tcg_hi = do_reduction_op(s, fpopcode, rn, esize, bits, vmap_hi, fpst); 7375 tcg_lo = do_reduction_op(s, fpopcode, rn, esize, bits, vmap_lo, fpst); 7376 tcg_res = tcg_temp_new_i32(); 7377 7378 switch (fpopcode) { 7379 case 0x0c: /* fmaxnmv half-precision */ 7380 gen_helper_advsimd_maxnumh(tcg_res, tcg_lo, tcg_hi, fpst); 7381 break; 7382 case 0x0f: /* fmaxv half-precision */ 7383 gen_helper_advsimd_maxh(tcg_res, tcg_lo, tcg_hi, fpst); 7384 break; 7385 case 0x1c: /* fminnmv half-precision */ 7386 gen_helper_advsimd_minnumh(tcg_res, tcg_lo, tcg_hi, fpst); 7387 break; 7388 case 0x1f: /* fminv half-precision */ 7389 gen_helper_advsimd_minh(tcg_res, tcg_lo, tcg_hi, fpst); 7390 break; 7391 case 0x2c: /* fmaxnmv */ 7392 gen_helper_vfp_maxnums(tcg_res, tcg_lo, tcg_hi, fpst); 7393 break; 7394 case 0x2f: /* fmaxv */ 7395 gen_helper_vfp_maxs(tcg_res, tcg_lo, tcg_hi, fpst); 7396 break; 7397 case 0x3c: /* fminnmv */ 7398 gen_helper_vfp_minnums(tcg_res, tcg_lo, tcg_hi, fpst); 7399 break; 7400 case 0x3f: /* fminv */ 7401 gen_helper_vfp_mins(tcg_res, tcg_lo, tcg_hi, fpst); 7402 break; 7403 default: 7404 g_assert_not_reached(); 7405 } 7406 return tcg_res; 7407 } 7408 } 7409 7410 /* AdvSIMD across lanes 7411 * 31 30 29 28 24 23 22 21 17 16 12 11 10 9 5 4 0 7412 * +---+---+---+-----------+------+-----------+--------+-----+------+------+ 7413 * | 0 | Q | U | 0 1 1 1 0 | size | 1 1 0 0 0 | opcode | 1 0 | Rn | Rd | 7414 * +---+---+---+-----------+------+-----------+--------+-----+------+------+ 7415 */ 7416 static void disas_simd_across_lanes(DisasContext *s, uint32_t insn) 7417 { 7418 int rd = extract32(insn, 0, 5); 7419 int rn = extract32(insn, 5, 5); 7420 int size = extract32(insn, 22, 2); 7421 int opcode = extract32(insn, 12, 5); 7422 bool is_q = extract32(insn, 30, 1); 7423 bool is_u = extract32(insn, 29, 1); 7424 bool is_fp = false; 7425 bool is_min = false; 7426 int esize; 7427 int elements; 7428 int i; 7429 TCGv_i64 tcg_res, tcg_elt; 7430 7431 switch (opcode) { 7432 case 0x1b: /* ADDV */ 7433 if (is_u) { 7434 unallocated_encoding(s); 7435 return; 7436 } 7437 /* fall through */ 7438 case 0x3: /* SADDLV, UADDLV */ 7439 case 0xa: /* SMAXV, UMAXV */ 7440 case 0x1a: /* SMINV, UMINV */ 7441 if (size == 3 || (size == 2 && !is_q)) { 7442 unallocated_encoding(s); 7443 return; 7444 } 7445 break; 7446 case 0xc: /* FMAXNMV, FMINNMV */ 7447 case 0xf: /* FMAXV, FMINV */ 7448 /* Bit 1 of size field encodes min vs max and the actual size 7449 * depends on the encoding of the U bit. If not set (and FP16 7450 * enabled) then we do half-precision float instead of single 7451 * precision. 7452 */ 7453 is_min = extract32(size, 1, 1); 7454 is_fp = true; 7455 if (!is_u && dc_isar_feature(aa64_fp16, s)) { 7456 size = 1; 7457 } else if (!is_u || !is_q || extract32(size, 0, 1)) { 7458 unallocated_encoding(s); 7459 return; 7460 } else { 7461 size = 2; 7462 } 7463 break; 7464 default: 7465 unallocated_encoding(s); 7466 return; 7467 } 7468 7469 if (!fp_access_check(s)) { 7470 return; 7471 } 7472 7473 esize = 8 << size; 7474 elements = (is_q ? 128 : 64) / esize; 7475 7476 tcg_res = tcg_temp_new_i64(); 7477 tcg_elt = tcg_temp_new_i64(); 7478 7479 /* These instructions operate across all lanes of a vector 7480 * to produce a single result. We can guarantee that a 64 7481 * bit intermediate is sufficient: 7482 * + for [US]ADDLV the maximum element size is 32 bits, and 7483 * the result type is 64 bits 7484 * + for FMAX*V, FMIN*V, ADDV the intermediate type is the 7485 * same as the element size, which is 32 bits at most 7486 * For the integer operations we can choose to work at 64 7487 * or 32 bits and truncate at the end; for simplicity 7488 * we use 64 bits always. The floating point 7489 * ops do require 32 bit intermediates, though. 7490 */ 7491 if (!is_fp) { 7492 read_vec_element(s, tcg_res, rn, 0, size | (is_u ? 0 : MO_SIGN)); 7493 7494 for (i = 1; i < elements; i++) { 7495 read_vec_element(s, tcg_elt, rn, i, size | (is_u ? 0 : MO_SIGN)); 7496 7497 switch (opcode) { 7498 case 0x03: /* SADDLV / UADDLV */ 7499 case 0x1b: /* ADDV */ 7500 tcg_gen_add_i64(tcg_res, tcg_res, tcg_elt); 7501 break; 7502 case 0x0a: /* SMAXV / UMAXV */ 7503 if (is_u) { 7504 tcg_gen_umax_i64(tcg_res, tcg_res, tcg_elt); 7505 } else { 7506 tcg_gen_smax_i64(tcg_res, tcg_res, tcg_elt); 7507 } 7508 break; 7509 case 0x1a: /* SMINV / UMINV */ 7510 if (is_u) { 7511 tcg_gen_umin_i64(tcg_res, tcg_res, tcg_elt); 7512 } else { 7513 tcg_gen_smin_i64(tcg_res, tcg_res, tcg_elt); 7514 } 7515 break; 7516 default: 7517 g_assert_not_reached(); 7518 } 7519 7520 } 7521 } else { 7522 /* Floating point vector reduction ops which work across 32 7523 * bit (single) or 16 bit (half-precision) intermediates. 7524 * Note that correct NaN propagation requires that we do these 7525 * operations in exactly the order specified by the pseudocode. 7526 */ 7527 TCGv_ptr fpst = fpstatus_ptr(size == MO_16 ? FPST_FPCR_F16 : FPST_FPCR); 7528 int fpopcode = opcode | is_min << 4 | is_u << 5; 7529 int vmap = (1 << elements) - 1; 7530 TCGv_i32 tcg_res32 = do_reduction_op(s, fpopcode, rn, esize, 7531 (is_q ? 128 : 64), vmap, fpst); 7532 tcg_gen_extu_i32_i64(tcg_res, tcg_res32); 7533 } 7534 7535 /* Now truncate the result to the width required for the final output */ 7536 if (opcode == 0x03) { 7537 /* SADDLV, UADDLV: result is 2*esize */ 7538 size++; 7539 } 7540 7541 switch (size) { 7542 case 0: 7543 tcg_gen_ext8u_i64(tcg_res, tcg_res); 7544 break; 7545 case 1: 7546 tcg_gen_ext16u_i64(tcg_res, tcg_res); 7547 break; 7548 case 2: 7549 tcg_gen_ext32u_i64(tcg_res, tcg_res); 7550 break; 7551 case 3: 7552 break; 7553 default: 7554 g_assert_not_reached(); 7555 } 7556 7557 write_fp_dreg(s, rd, tcg_res); 7558 } 7559 7560 /* DUP (Element, Vector) 7561 * 7562 * 31 30 29 21 20 16 15 10 9 5 4 0 7563 * +---+---+-------------------+--------+-------------+------+------+ 7564 * | 0 | Q | 0 0 1 1 1 0 0 0 0 | imm5 | 0 0 0 0 0 1 | Rn | Rd | 7565 * +---+---+-------------------+--------+-------------+------+------+ 7566 * 7567 * size: encoded in imm5 (see ARM ARM LowestSetBit()) 7568 */ 7569 static void handle_simd_dupe(DisasContext *s, int is_q, int rd, int rn, 7570 int imm5) 7571 { 7572 int size = ctz32(imm5); 7573 int index; 7574 7575 if (size > 3 || (size == 3 && !is_q)) { 7576 unallocated_encoding(s); 7577 return; 7578 } 7579 7580 if (!fp_access_check(s)) { 7581 return; 7582 } 7583 7584 index = imm5 >> (size + 1); 7585 tcg_gen_gvec_dup_mem(size, vec_full_reg_offset(s, rd), 7586 vec_reg_offset(s, rn, index, size), 7587 is_q ? 16 : 8, vec_full_reg_size(s)); 7588 } 7589 7590 /* DUP (element, scalar) 7591 * 31 21 20 16 15 10 9 5 4 0 7592 * +-----------------------+--------+-------------+------+------+ 7593 * | 0 1 0 1 1 1 1 0 0 0 0 | imm5 | 0 0 0 0 0 1 | Rn | Rd | 7594 * +-----------------------+--------+-------------+------+------+ 7595 */ 7596 static void handle_simd_dupes(DisasContext *s, int rd, int rn, 7597 int imm5) 7598 { 7599 int size = ctz32(imm5); 7600 int index; 7601 TCGv_i64 tmp; 7602 7603 if (size > 3) { 7604 unallocated_encoding(s); 7605 return; 7606 } 7607 7608 if (!fp_access_check(s)) { 7609 return; 7610 } 7611 7612 index = imm5 >> (size + 1); 7613 7614 /* This instruction just extracts the specified element and 7615 * zero-extends it into the bottom of the destination register. 7616 */ 7617 tmp = tcg_temp_new_i64(); 7618 read_vec_element(s, tmp, rn, index, size); 7619 write_fp_dreg(s, rd, tmp); 7620 } 7621 7622 /* DUP (General) 7623 * 7624 * 31 30 29 21 20 16 15 10 9 5 4 0 7625 * +---+---+-------------------+--------+-------------+------+------+ 7626 * | 0 | Q | 0 0 1 1 1 0 0 0 0 | imm5 | 0 0 0 0 1 1 | Rn | Rd | 7627 * +---+---+-------------------+--------+-------------+------+------+ 7628 * 7629 * size: encoded in imm5 (see ARM ARM LowestSetBit()) 7630 */ 7631 static void handle_simd_dupg(DisasContext *s, int is_q, int rd, int rn, 7632 int imm5) 7633 { 7634 int size = ctz32(imm5); 7635 uint32_t dofs, oprsz, maxsz; 7636 7637 if (size > 3 || ((size == 3) && !is_q)) { 7638 unallocated_encoding(s); 7639 return; 7640 } 7641 7642 if (!fp_access_check(s)) { 7643 return; 7644 } 7645 7646 dofs = vec_full_reg_offset(s, rd); 7647 oprsz = is_q ? 16 : 8; 7648 maxsz = vec_full_reg_size(s); 7649 7650 tcg_gen_gvec_dup_i64(size, dofs, oprsz, maxsz, cpu_reg(s, rn)); 7651 } 7652 7653 /* INS (Element) 7654 * 7655 * 31 21 20 16 15 14 11 10 9 5 4 0 7656 * +-----------------------+--------+------------+---+------+------+ 7657 * | 0 1 1 0 1 1 1 0 0 0 0 | imm5 | 0 | imm4 | 1 | Rn | Rd | 7658 * +-----------------------+--------+------------+---+------+------+ 7659 * 7660 * size: encoded in imm5 (see ARM ARM LowestSetBit()) 7661 * index: encoded in imm5<4:size+1> 7662 */ 7663 static void handle_simd_inse(DisasContext *s, int rd, int rn, 7664 int imm4, int imm5) 7665 { 7666 int size = ctz32(imm5); 7667 int src_index, dst_index; 7668 TCGv_i64 tmp; 7669 7670 if (size > 3) { 7671 unallocated_encoding(s); 7672 return; 7673 } 7674 7675 if (!fp_access_check(s)) { 7676 return; 7677 } 7678 7679 dst_index = extract32(imm5, 1+size, 5); 7680 src_index = extract32(imm4, size, 4); 7681 7682 tmp = tcg_temp_new_i64(); 7683 7684 read_vec_element(s, tmp, rn, src_index, size); 7685 write_vec_element(s, tmp, rd, dst_index, size); 7686 7687 /* INS is considered a 128-bit write for SVE. */ 7688 clear_vec_high(s, true, rd); 7689 } 7690 7691 7692 /* INS (General) 7693 * 7694 * 31 21 20 16 15 10 9 5 4 0 7695 * +-----------------------+--------+-------------+------+------+ 7696 * | 0 1 0 0 1 1 1 0 0 0 0 | imm5 | 0 0 0 1 1 1 | Rn | Rd | 7697 * +-----------------------+--------+-------------+------+------+ 7698 * 7699 * size: encoded in imm5 (see ARM ARM LowestSetBit()) 7700 * index: encoded in imm5<4:size+1> 7701 */ 7702 static void handle_simd_insg(DisasContext *s, int rd, int rn, int imm5) 7703 { 7704 int size = ctz32(imm5); 7705 int idx; 7706 7707 if (size > 3) { 7708 unallocated_encoding(s); 7709 return; 7710 } 7711 7712 if (!fp_access_check(s)) { 7713 return; 7714 } 7715 7716 idx = extract32(imm5, 1 + size, 4 - size); 7717 write_vec_element(s, cpu_reg(s, rn), rd, idx, size); 7718 7719 /* INS is considered a 128-bit write for SVE. */ 7720 clear_vec_high(s, true, rd); 7721 } 7722 7723 /* 7724 * UMOV (General) 7725 * SMOV (General) 7726 * 7727 * 31 30 29 21 20 16 15 12 10 9 5 4 0 7728 * +---+---+-------------------+--------+-------------+------+------+ 7729 * | 0 | Q | 0 0 1 1 1 0 0 0 0 | imm5 | 0 0 1 U 1 1 | Rn | Rd | 7730 * +---+---+-------------------+--------+-------------+------+------+ 7731 * 7732 * U: unsigned when set 7733 * size: encoded in imm5 (see ARM ARM LowestSetBit()) 7734 */ 7735 static void handle_simd_umov_smov(DisasContext *s, int is_q, int is_signed, 7736 int rn, int rd, int imm5) 7737 { 7738 int size = ctz32(imm5); 7739 int element; 7740 TCGv_i64 tcg_rd; 7741 7742 /* Check for UnallocatedEncodings */ 7743 if (is_signed) { 7744 if (size > 2 || (size == 2 && !is_q)) { 7745 unallocated_encoding(s); 7746 return; 7747 } 7748 } else { 7749 if (size > 3 7750 || (size < 3 && is_q) 7751 || (size == 3 && !is_q)) { 7752 unallocated_encoding(s); 7753 return; 7754 } 7755 } 7756 7757 if (!fp_access_check(s)) { 7758 return; 7759 } 7760 7761 element = extract32(imm5, 1+size, 4); 7762 7763 tcg_rd = cpu_reg(s, rd); 7764 read_vec_element(s, tcg_rd, rn, element, size | (is_signed ? MO_SIGN : 0)); 7765 if (is_signed && !is_q) { 7766 tcg_gen_ext32u_i64(tcg_rd, tcg_rd); 7767 } 7768 } 7769 7770 /* AdvSIMD copy 7771 * 31 30 29 28 21 20 16 15 14 11 10 9 5 4 0 7772 * +---+---+----+-----------------+------+---+------+---+------+------+ 7773 * | 0 | Q | op | 0 1 1 1 0 0 0 0 | imm5 | 0 | imm4 | 1 | Rn | Rd | 7774 * +---+---+----+-----------------+------+---+------+---+------+------+ 7775 */ 7776 static void disas_simd_copy(DisasContext *s, uint32_t insn) 7777 { 7778 int rd = extract32(insn, 0, 5); 7779 int rn = extract32(insn, 5, 5); 7780 int imm4 = extract32(insn, 11, 4); 7781 int op = extract32(insn, 29, 1); 7782 int is_q = extract32(insn, 30, 1); 7783 int imm5 = extract32(insn, 16, 5); 7784 7785 if (op) { 7786 if (is_q) { 7787 /* INS (element) */ 7788 handle_simd_inse(s, rd, rn, imm4, imm5); 7789 } else { 7790 unallocated_encoding(s); 7791 } 7792 } else { 7793 switch (imm4) { 7794 case 0: 7795 /* DUP (element - vector) */ 7796 handle_simd_dupe(s, is_q, rd, rn, imm5); 7797 break; 7798 case 1: 7799 /* DUP (general) */ 7800 handle_simd_dupg(s, is_q, rd, rn, imm5); 7801 break; 7802 case 3: 7803 if (is_q) { 7804 /* INS (general) */ 7805 handle_simd_insg(s, rd, rn, imm5); 7806 } else { 7807 unallocated_encoding(s); 7808 } 7809 break; 7810 case 5: 7811 case 7: 7812 /* UMOV/SMOV (is_q indicates 32/64; imm4 indicates signedness) */ 7813 handle_simd_umov_smov(s, is_q, (imm4 == 5), rn, rd, imm5); 7814 break; 7815 default: 7816 unallocated_encoding(s); 7817 break; 7818 } 7819 } 7820 } 7821 7822 /* AdvSIMD modified immediate 7823 * 31 30 29 28 19 18 16 15 12 11 10 9 5 4 0 7824 * +---+---+----+---------------------+-----+-------+----+---+-------+------+ 7825 * | 0 | Q | op | 0 1 1 1 1 0 0 0 0 0 | abc | cmode | o2 | 1 | defgh | Rd | 7826 * +---+---+----+---------------------+-----+-------+----+---+-------+------+ 7827 * 7828 * There are a number of operations that can be carried out here: 7829 * MOVI - move (shifted) imm into register 7830 * MVNI - move inverted (shifted) imm into register 7831 * ORR - bitwise OR of (shifted) imm with register 7832 * BIC - bitwise clear of (shifted) imm with register 7833 * With ARMv8.2 we also have: 7834 * FMOV half-precision 7835 */ 7836 static void disas_simd_mod_imm(DisasContext *s, uint32_t insn) 7837 { 7838 int rd = extract32(insn, 0, 5); 7839 int cmode = extract32(insn, 12, 4); 7840 int o2 = extract32(insn, 11, 1); 7841 uint64_t abcdefgh = extract32(insn, 5, 5) | (extract32(insn, 16, 3) << 5); 7842 bool is_neg = extract32(insn, 29, 1); 7843 bool is_q = extract32(insn, 30, 1); 7844 uint64_t imm = 0; 7845 7846 if (o2 != 0 || ((cmode == 0xf) && is_neg && !is_q)) { 7847 /* Check for FMOV (vector, immediate) - half-precision */ 7848 if (!(dc_isar_feature(aa64_fp16, s) && o2 && cmode == 0xf)) { 7849 unallocated_encoding(s); 7850 return; 7851 } 7852 } 7853 7854 if (!fp_access_check(s)) { 7855 return; 7856 } 7857 7858 if (cmode == 15 && o2 && !is_neg) { 7859 /* FMOV (vector, immediate) - half-precision */ 7860 imm = vfp_expand_imm(MO_16, abcdefgh); 7861 /* now duplicate across the lanes */ 7862 imm = dup_const(MO_16, imm); 7863 } else { 7864 imm = asimd_imm_const(abcdefgh, cmode, is_neg); 7865 } 7866 7867 if (!((cmode & 0x9) == 0x1 || (cmode & 0xd) == 0x9)) { 7868 /* MOVI or MVNI, with MVNI negation handled above. */ 7869 tcg_gen_gvec_dup_imm(MO_64, vec_full_reg_offset(s, rd), is_q ? 16 : 8, 7870 vec_full_reg_size(s), imm); 7871 } else { 7872 /* ORR or BIC, with BIC negation to AND handled above. */ 7873 if (is_neg) { 7874 gen_gvec_fn2i(s, is_q, rd, rd, imm, tcg_gen_gvec_andi, MO_64); 7875 } else { 7876 gen_gvec_fn2i(s, is_q, rd, rd, imm, tcg_gen_gvec_ori, MO_64); 7877 } 7878 } 7879 } 7880 7881 /* AdvSIMD scalar copy 7882 * 31 30 29 28 21 20 16 15 14 11 10 9 5 4 0 7883 * +-----+----+-----------------+------+---+------+---+------+------+ 7884 * | 0 1 | op | 1 1 1 1 0 0 0 0 | imm5 | 0 | imm4 | 1 | Rn | Rd | 7885 * +-----+----+-----------------+------+---+------+---+------+------+ 7886 */ 7887 static void disas_simd_scalar_copy(DisasContext *s, uint32_t insn) 7888 { 7889 int rd = extract32(insn, 0, 5); 7890 int rn = extract32(insn, 5, 5); 7891 int imm4 = extract32(insn, 11, 4); 7892 int imm5 = extract32(insn, 16, 5); 7893 int op = extract32(insn, 29, 1); 7894 7895 if (op != 0 || imm4 != 0) { 7896 unallocated_encoding(s); 7897 return; 7898 } 7899 7900 /* DUP (element, scalar) */ 7901 handle_simd_dupes(s, rd, rn, imm5); 7902 } 7903 7904 /* AdvSIMD scalar pairwise 7905 * 31 30 29 28 24 23 22 21 17 16 12 11 10 9 5 4 0 7906 * +-----+---+-----------+------+-----------+--------+-----+------+------+ 7907 * | 0 1 | U | 1 1 1 1 0 | size | 1 1 0 0 0 | opcode | 1 0 | Rn | Rd | 7908 * +-----+---+-----------+------+-----------+--------+-----+------+------+ 7909 */ 7910 static void disas_simd_scalar_pairwise(DisasContext *s, uint32_t insn) 7911 { 7912 int u = extract32(insn, 29, 1); 7913 int size = extract32(insn, 22, 2); 7914 int opcode = extract32(insn, 12, 5); 7915 int rn = extract32(insn, 5, 5); 7916 int rd = extract32(insn, 0, 5); 7917 TCGv_ptr fpst; 7918 7919 /* For some ops (the FP ones), size[1] is part of the encoding. 7920 * For ADDP strictly it is not but size[1] is always 1 for valid 7921 * encodings. 7922 */ 7923 opcode |= (extract32(size, 1, 1) << 5); 7924 7925 switch (opcode) { 7926 case 0x3b: /* ADDP */ 7927 if (u || size != 3) { 7928 unallocated_encoding(s); 7929 return; 7930 } 7931 if (!fp_access_check(s)) { 7932 return; 7933 } 7934 7935 fpst = NULL; 7936 break; 7937 case 0xc: /* FMAXNMP */ 7938 case 0xd: /* FADDP */ 7939 case 0xf: /* FMAXP */ 7940 case 0x2c: /* FMINNMP */ 7941 case 0x2f: /* FMINP */ 7942 /* FP op, size[0] is 32 or 64 bit*/ 7943 if (!u) { 7944 if (!dc_isar_feature(aa64_fp16, s)) { 7945 unallocated_encoding(s); 7946 return; 7947 } else { 7948 size = MO_16; 7949 } 7950 } else { 7951 size = extract32(size, 0, 1) ? MO_64 : MO_32; 7952 } 7953 7954 if (!fp_access_check(s)) { 7955 return; 7956 } 7957 7958 fpst = fpstatus_ptr(size == MO_16 ? FPST_FPCR_F16 : FPST_FPCR); 7959 break; 7960 default: 7961 unallocated_encoding(s); 7962 return; 7963 } 7964 7965 if (size == MO_64) { 7966 TCGv_i64 tcg_op1 = tcg_temp_new_i64(); 7967 TCGv_i64 tcg_op2 = tcg_temp_new_i64(); 7968 TCGv_i64 tcg_res = tcg_temp_new_i64(); 7969 7970 read_vec_element(s, tcg_op1, rn, 0, MO_64); 7971 read_vec_element(s, tcg_op2, rn, 1, MO_64); 7972 7973 switch (opcode) { 7974 case 0x3b: /* ADDP */ 7975 tcg_gen_add_i64(tcg_res, tcg_op1, tcg_op2); 7976 break; 7977 case 0xc: /* FMAXNMP */ 7978 gen_helper_vfp_maxnumd(tcg_res, tcg_op1, tcg_op2, fpst); 7979 break; 7980 case 0xd: /* FADDP */ 7981 gen_helper_vfp_addd(tcg_res, tcg_op1, tcg_op2, fpst); 7982 break; 7983 case 0xf: /* FMAXP */ 7984 gen_helper_vfp_maxd(tcg_res, tcg_op1, tcg_op2, fpst); 7985 break; 7986 case 0x2c: /* FMINNMP */ 7987 gen_helper_vfp_minnumd(tcg_res, tcg_op1, tcg_op2, fpst); 7988 break; 7989 case 0x2f: /* FMINP */ 7990 gen_helper_vfp_mind(tcg_res, tcg_op1, tcg_op2, fpst); 7991 break; 7992 default: 7993 g_assert_not_reached(); 7994 } 7995 7996 write_fp_dreg(s, rd, tcg_res); 7997 } else { 7998 TCGv_i32 tcg_op1 = tcg_temp_new_i32(); 7999 TCGv_i32 tcg_op2 = tcg_temp_new_i32(); 8000 TCGv_i32 tcg_res = tcg_temp_new_i32(); 8001 8002 read_vec_element_i32(s, tcg_op1, rn, 0, size); 8003 read_vec_element_i32(s, tcg_op2, rn, 1, size); 8004 8005 if (size == MO_16) { 8006 switch (opcode) { 8007 case 0xc: /* FMAXNMP */ 8008 gen_helper_advsimd_maxnumh(tcg_res, tcg_op1, tcg_op2, fpst); 8009 break; 8010 case 0xd: /* FADDP */ 8011 gen_helper_advsimd_addh(tcg_res, tcg_op1, tcg_op2, fpst); 8012 break; 8013 case 0xf: /* FMAXP */ 8014 gen_helper_advsimd_maxh(tcg_res, tcg_op1, tcg_op2, fpst); 8015 break; 8016 case 0x2c: /* FMINNMP */ 8017 gen_helper_advsimd_minnumh(tcg_res, tcg_op1, tcg_op2, fpst); 8018 break; 8019 case 0x2f: /* FMINP */ 8020 gen_helper_advsimd_minh(tcg_res, tcg_op1, tcg_op2, fpst); 8021 break; 8022 default: 8023 g_assert_not_reached(); 8024 } 8025 } else { 8026 switch (opcode) { 8027 case 0xc: /* FMAXNMP */ 8028 gen_helper_vfp_maxnums(tcg_res, tcg_op1, tcg_op2, fpst); 8029 break; 8030 case 0xd: /* FADDP */ 8031 gen_helper_vfp_adds(tcg_res, tcg_op1, tcg_op2, fpst); 8032 break; 8033 case 0xf: /* FMAXP */ 8034 gen_helper_vfp_maxs(tcg_res, tcg_op1, tcg_op2, fpst); 8035 break; 8036 case 0x2c: /* FMINNMP */ 8037 gen_helper_vfp_minnums(tcg_res, tcg_op1, tcg_op2, fpst); 8038 break; 8039 case 0x2f: /* FMINP */ 8040 gen_helper_vfp_mins(tcg_res, tcg_op1, tcg_op2, fpst); 8041 break; 8042 default: 8043 g_assert_not_reached(); 8044 } 8045 } 8046 8047 write_fp_sreg(s, rd, tcg_res); 8048 } 8049 } 8050 8051 /* 8052 * Common SSHR[RA]/USHR[RA] - Shift right (optional rounding/accumulate) 8053 * 8054 * This code is handles the common shifting code and is used by both 8055 * the vector and scalar code. 8056 */ 8057 static void handle_shri_with_rndacc(TCGv_i64 tcg_res, TCGv_i64 tcg_src, 8058 TCGv_i64 tcg_rnd, bool accumulate, 8059 bool is_u, int size, int shift) 8060 { 8061 bool extended_result = false; 8062 bool round = tcg_rnd != NULL; 8063 int ext_lshift = 0; 8064 TCGv_i64 tcg_src_hi; 8065 8066 if (round && size == 3) { 8067 extended_result = true; 8068 ext_lshift = 64 - shift; 8069 tcg_src_hi = tcg_temp_new_i64(); 8070 } else if (shift == 64) { 8071 if (!accumulate && is_u) { 8072 /* result is zero */ 8073 tcg_gen_movi_i64(tcg_res, 0); 8074 return; 8075 } 8076 } 8077 8078 /* Deal with the rounding step */ 8079 if (round) { 8080 if (extended_result) { 8081 TCGv_i64 tcg_zero = tcg_constant_i64(0); 8082 if (!is_u) { 8083 /* take care of sign extending tcg_res */ 8084 tcg_gen_sari_i64(tcg_src_hi, tcg_src, 63); 8085 tcg_gen_add2_i64(tcg_src, tcg_src_hi, 8086 tcg_src, tcg_src_hi, 8087 tcg_rnd, tcg_zero); 8088 } else { 8089 tcg_gen_add2_i64(tcg_src, tcg_src_hi, 8090 tcg_src, tcg_zero, 8091 tcg_rnd, tcg_zero); 8092 } 8093 } else { 8094 tcg_gen_add_i64(tcg_src, tcg_src, tcg_rnd); 8095 } 8096 } 8097 8098 /* Now do the shift right */ 8099 if (round && extended_result) { 8100 /* extended case, >64 bit precision required */ 8101 if (ext_lshift == 0) { 8102 /* special case, only high bits matter */ 8103 tcg_gen_mov_i64(tcg_src, tcg_src_hi); 8104 } else { 8105 tcg_gen_shri_i64(tcg_src, tcg_src, shift); 8106 tcg_gen_shli_i64(tcg_src_hi, tcg_src_hi, ext_lshift); 8107 tcg_gen_or_i64(tcg_src, tcg_src, tcg_src_hi); 8108 } 8109 } else { 8110 if (is_u) { 8111 if (shift == 64) { 8112 /* essentially shifting in 64 zeros */ 8113 tcg_gen_movi_i64(tcg_src, 0); 8114 } else { 8115 tcg_gen_shri_i64(tcg_src, tcg_src, shift); 8116 } 8117 } else { 8118 if (shift == 64) { 8119 /* effectively extending the sign-bit */ 8120 tcg_gen_sari_i64(tcg_src, tcg_src, 63); 8121 } else { 8122 tcg_gen_sari_i64(tcg_src, tcg_src, shift); 8123 } 8124 } 8125 } 8126 8127 if (accumulate) { 8128 tcg_gen_add_i64(tcg_res, tcg_res, tcg_src); 8129 } else { 8130 tcg_gen_mov_i64(tcg_res, tcg_src); 8131 } 8132 } 8133 8134 /* SSHR[RA]/USHR[RA] - Scalar shift right (optional rounding/accumulate) */ 8135 static void handle_scalar_simd_shri(DisasContext *s, 8136 bool is_u, int immh, int immb, 8137 int opcode, int rn, int rd) 8138 { 8139 const int size = 3; 8140 int immhb = immh << 3 | immb; 8141 int shift = 2 * (8 << size) - immhb; 8142 bool accumulate = false; 8143 bool round = false; 8144 bool insert = false; 8145 TCGv_i64 tcg_rn; 8146 TCGv_i64 tcg_rd; 8147 TCGv_i64 tcg_round; 8148 8149 if (!extract32(immh, 3, 1)) { 8150 unallocated_encoding(s); 8151 return; 8152 } 8153 8154 if (!fp_access_check(s)) { 8155 return; 8156 } 8157 8158 switch (opcode) { 8159 case 0x02: /* SSRA / USRA (accumulate) */ 8160 accumulate = true; 8161 break; 8162 case 0x04: /* SRSHR / URSHR (rounding) */ 8163 round = true; 8164 break; 8165 case 0x06: /* SRSRA / URSRA (accum + rounding) */ 8166 accumulate = round = true; 8167 break; 8168 case 0x08: /* SRI */ 8169 insert = true; 8170 break; 8171 } 8172 8173 if (round) { 8174 tcg_round = tcg_constant_i64(1ULL << (shift - 1)); 8175 } else { 8176 tcg_round = NULL; 8177 } 8178 8179 tcg_rn = read_fp_dreg(s, rn); 8180 tcg_rd = (accumulate || insert) ? read_fp_dreg(s, rd) : tcg_temp_new_i64(); 8181 8182 if (insert) { 8183 /* shift count same as element size is valid but does nothing; 8184 * special case to avoid potential shift by 64. 8185 */ 8186 int esize = 8 << size; 8187 if (shift != esize) { 8188 tcg_gen_shri_i64(tcg_rn, tcg_rn, shift); 8189 tcg_gen_deposit_i64(tcg_rd, tcg_rd, tcg_rn, 0, esize - shift); 8190 } 8191 } else { 8192 handle_shri_with_rndacc(tcg_rd, tcg_rn, tcg_round, 8193 accumulate, is_u, size, shift); 8194 } 8195 8196 write_fp_dreg(s, rd, tcg_rd); 8197 } 8198 8199 /* SHL/SLI - Scalar shift left */ 8200 static void handle_scalar_simd_shli(DisasContext *s, bool insert, 8201 int immh, int immb, int opcode, 8202 int rn, int rd) 8203 { 8204 int size = 32 - clz32(immh) - 1; 8205 int immhb = immh << 3 | immb; 8206 int shift = immhb - (8 << size); 8207 TCGv_i64 tcg_rn; 8208 TCGv_i64 tcg_rd; 8209 8210 if (!extract32(immh, 3, 1)) { 8211 unallocated_encoding(s); 8212 return; 8213 } 8214 8215 if (!fp_access_check(s)) { 8216 return; 8217 } 8218 8219 tcg_rn = read_fp_dreg(s, rn); 8220 tcg_rd = insert ? read_fp_dreg(s, rd) : tcg_temp_new_i64(); 8221 8222 if (insert) { 8223 tcg_gen_deposit_i64(tcg_rd, tcg_rd, tcg_rn, shift, 64 - shift); 8224 } else { 8225 tcg_gen_shli_i64(tcg_rd, tcg_rn, shift); 8226 } 8227 8228 write_fp_dreg(s, rd, tcg_rd); 8229 } 8230 8231 /* SQSHRN/SQSHRUN - Saturating (signed/unsigned) shift right with 8232 * (signed/unsigned) narrowing */ 8233 static void handle_vec_simd_sqshrn(DisasContext *s, bool is_scalar, bool is_q, 8234 bool is_u_shift, bool is_u_narrow, 8235 int immh, int immb, int opcode, 8236 int rn, int rd) 8237 { 8238 int immhb = immh << 3 | immb; 8239 int size = 32 - clz32(immh) - 1; 8240 int esize = 8 << size; 8241 int shift = (2 * esize) - immhb; 8242 int elements = is_scalar ? 1 : (64 / esize); 8243 bool round = extract32(opcode, 0, 1); 8244 MemOp ldop = (size + 1) | (is_u_shift ? 0 : MO_SIGN); 8245 TCGv_i64 tcg_rn, tcg_rd, tcg_round; 8246 TCGv_i32 tcg_rd_narrowed; 8247 TCGv_i64 tcg_final; 8248 8249 static NeonGenNarrowEnvFn * const signed_narrow_fns[4][2] = { 8250 { gen_helper_neon_narrow_sat_s8, 8251 gen_helper_neon_unarrow_sat8 }, 8252 { gen_helper_neon_narrow_sat_s16, 8253 gen_helper_neon_unarrow_sat16 }, 8254 { gen_helper_neon_narrow_sat_s32, 8255 gen_helper_neon_unarrow_sat32 }, 8256 { NULL, NULL }, 8257 }; 8258 static NeonGenNarrowEnvFn * const unsigned_narrow_fns[4] = { 8259 gen_helper_neon_narrow_sat_u8, 8260 gen_helper_neon_narrow_sat_u16, 8261 gen_helper_neon_narrow_sat_u32, 8262 NULL 8263 }; 8264 NeonGenNarrowEnvFn *narrowfn; 8265 8266 int i; 8267 8268 assert(size < 4); 8269 8270 if (extract32(immh, 3, 1)) { 8271 unallocated_encoding(s); 8272 return; 8273 } 8274 8275 if (!fp_access_check(s)) { 8276 return; 8277 } 8278 8279 if (is_u_shift) { 8280 narrowfn = unsigned_narrow_fns[size]; 8281 } else { 8282 narrowfn = signed_narrow_fns[size][is_u_narrow ? 1 : 0]; 8283 } 8284 8285 tcg_rn = tcg_temp_new_i64(); 8286 tcg_rd = tcg_temp_new_i64(); 8287 tcg_rd_narrowed = tcg_temp_new_i32(); 8288 tcg_final = tcg_temp_new_i64(); 8289 8290 if (round) { 8291 tcg_round = tcg_constant_i64(1ULL << (shift - 1)); 8292 } else { 8293 tcg_round = NULL; 8294 } 8295 8296 for (i = 0; i < elements; i++) { 8297 read_vec_element(s, tcg_rn, rn, i, ldop); 8298 handle_shri_with_rndacc(tcg_rd, tcg_rn, tcg_round, 8299 false, is_u_shift, size+1, shift); 8300 narrowfn(tcg_rd_narrowed, cpu_env, tcg_rd); 8301 tcg_gen_extu_i32_i64(tcg_rd, tcg_rd_narrowed); 8302 if (i == 0) { 8303 tcg_gen_mov_i64(tcg_final, tcg_rd); 8304 } else { 8305 tcg_gen_deposit_i64(tcg_final, tcg_final, tcg_rd, esize * i, esize); 8306 } 8307 } 8308 8309 if (!is_q) { 8310 write_vec_element(s, tcg_final, rd, 0, MO_64); 8311 } else { 8312 write_vec_element(s, tcg_final, rd, 1, MO_64); 8313 } 8314 clear_vec_high(s, is_q, rd); 8315 } 8316 8317 /* SQSHLU, UQSHL, SQSHL: saturating left shifts */ 8318 static void handle_simd_qshl(DisasContext *s, bool scalar, bool is_q, 8319 bool src_unsigned, bool dst_unsigned, 8320 int immh, int immb, int rn, int rd) 8321 { 8322 int immhb = immh << 3 | immb; 8323 int size = 32 - clz32(immh) - 1; 8324 int shift = immhb - (8 << size); 8325 int pass; 8326 8327 assert(immh != 0); 8328 assert(!(scalar && is_q)); 8329 8330 if (!scalar) { 8331 if (!is_q && extract32(immh, 3, 1)) { 8332 unallocated_encoding(s); 8333 return; 8334 } 8335 8336 /* Since we use the variable-shift helpers we must 8337 * replicate the shift count into each element of 8338 * the tcg_shift value. 8339 */ 8340 switch (size) { 8341 case 0: 8342 shift |= shift << 8; 8343 /* fall through */ 8344 case 1: 8345 shift |= shift << 16; 8346 break; 8347 case 2: 8348 case 3: 8349 break; 8350 default: 8351 g_assert_not_reached(); 8352 } 8353 } 8354 8355 if (!fp_access_check(s)) { 8356 return; 8357 } 8358 8359 if (size == 3) { 8360 TCGv_i64 tcg_shift = tcg_constant_i64(shift); 8361 static NeonGenTwo64OpEnvFn * const fns[2][2] = { 8362 { gen_helper_neon_qshl_s64, gen_helper_neon_qshlu_s64 }, 8363 { NULL, gen_helper_neon_qshl_u64 }, 8364 }; 8365 NeonGenTwo64OpEnvFn *genfn = fns[src_unsigned][dst_unsigned]; 8366 int maxpass = is_q ? 2 : 1; 8367 8368 for (pass = 0; pass < maxpass; pass++) { 8369 TCGv_i64 tcg_op = tcg_temp_new_i64(); 8370 8371 read_vec_element(s, tcg_op, rn, pass, MO_64); 8372 genfn(tcg_op, cpu_env, tcg_op, tcg_shift); 8373 write_vec_element(s, tcg_op, rd, pass, MO_64); 8374 } 8375 clear_vec_high(s, is_q, rd); 8376 } else { 8377 TCGv_i32 tcg_shift = tcg_constant_i32(shift); 8378 static NeonGenTwoOpEnvFn * const fns[2][2][3] = { 8379 { 8380 { gen_helper_neon_qshl_s8, 8381 gen_helper_neon_qshl_s16, 8382 gen_helper_neon_qshl_s32 }, 8383 { gen_helper_neon_qshlu_s8, 8384 gen_helper_neon_qshlu_s16, 8385 gen_helper_neon_qshlu_s32 } 8386 }, { 8387 { NULL, NULL, NULL }, 8388 { gen_helper_neon_qshl_u8, 8389 gen_helper_neon_qshl_u16, 8390 gen_helper_neon_qshl_u32 } 8391 } 8392 }; 8393 NeonGenTwoOpEnvFn *genfn = fns[src_unsigned][dst_unsigned][size]; 8394 MemOp memop = scalar ? size : MO_32; 8395 int maxpass = scalar ? 1 : is_q ? 4 : 2; 8396 8397 for (pass = 0; pass < maxpass; pass++) { 8398 TCGv_i32 tcg_op = tcg_temp_new_i32(); 8399 8400 read_vec_element_i32(s, tcg_op, rn, pass, memop); 8401 genfn(tcg_op, cpu_env, tcg_op, tcg_shift); 8402 if (scalar) { 8403 switch (size) { 8404 case 0: 8405 tcg_gen_ext8u_i32(tcg_op, tcg_op); 8406 break; 8407 case 1: 8408 tcg_gen_ext16u_i32(tcg_op, tcg_op); 8409 break; 8410 case 2: 8411 break; 8412 default: 8413 g_assert_not_reached(); 8414 } 8415 write_fp_sreg(s, rd, tcg_op); 8416 } else { 8417 write_vec_element_i32(s, tcg_op, rd, pass, MO_32); 8418 } 8419 } 8420 8421 if (!scalar) { 8422 clear_vec_high(s, is_q, rd); 8423 } 8424 } 8425 } 8426 8427 /* Common vector code for handling integer to FP conversion */ 8428 static void handle_simd_intfp_conv(DisasContext *s, int rd, int rn, 8429 int elements, int is_signed, 8430 int fracbits, int size) 8431 { 8432 TCGv_ptr tcg_fpst = fpstatus_ptr(size == MO_16 ? FPST_FPCR_F16 : FPST_FPCR); 8433 TCGv_i32 tcg_shift = NULL; 8434 8435 MemOp mop = size | (is_signed ? MO_SIGN : 0); 8436 int pass; 8437 8438 if (fracbits || size == MO_64) { 8439 tcg_shift = tcg_constant_i32(fracbits); 8440 } 8441 8442 if (size == MO_64) { 8443 TCGv_i64 tcg_int64 = tcg_temp_new_i64(); 8444 TCGv_i64 tcg_double = tcg_temp_new_i64(); 8445 8446 for (pass = 0; pass < elements; pass++) { 8447 read_vec_element(s, tcg_int64, rn, pass, mop); 8448 8449 if (is_signed) { 8450 gen_helper_vfp_sqtod(tcg_double, tcg_int64, 8451 tcg_shift, tcg_fpst); 8452 } else { 8453 gen_helper_vfp_uqtod(tcg_double, tcg_int64, 8454 tcg_shift, tcg_fpst); 8455 } 8456 if (elements == 1) { 8457 write_fp_dreg(s, rd, tcg_double); 8458 } else { 8459 write_vec_element(s, tcg_double, rd, pass, MO_64); 8460 } 8461 } 8462 } else { 8463 TCGv_i32 tcg_int32 = tcg_temp_new_i32(); 8464 TCGv_i32 tcg_float = tcg_temp_new_i32(); 8465 8466 for (pass = 0; pass < elements; pass++) { 8467 read_vec_element_i32(s, tcg_int32, rn, pass, mop); 8468 8469 switch (size) { 8470 case MO_32: 8471 if (fracbits) { 8472 if (is_signed) { 8473 gen_helper_vfp_sltos(tcg_float, tcg_int32, 8474 tcg_shift, tcg_fpst); 8475 } else { 8476 gen_helper_vfp_ultos(tcg_float, tcg_int32, 8477 tcg_shift, tcg_fpst); 8478 } 8479 } else { 8480 if (is_signed) { 8481 gen_helper_vfp_sitos(tcg_float, tcg_int32, tcg_fpst); 8482 } else { 8483 gen_helper_vfp_uitos(tcg_float, tcg_int32, tcg_fpst); 8484 } 8485 } 8486 break; 8487 case MO_16: 8488 if (fracbits) { 8489 if (is_signed) { 8490 gen_helper_vfp_sltoh(tcg_float, tcg_int32, 8491 tcg_shift, tcg_fpst); 8492 } else { 8493 gen_helper_vfp_ultoh(tcg_float, tcg_int32, 8494 tcg_shift, tcg_fpst); 8495 } 8496 } else { 8497 if (is_signed) { 8498 gen_helper_vfp_sitoh(tcg_float, tcg_int32, tcg_fpst); 8499 } else { 8500 gen_helper_vfp_uitoh(tcg_float, tcg_int32, tcg_fpst); 8501 } 8502 } 8503 break; 8504 default: 8505 g_assert_not_reached(); 8506 } 8507 8508 if (elements == 1) { 8509 write_fp_sreg(s, rd, tcg_float); 8510 } else { 8511 write_vec_element_i32(s, tcg_float, rd, pass, size); 8512 } 8513 } 8514 } 8515 8516 clear_vec_high(s, elements << size == 16, rd); 8517 } 8518 8519 /* UCVTF/SCVTF - Integer to FP conversion */ 8520 static void handle_simd_shift_intfp_conv(DisasContext *s, bool is_scalar, 8521 bool is_q, bool is_u, 8522 int immh, int immb, int opcode, 8523 int rn, int rd) 8524 { 8525 int size, elements, fracbits; 8526 int immhb = immh << 3 | immb; 8527 8528 if (immh & 8) { 8529 size = MO_64; 8530 if (!is_scalar && !is_q) { 8531 unallocated_encoding(s); 8532 return; 8533 } 8534 } else if (immh & 4) { 8535 size = MO_32; 8536 } else if (immh & 2) { 8537 size = MO_16; 8538 if (!dc_isar_feature(aa64_fp16, s)) { 8539 unallocated_encoding(s); 8540 return; 8541 } 8542 } else { 8543 /* immh == 0 would be a failure of the decode logic */ 8544 g_assert(immh == 1); 8545 unallocated_encoding(s); 8546 return; 8547 } 8548 8549 if (is_scalar) { 8550 elements = 1; 8551 } else { 8552 elements = (8 << is_q) >> size; 8553 } 8554 fracbits = (16 << size) - immhb; 8555 8556 if (!fp_access_check(s)) { 8557 return; 8558 } 8559 8560 handle_simd_intfp_conv(s, rd, rn, elements, !is_u, fracbits, size); 8561 } 8562 8563 /* FCVTZS, FVCVTZU - FP to fixedpoint conversion */ 8564 static void handle_simd_shift_fpint_conv(DisasContext *s, bool is_scalar, 8565 bool is_q, bool is_u, 8566 int immh, int immb, int rn, int rd) 8567 { 8568 int immhb = immh << 3 | immb; 8569 int pass, size, fracbits; 8570 TCGv_ptr tcg_fpstatus; 8571 TCGv_i32 tcg_rmode, tcg_shift; 8572 8573 if (immh & 0x8) { 8574 size = MO_64; 8575 if (!is_scalar && !is_q) { 8576 unallocated_encoding(s); 8577 return; 8578 } 8579 } else if (immh & 0x4) { 8580 size = MO_32; 8581 } else if (immh & 0x2) { 8582 size = MO_16; 8583 if (!dc_isar_feature(aa64_fp16, s)) { 8584 unallocated_encoding(s); 8585 return; 8586 } 8587 } else { 8588 /* Should have split out AdvSIMD modified immediate earlier. */ 8589 assert(immh == 1); 8590 unallocated_encoding(s); 8591 return; 8592 } 8593 8594 if (!fp_access_check(s)) { 8595 return; 8596 } 8597 8598 assert(!(is_scalar && is_q)); 8599 8600 tcg_fpstatus = fpstatus_ptr(size == MO_16 ? FPST_FPCR_F16 : FPST_FPCR); 8601 tcg_rmode = gen_set_rmode(FPROUNDING_ZERO, tcg_fpstatus); 8602 fracbits = (16 << size) - immhb; 8603 tcg_shift = tcg_constant_i32(fracbits); 8604 8605 if (size == MO_64) { 8606 int maxpass = is_scalar ? 1 : 2; 8607 8608 for (pass = 0; pass < maxpass; pass++) { 8609 TCGv_i64 tcg_op = tcg_temp_new_i64(); 8610 8611 read_vec_element(s, tcg_op, rn, pass, MO_64); 8612 if (is_u) { 8613 gen_helper_vfp_touqd(tcg_op, tcg_op, tcg_shift, tcg_fpstatus); 8614 } else { 8615 gen_helper_vfp_tosqd(tcg_op, tcg_op, tcg_shift, tcg_fpstatus); 8616 } 8617 write_vec_element(s, tcg_op, rd, pass, MO_64); 8618 } 8619 clear_vec_high(s, is_q, rd); 8620 } else { 8621 void (*fn)(TCGv_i32, TCGv_i32, TCGv_i32, TCGv_ptr); 8622 int maxpass = is_scalar ? 1 : ((8 << is_q) >> size); 8623 8624 switch (size) { 8625 case MO_16: 8626 if (is_u) { 8627 fn = gen_helper_vfp_touhh; 8628 } else { 8629 fn = gen_helper_vfp_toshh; 8630 } 8631 break; 8632 case MO_32: 8633 if (is_u) { 8634 fn = gen_helper_vfp_touls; 8635 } else { 8636 fn = gen_helper_vfp_tosls; 8637 } 8638 break; 8639 default: 8640 g_assert_not_reached(); 8641 } 8642 8643 for (pass = 0; pass < maxpass; pass++) { 8644 TCGv_i32 tcg_op = tcg_temp_new_i32(); 8645 8646 read_vec_element_i32(s, tcg_op, rn, pass, size); 8647 fn(tcg_op, tcg_op, tcg_shift, tcg_fpstatus); 8648 if (is_scalar) { 8649 write_fp_sreg(s, rd, tcg_op); 8650 } else { 8651 write_vec_element_i32(s, tcg_op, rd, pass, size); 8652 } 8653 } 8654 if (!is_scalar) { 8655 clear_vec_high(s, is_q, rd); 8656 } 8657 } 8658 8659 gen_restore_rmode(tcg_rmode, tcg_fpstatus); 8660 } 8661 8662 /* AdvSIMD scalar shift by immediate 8663 * 31 30 29 28 23 22 19 18 16 15 11 10 9 5 4 0 8664 * +-----+---+-------------+------+------+--------+---+------+------+ 8665 * | 0 1 | U | 1 1 1 1 1 0 | immh | immb | opcode | 1 | Rn | Rd | 8666 * +-----+---+-------------+------+------+--------+---+------+------+ 8667 * 8668 * This is the scalar version so it works on a fixed sized registers 8669 */ 8670 static void disas_simd_scalar_shift_imm(DisasContext *s, uint32_t insn) 8671 { 8672 int rd = extract32(insn, 0, 5); 8673 int rn = extract32(insn, 5, 5); 8674 int opcode = extract32(insn, 11, 5); 8675 int immb = extract32(insn, 16, 3); 8676 int immh = extract32(insn, 19, 4); 8677 bool is_u = extract32(insn, 29, 1); 8678 8679 if (immh == 0) { 8680 unallocated_encoding(s); 8681 return; 8682 } 8683 8684 switch (opcode) { 8685 case 0x08: /* SRI */ 8686 if (!is_u) { 8687 unallocated_encoding(s); 8688 return; 8689 } 8690 /* fall through */ 8691 case 0x00: /* SSHR / USHR */ 8692 case 0x02: /* SSRA / USRA */ 8693 case 0x04: /* SRSHR / URSHR */ 8694 case 0x06: /* SRSRA / URSRA */ 8695 handle_scalar_simd_shri(s, is_u, immh, immb, opcode, rn, rd); 8696 break; 8697 case 0x0a: /* SHL / SLI */ 8698 handle_scalar_simd_shli(s, is_u, immh, immb, opcode, rn, rd); 8699 break; 8700 case 0x1c: /* SCVTF, UCVTF */ 8701 handle_simd_shift_intfp_conv(s, true, false, is_u, immh, immb, 8702 opcode, rn, rd); 8703 break; 8704 case 0x10: /* SQSHRUN, SQSHRUN2 */ 8705 case 0x11: /* SQRSHRUN, SQRSHRUN2 */ 8706 if (!is_u) { 8707 unallocated_encoding(s); 8708 return; 8709 } 8710 handle_vec_simd_sqshrn(s, true, false, false, true, 8711 immh, immb, opcode, rn, rd); 8712 break; 8713 case 0x12: /* SQSHRN, SQSHRN2, UQSHRN */ 8714 case 0x13: /* SQRSHRN, SQRSHRN2, UQRSHRN, UQRSHRN2 */ 8715 handle_vec_simd_sqshrn(s, true, false, is_u, is_u, 8716 immh, immb, opcode, rn, rd); 8717 break; 8718 case 0xc: /* SQSHLU */ 8719 if (!is_u) { 8720 unallocated_encoding(s); 8721 return; 8722 } 8723 handle_simd_qshl(s, true, false, false, true, immh, immb, rn, rd); 8724 break; 8725 case 0xe: /* SQSHL, UQSHL */ 8726 handle_simd_qshl(s, true, false, is_u, is_u, immh, immb, rn, rd); 8727 break; 8728 case 0x1f: /* FCVTZS, FCVTZU */ 8729 handle_simd_shift_fpint_conv(s, true, false, is_u, immh, immb, rn, rd); 8730 break; 8731 default: 8732 unallocated_encoding(s); 8733 break; 8734 } 8735 } 8736 8737 /* AdvSIMD scalar three different 8738 * 31 30 29 28 24 23 22 21 20 16 15 12 11 10 9 5 4 0 8739 * +-----+---+-----------+------+---+------+--------+-----+------+------+ 8740 * | 0 1 | U | 1 1 1 1 0 | size | 1 | Rm | opcode | 0 0 | Rn | Rd | 8741 * +-----+---+-----------+------+---+------+--------+-----+------+------+ 8742 */ 8743 static void disas_simd_scalar_three_reg_diff(DisasContext *s, uint32_t insn) 8744 { 8745 bool is_u = extract32(insn, 29, 1); 8746 int size = extract32(insn, 22, 2); 8747 int opcode = extract32(insn, 12, 4); 8748 int rm = extract32(insn, 16, 5); 8749 int rn = extract32(insn, 5, 5); 8750 int rd = extract32(insn, 0, 5); 8751 8752 if (is_u) { 8753 unallocated_encoding(s); 8754 return; 8755 } 8756 8757 switch (opcode) { 8758 case 0x9: /* SQDMLAL, SQDMLAL2 */ 8759 case 0xb: /* SQDMLSL, SQDMLSL2 */ 8760 case 0xd: /* SQDMULL, SQDMULL2 */ 8761 if (size == 0 || size == 3) { 8762 unallocated_encoding(s); 8763 return; 8764 } 8765 break; 8766 default: 8767 unallocated_encoding(s); 8768 return; 8769 } 8770 8771 if (!fp_access_check(s)) { 8772 return; 8773 } 8774 8775 if (size == 2) { 8776 TCGv_i64 tcg_op1 = tcg_temp_new_i64(); 8777 TCGv_i64 tcg_op2 = tcg_temp_new_i64(); 8778 TCGv_i64 tcg_res = tcg_temp_new_i64(); 8779 8780 read_vec_element(s, tcg_op1, rn, 0, MO_32 | MO_SIGN); 8781 read_vec_element(s, tcg_op2, rm, 0, MO_32 | MO_SIGN); 8782 8783 tcg_gen_mul_i64(tcg_res, tcg_op1, tcg_op2); 8784 gen_helper_neon_addl_saturate_s64(tcg_res, cpu_env, tcg_res, tcg_res); 8785 8786 switch (opcode) { 8787 case 0xd: /* SQDMULL, SQDMULL2 */ 8788 break; 8789 case 0xb: /* SQDMLSL, SQDMLSL2 */ 8790 tcg_gen_neg_i64(tcg_res, tcg_res); 8791 /* fall through */ 8792 case 0x9: /* SQDMLAL, SQDMLAL2 */ 8793 read_vec_element(s, tcg_op1, rd, 0, MO_64); 8794 gen_helper_neon_addl_saturate_s64(tcg_res, cpu_env, 8795 tcg_res, tcg_op1); 8796 break; 8797 default: 8798 g_assert_not_reached(); 8799 } 8800 8801 write_fp_dreg(s, rd, tcg_res); 8802 } else { 8803 TCGv_i32 tcg_op1 = read_fp_hreg(s, rn); 8804 TCGv_i32 tcg_op2 = read_fp_hreg(s, rm); 8805 TCGv_i64 tcg_res = tcg_temp_new_i64(); 8806 8807 gen_helper_neon_mull_s16(tcg_res, tcg_op1, tcg_op2); 8808 gen_helper_neon_addl_saturate_s32(tcg_res, cpu_env, tcg_res, tcg_res); 8809 8810 switch (opcode) { 8811 case 0xd: /* SQDMULL, SQDMULL2 */ 8812 break; 8813 case 0xb: /* SQDMLSL, SQDMLSL2 */ 8814 gen_helper_neon_negl_u32(tcg_res, tcg_res); 8815 /* fall through */ 8816 case 0x9: /* SQDMLAL, SQDMLAL2 */ 8817 { 8818 TCGv_i64 tcg_op3 = tcg_temp_new_i64(); 8819 read_vec_element(s, tcg_op3, rd, 0, MO_32); 8820 gen_helper_neon_addl_saturate_s32(tcg_res, cpu_env, 8821 tcg_res, tcg_op3); 8822 break; 8823 } 8824 default: 8825 g_assert_not_reached(); 8826 } 8827 8828 tcg_gen_ext32u_i64(tcg_res, tcg_res); 8829 write_fp_dreg(s, rd, tcg_res); 8830 } 8831 } 8832 8833 static void handle_3same_64(DisasContext *s, int opcode, bool u, 8834 TCGv_i64 tcg_rd, TCGv_i64 tcg_rn, TCGv_i64 tcg_rm) 8835 { 8836 /* Handle 64x64->64 opcodes which are shared between the scalar 8837 * and vector 3-same groups. We cover every opcode where size == 3 8838 * is valid in either the three-reg-same (integer, not pairwise) 8839 * or scalar-three-reg-same groups. 8840 */ 8841 TCGCond cond; 8842 8843 switch (opcode) { 8844 case 0x1: /* SQADD */ 8845 if (u) { 8846 gen_helper_neon_qadd_u64(tcg_rd, cpu_env, tcg_rn, tcg_rm); 8847 } else { 8848 gen_helper_neon_qadd_s64(tcg_rd, cpu_env, tcg_rn, tcg_rm); 8849 } 8850 break; 8851 case 0x5: /* SQSUB */ 8852 if (u) { 8853 gen_helper_neon_qsub_u64(tcg_rd, cpu_env, tcg_rn, tcg_rm); 8854 } else { 8855 gen_helper_neon_qsub_s64(tcg_rd, cpu_env, tcg_rn, tcg_rm); 8856 } 8857 break; 8858 case 0x6: /* CMGT, CMHI */ 8859 /* 64 bit integer comparison, result = test ? (2^64 - 1) : 0. 8860 * We implement this using setcond (test) and then negating. 8861 */ 8862 cond = u ? TCG_COND_GTU : TCG_COND_GT; 8863 do_cmop: 8864 tcg_gen_setcond_i64(cond, tcg_rd, tcg_rn, tcg_rm); 8865 tcg_gen_neg_i64(tcg_rd, tcg_rd); 8866 break; 8867 case 0x7: /* CMGE, CMHS */ 8868 cond = u ? TCG_COND_GEU : TCG_COND_GE; 8869 goto do_cmop; 8870 case 0x11: /* CMTST, CMEQ */ 8871 if (u) { 8872 cond = TCG_COND_EQ; 8873 goto do_cmop; 8874 } 8875 gen_cmtst_i64(tcg_rd, tcg_rn, tcg_rm); 8876 break; 8877 case 0x8: /* SSHL, USHL */ 8878 if (u) { 8879 gen_ushl_i64(tcg_rd, tcg_rn, tcg_rm); 8880 } else { 8881 gen_sshl_i64(tcg_rd, tcg_rn, tcg_rm); 8882 } 8883 break; 8884 case 0x9: /* SQSHL, UQSHL */ 8885 if (u) { 8886 gen_helper_neon_qshl_u64(tcg_rd, cpu_env, tcg_rn, tcg_rm); 8887 } else { 8888 gen_helper_neon_qshl_s64(tcg_rd, cpu_env, tcg_rn, tcg_rm); 8889 } 8890 break; 8891 case 0xa: /* SRSHL, URSHL */ 8892 if (u) { 8893 gen_helper_neon_rshl_u64(tcg_rd, tcg_rn, tcg_rm); 8894 } else { 8895 gen_helper_neon_rshl_s64(tcg_rd, tcg_rn, tcg_rm); 8896 } 8897 break; 8898 case 0xb: /* SQRSHL, UQRSHL */ 8899 if (u) { 8900 gen_helper_neon_qrshl_u64(tcg_rd, cpu_env, tcg_rn, tcg_rm); 8901 } else { 8902 gen_helper_neon_qrshl_s64(tcg_rd, cpu_env, tcg_rn, tcg_rm); 8903 } 8904 break; 8905 case 0x10: /* ADD, SUB */ 8906 if (u) { 8907 tcg_gen_sub_i64(tcg_rd, tcg_rn, tcg_rm); 8908 } else { 8909 tcg_gen_add_i64(tcg_rd, tcg_rn, tcg_rm); 8910 } 8911 break; 8912 default: 8913 g_assert_not_reached(); 8914 } 8915 } 8916 8917 /* Handle the 3-same-operands float operations; shared by the scalar 8918 * and vector encodings. The caller must filter out any encodings 8919 * not allocated for the encoding it is dealing with. 8920 */ 8921 static void handle_3same_float(DisasContext *s, int size, int elements, 8922 int fpopcode, int rd, int rn, int rm) 8923 { 8924 int pass; 8925 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR); 8926 8927 for (pass = 0; pass < elements; pass++) { 8928 if (size) { 8929 /* Double */ 8930 TCGv_i64 tcg_op1 = tcg_temp_new_i64(); 8931 TCGv_i64 tcg_op2 = tcg_temp_new_i64(); 8932 TCGv_i64 tcg_res = tcg_temp_new_i64(); 8933 8934 read_vec_element(s, tcg_op1, rn, pass, MO_64); 8935 read_vec_element(s, tcg_op2, rm, pass, MO_64); 8936 8937 switch (fpopcode) { 8938 case 0x39: /* FMLS */ 8939 /* As usual for ARM, separate negation for fused multiply-add */ 8940 gen_helper_vfp_negd(tcg_op1, tcg_op1); 8941 /* fall through */ 8942 case 0x19: /* FMLA */ 8943 read_vec_element(s, tcg_res, rd, pass, MO_64); 8944 gen_helper_vfp_muladdd(tcg_res, tcg_op1, tcg_op2, 8945 tcg_res, fpst); 8946 break; 8947 case 0x18: /* FMAXNM */ 8948 gen_helper_vfp_maxnumd(tcg_res, tcg_op1, tcg_op2, fpst); 8949 break; 8950 case 0x1a: /* FADD */ 8951 gen_helper_vfp_addd(tcg_res, tcg_op1, tcg_op2, fpst); 8952 break; 8953 case 0x1b: /* FMULX */ 8954 gen_helper_vfp_mulxd(tcg_res, tcg_op1, tcg_op2, fpst); 8955 break; 8956 case 0x1c: /* FCMEQ */ 8957 gen_helper_neon_ceq_f64(tcg_res, tcg_op1, tcg_op2, fpst); 8958 break; 8959 case 0x1e: /* FMAX */ 8960 gen_helper_vfp_maxd(tcg_res, tcg_op1, tcg_op2, fpst); 8961 break; 8962 case 0x1f: /* FRECPS */ 8963 gen_helper_recpsf_f64(tcg_res, tcg_op1, tcg_op2, fpst); 8964 break; 8965 case 0x38: /* FMINNM */ 8966 gen_helper_vfp_minnumd(tcg_res, tcg_op1, tcg_op2, fpst); 8967 break; 8968 case 0x3a: /* FSUB */ 8969 gen_helper_vfp_subd(tcg_res, tcg_op1, tcg_op2, fpst); 8970 break; 8971 case 0x3e: /* FMIN */ 8972 gen_helper_vfp_mind(tcg_res, tcg_op1, tcg_op2, fpst); 8973 break; 8974 case 0x3f: /* FRSQRTS */ 8975 gen_helper_rsqrtsf_f64(tcg_res, tcg_op1, tcg_op2, fpst); 8976 break; 8977 case 0x5b: /* FMUL */ 8978 gen_helper_vfp_muld(tcg_res, tcg_op1, tcg_op2, fpst); 8979 break; 8980 case 0x5c: /* FCMGE */ 8981 gen_helper_neon_cge_f64(tcg_res, tcg_op1, tcg_op2, fpst); 8982 break; 8983 case 0x5d: /* FACGE */ 8984 gen_helper_neon_acge_f64(tcg_res, tcg_op1, tcg_op2, fpst); 8985 break; 8986 case 0x5f: /* FDIV */ 8987 gen_helper_vfp_divd(tcg_res, tcg_op1, tcg_op2, fpst); 8988 break; 8989 case 0x7a: /* FABD */ 8990 gen_helper_vfp_subd(tcg_res, tcg_op1, tcg_op2, fpst); 8991 gen_helper_vfp_absd(tcg_res, tcg_res); 8992 break; 8993 case 0x7c: /* FCMGT */ 8994 gen_helper_neon_cgt_f64(tcg_res, tcg_op1, tcg_op2, fpst); 8995 break; 8996 case 0x7d: /* FACGT */ 8997 gen_helper_neon_acgt_f64(tcg_res, tcg_op1, tcg_op2, fpst); 8998 break; 8999 default: 9000 g_assert_not_reached(); 9001 } 9002 9003 write_vec_element(s, tcg_res, rd, pass, MO_64); 9004 } else { 9005 /* Single */ 9006 TCGv_i32 tcg_op1 = tcg_temp_new_i32(); 9007 TCGv_i32 tcg_op2 = tcg_temp_new_i32(); 9008 TCGv_i32 tcg_res = tcg_temp_new_i32(); 9009 9010 read_vec_element_i32(s, tcg_op1, rn, pass, MO_32); 9011 read_vec_element_i32(s, tcg_op2, rm, pass, MO_32); 9012 9013 switch (fpopcode) { 9014 case 0x39: /* FMLS */ 9015 /* As usual for ARM, separate negation for fused multiply-add */ 9016 gen_helper_vfp_negs(tcg_op1, tcg_op1); 9017 /* fall through */ 9018 case 0x19: /* FMLA */ 9019 read_vec_element_i32(s, tcg_res, rd, pass, MO_32); 9020 gen_helper_vfp_muladds(tcg_res, tcg_op1, tcg_op2, 9021 tcg_res, fpst); 9022 break; 9023 case 0x1a: /* FADD */ 9024 gen_helper_vfp_adds(tcg_res, tcg_op1, tcg_op2, fpst); 9025 break; 9026 case 0x1b: /* FMULX */ 9027 gen_helper_vfp_mulxs(tcg_res, tcg_op1, tcg_op2, fpst); 9028 break; 9029 case 0x1c: /* FCMEQ */ 9030 gen_helper_neon_ceq_f32(tcg_res, tcg_op1, tcg_op2, fpst); 9031 break; 9032 case 0x1e: /* FMAX */ 9033 gen_helper_vfp_maxs(tcg_res, tcg_op1, tcg_op2, fpst); 9034 break; 9035 case 0x1f: /* FRECPS */ 9036 gen_helper_recpsf_f32(tcg_res, tcg_op1, tcg_op2, fpst); 9037 break; 9038 case 0x18: /* FMAXNM */ 9039 gen_helper_vfp_maxnums(tcg_res, tcg_op1, tcg_op2, fpst); 9040 break; 9041 case 0x38: /* FMINNM */ 9042 gen_helper_vfp_minnums(tcg_res, tcg_op1, tcg_op2, fpst); 9043 break; 9044 case 0x3a: /* FSUB */ 9045 gen_helper_vfp_subs(tcg_res, tcg_op1, tcg_op2, fpst); 9046 break; 9047 case 0x3e: /* FMIN */ 9048 gen_helper_vfp_mins(tcg_res, tcg_op1, tcg_op2, fpst); 9049 break; 9050 case 0x3f: /* FRSQRTS */ 9051 gen_helper_rsqrtsf_f32(tcg_res, tcg_op1, tcg_op2, fpst); 9052 break; 9053 case 0x5b: /* FMUL */ 9054 gen_helper_vfp_muls(tcg_res, tcg_op1, tcg_op2, fpst); 9055 break; 9056 case 0x5c: /* FCMGE */ 9057 gen_helper_neon_cge_f32(tcg_res, tcg_op1, tcg_op2, fpst); 9058 break; 9059 case 0x5d: /* FACGE */ 9060 gen_helper_neon_acge_f32(tcg_res, tcg_op1, tcg_op2, fpst); 9061 break; 9062 case 0x5f: /* FDIV */ 9063 gen_helper_vfp_divs(tcg_res, tcg_op1, tcg_op2, fpst); 9064 break; 9065 case 0x7a: /* FABD */ 9066 gen_helper_vfp_subs(tcg_res, tcg_op1, tcg_op2, fpst); 9067 gen_helper_vfp_abss(tcg_res, tcg_res); 9068 break; 9069 case 0x7c: /* FCMGT */ 9070 gen_helper_neon_cgt_f32(tcg_res, tcg_op1, tcg_op2, fpst); 9071 break; 9072 case 0x7d: /* FACGT */ 9073 gen_helper_neon_acgt_f32(tcg_res, tcg_op1, tcg_op2, fpst); 9074 break; 9075 default: 9076 g_assert_not_reached(); 9077 } 9078 9079 if (elements == 1) { 9080 /* scalar single so clear high part */ 9081 TCGv_i64 tcg_tmp = tcg_temp_new_i64(); 9082 9083 tcg_gen_extu_i32_i64(tcg_tmp, tcg_res); 9084 write_vec_element(s, tcg_tmp, rd, pass, MO_64); 9085 } else { 9086 write_vec_element_i32(s, tcg_res, rd, pass, MO_32); 9087 } 9088 } 9089 } 9090 9091 clear_vec_high(s, elements * (size ? 8 : 4) > 8, rd); 9092 } 9093 9094 /* AdvSIMD scalar three same 9095 * 31 30 29 28 24 23 22 21 20 16 15 11 10 9 5 4 0 9096 * +-----+---+-----------+------+---+------+--------+---+------+------+ 9097 * | 0 1 | U | 1 1 1 1 0 | size | 1 | Rm | opcode | 1 | Rn | Rd | 9098 * +-----+---+-----------+------+---+------+--------+---+------+------+ 9099 */ 9100 static void disas_simd_scalar_three_reg_same(DisasContext *s, uint32_t insn) 9101 { 9102 int rd = extract32(insn, 0, 5); 9103 int rn = extract32(insn, 5, 5); 9104 int opcode = extract32(insn, 11, 5); 9105 int rm = extract32(insn, 16, 5); 9106 int size = extract32(insn, 22, 2); 9107 bool u = extract32(insn, 29, 1); 9108 TCGv_i64 tcg_rd; 9109 9110 if (opcode >= 0x18) { 9111 /* Floating point: U, size[1] and opcode indicate operation */ 9112 int fpopcode = opcode | (extract32(size, 1, 1) << 5) | (u << 6); 9113 switch (fpopcode) { 9114 case 0x1b: /* FMULX */ 9115 case 0x1f: /* FRECPS */ 9116 case 0x3f: /* FRSQRTS */ 9117 case 0x5d: /* FACGE */ 9118 case 0x7d: /* FACGT */ 9119 case 0x1c: /* FCMEQ */ 9120 case 0x5c: /* FCMGE */ 9121 case 0x7c: /* FCMGT */ 9122 case 0x7a: /* FABD */ 9123 break; 9124 default: 9125 unallocated_encoding(s); 9126 return; 9127 } 9128 9129 if (!fp_access_check(s)) { 9130 return; 9131 } 9132 9133 handle_3same_float(s, extract32(size, 0, 1), 1, fpopcode, rd, rn, rm); 9134 return; 9135 } 9136 9137 switch (opcode) { 9138 case 0x1: /* SQADD, UQADD */ 9139 case 0x5: /* SQSUB, UQSUB */ 9140 case 0x9: /* SQSHL, UQSHL */ 9141 case 0xb: /* SQRSHL, UQRSHL */ 9142 break; 9143 case 0x8: /* SSHL, USHL */ 9144 case 0xa: /* SRSHL, URSHL */ 9145 case 0x6: /* CMGT, CMHI */ 9146 case 0x7: /* CMGE, CMHS */ 9147 case 0x11: /* CMTST, CMEQ */ 9148 case 0x10: /* ADD, SUB (vector) */ 9149 if (size != 3) { 9150 unallocated_encoding(s); 9151 return; 9152 } 9153 break; 9154 case 0x16: /* SQDMULH, SQRDMULH (vector) */ 9155 if (size != 1 && size != 2) { 9156 unallocated_encoding(s); 9157 return; 9158 } 9159 break; 9160 default: 9161 unallocated_encoding(s); 9162 return; 9163 } 9164 9165 if (!fp_access_check(s)) { 9166 return; 9167 } 9168 9169 tcg_rd = tcg_temp_new_i64(); 9170 9171 if (size == 3) { 9172 TCGv_i64 tcg_rn = read_fp_dreg(s, rn); 9173 TCGv_i64 tcg_rm = read_fp_dreg(s, rm); 9174 9175 handle_3same_64(s, opcode, u, tcg_rd, tcg_rn, tcg_rm); 9176 } else { 9177 /* Do a single operation on the lowest element in the vector. 9178 * We use the standard Neon helpers and rely on 0 OP 0 == 0 with 9179 * no side effects for all these operations. 9180 * OPTME: special-purpose helpers would avoid doing some 9181 * unnecessary work in the helper for the 8 and 16 bit cases. 9182 */ 9183 NeonGenTwoOpEnvFn *genenvfn; 9184 TCGv_i32 tcg_rn = tcg_temp_new_i32(); 9185 TCGv_i32 tcg_rm = tcg_temp_new_i32(); 9186 TCGv_i32 tcg_rd32 = tcg_temp_new_i32(); 9187 9188 read_vec_element_i32(s, tcg_rn, rn, 0, size); 9189 read_vec_element_i32(s, tcg_rm, rm, 0, size); 9190 9191 switch (opcode) { 9192 case 0x1: /* SQADD, UQADD */ 9193 { 9194 static NeonGenTwoOpEnvFn * const fns[3][2] = { 9195 { gen_helper_neon_qadd_s8, gen_helper_neon_qadd_u8 }, 9196 { gen_helper_neon_qadd_s16, gen_helper_neon_qadd_u16 }, 9197 { gen_helper_neon_qadd_s32, gen_helper_neon_qadd_u32 }, 9198 }; 9199 genenvfn = fns[size][u]; 9200 break; 9201 } 9202 case 0x5: /* SQSUB, UQSUB */ 9203 { 9204 static NeonGenTwoOpEnvFn * const fns[3][2] = { 9205 { gen_helper_neon_qsub_s8, gen_helper_neon_qsub_u8 }, 9206 { gen_helper_neon_qsub_s16, gen_helper_neon_qsub_u16 }, 9207 { gen_helper_neon_qsub_s32, gen_helper_neon_qsub_u32 }, 9208 }; 9209 genenvfn = fns[size][u]; 9210 break; 9211 } 9212 case 0x9: /* SQSHL, UQSHL */ 9213 { 9214 static NeonGenTwoOpEnvFn * const fns[3][2] = { 9215 { gen_helper_neon_qshl_s8, gen_helper_neon_qshl_u8 }, 9216 { gen_helper_neon_qshl_s16, gen_helper_neon_qshl_u16 }, 9217 { gen_helper_neon_qshl_s32, gen_helper_neon_qshl_u32 }, 9218 }; 9219 genenvfn = fns[size][u]; 9220 break; 9221 } 9222 case 0xb: /* SQRSHL, UQRSHL */ 9223 { 9224 static NeonGenTwoOpEnvFn * const fns[3][2] = { 9225 { gen_helper_neon_qrshl_s8, gen_helper_neon_qrshl_u8 }, 9226 { gen_helper_neon_qrshl_s16, gen_helper_neon_qrshl_u16 }, 9227 { gen_helper_neon_qrshl_s32, gen_helper_neon_qrshl_u32 }, 9228 }; 9229 genenvfn = fns[size][u]; 9230 break; 9231 } 9232 case 0x16: /* SQDMULH, SQRDMULH */ 9233 { 9234 static NeonGenTwoOpEnvFn * const fns[2][2] = { 9235 { gen_helper_neon_qdmulh_s16, gen_helper_neon_qrdmulh_s16 }, 9236 { gen_helper_neon_qdmulh_s32, gen_helper_neon_qrdmulh_s32 }, 9237 }; 9238 assert(size == 1 || size == 2); 9239 genenvfn = fns[size - 1][u]; 9240 break; 9241 } 9242 default: 9243 g_assert_not_reached(); 9244 } 9245 9246 genenvfn(tcg_rd32, cpu_env, tcg_rn, tcg_rm); 9247 tcg_gen_extu_i32_i64(tcg_rd, tcg_rd32); 9248 } 9249 9250 write_fp_dreg(s, rd, tcg_rd); 9251 } 9252 9253 /* AdvSIMD scalar three same FP16 9254 * 31 30 29 28 24 23 22 21 20 16 15 14 13 11 10 9 5 4 0 9255 * +-----+---+-----------+---+-----+------+-----+--------+---+----+----+ 9256 * | 0 1 | U | 1 1 1 1 0 | a | 1 0 | Rm | 0 0 | opcode | 1 | Rn | Rd | 9257 * +-----+---+-----------+---+-----+------+-----+--------+---+----+----+ 9258 * v: 0101 1110 0100 0000 0000 0100 0000 0000 => 5e400400 9259 * m: 1101 1111 0110 0000 1100 0100 0000 0000 => df60c400 9260 */ 9261 static void disas_simd_scalar_three_reg_same_fp16(DisasContext *s, 9262 uint32_t insn) 9263 { 9264 int rd = extract32(insn, 0, 5); 9265 int rn = extract32(insn, 5, 5); 9266 int opcode = extract32(insn, 11, 3); 9267 int rm = extract32(insn, 16, 5); 9268 bool u = extract32(insn, 29, 1); 9269 bool a = extract32(insn, 23, 1); 9270 int fpopcode = opcode | (a << 3) | (u << 4); 9271 TCGv_ptr fpst; 9272 TCGv_i32 tcg_op1; 9273 TCGv_i32 tcg_op2; 9274 TCGv_i32 tcg_res; 9275 9276 switch (fpopcode) { 9277 case 0x03: /* FMULX */ 9278 case 0x04: /* FCMEQ (reg) */ 9279 case 0x07: /* FRECPS */ 9280 case 0x0f: /* FRSQRTS */ 9281 case 0x14: /* FCMGE (reg) */ 9282 case 0x15: /* FACGE */ 9283 case 0x1a: /* FABD */ 9284 case 0x1c: /* FCMGT (reg) */ 9285 case 0x1d: /* FACGT */ 9286 break; 9287 default: 9288 unallocated_encoding(s); 9289 return; 9290 } 9291 9292 if (!dc_isar_feature(aa64_fp16, s)) { 9293 unallocated_encoding(s); 9294 } 9295 9296 if (!fp_access_check(s)) { 9297 return; 9298 } 9299 9300 fpst = fpstatus_ptr(FPST_FPCR_F16); 9301 9302 tcg_op1 = read_fp_hreg(s, rn); 9303 tcg_op2 = read_fp_hreg(s, rm); 9304 tcg_res = tcg_temp_new_i32(); 9305 9306 switch (fpopcode) { 9307 case 0x03: /* FMULX */ 9308 gen_helper_advsimd_mulxh(tcg_res, tcg_op1, tcg_op2, fpst); 9309 break; 9310 case 0x04: /* FCMEQ (reg) */ 9311 gen_helper_advsimd_ceq_f16(tcg_res, tcg_op1, tcg_op2, fpst); 9312 break; 9313 case 0x07: /* FRECPS */ 9314 gen_helper_recpsf_f16(tcg_res, tcg_op1, tcg_op2, fpst); 9315 break; 9316 case 0x0f: /* FRSQRTS */ 9317 gen_helper_rsqrtsf_f16(tcg_res, tcg_op1, tcg_op2, fpst); 9318 break; 9319 case 0x14: /* FCMGE (reg) */ 9320 gen_helper_advsimd_cge_f16(tcg_res, tcg_op1, tcg_op2, fpst); 9321 break; 9322 case 0x15: /* FACGE */ 9323 gen_helper_advsimd_acge_f16(tcg_res, tcg_op1, tcg_op2, fpst); 9324 break; 9325 case 0x1a: /* FABD */ 9326 gen_helper_advsimd_subh(tcg_res, tcg_op1, tcg_op2, fpst); 9327 tcg_gen_andi_i32(tcg_res, tcg_res, 0x7fff); 9328 break; 9329 case 0x1c: /* FCMGT (reg) */ 9330 gen_helper_advsimd_cgt_f16(tcg_res, tcg_op1, tcg_op2, fpst); 9331 break; 9332 case 0x1d: /* FACGT */ 9333 gen_helper_advsimd_acgt_f16(tcg_res, tcg_op1, tcg_op2, fpst); 9334 break; 9335 default: 9336 g_assert_not_reached(); 9337 } 9338 9339 write_fp_sreg(s, rd, tcg_res); 9340 } 9341 9342 /* AdvSIMD scalar three same extra 9343 * 31 30 29 28 24 23 22 21 20 16 15 14 11 10 9 5 4 0 9344 * +-----+---+-----------+------+---+------+---+--------+---+----+----+ 9345 * | 0 1 | U | 1 1 1 1 0 | size | 0 | Rm | 1 | opcode | 1 | Rn | Rd | 9346 * +-----+---+-----------+------+---+------+---+--------+---+----+----+ 9347 */ 9348 static void disas_simd_scalar_three_reg_same_extra(DisasContext *s, 9349 uint32_t insn) 9350 { 9351 int rd = extract32(insn, 0, 5); 9352 int rn = extract32(insn, 5, 5); 9353 int opcode = extract32(insn, 11, 4); 9354 int rm = extract32(insn, 16, 5); 9355 int size = extract32(insn, 22, 2); 9356 bool u = extract32(insn, 29, 1); 9357 TCGv_i32 ele1, ele2, ele3; 9358 TCGv_i64 res; 9359 bool feature; 9360 9361 switch (u * 16 + opcode) { 9362 case 0x10: /* SQRDMLAH (vector) */ 9363 case 0x11: /* SQRDMLSH (vector) */ 9364 if (size != 1 && size != 2) { 9365 unallocated_encoding(s); 9366 return; 9367 } 9368 feature = dc_isar_feature(aa64_rdm, s); 9369 break; 9370 default: 9371 unallocated_encoding(s); 9372 return; 9373 } 9374 if (!feature) { 9375 unallocated_encoding(s); 9376 return; 9377 } 9378 if (!fp_access_check(s)) { 9379 return; 9380 } 9381 9382 /* Do a single operation on the lowest element in the vector. 9383 * We use the standard Neon helpers and rely on 0 OP 0 == 0 9384 * with no side effects for all these operations. 9385 * OPTME: special-purpose helpers would avoid doing some 9386 * unnecessary work in the helper for the 16 bit cases. 9387 */ 9388 ele1 = tcg_temp_new_i32(); 9389 ele2 = tcg_temp_new_i32(); 9390 ele3 = tcg_temp_new_i32(); 9391 9392 read_vec_element_i32(s, ele1, rn, 0, size); 9393 read_vec_element_i32(s, ele2, rm, 0, size); 9394 read_vec_element_i32(s, ele3, rd, 0, size); 9395 9396 switch (opcode) { 9397 case 0x0: /* SQRDMLAH */ 9398 if (size == 1) { 9399 gen_helper_neon_qrdmlah_s16(ele3, cpu_env, ele1, ele2, ele3); 9400 } else { 9401 gen_helper_neon_qrdmlah_s32(ele3, cpu_env, ele1, ele2, ele3); 9402 } 9403 break; 9404 case 0x1: /* SQRDMLSH */ 9405 if (size == 1) { 9406 gen_helper_neon_qrdmlsh_s16(ele3, cpu_env, ele1, ele2, ele3); 9407 } else { 9408 gen_helper_neon_qrdmlsh_s32(ele3, cpu_env, ele1, ele2, ele3); 9409 } 9410 break; 9411 default: 9412 g_assert_not_reached(); 9413 } 9414 9415 res = tcg_temp_new_i64(); 9416 tcg_gen_extu_i32_i64(res, ele3); 9417 write_fp_dreg(s, rd, res); 9418 } 9419 9420 static void handle_2misc_64(DisasContext *s, int opcode, bool u, 9421 TCGv_i64 tcg_rd, TCGv_i64 tcg_rn, 9422 TCGv_i32 tcg_rmode, TCGv_ptr tcg_fpstatus) 9423 { 9424 /* Handle 64->64 opcodes which are shared between the scalar and 9425 * vector 2-reg-misc groups. We cover every integer opcode where size == 3 9426 * is valid in either group and also the double-precision fp ops. 9427 * The caller only need provide tcg_rmode and tcg_fpstatus if the op 9428 * requires them. 9429 */ 9430 TCGCond cond; 9431 9432 switch (opcode) { 9433 case 0x4: /* CLS, CLZ */ 9434 if (u) { 9435 tcg_gen_clzi_i64(tcg_rd, tcg_rn, 64); 9436 } else { 9437 tcg_gen_clrsb_i64(tcg_rd, tcg_rn); 9438 } 9439 break; 9440 case 0x5: /* NOT */ 9441 /* This opcode is shared with CNT and RBIT but we have earlier 9442 * enforced that size == 3 if and only if this is the NOT insn. 9443 */ 9444 tcg_gen_not_i64(tcg_rd, tcg_rn); 9445 break; 9446 case 0x7: /* SQABS, SQNEG */ 9447 if (u) { 9448 gen_helper_neon_qneg_s64(tcg_rd, cpu_env, tcg_rn); 9449 } else { 9450 gen_helper_neon_qabs_s64(tcg_rd, cpu_env, tcg_rn); 9451 } 9452 break; 9453 case 0xa: /* CMLT */ 9454 /* 64 bit integer comparison against zero, result is 9455 * test ? (2^64 - 1) : 0. We implement via setcond(!test) and 9456 * subtracting 1. 9457 */ 9458 cond = TCG_COND_LT; 9459 do_cmop: 9460 tcg_gen_setcondi_i64(cond, tcg_rd, tcg_rn, 0); 9461 tcg_gen_neg_i64(tcg_rd, tcg_rd); 9462 break; 9463 case 0x8: /* CMGT, CMGE */ 9464 cond = u ? TCG_COND_GE : TCG_COND_GT; 9465 goto do_cmop; 9466 case 0x9: /* CMEQ, CMLE */ 9467 cond = u ? TCG_COND_LE : TCG_COND_EQ; 9468 goto do_cmop; 9469 case 0xb: /* ABS, NEG */ 9470 if (u) { 9471 tcg_gen_neg_i64(tcg_rd, tcg_rn); 9472 } else { 9473 tcg_gen_abs_i64(tcg_rd, tcg_rn); 9474 } 9475 break; 9476 case 0x2f: /* FABS */ 9477 gen_helper_vfp_absd(tcg_rd, tcg_rn); 9478 break; 9479 case 0x6f: /* FNEG */ 9480 gen_helper_vfp_negd(tcg_rd, tcg_rn); 9481 break; 9482 case 0x7f: /* FSQRT */ 9483 gen_helper_vfp_sqrtd(tcg_rd, tcg_rn, cpu_env); 9484 break; 9485 case 0x1a: /* FCVTNS */ 9486 case 0x1b: /* FCVTMS */ 9487 case 0x1c: /* FCVTAS */ 9488 case 0x3a: /* FCVTPS */ 9489 case 0x3b: /* FCVTZS */ 9490 gen_helper_vfp_tosqd(tcg_rd, tcg_rn, tcg_constant_i32(0), tcg_fpstatus); 9491 break; 9492 case 0x5a: /* FCVTNU */ 9493 case 0x5b: /* FCVTMU */ 9494 case 0x5c: /* FCVTAU */ 9495 case 0x7a: /* FCVTPU */ 9496 case 0x7b: /* FCVTZU */ 9497 gen_helper_vfp_touqd(tcg_rd, tcg_rn, tcg_constant_i32(0), tcg_fpstatus); 9498 break; 9499 case 0x18: /* FRINTN */ 9500 case 0x19: /* FRINTM */ 9501 case 0x38: /* FRINTP */ 9502 case 0x39: /* FRINTZ */ 9503 case 0x58: /* FRINTA */ 9504 case 0x79: /* FRINTI */ 9505 gen_helper_rintd(tcg_rd, tcg_rn, tcg_fpstatus); 9506 break; 9507 case 0x59: /* FRINTX */ 9508 gen_helper_rintd_exact(tcg_rd, tcg_rn, tcg_fpstatus); 9509 break; 9510 case 0x1e: /* FRINT32Z */ 9511 case 0x5e: /* FRINT32X */ 9512 gen_helper_frint32_d(tcg_rd, tcg_rn, tcg_fpstatus); 9513 break; 9514 case 0x1f: /* FRINT64Z */ 9515 case 0x5f: /* FRINT64X */ 9516 gen_helper_frint64_d(tcg_rd, tcg_rn, tcg_fpstatus); 9517 break; 9518 default: 9519 g_assert_not_reached(); 9520 } 9521 } 9522 9523 static void handle_2misc_fcmp_zero(DisasContext *s, int opcode, 9524 bool is_scalar, bool is_u, bool is_q, 9525 int size, int rn, int rd) 9526 { 9527 bool is_double = (size == MO_64); 9528 TCGv_ptr fpst; 9529 9530 if (!fp_access_check(s)) { 9531 return; 9532 } 9533 9534 fpst = fpstatus_ptr(size == MO_16 ? FPST_FPCR_F16 : FPST_FPCR); 9535 9536 if (is_double) { 9537 TCGv_i64 tcg_op = tcg_temp_new_i64(); 9538 TCGv_i64 tcg_zero = tcg_constant_i64(0); 9539 TCGv_i64 tcg_res = tcg_temp_new_i64(); 9540 NeonGenTwoDoubleOpFn *genfn; 9541 bool swap = false; 9542 int pass; 9543 9544 switch (opcode) { 9545 case 0x2e: /* FCMLT (zero) */ 9546 swap = true; 9547 /* fallthrough */ 9548 case 0x2c: /* FCMGT (zero) */ 9549 genfn = gen_helper_neon_cgt_f64; 9550 break; 9551 case 0x2d: /* FCMEQ (zero) */ 9552 genfn = gen_helper_neon_ceq_f64; 9553 break; 9554 case 0x6d: /* FCMLE (zero) */ 9555 swap = true; 9556 /* fall through */ 9557 case 0x6c: /* FCMGE (zero) */ 9558 genfn = gen_helper_neon_cge_f64; 9559 break; 9560 default: 9561 g_assert_not_reached(); 9562 } 9563 9564 for (pass = 0; pass < (is_scalar ? 1 : 2); pass++) { 9565 read_vec_element(s, tcg_op, rn, pass, MO_64); 9566 if (swap) { 9567 genfn(tcg_res, tcg_zero, tcg_op, fpst); 9568 } else { 9569 genfn(tcg_res, tcg_op, tcg_zero, fpst); 9570 } 9571 write_vec_element(s, tcg_res, rd, pass, MO_64); 9572 } 9573 9574 clear_vec_high(s, !is_scalar, rd); 9575 } else { 9576 TCGv_i32 tcg_op = tcg_temp_new_i32(); 9577 TCGv_i32 tcg_zero = tcg_constant_i32(0); 9578 TCGv_i32 tcg_res = tcg_temp_new_i32(); 9579 NeonGenTwoSingleOpFn *genfn; 9580 bool swap = false; 9581 int pass, maxpasses; 9582 9583 if (size == MO_16) { 9584 switch (opcode) { 9585 case 0x2e: /* FCMLT (zero) */ 9586 swap = true; 9587 /* fall through */ 9588 case 0x2c: /* FCMGT (zero) */ 9589 genfn = gen_helper_advsimd_cgt_f16; 9590 break; 9591 case 0x2d: /* FCMEQ (zero) */ 9592 genfn = gen_helper_advsimd_ceq_f16; 9593 break; 9594 case 0x6d: /* FCMLE (zero) */ 9595 swap = true; 9596 /* fall through */ 9597 case 0x6c: /* FCMGE (zero) */ 9598 genfn = gen_helper_advsimd_cge_f16; 9599 break; 9600 default: 9601 g_assert_not_reached(); 9602 } 9603 } else { 9604 switch (opcode) { 9605 case 0x2e: /* FCMLT (zero) */ 9606 swap = true; 9607 /* fall through */ 9608 case 0x2c: /* FCMGT (zero) */ 9609 genfn = gen_helper_neon_cgt_f32; 9610 break; 9611 case 0x2d: /* FCMEQ (zero) */ 9612 genfn = gen_helper_neon_ceq_f32; 9613 break; 9614 case 0x6d: /* FCMLE (zero) */ 9615 swap = true; 9616 /* fall through */ 9617 case 0x6c: /* FCMGE (zero) */ 9618 genfn = gen_helper_neon_cge_f32; 9619 break; 9620 default: 9621 g_assert_not_reached(); 9622 } 9623 } 9624 9625 if (is_scalar) { 9626 maxpasses = 1; 9627 } else { 9628 int vector_size = 8 << is_q; 9629 maxpasses = vector_size >> size; 9630 } 9631 9632 for (pass = 0; pass < maxpasses; pass++) { 9633 read_vec_element_i32(s, tcg_op, rn, pass, size); 9634 if (swap) { 9635 genfn(tcg_res, tcg_zero, tcg_op, fpst); 9636 } else { 9637 genfn(tcg_res, tcg_op, tcg_zero, fpst); 9638 } 9639 if (is_scalar) { 9640 write_fp_sreg(s, rd, tcg_res); 9641 } else { 9642 write_vec_element_i32(s, tcg_res, rd, pass, size); 9643 } 9644 } 9645 9646 if (!is_scalar) { 9647 clear_vec_high(s, is_q, rd); 9648 } 9649 } 9650 } 9651 9652 static void handle_2misc_reciprocal(DisasContext *s, int opcode, 9653 bool is_scalar, bool is_u, bool is_q, 9654 int size, int rn, int rd) 9655 { 9656 bool is_double = (size == 3); 9657 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR); 9658 9659 if (is_double) { 9660 TCGv_i64 tcg_op = tcg_temp_new_i64(); 9661 TCGv_i64 tcg_res = tcg_temp_new_i64(); 9662 int pass; 9663 9664 for (pass = 0; pass < (is_scalar ? 1 : 2); pass++) { 9665 read_vec_element(s, tcg_op, rn, pass, MO_64); 9666 switch (opcode) { 9667 case 0x3d: /* FRECPE */ 9668 gen_helper_recpe_f64(tcg_res, tcg_op, fpst); 9669 break; 9670 case 0x3f: /* FRECPX */ 9671 gen_helper_frecpx_f64(tcg_res, tcg_op, fpst); 9672 break; 9673 case 0x7d: /* FRSQRTE */ 9674 gen_helper_rsqrte_f64(tcg_res, tcg_op, fpst); 9675 break; 9676 default: 9677 g_assert_not_reached(); 9678 } 9679 write_vec_element(s, tcg_res, rd, pass, MO_64); 9680 } 9681 clear_vec_high(s, !is_scalar, rd); 9682 } else { 9683 TCGv_i32 tcg_op = tcg_temp_new_i32(); 9684 TCGv_i32 tcg_res = tcg_temp_new_i32(); 9685 int pass, maxpasses; 9686 9687 if (is_scalar) { 9688 maxpasses = 1; 9689 } else { 9690 maxpasses = is_q ? 4 : 2; 9691 } 9692 9693 for (pass = 0; pass < maxpasses; pass++) { 9694 read_vec_element_i32(s, tcg_op, rn, pass, MO_32); 9695 9696 switch (opcode) { 9697 case 0x3c: /* URECPE */ 9698 gen_helper_recpe_u32(tcg_res, tcg_op); 9699 break; 9700 case 0x3d: /* FRECPE */ 9701 gen_helper_recpe_f32(tcg_res, tcg_op, fpst); 9702 break; 9703 case 0x3f: /* FRECPX */ 9704 gen_helper_frecpx_f32(tcg_res, tcg_op, fpst); 9705 break; 9706 case 0x7d: /* FRSQRTE */ 9707 gen_helper_rsqrte_f32(tcg_res, tcg_op, fpst); 9708 break; 9709 default: 9710 g_assert_not_reached(); 9711 } 9712 9713 if (is_scalar) { 9714 write_fp_sreg(s, rd, tcg_res); 9715 } else { 9716 write_vec_element_i32(s, tcg_res, rd, pass, MO_32); 9717 } 9718 } 9719 if (!is_scalar) { 9720 clear_vec_high(s, is_q, rd); 9721 } 9722 } 9723 } 9724 9725 static void handle_2misc_narrow(DisasContext *s, bool scalar, 9726 int opcode, bool u, bool is_q, 9727 int size, int rn, int rd) 9728 { 9729 /* Handle 2-reg-misc ops which are narrowing (so each 2*size element 9730 * in the source becomes a size element in the destination). 9731 */ 9732 int pass; 9733 TCGv_i32 tcg_res[2]; 9734 int destelt = is_q ? 2 : 0; 9735 int passes = scalar ? 1 : 2; 9736 9737 if (scalar) { 9738 tcg_res[1] = tcg_constant_i32(0); 9739 } 9740 9741 for (pass = 0; pass < passes; pass++) { 9742 TCGv_i64 tcg_op = tcg_temp_new_i64(); 9743 NeonGenNarrowFn *genfn = NULL; 9744 NeonGenNarrowEnvFn *genenvfn = NULL; 9745 9746 if (scalar) { 9747 read_vec_element(s, tcg_op, rn, pass, size + 1); 9748 } else { 9749 read_vec_element(s, tcg_op, rn, pass, MO_64); 9750 } 9751 tcg_res[pass] = tcg_temp_new_i32(); 9752 9753 switch (opcode) { 9754 case 0x12: /* XTN, SQXTUN */ 9755 { 9756 static NeonGenNarrowFn * const xtnfns[3] = { 9757 gen_helper_neon_narrow_u8, 9758 gen_helper_neon_narrow_u16, 9759 tcg_gen_extrl_i64_i32, 9760 }; 9761 static NeonGenNarrowEnvFn * const sqxtunfns[3] = { 9762 gen_helper_neon_unarrow_sat8, 9763 gen_helper_neon_unarrow_sat16, 9764 gen_helper_neon_unarrow_sat32, 9765 }; 9766 if (u) { 9767 genenvfn = sqxtunfns[size]; 9768 } else { 9769 genfn = xtnfns[size]; 9770 } 9771 break; 9772 } 9773 case 0x14: /* SQXTN, UQXTN */ 9774 { 9775 static NeonGenNarrowEnvFn * const fns[3][2] = { 9776 { gen_helper_neon_narrow_sat_s8, 9777 gen_helper_neon_narrow_sat_u8 }, 9778 { gen_helper_neon_narrow_sat_s16, 9779 gen_helper_neon_narrow_sat_u16 }, 9780 { gen_helper_neon_narrow_sat_s32, 9781 gen_helper_neon_narrow_sat_u32 }, 9782 }; 9783 genenvfn = fns[size][u]; 9784 break; 9785 } 9786 case 0x16: /* FCVTN, FCVTN2 */ 9787 /* 32 bit to 16 bit or 64 bit to 32 bit float conversion */ 9788 if (size == 2) { 9789 gen_helper_vfp_fcvtsd(tcg_res[pass], tcg_op, cpu_env); 9790 } else { 9791 TCGv_i32 tcg_lo = tcg_temp_new_i32(); 9792 TCGv_i32 tcg_hi = tcg_temp_new_i32(); 9793 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR); 9794 TCGv_i32 ahp = get_ahp_flag(); 9795 9796 tcg_gen_extr_i64_i32(tcg_lo, tcg_hi, tcg_op); 9797 gen_helper_vfp_fcvt_f32_to_f16(tcg_lo, tcg_lo, fpst, ahp); 9798 gen_helper_vfp_fcvt_f32_to_f16(tcg_hi, tcg_hi, fpst, ahp); 9799 tcg_gen_deposit_i32(tcg_res[pass], tcg_lo, tcg_hi, 16, 16); 9800 } 9801 break; 9802 case 0x36: /* BFCVTN, BFCVTN2 */ 9803 { 9804 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR); 9805 gen_helper_bfcvt_pair(tcg_res[pass], tcg_op, fpst); 9806 } 9807 break; 9808 case 0x56: /* FCVTXN, FCVTXN2 */ 9809 /* 64 bit to 32 bit float conversion 9810 * with von Neumann rounding (round to odd) 9811 */ 9812 assert(size == 2); 9813 gen_helper_fcvtx_f64_to_f32(tcg_res[pass], tcg_op, cpu_env); 9814 break; 9815 default: 9816 g_assert_not_reached(); 9817 } 9818 9819 if (genfn) { 9820 genfn(tcg_res[pass], tcg_op); 9821 } else if (genenvfn) { 9822 genenvfn(tcg_res[pass], cpu_env, tcg_op); 9823 } 9824 } 9825 9826 for (pass = 0; pass < 2; pass++) { 9827 write_vec_element_i32(s, tcg_res[pass], rd, destelt + pass, MO_32); 9828 } 9829 clear_vec_high(s, is_q, rd); 9830 } 9831 9832 /* Remaining saturating accumulating ops */ 9833 static void handle_2misc_satacc(DisasContext *s, bool is_scalar, bool is_u, 9834 bool is_q, int size, int rn, int rd) 9835 { 9836 bool is_double = (size == 3); 9837 9838 if (is_double) { 9839 TCGv_i64 tcg_rn = tcg_temp_new_i64(); 9840 TCGv_i64 tcg_rd = tcg_temp_new_i64(); 9841 int pass; 9842 9843 for (pass = 0; pass < (is_scalar ? 1 : 2); pass++) { 9844 read_vec_element(s, tcg_rn, rn, pass, MO_64); 9845 read_vec_element(s, tcg_rd, rd, pass, MO_64); 9846 9847 if (is_u) { /* USQADD */ 9848 gen_helper_neon_uqadd_s64(tcg_rd, cpu_env, tcg_rn, tcg_rd); 9849 } else { /* SUQADD */ 9850 gen_helper_neon_sqadd_u64(tcg_rd, cpu_env, tcg_rn, tcg_rd); 9851 } 9852 write_vec_element(s, tcg_rd, rd, pass, MO_64); 9853 } 9854 clear_vec_high(s, !is_scalar, rd); 9855 } else { 9856 TCGv_i32 tcg_rn = tcg_temp_new_i32(); 9857 TCGv_i32 tcg_rd = tcg_temp_new_i32(); 9858 int pass, maxpasses; 9859 9860 if (is_scalar) { 9861 maxpasses = 1; 9862 } else { 9863 maxpasses = is_q ? 4 : 2; 9864 } 9865 9866 for (pass = 0; pass < maxpasses; pass++) { 9867 if (is_scalar) { 9868 read_vec_element_i32(s, tcg_rn, rn, pass, size); 9869 read_vec_element_i32(s, tcg_rd, rd, pass, size); 9870 } else { 9871 read_vec_element_i32(s, tcg_rn, rn, pass, MO_32); 9872 read_vec_element_i32(s, tcg_rd, rd, pass, MO_32); 9873 } 9874 9875 if (is_u) { /* USQADD */ 9876 switch (size) { 9877 case 0: 9878 gen_helper_neon_uqadd_s8(tcg_rd, cpu_env, tcg_rn, tcg_rd); 9879 break; 9880 case 1: 9881 gen_helper_neon_uqadd_s16(tcg_rd, cpu_env, tcg_rn, tcg_rd); 9882 break; 9883 case 2: 9884 gen_helper_neon_uqadd_s32(tcg_rd, cpu_env, tcg_rn, tcg_rd); 9885 break; 9886 default: 9887 g_assert_not_reached(); 9888 } 9889 } else { /* SUQADD */ 9890 switch (size) { 9891 case 0: 9892 gen_helper_neon_sqadd_u8(tcg_rd, cpu_env, tcg_rn, tcg_rd); 9893 break; 9894 case 1: 9895 gen_helper_neon_sqadd_u16(tcg_rd, cpu_env, tcg_rn, tcg_rd); 9896 break; 9897 case 2: 9898 gen_helper_neon_sqadd_u32(tcg_rd, cpu_env, tcg_rn, tcg_rd); 9899 break; 9900 default: 9901 g_assert_not_reached(); 9902 } 9903 } 9904 9905 if (is_scalar) { 9906 write_vec_element(s, tcg_constant_i64(0), rd, 0, MO_64); 9907 } 9908 write_vec_element_i32(s, tcg_rd, rd, pass, MO_32); 9909 } 9910 clear_vec_high(s, is_q, rd); 9911 } 9912 } 9913 9914 /* AdvSIMD scalar two reg misc 9915 * 31 30 29 28 24 23 22 21 17 16 12 11 10 9 5 4 0 9916 * +-----+---+-----------+------+-----------+--------+-----+------+------+ 9917 * | 0 1 | U | 1 1 1 1 0 | size | 1 0 0 0 0 | opcode | 1 0 | Rn | Rd | 9918 * +-----+---+-----------+------+-----------+--------+-----+------+------+ 9919 */ 9920 static void disas_simd_scalar_two_reg_misc(DisasContext *s, uint32_t insn) 9921 { 9922 int rd = extract32(insn, 0, 5); 9923 int rn = extract32(insn, 5, 5); 9924 int opcode = extract32(insn, 12, 5); 9925 int size = extract32(insn, 22, 2); 9926 bool u = extract32(insn, 29, 1); 9927 bool is_fcvt = false; 9928 int rmode; 9929 TCGv_i32 tcg_rmode; 9930 TCGv_ptr tcg_fpstatus; 9931 9932 switch (opcode) { 9933 case 0x3: /* USQADD / SUQADD*/ 9934 if (!fp_access_check(s)) { 9935 return; 9936 } 9937 handle_2misc_satacc(s, true, u, false, size, rn, rd); 9938 return; 9939 case 0x7: /* SQABS / SQNEG */ 9940 break; 9941 case 0xa: /* CMLT */ 9942 if (u) { 9943 unallocated_encoding(s); 9944 return; 9945 } 9946 /* fall through */ 9947 case 0x8: /* CMGT, CMGE */ 9948 case 0x9: /* CMEQ, CMLE */ 9949 case 0xb: /* ABS, NEG */ 9950 if (size != 3) { 9951 unallocated_encoding(s); 9952 return; 9953 } 9954 break; 9955 case 0x12: /* SQXTUN */ 9956 if (!u) { 9957 unallocated_encoding(s); 9958 return; 9959 } 9960 /* fall through */ 9961 case 0x14: /* SQXTN, UQXTN */ 9962 if (size == 3) { 9963 unallocated_encoding(s); 9964 return; 9965 } 9966 if (!fp_access_check(s)) { 9967 return; 9968 } 9969 handle_2misc_narrow(s, true, opcode, u, false, size, rn, rd); 9970 return; 9971 case 0xc ... 0xf: 9972 case 0x16 ... 0x1d: 9973 case 0x1f: 9974 /* Floating point: U, size[1] and opcode indicate operation; 9975 * size[0] indicates single or double precision. 9976 */ 9977 opcode |= (extract32(size, 1, 1) << 5) | (u << 6); 9978 size = extract32(size, 0, 1) ? 3 : 2; 9979 switch (opcode) { 9980 case 0x2c: /* FCMGT (zero) */ 9981 case 0x2d: /* FCMEQ (zero) */ 9982 case 0x2e: /* FCMLT (zero) */ 9983 case 0x6c: /* FCMGE (zero) */ 9984 case 0x6d: /* FCMLE (zero) */ 9985 handle_2misc_fcmp_zero(s, opcode, true, u, true, size, rn, rd); 9986 return; 9987 case 0x1d: /* SCVTF */ 9988 case 0x5d: /* UCVTF */ 9989 { 9990 bool is_signed = (opcode == 0x1d); 9991 if (!fp_access_check(s)) { 9992 return; 9993 } 9994 handle_simd_intfp_conv(s, rd, rn, 1, is_signed, 0, size); 9995 return; 9996 } 9997 case 0x3d: /* FRECPE */ 9998 case 0x3f: /* FRECPX */ 9999 case 0x7d: /* FRSQRTE */ 10000 if (!fp_access_check(s)) { 10001 return; 10002 } 10003 handle_2misc_reciprocal(s, opcode, true, u, true, size, rn, rd); 10004 return; 10005 case 0x1a: /* FCVTNS */ 10006 case 0x1b: /* FCVTMS */ 10007 case 0x3a: /* FCVTPS */ 10008 case 0x3b: /* FCVTZS */ 10009 case 0x5a: /* FCVTNU */ 10010 case 0x5b: /* FCVTMU */ 10011 case 0x7a: /* FCVTPU */ 10012 case 0x7b: /* FCVTZU */ 10013 is_fcvt = true; 10014 rmode = extract32(opcode, 5, 1) | (extract32(opcode, 0, 1) << 1); 10015 break; 10016 case 0x1c: /* FCVTAS */ 10017 case 0x5c: /* FCVTAU */ 10018 /* TIEAWAY doesn't fit in the usual rounding mode encoding */ 10019 is_fcvt = true; 10020 rmode = FPROUNDING_TIEAWAY; 10021 break; 10022 case 0x56: /* FCVTXN, FCVTXN2 */ 10023 if (size == 2) { 10024 unallocated_encoding(s); 10025 return; 10026 } 10027 if (!fp_access_check(s)) { 10028 return; 10029 } 10030 handle_2misc_narrow(s, true, opcode, u, false, size - 1, rn, rd); 10031 return; 10032 default: 10033 unallocated_encoding(s); 10034 return; 10035 } 10036 break; 10037 default: 10038 unallocated_encoding(s); 10039 return; 10040 } 10041 10042 if (!fp_access_check(s)) { 10043 return; 10044 } 10045 10046 if (is_fcvt) { 10047 tcg_fpstatus = fpstatus_ptr(FPST_FPCR); 10048 tcg_rmode = gen_set_rmode(rmode, tcg_fpstatus); 10049 } else { 10050 tcg_fpstatus = NULL; 10051 tcg_rmode = NULL; 10052 } 10053 10054 if (size == 3) { 10055 TCGv_i64 tcg_rn = read_fp_dreg(s, rn); 10056 TCGv_i64 tcg_rd = tcg_temp_new_i64(); 10057 10058 handle_2misc_64(s, opcode, u, tcg_rd, tcg_rn, tcg_rmode, tcg_fpstatus); 10059 write_fp_dreg(s, rd, tcg_rd); 10060 } else { 10061 TCGv_i32 tcg_rn = tcg_temp_new_i32(); 10062 TCGv_i32 tcg_rd = tcg_temp_new_i32(); 10063 10064 read_vec_element_i32(s, tcg_rn, rn, 0, size); 10065 10066 switch (opcode) { 10067 case 0x7: /* SQABS, SQNEG */ 10068 { 10069 NeonGenOneOpEnvFn *genfn; 10070 static NeonGenOneOpEnvFn * const fns[3][2] = { 10071 { gen_helper_neon_qabs_s8, gen_helper_neon_qneg_s8 }, 10072 { gen_helper_neon_qabs_s16, gen_helper_neon_qneg_s16 }, 10073 { gen_helper_neon_qabs_s32, gen_helper_neon_qneg_s32 }, 10074 }; 10075 genfn = fns[size][u]; 10076 genfn(tcg_rd, cpu_env, tcg_rn); 10077 break; 10078 } 10079 case 0x1a: /* FCVTNS */ 10080 case 0x1b: /* FCVTMS */ 10081 case 0x1c: /* FCVTAS */ 10082 case 0x3a: /* FCVTPS */ 10083 case 0x3b: /* FCVTZS */ 10084 gen_helper_vfp_tosls(tcg_rd, tcg_rn, tcg_constant_i32(0), 10085 tcg_fpstatus); 10086 break; 10087 case 0x5a: /* FCVTNU */ 10088 case 0x5b: /* FCVTMU */ 10089 case 0x5c: /* FCVTAU */ 10090 case 0x7a: /* FCVTPU */ 10091 case 0x7b: /* FCVTZU */ 10092 gen_helper_vfp_touls(tcg_rd, tcg_rn, tcg_constant_i32(0), 10093 tcg_fpstatus); 10094 break; 10095 default: 10096 g_assert_not_reached(); 10097 } 10098 10099 write_fp_sreg(s, rd, tcg_rd); 10100 } 10101 10102 if (is_fcvt) { 10103 gen_restore_rmode(tcg_rmode, tcg_fpstatus); 10104 } 10105 } 10106 10107 /* SSHR[RA]/USHR[RA] - Vector shift right (optional rounding/accumulate) */ 10108 static void handle_vec_simd_shri(DisasContext *s, bool is_q, bool is_u, 10109 int immh, int immb, int opcode, int rn, int rd) 10110 { 10111 int size = 32 - clz32(immh) - 1; 10112 int immhb = immh << 3 | immb; 10113 int shift = 2 * (8 << size) - immhb; 10114 GVecGen2iFn *gvec_fn; 10115 10116 if (extract32(immh, 3, 1) && !is_q) { 10117 unallocated_encoding(s); 10118 return; 10119 } 10120 tcg_debug_assert(size <= 3); 10121 10122 if (!fp_access_check(s)) { 10123 return; 10124 } 10125 10126 switch (opcode) { 10127 case 0x02: /* SSRA / USRA (accumulate) */ 10128 gvec_fn = is_u ? gen_gvec_usra : gen_gvec_ssra; 10129 break; 10130 10131 case 0x08: /* SRI */ 10132 gvec_fn = gen_gvec_sri; 10133 break; 10134 10135 case 0x00: /* SSHR / USHR */ 10136 if (is_u) { 10137 if (shift == 8 << size) { 10138 /* Shift count the same size as element size produces zero. */ 10139 tcg_gen_gvec_dup_imm(size, vec_full_reg_offset(s, rd), 10140 is_q ? 16 : 8, vec_full_reg_size(s), 0); 10141 return; 10142 } 10143 gvec_fn = tcg_gen_gvec_shri; 10144 } else { 10145 /* Shift count the same size as element size produces all sign. */ 10146 if (shift == 8 << size) { 10147 shift -= 1; 10148 } 10149 gvec_fn = tcg_gen_gvec_sari; 10150 } 10151 break; 10152 10153 case 0x04: /* SRSHR / URSHR (rounding) */ 10154 gvec_fn = is_u ? gen_gvec_urshr : gen_gvec_srshr; 10155 break; 10156 10157 case 0x06: /* SRSRA / URSRA (accum + rounding) */ 10158 gvec_fn = is_u ? gen_gvec_ursra : gen_gvec_srsra; 10159 break; 10160 10161 default: 10162 g_assert_not_reached(); 10163 } 10164 10165 gen_gvec_fn2i(s, is_q, rd, rn, shift, gvec_fn, size); 10166 } 10167 10168 /* SHL/SLI - Vector shift left */ 10169 static void handle_vec_simd_shli(DisasContext *s, bool is_q, bool insert, 10170 int immh, int immb, int opcode, int rn, int rd) 10171 { 10172 int size = 32 - clz32(immh) - 1; 10173 int immhb = immh << 3 | immb; 10174 int shift = immhb - (8 << size); 10175 10176 /* Range of size is limited by decode: immh is a non-zero 4 bit field */ 10177 assert(size >= 0 && size <= 3); 10178 10179 if (extract32(immh, 3, 1) && !is_q) { 10180 unallocated_encoding(s); 10181 return; 10182 } 10183 10184 if (!fp_access_check(s)) { 10185 return; 10186 } 10187 10188 if (insert) { 10189 gen_gvec_fn2i(s, is_q, rd, rn, shift, gen_gvec_sli, size); 10190 } else { 10191 gen_gvec_fn2i(s, is_q, rd, rn, shift, tcg_gen_gvec_shli, size); 10192 } 10193 } 10194 10195 /* USHLL/SHLL - Vector shift left with widening */ 10196 static void handle_vec_simd_wshli(DisasContext *s, bool is_q, bool is_u, 10197 int immh, int immb, int opcode, int rn, int rd) 10198 { 10199 int size = 32 - clz32(immh) - 1; 10200 int immhb = immh << 3 | immb; 10201 int shift = immhb - (8 << size); 10202 int dsize = 64; 10203 int esize = 8 << size; 10204 int elements = dsize/esize; 10205 TCGv_i64 tcg_rn = tcg_temp_new_i64(); 10206 TCGv_i64 tcg_rd = tcg_temp_new_i64(); 10207 int i; 10208 10209 if (size >= 3) { 10210 unallocated_encoding(s); 10211 return; 10212 } 10213 10214 if (!fp_access_check(s)) { 10215 return; 10216 } 10217 10218 /* For the LL variants the store is larger than the load, 10219 * so if rd == rn we would overwrite parts of our input. 10220 * So load everything right now and use shifts in the main loop. 10221 */ 10222 read_vec_element(s, tcg_rn, rn, is_q ? 1 : 0, MO_64); 10223 10224 for (i = 0; i < elements; i++) { 10225 tcg_gen_shri_i64(tcg_rd, tcg_rn, i * esize); 10226 ext_and_shift_reg(tcg_rd, tcg_rd, size | (!is_u << 2), 0); 10227 tcg_gen_shli_i64(tcg_rd, tcg_rd, shift); 10228 write_vec_element(s, tcg_rd, rd, i, size + 1); 10229 } 10230 } 10231 10232 /* SHRN/RSHRN - Shift right with narrowing (and potential rounding) */ 10233 static void handle_vec_simd_shrn(DisasContext *s, bool is_q, 10234 int immh, int immb, int opcode, int rn, int rd) 10235 { 10236 int immhb = immh << 3 | immb; 10237 int size = 32 - clz32(immh) - 1; 10238 int dsize = 64; 10239 int esize = 8 << size; 10240 int elements = dsize/esize; 10241 int shift = (2 * esize) - immhb; 10242 bool round = extract32(opcode, 0, 1); 10243 TCGv_i64 tcg_rn, tcg_rd, tcg_final; 10244 TCGv_i64 tcg_round; 10245 int i; 10246 10247 if (extract32(immh, 3, 1)) { 10248 unallocated_encoding(s); 10249 return; 10250 } 10251 10252 if (!fp_access_check(s)) { 10253 return; 10254 } 10255 10256 tcg_rn = tcg_temp_new_i64(); 10257 tcg_rd = tcg_temp_new_i64(); 10258 tcg_final = tcg_temp_new_i64(); 10259 read_vec_element(s, tcg_final, rd, is_q ? 1 : 0, MO_64); 10260 10261 if (round) { 10262 tcg_round = tcg_constant_i64(1ULL << (shift - 1)); 10263 } else { 10264 tcg_round = NULL; 10265 } 10266 10267 for (i = 0; i < elements; i++) { 10268 read_vec_element(s, tcg_rn, rn, i, size+1); 10269 handle_shri_with_rndacc(tcg_rd, tcg_rn, tcg_round, 10270 false, true, size+1, shift); 10271 10272 tcg_gen_deposit_i64(tcg_final, tcg_final, tcg_rd, esize * i, esize); 10273 } 10274 10275 if (!is_q) { 10276 write_vec_element(s, tcg_final, rd, 0, MO_64); 10277 } else { 10278 write_vec_element(s, tcg_final, rd, 1, MO_64); 10279 } 10280 10281 clear_vec_high(s, is_q, rd); 10282 } 10283 10284 10285 /* AdvSIMD shift by immediate 10286 * 31 30 29 28 23 22 19 18 16 15 11 10 9 5 4 0 10287 * +---+---+---+-------------+------+------+--------+---+------+------+ 10288 * | 0 | Q | U | 0 1 1 1 1 0 | immh | immb | opcode | 1 | Rn | Rd | 10289 * +---+---+---+-------------+------+------+--------+---+------+------+ 10290 */ 10291 static void disas_simd_shift_imm(DisasContext *s, uint32_t insn) 10292 { 10293 int rd = extract32(insn, 0, 5); 10294 int rn = extract32(insn, 5, 5); 10295 int opcode = extract32(insn, 11, 5); 10296 int immb = extract32(insn, 16, 3); 10297 int immh = extract32(insn, 19, 4); 10298 bool is_u = extract32(insn, 29, 1); 10299 bool is_q = extract32(insn, 30, 1); 10300 10301 /* data_proc_simd[] has sent immh == 0 to disas_simd_mod_imm. */ 10302 assert(immh != 0); 10303 10304 switch (opcode) { 10305 case 0x08: /* SRI */ 10306 if (!is_u) { 10307 unallocated_encoding(s); 10308 return; 10309 } 10310 /* fall through */ 10311 case 0x00: /* SSHR / USHR */ 10312 case 0x02: /* SSRA / USRA (accumulate) */ 10313 case 0x04: /* SRSHR / URSHR (rounding) */ 10314 case 0x06: /* SRSRA / URSRA (accum + rounding) */ 10315 handle_vec_simd_shri(s, is_q, is_u, immh, immb, opcode, rn, rd); 10316 break; 10317 case 0x0a: /* SHL / SLI */ 10318 handle_vec_simd_shli(s, is_q, is_u, immh, immb, opcode, rn, rd); 10319 break; 10320 case 0x10: /* SHRN */ 10321 case 0x11: /* RSHRN / SQRSHRUN */ 10322 if (is_u) { 10323 handle_vec_simd_sqshrn(s, false, is_q, false, true, immh, immb, 10324 opcode, rn, rd); 10325 } else { 10326 handle_vec_simd_shrn(s, is_q, immh, immb, opcode, rn, rd); 10327 } 10328 break; 10329 case 0x12: /* SQSHRN / UQSHRN */ 10330 case 0x13: /* SQRSHRN / UQRSHRN */ 10331 handle_vec_simd_sqshrn(s, false, is_q, is_u, is_u, immh, immb, 10332 opcode, rn, rd); 10333 break; 10334 case 0x14: /* SSHLL / USHLL */ 10335 handle_vec_simd_wshli(s, is_q, is_u, immh, immb, opcode, rn, rd); 10336 break; 10337 case 0x1c: /* SCVTF / UCVTF */ 10338 handle_simd_shift_intfp_conv(s, false, is_q, is_u, immh, immb, 10339 opcode, rn, rd); 10340 break; 10341 case 0xc: /* SQSHLU */ 10342 if (!is_u) { 10343 unallocated_encoding(s); 10344 return; 10345 } 10346 handle_simd_qshl(s, false, is_q, false, true, immh, immb, rn, rd); 10347 break; 10348 case 0xe: /* SQSHL, UQSHL */ 10349 handle_simd_qshl(s, false, is_q, is_u, is_u, immh, immb, rn, rd); 10350 break; 10351 case 0x1f: /* FCVTZS/ FCVTZU */ 10352 handle_simd_shift_fpint_conv(s, false, is_q, is_u, immh, immb, rn, rd); 10353 return; 10354 default: 10355 unallocated_encoding(s); 10356 return; 10357 } 10358 } 10359 10360 /* Generate code to do a "long" addition or subtraction, ie one done in 10361 * TCGv_i64 on vector lanes twice the width specified by size. 10362 */ 10363 static void gen_neon_addl(int size, bool is_sub, TCGv_i64 tcg_res, 10364 TCGv_i64 tcg_op1, TCGv_i64 tcg_op2) 10365 { 10366 static NeonGenTwo64OpFn * const fns[3][2] = { 10367 { gen_helper_neon_addl_u16, gen_helper_neon_subl_u16 }, 10368 { gen_helper_neon_addl_u32, gen_helper_neon_subl_u32 }, 10369 { tcg_gen_add_i64, tcg_gen_sub_i64 }, 10370 }; 10371 NeonGenTwo64OpFn *genfn; 10372 assert(size < 3); 10373 10374 genfn = fns[size][is_sub]; 10375 genfn(tcg_res, tcg_op1, tcg_op2); 10376 } 10377 10378 static void handle_3rd_widening(DisasContext *s, int is_q, int is_u, int size, 10379 int opcode, int rd, int rn, int rm) 10380 { 10381 /* 3-reg-different widening insns: 64 x 64 -> 128 */ 10382 TCGv_i64 tcg_res[2]; 10383 int pass, accop; 10384 10385 tcg_res[0] = tcg_temp_new_i64(); 10386 tcg_res[1] = tcg_temp_new_i64(); 10387 10388 /* Does this op do an adding accumulate, a subtracting accumulate, 10389 * or no accumulate at all? 10390 */ 10391 switch (opcode) { 10392 case 5: 10393 case 8: 10394 case 9: 10395 accop = 1; 10396 break; 10397 case 10: 10398 case 11: 10399 accop = -1; 10400 break; 10401 default: 10402 accop = 0; 10403 break; 10404 } 10405 10406 if (accop != 0) { 10407 read_vec_element(s, tcg_res[0], rd, 0, MO_64); 10408 read_vec_element(s, tcg_res[1], rd, 1, MO_64); 10409 } 10410 10411 /* size == 2 means two 32x32->64 operations; this is worth special 10412 * casing because we can generally handle it inline. 10413 */ 10414 if (size == 2) { 10415 for (pass = 0; pass < 2; pass++) { 10416 TCGv_i64 tcg_op1 = tcg_temp_new_i64(); 10417 TCGv_i64 tcg_op2 = tcg_temp_new_i64(); 10418 TCGv_i64 tcg_passres; 10419 MemOp memop = MO_32 | (is_u ? 0 : MO_SIGN); 10420 10421 int elt = pass + is_q * 2; 10422 10423 read_vec_element(s, tcg_op1, rn, elt, memop); 10424 read_vec_element(s, tcg_op2, rm, elt, memop); 10425 10426 if (accop == 0) { 10427 tcg_passres = tcg_res[pass]; 10428 } else { 10429 tcg_passres = tcg_temp_new_i64(); 10430 } 10431 10432 switch (opcode) { 10433 case 0: /* SADDL, SADDL2, UADDL, UADDL2 */ 10434 tcg_gen_add_i64(tcg_passres, tcg_op1, tcg_op2); 10435 break; 10436 case 2: /* SSUBL, SSUBL2, USUBL, USUBL2 */ 10437 tcg_gen_sub_i64(tcg_passres, tcg_op1, tcg_op2); 10438 break; 10439 case 5: /* SABAL, SABAL2, UABAL, UABAL2 */ 10440 case 7: /* SABDL, SABDL2, UABDL, UABDL2 */ 10441 { 10442 TCGv_i64 tcg_tmp1 = tcg_temp_new_i64(); 10443 TCGv_i64 tcg_tmp2 = tcg_temp_new_i64(); 10444 10445 tcg_gen_sub_i64(tcg_tmp1, tcg_op1, tcg_op2); 10446 tcg_gen_sub_i64(tcg_tmp2, tcg_op2, tcg_op1); 10447 tcg_gen_movcond_i64(is_u ? TCG_COND_GEU : TCG_COND_GE, 10448 tcg_passres, 10449 tcg_op1, tcg_op2, tcg_tmp1, tcg_tmp2); 10450 break; 10451 } 10452 case 8: /* SMLAL, SMLAL2, UMLAL, UMLAL2 */ 10453 case 10: /* SMLSL, SMLSL2, UMLSL, UMLSL2 */ 10454 case 12: /* UMULL, UMULL2, SMULL, SMULL2 */ 10455 tcg_gen_mul_i64(tcg_passres, tcg_op1, tcg_op2); 10456 break; 10457 case 9: /* SQDMLAL, SQDMLAL2 */ 10458 case 11: /* SQDMLSL, SQDMLSL2 */ 10459 case 13: /* SQDMULL, SQDMULL2 */ 10460 tcg_gen_mul_i64(tcg_passres, tcg_op1, tcg_op2); 10461 gen_helper_neon_addl_saturate_s64(tcg_passres, cpu_env, 10462 tcg_passres, tcg_passres); 10463 break; 10464 default: 10465 g_assert_not_reached(); 10466 } 10467 10468 if (opcode == 9 || opcode == 11) { 10469 /* saturating accumulate ops */ 10470 if (accop < 0) { 10471 tcg_gen_neg_i64(tcg_passres, tcg_passres); 10472 } 10473 gen_helper_neon_addl_saturate_s64(tcg_res[pass], cpu_env, 10474 tcg_res[pass], tcg_passres); 10475 } else if (accop > 0) { 10476 tcg_gen_add_i64(tcg_res[pass], tcg_res[pass], tcg_passres); 10477 } else if (accop < 0) { 10478 tcg_gen_sub_i64(tcg_res[pass], tcg_res[pass], tcg_passres); 10479 } 10480 } 10481 } else { 10482 /* size 0 or 1, generally helper functions */ 10483 for (pass = 0; pass < 2; pass++) { 10484 TCGv_i32 tcg_op1 = tcg_temp_new_i32(); 10485 TCGv_i32 tcg_op2 = tcg_temp_new_i32(); 10486 TCGv_i64 tcg_passres; 10487 int elt = pass + is_q * 2; 10488 10489 read_vec_element_i32(s, tcg_op1, rn, elt, MO_32); 10490 read_vec_element_i32(s, tcg_op2, rm, elt, MO_32); 10491 10492 if (accop == 0) { 10493 tcg_passres = tcg_res[pass]; 10494 } else { 10495 tcg_passres = tcg_temp_new_i64(); 10496 } 10497 10498 switch (opcode) { 10499 case 0: /* SADDL, SADDL2, UADDL, UADDL2 */ 10500 case 2: /* SSUBL, SSUBL2, USUBL, USUBL2 */ 10501 { 10502 TCGv_i64 tcg_op2_64 = tcg_temp_new_i64(); 10503 static NeonGenWidenFn * const widenfns[2][2] = { 10504 { gen_helper_neon_widen_s8, gen_helper_neon_widen_u8 }, 10505 { gen_helper_neon_widen_s16, gen_helper_neon_widen_u16 }, 10506 }; 10507 NeonGenWidenFn *widenfn = widenfns[size][is_u]; 10508 10509 widenfn(tcg_op2_64, tcg_op2); 10510 widenfn(tcg_passres, tcg_op1); 10511 gen_neon_addl(size, (opcode == 2), tcg_passres, 10512 tcg_passres, tcg_op2_64); 10513 break; 10514 } 10515 case 5: /* SABAL, SABAL2, UABAL, UABAL2 */ 10516 case 7: /* SABDL, SABDL2, UABDL, UABDL2 */ 10517 if (size == 0) { 10518 if (is_u) { 10519 gen_helper_neon_abdl_u16(tcg_passres, tcg_op1, tcg_op2); 10520 } else { 10521 gen_helper_neon_abdl_s16(tcg_passres, tcg_op1, tcg_op2); 10522 } 10523 } else { 10524 if (is_u) { 10525 gen_helper_neon_abdl_u32(tcg_passres, tcg_op1, tcg_op2); 10526 } else { 10527 gen_helper_neon_abdl_s32(tcg_passres, tcg_op1, tcg_op2); 10528 } 10529 } 10530 break; 10531 case 8: /* SMLAL, SMLAL2, UMLAL, UMLAL2 */ 10532 case 10: /* SMLSL, SMLSL2, UMLSL, UMLSL2 */ 10533 case 12: /* UMULL, UMULL2, SMULL, SMULL2 */ 10534 if (size == 0) { 10535 if (is_u) { 10536 gen_helper_neon_mull_u8(tcg_passres, tcg_op1, tcg_op2); 10537 } else { 10538 gen_helper_neon_mull_s8(tcg_passres, tcg_op1, tcg_op2); 10539 } 10540 } else { 10541 if (is_u) { 10542 gen_helper_neon_mull_u16(tcg_passres, tcg_op1, tcg_op2); 10543 } else { 10544 gen_helper_neon_mull_s16(tcg_passres, tcg_op1, tcg_op2); 10545 } 10546 } 10547 break; 10548 case 9: /* SQDMLAL, SQDMLAL2 */ 10549 case 11: /* SQDMLSL, SQDMLSL2 */ 10550 case 13: /* SQDMULL, SQDMULL2 */ 10551 assert(size == 1); 10552 gen_helper_neon_mull_s16(tcg_passres, tcg_op1, tcg_op2); 10553 gen_helper_neon_addl_saturate_s32(tcg_passres, cpu_env, 10554 tcg_passres, tcg_passres); 10555 break; 10556 default: 10557 g_assert_not_reached(); 10558 } 10559 10560 if (accop != 0) { 10561 if (opcode == 9 || opcode == 11) { 10562 /* saturating accumulate ops */ 10563 if (accop < 0) { 10564 gen_helper_neon_negl_u32(tcg_passres, tcg_passres); 10565 } 10566 gen_helper_neon_addl_saturate_s32(tcg_res[pass], cpu_env, 10567 tcg_res[pass], 10568 tcg_passres); 10569 } else { 10570 gen_neon_addl(size, (accop < 0), tcg_res[pass], 10571 tcg_res[pass], tcg_passres); 10572 } 10573 } 10574 } 10575 } 10576 10577 write_vec_element(s, tcg_res[0], rd, 0, MO_64); 10578 write_vec_element(s, tcg_res[1], rd, 1, MO_64); 10579 } 10580 10581 static void handle_3rd_wide(DisasContext *s, int is_q, int is_u, int size, 10582 int opcode, int rd, int rn, int rm) 10583 { 10584 TCGv_i64 tcg_res[2]; 10585 int part = is_q ? 2 : 0; 10586 int pass; 10587 10588 for (pass = 0; pass < 2; pass++) { 10589 TCGv_i64 tcg_op1 = tcg_temp_new_i64(); 10590 TCGv_i32 tcg_op2 = tcg_temp_new_i32(); 10591 TCGv_i64 tcg_op2_wide = tcg_temp_new_i64(); 10592 static NeonGenWidenFn * const widenfns[3][2] = { 10593 { gen_helper_neon_widen_s8, gen_helper_neon_widen_u8 }, 10594 { gen_helper_neon_widen_s16, gen_helper_neon_widen_u16 }, 10595 { tcg_gen_ext_i32_i64, tcg_gen_extu_i32_i64 }, 10596 }; 10597 NeonGenWidenFn *widenfn = widenfns[size][is_u]; 10598 10599 read_vec_element(s, tcg_op1, rn, pass, MO_64); 10600 read_vec_element_i32(s, tcg_op2, rm, part + pass, MO_32); 10601 widenfn(tcg_op2_wide, tcg_op2); 10602 tcg_res[pass] = tcg_temp_new_i64(); 10603 gen_neon_addl(size, (opcode == 3), 10604 tcg_res[pass], tcg_op1, tcg_op2_wide); 10605 } 10606 10607 for (pass = 0; pass < 2; pass++) { 10608 write_vec_element(s, tcg_res[pass], rd, pass, MO_64); 10609 } 10610 } 10611 10612 static void do_narrow_round_high_u32(TCGv_i32 res, TCGv_i64 in) 10613 { 10614 tcg_gen_addi_i64(in, in, 1U << 31); 10615 tcg_gen_extrh_i64_i32(res, in); 10616 } 10617 10618 static void handle_3rd_narrowing(DisasContext *s, int is_q, int is_u, int size, 10619 int opcode, int rd, int rn, int rm) 10620 { 10621 TCGv_i32 tcg_res[2]; 10622 int part = is_q ? 2 : 0; 10623 int pass; 10624 10625 for (pass = 0; pass < 2; pass++) { 10626 TCGv_i64 tcg_op1 = tcg_temp_new_i64(); 10627 TCGv_i64 tcg_op2 = tcg_temp_new_i64(); 10628 TCGv_i64 tcg_wideres = tcg_temp_new_i64(); 10629 static NeonGenNarrowFn * const narrowfns[3][2] = { 10630 { gen_helper_neon_narrow_high_u8, 10631 gen_helper_neon_narrow_round_high_u8 }, 10632 { gen_helper_neon_narrow_high_u16, 10633 gen_helper_neon_narrow_round_high_u16 }, 10634 { tcg_gen_extrh_i64_i32, do_narrow_round_high_u32 }, 10635 }; 10636 NeonGenNarrowFn *gennarrow = narrowfns[size][is_u]; 10637 10638 read_vec_element(s, tcg_op1, rn, pass, MO_64); 10639 read_vec_element(s, tcg_op2, rm, pass, MO_64); 10640 10641 gen_neon_addl(size, (opcode == 6), tcg_wideres, tcg_op1, tcg_op2); 10642 10643 tcg_res[pass] = tcg_temp_new_i32(); 10644 gennarrow(tcg_res[pass], tcg_wideres); 10645 } 10646 10647 for (pass = 0; pass < 2; pass++) { 10648 write_vec_element_i32(s, tcg_res[pass], rd, pass + part, MO_32); 10649 } 10650 clear_vec_high(s, is_q, rd); 10651 } 10652 10653 /* AdvSIMD three different 10654 * 31 30 29 28 24 23 22 21 20 16 15 12 11 10 9 5 4 0 10655 * +---+---+---+-----------+------+---+------+--------+-----+------+------+ 10656 * | 0 | Q | U | 0 1 1 1 0 | size | 1 | Rm | opcode | 0 0 | Rn | Rd | 10657 * +---+---+---+-----------+------+---+------+--------+-----+------+------+ 10658 */ 10659 static void disas_simd_three_reg_diff(DisasContext *s, uint32_t insn) 10660 { 10661 /* Instructions in this group fall into three basic classes 10662 * (in each case with the operation working on each element in 10663 * the input vectors): 10664 * (1) widening 64 x 64 -> 128 (with possibly Vd as an extra 10665 * 128 bit input) 10666 * (2) wide 64 x 128 -> 128 10667 * (3) narrowing 128 x 128 -> 64 10668 * Here we do initial decode, catch unallocated cases and 10669 * dispatch to separate functions for each class. 10670 */ 10671 int is_q = extract32(insn, 30, 1); 10672 int is_u = extract32(insn, 29, 1); 10673 int size = extract32(insn, 22, 2); 10674 int opcode = extract32(insn, 12, 4); 10675 int rm = extract32(insn, 16, 5); 10676 int rn = extract32(insn, 5, 5); 10677 int rd = extract32(insn, 0, 5); 10678 10679 switch (opcode) { 10680 case 1: /* SADDW, SADDW2, UADDW, UADDW2 */ 10681 case 3: /* SSUBW, SSUBW2, USUBW, USUBW2 */ 10682 /* 64 x 128 -> 128 */ 10683 if (size == 3) { 10684 unallocated_encoding(s); 10685 return; 10686 } 10687 if (!fp_access_check(s)) { 10688 return; 10689 } 10690 handle_3rd_wide(s, is_q, is_u, size, opcode, rd, rn, rm); 10691 break; 10692 case 4: /* ADDHN, ADDHN2, RADDHN, RADDHN2 */ 10693 case 6: /* SUBHN, SUBHN2, RSUBHN, RSUBHN2 */ 10694 /* 128 x 128 -> 64 */ 10695 if (size == 3) { 10696 unallocated_encoding(s); 10697 return; 10698 } 10699 if (!fp_access_check(s)) { 10700 return; 10701 } 10702 handle_3rd_narrowing(s, is_q, is_u, size, opcode, rd, rn, rm); 10703 break; 10704 case 14: /* PMULL, PMULL2 */ 10705 if (is_u) { 10706 unallocated_encoding(s); 10707 return; 10708 } 10709 switch (size) { 10710 case 0: /* PMULL.P8 */ 10711 if (!fp_access_check(s)) { 10712 return; 10713 } 10714 /* The Q field specifies lo/hi half input for this insn. */ 10715 gen_gvec_op3_ool(s, true, rd, rn, rm, is_q, 10716 gen_helper_neon_pmull_h); 10717 break; 10718 10719 case 3: /* PMULL.P64 */ 10720 if (!dc_isar_feature(aa64_pmull, s)) { 10721 unallocated_encoding(s); 10722 return; 10723 } 10724 if (!fp_access_check(s)) { 10725 return; 10726 } 10727 /* The Q field specifies lo/hi half input for this insn. */ 10728 gen_gvec_op3_ool(s, true, rd, rn, rm, is_q, 10729 gen_helper_gvec_pmull_q); 10730 break; 10731 10732 default: 10733 unallocated_encoding(s); 10734 break; 10735 } 10736 return; 10737 case 9: /* SQDMLAL, SQDMLAL2 */ 10738 case 11: /* SQDMLSL, SQDMLSL2 */ 10739 case 13: /* SQDMULL, SQDMULL2 */ 10740 if (is_u || size == 0) { 10741 unallocated_encoding(s); 10742 return; 10743 } 10744 /* fall through */ 10745 case 0: /* SADDL, SADDL2, UADDL, UADDL2 */ 10746 case 2: /* SSUBL, SSUBL2, USUBL, USUBL2 */ 10747 case 5: /* SABAL, SABAL2, UABAL, UABAL2 */ 10748 case 7: /* SABDL, SABDL2, UABDL, UABDL2 */ 10749 case 8: /* SMLAL, SMLAL2, UMLAL, UMLAL2 */ 10750 case 10: /* SMLSL, SMLSL2, UMLSL, UMLSL2 */ 10751 case 12: /* SMULL, SMULL2, UMULL, UMULL2 */ 10752 /* 64 x 64 -> 128 */ 10753 if (size == 3) { 10754 unallocated_encoding(s); 10755 return; 10756 } 10757 if (!fp_access_check(s)) { 10758 return; 10759 } 10760 10761 handle_3rd_widening(s, is_q, is_u, size, opcode, rd, rn, rm); 10762 break; 10763 default: 10764 /* opcode 15 not allocated */ 10765 unallocated_encoding(s); 10766 break; 10767 } 10768 } 10769 10770 /* Logic op (opcode == 3) subgroup of C3.6.16. */ 10771 static void disas_simd_3same_logic(DisasContext *s, uint32_t insn) 10772 { 10773 int rd = extract32(insn, 0, 5); 10774 int rn = extract32(insn, 5, 5); 10775 int rm = extract32(insn, 16, 5); 10776 int size = extract32(insn, 22, 2); 10777 bool is_u = extract32(insn, 29, 1); 10778 bool is_q = extract32(insn, 30, 1); 10779 10780 if (!fp_access_check(s)) { 10781 return; 10782 } 10783 10784 switch (size + 4 * is_u) { 10785 case 0: /* AND */ 10786 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_and, 0); 10787 return; 10788 case 1: /* BIC */ 10789 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_andc, 0); 10790 return; 10791 case 2: /* ORR */ 10792 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_or, 0); 10793 return; 10794 case 3: /* ORN */ 10795 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_orc, 0); 10796 return; 10797 case 4: /* EOR */ 10798 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_xor, 0); 10799 return; 10800 10801 case 5: /* BSL bitwise select */ 10802 gen_gvec_fn4(s, is_q, rd, rd, rn, rm, tcg_gen_gvec_bitsel, 0); 10803 return; 10804 case 6: /* BIT, bitwise insert if true */ 10805 gen_gvec_fn4(s, is_q, rd, rm, rn, rd, tcg_gen_gvec_bitsel, 0); 10806 return; 10807 case 7: /* BIF, bitwise insert if false */ 10808 gen_gvec_fn4(s, is_q, rd, rm, rd, rn, tcg_gen_gvec_bitsel, 0); 10809 return; 10810 10811 default: 10812 g_assert_not_reached(); 10813 } 10814 } 10815 10816 /* Pairwise op subgroup of C3.6.16. 10817 * 10818 * This is called directly or via the handle_3same_float for float pairwise 10819 * operations where the opcode and size are calculated differently. 10820 */ 10821 static void handle_simd_3same_pair(DisasContext *s, int is_q, int u, int opcode, 10822 int size, int rn, int rm, int rd) 10823 { 10824 TCGv_ptr fpst; 10825 int pass; 10826 10827 /* Floating point operations need fpst */ 10828 if (opcode >= 0x58) { 10829 fpst = fpstatus_ptr(FPST_FPCR); 10830 } else { 10831 fpst = NULL; 10832 } 10833 10834 if (!fp_access_check(s)) { 10835 return; 10836 } 10837 10838 /* These operations work on the concatenated rm:rn, with each pair of 10839 * adjacent elements being operated on to produce an element in the result. 10840 */ 10841 if (size == 3) { 10842 TCGv_i64 tcg_res[2]; 10843 10844 for (pass = 0; pass < 2; pass++) { 10845 TCGv_i64 tcg_op1 = tcg_temp_new_i64(); 10846 TCGv_i64 tcg_op2 = tcg_temp_new_i64(); 10847 int passreg = (pass == 0) ? rn : rm; 10848 10849 read_vec_element(s, tcg_op1, passreg, 0, MO_64); 10850 read_vec_element(s, tcg_op2, passreg, 1, MO_64); 10851 tcg_res[pass] = tcg_temp_new_i64(); 10852 10853 switch (opcode) { 10854 case 0x17: /* ADDP */ 10855 tcg_gen_add_i64(tcg_res[pass], tcg_op1, tcg_op2); 10856 break; 10857 case 0x58: /* FMAXNMP */ 10858 gen_helper_vfp_maxnumd(tcg_res[pass], tcg_op1, tcg_op2, fpst); 10859 break; 10860 case 0x5a: /* FADDP */ 10861 gen_helper_vfp_addd(tcg_res[pass], tcg_op1, tcg_op2, fpst); 10862 break; 10863 case 0x5e: /* FMAXP */ 10864 gen_helper_vfp_maxd(tcg_res[pass], tcg_op1, tcg_op2, fpst); 10865 break; 10866 case 0x78: /* FMINNMP */ 10867 gen_helper_vfp_minnumd(tcg_res[pass], tcg_op1, tcg_op2, fpst); 10868 break; 10869 case 0x7e: /* FMINP */ 10870 gen_helper_vfp_mind(tcg_res[pass], tcg_op1, tcg_op2, fpst); 10871 break; 10872 default: 10873 g_assert_not_reached(); 10874 } 10875 } 10876 10877 for (pass = 0; pass < 2; pass++) { 10878 write_vec_element(s, tcg_res[pass], rd, pass, MO_64); 10879 } 10880 } else { 10881 int maxpass = is_q ? 4 : 2; 10882 TCGv_i32 tcg_res[4]; 10883 10884 for (pass = 0; pass < maxpass; pass++) { 10885 TCGv_i32 tcg_op1 = tcg_temp_new_i32(); 10886 TCGv_i32 tcg_op2 = tcg_temp_new_i32(); 10887 NeonGenTwoOpFn *genfn = NULL; 10888 int passreg = pass < (maxpass / 2) ? rn : rm; 10889 int passelt = (is_q && (pass & 1)) ? 2 : 0; 10890 10891 read_vec_element_i32(s, tcg_op1, passreg, passelt, MO_32); 10892 read_vec_element_i32(s, tcg_op2, passreg, passelt + 1, MO_32); 10893 tcg_res[pass] = tcg_temp_new_i32(); 10894 10895 switch (opcode) { 10896 case 0x17: /* ADDP */ 10897 { 10898 static NeonGenTwoOpFn * const fns[3] = { 10899 gen_helper_neon_padd_u8, 10900 gen_helper_neon_padd_u16, 10901 tcg_gen_add_i32, 10902 }; 10903 genfn = fns[size]; 10904 break; 10905 } 10906 case 0x14: /* SMAXP, UMAXP */ 10907 { 10908 static NeonGenTwoOpFn * const fns[3][2] = { 10909 { gen_helper_neon_pmax_s8, gen_helper_neon_pmax_u8 }, 10910 { gen_helper_neon_pmax_s16, gen_helper_neon_pmax_u16 }, 10911 { tcg_gen_smax_i32, tcg_gen_umax_i32 }, 10912 }; 10913 genfn = fns[size][u]; 10914 break; 10915 } 10916 case 0x15: /* SMINP, UMINP */ 10917 { 10918 static NeonGenTwoOpFn * const fns[3][2] = { 10919 { gen_helper_neon_pmin_s8, gen_helper_neon_pmin_u8 }, 10920 { gen_helper_neon_pmin_s16, gen_helper_neon_pmin_u16 }, 10921 { tcg_gen_smin_i32, tcg_gen_umin_i32 }, 10922 }; 10923 genfn = fns[size][u]; 10924 break; 10925 } 10926 /* The FP operations are all on single floats (32 bit) */ 10927 case 0x58: /* FMAXNMP */ 10928 gen_helper_vfp_maxnums(tcg_res[pass], tcg_op1, tcg_op2, fpst); 10929 break; 10930 case 0x5a: /* FADDP */ 10931 gen_helper_vfp_adds(tcg_res[pass], tcg_op1, tcg_op2, fpst); 10932 break; 10933 case 0x5e: /* FMAXP */ 10934 gen_helper_vfp_maxs(tcg_res[pass], tcg_op1, tcg_op2, fpst); 10935 break; 10936 case 0x78: /* FMINNMP */ 10937 gen_helper_vfp_minnums(tcg_res[pass], tcg_op1, tcg_op2, fpst); 10938 break; 10939 case 0x7e: /* FMINP */ 10940 gen_helper_vfp_mins(tcg_res[pass], tcg_op1, tcg_op2, fpst); 10941 break; 10942 default: 10943 g_assert_not_reached(); 10944 } 10945 10946 /* FP ops called directly, otherwise call now */ 10947 if (genfn) { 10948 genfn(tcg_res[pass], tcg_op1, tcg_op2); 10949 } 10950 } 10951 10952 for (pass = 0; pass < maxpass; pass++) { 10953 write_vec_element_i32(s, tcg_res[pass], rd, pass, MO_32); 10954 } 10955 clear_vec_high(s, is_q, rd); 10956 } 10957 } 10958 10959 /* Floating point op subgroup of C3.6.16. */ 10960 static void disas_simd_3same_float(DisasContext *s, uint32_t insn) 10961 { 10962 /* For floating point ops, the U, size[1] and opcode bits 10963 * together indicate the operation. size[0] indicates single 10964 * or double. 10965 */ 10966 int fpopcode = extract32(insn, 11, 5) 10967 | (extract32(insn, 23, 1) << 5) 10968 | (extract32(insn, 29, 1) << 6); 10969 int is_q = extract32(insn, 30, 1); 10970 int size = extract32(insn, 22, 1); 10971 int rm = extract32(insn, 16, 5); 10972 int rn = extract32(insn, 5, 5); 10973 int rd = extract32(insn, 0, 5); 10974 10975 int datasize = is_q ? 128 : 64; 10976 int esize = 32 << size; 10977 int elements = datasize / esize; 10978 10979 if (size == 1 && !is_q) { 10980 unallocated_encoding(s); 10981 return; 10982 } 10983 10984 switch (fpopcode) { 10985 case 0x58: /* FMAXNMP */ 10986 case 0x5a: /* FADDP */ 10987 case 0x5e: /* FMAXP */ 10988 case 0x78: /* FMINNMP */ 10989 case 0x7e: /* FMINP */ 10990 if (size && !is_q) { 10991 unallocated_encoding(s); 10992 return; 10993 } 10994 handle_simd_3same_pair(s, is_q, 0, fpopcode, size ? MO_64 : MO_32, 10995 rn, rm, rd); 10996 return; 10997 case 0x1b: /* FMULX */ 10998 case 0x1f: /* FRECPS */ 10999 case 0x3f: /* FRSQRTS */ 11000 case 0x5d: /* FACGE */ 11001 case 0x7d: /* FACGT */ 11002 case 0x19: /* FMLA */ 11003 case 0x39: /* FMLS */ 11004 case 0x18: /* FMAXNM */ 11005 case 0x1a: /* FADD */ 11006 case 0x1c: /* FCMEQ */ 11007 case 0x1e: /* FMAX */ 11008 case 0x38: /* FMINNM */ 11009 case 0x3a: /* FSUB */ 11010 case 0x3e: /* FMIN */ 11011 case 0x5b: /* FMUL */ 11012 case 0x5c: /* FCMGE */ 11013 case 0x5f: /* FDIV */ 11014 case 0x7a: /* FABD */ 11015 case 0x7c: /* FCMGT */ 11016 if (!fp_access_check(s)) { 11017 return; 11018 } 11019 handle_3same_float(s, size, elements, fpopcode, rd, rn, rm); 11020 return; 11021 11022 case 0x1d: /* FMLAL */ 11023 case 0x3d: /* FMLSL */ 11024 case 0x59: /* FMLAL2 */ 11025 case 0x79: /* FMLSL2 */ 11026 if (size & 1 || !dc_isar_feature(aa64_fhm, s)) { 11027 unallocated_encoding(s); 11028 return; 11029 } 11030 if (fp_access_check(s)) { 11031 int is_s = extract32(insn, 23, 1); 11032 int is_2 = extract32(insn, 29, 1); 11033 int data = (is_2 << 1) | is_s; 11034 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd), 11035 vec_full_reg_offset(s, rn), 11036 vec_full_reg_offset(s, rm), cpu_env, 11037 is_q ? 16 : 8, vec_full_reg_size(s), 11038 data, gen_helper_gvec_fmlal_a64); 11039 } 11040 return; 11041 11042 default: 11043 unallocated_encoding(s); 11044 return; 11045 } 11046 } 11047 11048 /* Integer op subgroup of C3.6.16. */ 11049 static void disas_simd_3same_int(DisasContext *s, uint32_t insn) 11050 { 11051 int is_q = extract32(insn, 30, 1); 11052 int u = extract32(insn, 29, 1); 11053 int size = extract32(insn, 22, 2); 11054 int opcode = extract32(insn, 11, 5); 11055 int rm = extract32(insn, 16, 5); 11056 int rn = extract32(insn, 5, 5); 11057 int rd = extract32(insn, 0, 5); 11058 int pass; 11059 TCGCond cond; 11060 11061 switch (opcode) { 11062 case 0x13: /* MUL, PMUL */ 11063 if (u && size != 0) { 11064 unallocated_encoding(s); 11065 return; 11066 } 11067 /* fall through */ 11068 case 0x0: /* SHADD, UHADD */ 11069 case 0x2: /* SRHADD, URHADD */ 11070 case 0x4: /* SHSUB, UHSUB */ 11071 case 0xc: /* SMAX, UMAX */ 11072 case 0xd: /* SMIN, UMIN */ 11073 case 0xe: /* SABD, UABD */ 11074 case 0xf: /* SABA, UABA */ 11075 case 0x12: /* MLA, MLS */ 11076 if (size == 3) { 11077 unallocated_encoding(s); 11078 return; 11079 } 11080 break; 11081 case 0x16: /* SQDMULH, SQRDMULH */ 11082 if (size == 0 || size == 3) { 11083 unallocated_encoding(s); 11084 return; 11085 } 11086 break; 11087 default: 11088 if (size == 3 && !is_q) { 11089 unallocated_encoding(s); 11090 return; 11091 } 11092 break; 11093 } 11094 11095 if (!fp_access_check(s)) { 11096 return; 11097 } 11098 11099 switch (opcode) { 11100 case 0x01: /* SQADD, UQADD */ 11101 if (u) { 11102 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_uqadd_qc, size); 11103 } else { 11104 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_sqadd_qc, size); 11105 } 11106 return; 11107 case 0x05: /* SQSUB, UQSUB */ 11108 if (u) { 11109 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_uqsub_qc, size); 11110 } else { 11111 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_sqsub_qc, size); 11112 } 11113 return; 11114 case 0x08: /* SSHL, USHL */ 11115 if (u) { 11116 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_ushl, size); 11117 } else { 11118 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_sshl, size); 11119 } 11120 return; 11121 case 0x0c: /* SMAX, UMAX */ 11122 if (u) { 11123 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_umax, size); 11124 } else { 11125 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_smax, size); 11126 } 11127 return; 11128 case 0x0d: /* SMIN, UMIN */ 11129 if (u) { 11130 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_umin, size); 11131 } else { 11132 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_smin, size); 11133 } 11134 return; 11135 case 0xe: /* SABD, UABD */ 11136 if (u) { 11137 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_uabd, size); 11138 } else { 11139 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_sabd, size); 11140 } 11141 return; 11142 case 0xf: /* SABA, UABA */ 11143 if (u) { 11144 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_uaba, size); 11145 } else { 11146 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_saba, size); 11147 } 11148 return; 11149 case 0x10: /* ADD, SUB */ 11150 if (u) { 11151 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_sub, size); 11152 } else { 11153 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_add, size); 11154 } 11155 return; 11156 case 0x13: /* MUL, PMUL */ 11157 if (!u) { /* MUL */ 11158 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_mul, size); 11159 } else { /* PMUL */ 11160 gen_gvec_op3_ool(s, is_q, rd, rn, rm, 0, gen_helper_gvec_pmul_b); 11161 } 11162 return; 11163 case 0x12: /* MLA, MLS */ 11164 if (u) { 11165 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_mls, size); 11166 } else { 11167 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_mla, size); 11168 } 11169 return; 11170 case 0x16: /* SQDMULH, SQRDMULH */ 11171 { 11172 static gen_helper_gvec_3_ptr * const fns[2][2] = { 11173 { gen_helper_neon_sqdmulh_h, gen_helper_neon_sqrdmulh_h }, 11174 { gen_helper_neon_sqdmulh_s, gen_helper_neon_sqrdmulh_s }, 11175 }; 11176 gen_gvec_op3_qc(s, is_q, rd, rn, rm, fns[size - 1][u]); 11177 } 11178 return; 11179 case 0x11: 11180 if (!u) { /* CMTST */ 11181 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_cmtst, size); 11182 return; 11183 } 11184 /* else CMEQ */ 11185 cond = TCG_COND_EQ; 11186 goto do_gvec_cmp; 11187 case 0x06: /* CMGT, CMHI */ 11188 cond = u ? TCG_COND_GTU : TCG_COND_GT; 11189 goto do_gvec_cmp; 11190 case 0x07: /* CMGE, CMHS */ 11191 cond = u ? TCG_COND_GEU : TCG_COND_GE; 11192 do_gvec_cmp: 11193 tcg_gen_gvec_cmp(cond, size, vec_full_reg_offset(s, rd), 11194 vec_full_reg_offset(s, rn), 11195 vec_full_reg_offset(s, rm), 11196 is_q ? 16 : 8, vec_full_reg_size(s)); 11197 return; 11198 } 11199 11200 if (size == 3) { 11201 assert(is_q); 11202 for (pass = 0; pass < 2; pass++) { 11203 TCGv_i64 tcg_op1 = tcg_temp_new_i64(); 11204 TCGv_i64 tcg_op2 = tcg_temp_new_i64(); 11205 TCGv_i64 tcg_res = tcg_temp_new_i64(); 11206 11207 read_vec_element(s, tcg_op1, rn, pass, MO_64); 11208 read_vec_element(s, tcg_op2, rm, pass, MO_64); 11209 11210 handle_3same_64(s, opcode, u, tcg_res, tcg_op1, tcg_op2); 11211 11212 write_vec_element(s, tcg_res, rd, pass, MO_64); 11213 } 11214 } else { 11215 for (pass = 0; pass < (is_q ? 4 : 2); pass++) { 11216 TCGv_i32 tcg_op1 = tcg_temp_new_i32(); 11217 TCGv_i32 tcg_op2 = tcg_temp_new_i32(); 11218 TCGv_i32 tcg_res = tcg_temp_new_i32(); 11219 NeonGenTwoOpFn *genfn = NULL; 11220 NeonGenTwoOpEnvFn *genenvfn = NULL; 11221 11222 read_vec_element_i32(s, tcg_op1, rn, pass, MO_32); 11223 read_vec_element_i32(s, tcg_op2, rm, pass, MO_32); 11224 11225 switch (opcode) { 11226 case 0x0: /* SHADD, UHADD */ 11227 { 11228 static NeonGenTwoOpFn * const fns[3][2] = { 11229 { gen_helper_neon_hadd_s8, gen_helper_neon_hadd_u8 }, 11230 { gen_helper_neon_hadd_s16, gen_helper_neon_hadd_u16 }, 11231 { gen_helper_neon_hadd_s32, gen_helper_neon_hadd_u32 }, 11232 }; 11233 genfn = fns[size][u]; 11234 break; 11235 } 11236 case 0x2: /* SRHADD, URHADD */ 11237 { 11238 static NeonGenTwoOpFn * const fns[3][2] = { 11239 { gen_helper_neon_rhadd_s8, gen_helper_neon_rhadd_u8 }, 11240 { gen_helper_neon_rhadd_s16, gen_helper_neon_rhadd_u16 }, 11241 { gen_helper_neon_rhadd_s32, gen_helper_neon_rhadd_u32 }, 11242 }; 11243 genfn = fns[size][u]; 11244 break; 11245 } 11246 case 0x4: /* SHSUB, UHSUB */ 11247 { 11248 static NeonGenTwoOpFn * const fns[3][2] = { 11249 { gen_helper_neon_hsub_s8, gen_helper_neon_hsub_u8 }, 11250 { gen_helper_neon_hsub_s16, gen_helper_neon_hsub_u16 }, 11251 { gen_helper_neon_hsub_s32, gen_helper_neon_hsub_u32 }, 11252 }; 11253 genfn = fns[size][u]; 11254 break; 11255 } 11256 case 0x9: /* SQSHL, UQSHL */ 11257 { 11258 static NeonGenTwoOpEnvFn * const fns[3][2] = { 11259 { gen_helper_neon_qshl_s8, gen_helper_neon_qshl_u8 }, 11260 { gen_helper_neon_qshl_s16, gen_helper_neon_qshl_u16 }, 11261 { gen_helper_neon_qshl_s32, gen_helper_neon_qshl_u32 }, 11262 }; 11263 genenvfn = fns[size][u]; 11264 break; 11265 } 11266 case 0xa: /* SRSHL, URSHL */ 11267 { 11268 static NeonGenTwoOpFn * const fns[3][2] = { 11269 { gen_helper_neon_rshl_s8, gen_helper_neon_rshl_u8 }, 11270 { gen_helper_neon_rshl_s16, gen_helper_neon_rshl_u16 }, 11271 { gen_helper_neon_rshl_s32, gen_helper_neon_rshl_u32 }, 11272 }; 11273 genfn = fns[size][u]; 11274 break; 11275 } 11276 case 0xb: /* SQRSHL, UQRSHL */ 11277 { 11278 static NeonGenTwoOpEnvFn * const fns[3][2] = { 11279 { gen_helper_neon_qrshl_s8, gen_helper_neon_qrshl_u8 }, 11280 { gen_helper_neon_qrshl_s16, gen_helper_neon_qrshl_u16 }, 11281 { gen_helper_neon_qrshl_s32, gen_helper_neon_qrshl_u32 }, 11282 }; 11283 genenvfn = fns[size][u]; 11284 break; 11285 } 11286 default: 11287 g_assert_not_reached(); 11288 } 11289 11290 if (genenvfn) { 11291 genenvfn(tcg_res, cpu_env, tcg_op1, tcg_op2); 11292 } else { 11293 genfn(tcg_res, tcg_op1, tcg_op2); 11294 } 11295 11296 write_vec_element_i32(s, tcg_res, rd, pass, MO_32); 11297 } 11298 } 11299 clear_vec_high(s, is_q, rd); 11300 } 11301 11302 /* AdvSIMD three same 11303 * 31 30 29 28 24 23 22 21 20 16 15 11 10 9 5 4 0 11304 * +---+---+---+-----------+------+---+------+--------+---+------+------+ 11305 * | 0 | Q | U | 0 1 1 1 0 | size | 1 | Rm | opcode | 1 | Rn | Rd | 11306 * +---+---+---+-----------+------+---+------+--------+---+------+------+ 11307 */ 11308 static void disas_simd_three_reg_same(DisasContext *s, uint32_t insn) 11309 { 11310 int opcode = extract32(insn, 11, 5); 11311 11312 switch (opcode) { 11313 case 0x3: /* logic ops */ 11314 disas_simd_3same_logic(s, insn); 11315 break; 11316 case 0x17: /* ADDP */ 11317 case 0x14: /* SMAXP, UMAXP */ 11318 case 0x15: /* SMINP, UMINP */ 11319 { 11320 /* Pairwise operations */ 11321 int is_q = extract32(insn, 30, 1); 11322 int u = extract32(insn, 29, 1); 11323 int size = extract32(insn, 22, 2); 11324 int rm = extract32(insn, 16, 5); 11325 int rn = extract32(insn, 5, 5); 11326 int rd = extract32(insn, 0, 5); 11327 if (opcode == 0x17) { 11328 if (u || (size == 3 && !is_q)) { 11329 unallocated_encoding(s); 11330 return; 11331 } 11332 } else { 11333 if (size == 3) { 11334 unallocated_encoding(s); 11335 return; 11336 } 11337 } 11338 handle_simd_3same_pair(s, is_q, u, opcode, size, rn, rm, rd); 11339 break; 11340 } 11341 case 0x18 ... 0x31: 11342 /* floating point ops, sz[1] and U are part of opcode */ 11343 disas_simd_3same_float(s, insn); 11344 break; 11345 default: 11346 disas_simd_3same_int(s, insn); 11347 break; 11348 } 11349 } 11350 11351 /* 11352 * Advanced SIMD three same (ARMv8.2 FP16 variants) 11353 * 11354 * 31 30 29 28 24 23 22 21 20 16 15 14 13 11 10 9 5 4 0 11355 * +---+---+---+-----------+---------+------+-----+--------+---+------+------+ 11356 * | 0 | Q | U | 0 1 1 1 0 | a | 1 0 | Rm | 0 0 | opcode | 1 | Rn | Rd | 11357 * +---+---+---+-----------+---------+------+-----+--------+---+------+------+ 11358 * 11359 * This includes FMULX, FCMEQ (register), FRECPS, FRSQRTS, FCMGE 11360 * (register), FACGE, FABD, FCMGT (register) and FACGT. 11361 * 11362 */ 11363 static void disas_simd_three_reg_same_fp16(DisasContext *s, uint32_t insn) 11364 { 11365 int opcode = extract32(insn, 11, 3); 11366 int u = extract32(insn, 29, 1); 11367 int a = extract32(insn, 23, 1); 11368 int is_q = extract32(insn, 30, 1); 11369 int rm = extract32(insn, 16, 5); 11370 int rn = extract32(insn, 5, 5); 11371 int rd = extract32(insn, 0, 5); 11372 /* 11373 * For these floating point ops, the U, a and opcode bits 11374 * together indicate the operation. 11375 */ 11376 int fpopcode = opcode | (a << 3) | (u << 4); 11377 int datasize = is_q ? 128 : 64; 11378 int elements = datasize / 16; 11379 bool pairwise; 11380 TCGv_ptr fpst; 11381 int pass; 11382 11383 switch (fpopcode) { 11384 case 0x0: /* FMAXNM */ 11385 case 0x1: /* FMLA */ 11386 case 0x2: /* FADD */ 11387 case 0x3: /* FMULX */ 11388 case 0x4: /* FCMEQ */ 11389 case 0x6: /* FMAX */ 11390 case 0x7: /* FRECPS */ 11391 case 0x8: /* FMINNM */ 11392 case 0x9: /* FMLS */ 11393 case 0xa: /* FSUB */ 11394 case 0xe: /* FMIN */ 11395 case 0xf: /* FRSQRTS */ 11396 case 0x13: /* FMUL */ 11397 case 0x14: /* FCMGE */ 11398 case 0x15: /* FACGE */ 11399 case 0x17: /* FDIV */ 11400 case 0x1a: /* FABD */ 11401 case 0x1c: /* FCMGT */ 11402 case 0x1d: /* FACGT */ 11403 pairwise = false; 11404 break; 11405 case 0x10: /* FMAXNMP */ 11406 case 0x12: /* FADDP */ 11407 case 0x16: /* FMAXP */ 11408 case 0x18: /* FMINNMP */ 11409 case 0x1e: /* FMINP */ 11410 pairwise = true; 11411 break; 11412 default: 11413 unallocated_encoding(s); 11414 return; 11415 } 11416 11417 if (!dc_isar_feature(aa64_fp16, s)) { 11418 unallocated_encoding(s); 11419 return; 11420 } 11421 11422 if (!fp_access_check(s)) { 11423 return; 11424 } 11425 11426 fpst = fpstatus_ptr(FPST_FPCR_F16); 11427 11428 if (pairwise) { 11429 int maxpass = is_q ? 8 : 4; 11430 TCGv_i32 tcg_op1 = tcg_temp_new_i32(); 11431 TCGv_i32 tcg_op2 = tcg_temp_new_i32(); 11432 TCGv_i32 tcg_res[8]; 11433 11434 for (pass = 0; pass < maxpass; pass++) { 11435 int passreg = pass < (maxpass / 2) ? rn : rm; 11436 int passelt = (pass << 1) & (maxpass - 1); 11437 11438 read_vec_element_i32(s, tcg_op1, passreg, passelt, MO_16); 11439 read_vec_element_i32(s, tcg_op2, passreg, passelt + 1, MO_16); 11440 tcg_res[pass] = tcg_temp_new_i32(); 11441 11442 switch (fpopcode) { 11443 case 0x10: /* FMAXNMP */ 11444 gen_helper_advsimd_maxnumh(tcg_res[pass], tcg_op1, tcg_op2, 11445 fpst); 11446 break; 11447 case 0x12: /* FADDP */ 11448 gen_helper_advsimd_addh(tcg_res[pass], tcg_op1, tcg_op2, fpst); 11449 break; 11450 case 0x16: /* FMAXP */ 11451 gen_helper_advsimd_maxh(tcg_res[pass], tcg_op1, tcg_op2, fpst); 11452 break; 11453 case 0x18: /* FMINNMP */ 11454 gen_helper_advsimd_minnumh(tcg_res[pass], tcg_op1, tcg_op2, 11455 fpst); 11456 break; 11457 case 0x1e: /* FMINP */ 11458 gen_helper_advsimd_minh(tcg_res[pass], tcg_op1, tcg_op2, fpst); 11459 break; 11460 default: 11461 g_assert_not_reached(); 11462 } 11463 } 11464 11465 for (pass = 0; pass < maxpass; pass++) { 11466 write_vec_element_i32(s, tcg_res[pass], rd, pass, MO_16); 11467 } 11468 } else { 11469 for (pass = 0; pass < elements; pass++) { 11470 TCGv_i32 tcg_op1 = tcg_temp_new_i32(); 11471 TCGv_i32 tcg_op2 = tcg_temp_new_i32(); 11472 TCGv_i32 tcg_res = tcg_temp_new_i32(); 11473 11474 read_vec_element_i32(s, tcg_op1, rn, pass, MO_16); 11475 read_vec_element_i32(s, tcg_op2, rm, pass, MO_16); 11476 11477 switch (fpopcode) { 11478 case 0x0: /* FMAXNM */ 11479 gen_helper_advsimd_maxnumh(tcg_res, tcg_op1, tcg_op2, fpst); 11480 break; 11481 case 0x1: /* FMLA */ 11482 read_vec_element_i32(s, tcg_res, rd, pass, MO_16); 11483 gen_helper_advsimd_muladdh(tcg_res, tcg_op1, tcg_op2, tcg_res, 11484 fpst); 11485 break; 11486 case 0x2: /* FADD */ 11487 gen_helper_advsimd_addh(tcg_res, tcg_op1, tcg_op2, fpst); 11488 break; 11489 case 0x3: /* FMULX */ 11490 gen_helper_advsimd_mulxh(tcg_res, tcg_op1, tcg_op2, fpst); 11491 break; 11492 case 0x4: /* FCMEQ */ 11493 gen_helper_advsimd_ceq_f16(tcg_res, tcg_op1, tcg_op2, fpst); 11494 break; 11495 case 0x6: /* FMAX */ 11496 gen_helper_advsimd_maxh(tcg_res, tcg_op1, tcg_op2, fpst); 11497 break; 11498 case 0x7: /* FRECPS */ 11499 gen_helper_recpsf_f16(tcg_res, tcg_op1, tcg_op2, fpst); 11500 break; 11501 case 0x8: /* FMINNM */ 11502 gen_helper_advsimd_minnumh(tcg_res, tcg_op1, tcg_op2, fpst); 11503 break; 11504 case 0x9: /* FMLS */ 11505 /* As usual for ARM, separate negation for fused multiply-add */ 11506 tcg_gen_xori_i32(tcg_op1, tcg_op1, 0x8000); 11507 read_vec_element_i32(s, tcg_res, rd, pass, MO_16); 11508 gen_helper_advsimd_muladdh(tcg_res, tcg_op1, tcg_op2, tcg_res, 11509 fpst); 11510 break; 11511 case 0xa: /* FSUB */ 11512 gen_helper_advsimd_subh(tcg_res, tcg_op1, tcg_op2, fpst); 11513 break; 11514 case 0xe: /* FMIN */ 11515 gen_helper_advsimd_minh(tcg_res, tcg_op1, tcg_op2, fpst); 11516 break; 11517 case 0xf: /* FRSQRTS */ 11518 gen_helper_rsqrtsf_f16(tcg_res, tcg_op1, tcg_op2, fpst); 11519 break; 11520 case 0x13: /* FMUL */ 11521 gen_helper_advsimd_mulh(tcg_res, tcg_op1, tcg_op2, fpst); 11522 break; 11523 case 0x14: /* FCMGE */ 11524 gen_helper_advsimd_cge_f16(tcg_res, tcg_op1, tcg_op2, fpst); 11525 break; 11526 case 0x15: /* FACGE */ 11527 gen_helper_advsimd_acge_f16(tcg_res, tcg_op1, tcg_op2, fpst); 11528 break; 11529 case 0x17: /* FDIV */ 11530 gen_helper_advsimd_divh(tcg_res, tcg_op1, tcg_op2, fpst); 11531 break; 11532 case 0x1a: /* FABD */ 11533 gen_helper_advsimd_subh(tcg_res, tcg_op1, tcg_op2, fpst); 11534 tcg_gen_andi_i32(tcg_res, tcg_res, 0x7fff); 11535 break; 11536 case 0x1c: /* FCMGT */ 11537 gen_helper_advsimd_cgt_f16(tcg_res, tcg_op1, tcg_op2, fpst); 11538 break; 11539 case 0x1d: /* FACGT */ 11540 gen_helper_advsimd_acgt_f16(tcg_res, tcg_op1, tcg_op2, fpst); 11541 break; 11542 default: 11543 g_assert_not_reached(); 11544 } 11545 11546 write_vec_element_i32(s, tcg_res, rd, pass, MO_16); 11547 } 11548 } 11549 11550 clear_vec_high(s, is_q, rd); 11551 } 11552 11553 /* AdvSIMD three same extra 11554 * 31 30 29 28 24 23 22 21 20 16 15 14 11 10 9 5 4 0 11555 * +---+---+---+-----------+------+---+------+---+--------+---+----+----+ 11556 * | 0 | Q | U | 0 1 1 1 0 | size | 0 | Rm | 1 | opcode | 1 | Rn | Rd | 11557 * +---+---+---+-----------+------+---+------+---+--------+---+----+----+ 11558 */ 11559 static void disas_simd_three_reg_same_extra(DisasContext *s, uint32_t insn) 11560 { 11561 int rd = extract32(insn, 0, 5); 11562 int rn = extract32(insn, 5, 5); 11563 int opcode = extract32(insn, 11, 4); 11564 int rm = extract32(insn, 16, 5); 11565 int size = extract32(insn, 22, 2); 11566 bool u = extract32(insn, 29, 1); 11567 bool is_q = extract32(insn, 30, 1); 11568 bool feature; 11569 int rot; 11570 11571 switch (u * 16 + opcode) { 11572 case 0x10: /* SQRDMLAH (vector) */ 11573 case 0x11: /* SQRDMLSH (vector) */ 11574 if (size != 1 && size != 2) { 11575 unallocated_encoding(s); 11576 return; 11577 } 11578 feature = dc_isar_feature(aa64_rdm, s); 11579 break; 11580 case 0x02: /* SDOT (vector) */ 11581 case 0x12: /* UDOT (vector) */ 11582 if (size != MO_32) { 11583 unallocated_encoding(s); 11584 return; 11585 } 11586 feature = dc_isar_feature(aa64_dp, s); 11587 break; 11588 case 0x03: /* USDOT */ 11589 if (size != MO_32) { 11590 unallocated_encoding(s); 11591 return; 11592 } 11593 feature = dc_isar_feature(aa64_i8mm, s); 11594 break; 11595 case 0x04: /* SMMLA */ 11596 case 0x14: /* UMMLA */ 11597 case 0x05: /* USMMLA */ 11598 if (!is_q || size != MO_32) { 11599 unallocated_encoding(s); 11600 return; 11601 } 11602 feature = dc_isar_feature(aa64_i8mm, s); 11603 break; 11604 case 0x18: /* FCMLA, #0 */ 11605 case 0x19: /* FCMLA, #90 */ 11606 case 0x1a: /* FCMLA, #180 */ 11607 case 0x1b: /* FCMLA, #270 */ 11608 case 0x1c: /* FCADD, #90 */ 11609 case 0x1e: /* FCADD, #270 */ 11610 if (size == 0 11611 || (size == 1 && !dc_isar_feature(aa64_fp16, s)) 11612 || (size == 3 && !is_q)) { 11613 unallocated_encoding(s); 11614 return; 11615 } 11616 feature = dc_isar_feature(aa64_fcma, s); 11617 break; 11618 case 0x1d: /* BFMMLA */ 11619 if (size != MO_16 || !is_q) { 11620 unallocated_encoding(s); 11621 return; 11622 } 11623 feature = dc_isar_feature(aa64_bf16, s); 11624 break; 11625 case 0x1f: 11626 switch (size) { 11627 case 1: /* BFDOT */ 11628 case 3: /* BFMLAL{B,T} */ 11629 feature = dc_isar_feature(aa64_bf16, s); 11630 break; 11631 default: 11632 unallocated_encoding(s); 11633 return; 11634 } 11635 break; 11636 default: 11637 unallocated_encoding(s); 11638 return; 11639 } 11640 if (!feature) { 11641 unallocated_encoding(s); 11642 return; 11643 } 11644 if (!fp_access_check(s)) { 11645 return; 11646 } 11647 11648 switch (opcode) { 11649 case 0x0: /* SQRDMLAH (vector) */ 11650 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_sqrdmlah_qc, size); 11651 return; 11652 11653 case 0x1: /* SQRDMLSH (vector) */ 11654 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_sqrdmlsh_qc, size); 11655 return; 11656 11657 case 0x2: /* SDOT / UDOT */ 11658 gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, 0, 11659 u ? gen_helper_gvec_udot_b : gen_helper_gvec_sdot_b); 11660 return; 11661 11662 case 0x3: /* USDOT */ 11663 gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, 0, gen_helper_gvec_usdot_b); 11664 return; 11665 11666 case 0x04: /* SMMLA, UMMLA */ 11667 gen_gvec_op4_ool(s, 1, rd, rn, rm, rd, 0, 11668 u ? gen_helper_gvec_ummla_b 11669 : gen_helper_gvec_smmla_b); 11670 return; 11671 case 0x05: /* USMMLA */ 11672 gen_gvec_op4_ool(s, 1, rd, rn, rm, rd, 0, gen_helper_gvec_usmmla_b); 11673 return; 11674 11675 case 0x8: /* FCMLA, #0 */ 11676 case 0x9: /* FCMLA, #90 */ 11677 case 0xa: /* FCMLA, #180 */ 11678 case 0xb: /* FCMLA, #270 */ 11679 rot = extract32(opcode, 0, 2); 11680 switch (size) { 11681 case 1: 11682 gen_gvec_op4_fpst(s, is_q, rd, rn, rm, rd, true, rot, 11683 gen_helper_gvec_fcmlah); 11684 break; 11685 case 2: 11686 gen_gvec_op4_fpst(s, is_q, rd, rn, rm, rd, false, rot, 11687 gen_helper_gvec_fcmlas); 11688 break; 11689 case 3: 11690 gen_gvec_op4_fpst(s, is_q, rd, rn, rm, rd, false, rot, 11691 gen_helper_gvec_fcmlad); 11692 break; 11693 default: 11694 g_assert_not_reached(); 11695 } 11696 return; 11697 11698 case 0xc: /* FCADD, #90 */ 11699 case 0xe: /* FCADD, #270 */ 11700 rot = extract32(opcode, 1, 1); 11701 switch (size) { 11702 case 1: 11703 gen_gvec_op3_fpst(s, is_q, rd, rn, rm, size == 1, rot, 11704 gen_helper_gvec_fcaddh); 11705 break; 11706 case 2: 11707 gen_gvec_op3_fpst(s, is_q, rd, rn, rm, size == 1, rot, 11708 gen_helper_gvec_fcadds); 11709 break; 11710 case 3: 11711 gen_gvec_op3_fpst(s, is_q, rd, rn, rm, size == 1, rot, 11712 gen_helper_gvec_fcaddd); 11713 break; 11714 default: 11715 g_assert_not_reached(); 11716 } 11717 return; 11718 11719 case 0xd: /* BFMMLA */ 11720 gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, 0, gen_helper_gvec_bfmmla); 11721 return; 11722 case 0xf: 11723 switch (size) { 11724 case 1: /* BFDOT */ 11725 gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, 0, gen_helper_gvec_bfdot); 11726 break; 11727 case 3: /* BFMLAL{B,T} */ 11728 gen_gvec_op4_fpst(s, 1, rd, rn, rm, rd, false, is_q, 11729 gen_helper_gvec_bfmlal); 11730 break; 11731 default: 11732 g_assert_not_reached(); 11733 } 11734 return; 11735 11736 default: 11737 g_assert_not_reached(); 11738 } 11739 } 11740 11741 static void handle_2misc_widening(DisasContext *s, int opcode, bool is_q, 11742 int size, int rn, int rd) 11743 { 11744 /* Handle 2-reg-misc ops which are widening (so each size element 11745 * in the source becomes a 2*size element in the destination. 11746 * The only instruction like this is FCVTL. 11747 */ 11748 int pass; 11749 11750 if (size == 3) { 11751 /* 32 -> 64 bit fp conversion */ 11752 TCGv_i64 tcg_res[2]; 11753 int srcelt = is_q ? 2 : 0; 11754 11755 for (pass = 0; pass < 2; pass++) { 11756 TCGv_i32 tcg_op = tcg_temp_new_i32(); 11757 tcg_res[pass] = tcg_temp_new_i64(); 11758 11759 read_vec_element_i32(s, tcg_op, rn, srcelt + pass, MO_32); 11760 gen_helper_vfp_fcvtds(tcg_res[pass], tcg_op, cpu_env); 11761 } 11762 for (pass = 0; pass < 2; pass++) { 11763 write_vec_element(s, tcg_res[pass], rd, pass, MO_64); 11764 } 11765 } else { 11766 /* 16 -> 32 bit fp conversion */ 11767 int srcelt = is_q ? 4 : 0; 11768 TCGv_i32 tcg_res[4]; 11769 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR); 11770 TCGv_i32 ahp = get_ahp_flag(); 11771 11772 for (pass = 0; pass < 4; pass++) { 11773 tcg_res[pass] = tcg_temp_new_i32(); 11774 11775 read_vec_element_i32(s, tcg_res[pass], rn, srcelt + pass, MO_16); 11776 gen_helper_vfp_fcvt_f16_to_f32(tcg_res[pass], tcg_res[pass], 11777 fpst, ahp); 11778 } 11779 for (pass = 0; pass < 4; pass++) { 11780 write_vec_element_i32(s, tcg_res[pass], rd, pass, MO_32); 11781 } 11782 } 11783 } 11784 11785 static void handle_rev(DisasContext *s, int opcode, bool u, 11786 bool is_q, int size, int rn, int rd) 11787 { 11788 int op = (opcode << 1) | u; 11789 int opsz = op + size; 11790 int grp_size = 3 - opsz; 11791 int dsize = is_q ? 128 : 64; 11792 int i; 11793 11794 if (opsz >= 3) { 11795 unallocated_encoding(s); 11796 return; 11797 } 11798 11799 if (!fp_access_check(s)) { 11800 return; 11801 } 11802 11803 if (size == 0) { 11804 /* Special case bytes, use bswap op on each group of elements */ 11805 int groups = dsize / (8 << grp_size); 11806 11807 for (i = 0; i < groups; i++) { 11808 TCGv_i64 tcg_tmp = tcg_temp_new_i64(); 11809 11810 read_vec_element(s, tcg_tmp, rn, i, grp_size); 11811 switch (grp_size) { 11812 case MO_16: 11813 tcg_gen_bswap16_i64(tcg_tmp, tcg_tmp, TCG_BSWAP_IZ); 11814 break; 11815 case MO_32: 11816 tcg_gen_bswap32_i64(tcg_tmp, tcg_tmp, TCG_BSWAP_IZ); 11817 break; 11818 case MO_64: 11819 tcg_gen_bswap64_i64(tcg_tmp, tcg_tmp); 11820 break; 11821 default: 11822 g_assert_not_reached(); 11823 } 11824 write_vec_element(s, tcg_tmp, rd, i, grp_size); 11825 } 11826 clear_vec_high(s, is_q, rd); 11827 } else { 11828 int revmask = (1 << grp_size) - 1; 11829 int esize = 8 << size; 11830 int elements = dsize / esize; 11831 TCGv_i64 tcg_rn = tcg_temp_new_i64(); 11832 TCGv_i64 tcg_rd[2]; 11833 11834 for (i = 0; i < 2; i++) { 11835 tcg_rd[i] = tcg_temp_new_i64(); 11836 tcg_gen_movi_i64(tcg_rd[i], 0); 11837 } 11838 11839 for (i = 0; i < elements; i++) { 11840 int e_rev = (i & 0xf) ^ revmask; 11841 int w = (e_rev * esize) / 64; 11842 int o = (e_rev * esize) % 64; 11843 11844 read_vec_element(s, tcg_rn, rn, i, size); 11845 tcg_gen_deposit_i64(tcg_rd[w], tcg_rd[w], tcg_rn, o, esize); 11846 } 11847 11848 for (i = 0; i < 2; i++) { 11849 write_vec_element(s, tcg_rd[i], rd, i, MO_64); 11850 } 11851 clear_vec_high(s, true, rd); 11852 } 11853 } 11854 11855 static void handle_2misc_pairwise(DisasContext *s, int opcode, bool u, 11856 bool is_q, int size, int rn, int rd) 11857 { 11858 /* Implement the pairwise operations from 2-misc: 11859 * SADDLP, UADDLP, SADALP, UADALP. 11860 * These all add pairs of elements in the input to produce a 11861 * double-width result element in the output (possibly accumulating). 11862 */ 11863 bool accum = (opcode == 0x6); 11864 int maxpass = is_q ? 2 : 1; 11865 int pass; 11866 TCGv_i64 tcg_res[2]; 11867 11868 if (size == 2) { 11869 /* 32 + 32 -> 64 op */ 11870 MemOp memop = size + (u ? 0 : MO_SIGN); 11871 11872 for (pass = 0; pass < maxpass; pass++) { 11873 TCGv_i64 tcg_op1 = tcg_temp_new_i64(); 11874 TCGv_i64 tcg_op2 = tcg_temp_new_i64(); 11875 11876 tcg_res[pass] = tcg_temp_new_i64(); 11877 11878 read_vec_element(s, tcg_op1, rn, pass * 2, memop); 11879 read_vec_element(s, tcg_op2, rn, pass * 2 + 1, memop); 11880 tcg_gen_add_i64(tcg_res[pass], tcg_op1, tcg_op2); 11881 if (accum) { 11882 read_vec_element(s, tcg_op1, rd, pass, MO_64); 11883 tcg_gen_add_i64(tcg_res[pass], tcg_res[pass], tcg_op1); 11884 } 11885 } 11886 } else { 11887 for (pass = 0; pass < maxpass; pass++) { 11888 TCGv_i64 tcg_op = tcg_temp_new_i64(); 11889 NeonGenOne64OpFn *genfn; 11890 static NeonGenOne64OpFn * const fns[2][2] = { 11891 { gen_helper_neon_addlp_s8, gen_helper_neon_addlp_u8 }, 11892 { gen_helper_neon_addlp_s16, gen_helper_neon_addlp_u16 }, 11893 }; 11894 11895 genfn = fns[size][u]; 11896 11897 tcg_res[pass] = tcg_temp_new_i64(); 11898 11899 read_vec_element(s, tcg_op, rn, pass, MO_64); 11900 genfn(tcg_res[pass], tcg_op); 11901 11902 if (accum) { 11903 read_vec_element(s, tcg_op, rd, pass, MO_64); 11904 if (size == 0) { 11905 gen_helper_neon_addl_u16(tcg_res[pass], 11906 tcg_res[pass], tcg_op); 11907 } else { 11908 gen_helper_neon_addl_u32(tcg_res[pass], 11909 tcg_res[pass], tcg_op); 11910 } 11911 } 11912 } 11913 } 11914 if (!is_q) { 11915 tcg_res[1] = tcg_constant_i64(0); 11916 } 11917 for (pass = 0; pass < 2; pass++) { 11918 write_vec_element(s, tcg_res[pass], rd, pass, MO_64); 11919 } 11920 } 11921 11922 static void handle_shll(DisasContext *s, bool is_q, int size, int rn, int rd) 11923 { 11924 /* Implement SHLL and SHLL2 */ 11925 int pass; 11926 int part = is_q ? 2 : 0; 11927 TCGv_i64 tcg_res[2]; 11928 11929 for (pass = 0; pass < 2; pass++) { 11930 static NeonGenWidenFn * const widenfns[3] = { 11931 gen_helper_neon_widen_u8, 11932 gen_helper_neon_widen_u16, 11933 tcg_gen_extu_i32_i64, 11934 }; 11935 NeonGenWidenFn *widenfn = widenfns[size]; 11936 TCGv_i32 tcg_op = tcg_temp_new_i32(); 11937 11938 read_vec_element_i32(s, tcg_op, rn, part + pass, MO_32); 11939 tcg_res[pass] = tcg_temp_new_i64(); 11940 widenfn(tcg_res[pass], tcg_op); 11941 tcg_gen_shli_i64(tcg_res[pass], tcg_res[pass], 8 << size); 11942 } 11943 11944 for (pass = 0; pass < 2; pass++) { 11945 write_vec_element(s, tcg_res[pass], rd, pass, MO_64); 11946 } 11947 } 11948 11949 /* AdvSIMD two reg misc 11950 * 31 30 29 28 24 23 22 21 17 16 12 11 10 9 5 4 0 11951 * +---+---+---+-----------+------+-----------+--------+-----+------+------+ 11952 * | 0 | Q | U | 0 1 1 1 0 | size | 1 0 0 0 0 | opcode | 1 0 | Rn | Rd | 11953 * +---+---+---+-----------+------+-----------+--------+-----+------+------+ 11954 */ 11955 static void disas_simd_two_reg_misc(DisasContext *s, uint32_t insn) 11956 { 11957 int size = extract32(insn, 22, 2); 11958 int opcode = extract32(insn, 12, 5); 11959 bool u = extract32(insn, 29, 1); 11960 bool is_q = extract32(insn, 30, 1); 11961 int rn = extract32(insn, 5, 5); 11962 int rd = extract32(insn, 0, 5); 11963 bool need_fpstatus = false; 11964 int rmode = -1; 11965 TCGv_i32 tcg_rmode; 11966 TCGv_ptr tcg_fpstatus; 11967 11968 switch (opcode) { 11969 case 0x0: /* REV64, REV32 */ 11970 case 0x1: /* REV16 */ 11971 handle_rev(s, opcode, u, is_q, size, rn, rd); 11972 return; 11973 case 0x5: /* CNT, NOT, RBIT */ 11974 if (u && size == 0) { 11975 /* NOT */ 11976 break; 11977 } else if (u && size == 1) { 11978 /* RBIT */ 11979 break; 11980 } else if (!u && size == 0) { 11981 /* CNT */ 11982 break; 11983 } 11984 unallocated_encoding(s); 11985 return; 11986 case 0x12: /* XTN, XTN2, SQXTUN, SQXTUN2 */ 11987 case 0x14: /* SQXTN, SQXTN2, UQXTN, UQXTN2 */ 11988 if (size == 3) { 11989 unallocated_encoding(s); 11990 return; 11991 } 11992 if (!fp_access_check(s)) { 11993 return; 11994 } 11995 11996 handle_2misc_narrow(s, false, opcode, u, is_q, size, rn, rd); 11997 return; 11998 case 0x4: /* CLS, CLZ */ 11999 if (size == 3) { 12000 unallocated_encoding(s); 12001 return; 12002 } 12003 break; 12004 case 0x2: /* SADDLP, UADDLP */ 12005 case 0x6: /* SADALP, UADALP */ 12006 if (size == 3) { 12007 unallocated_encoding(s); 12008 return; 12009 } 12010 if (!fp_access_check(s)) { 12011 return; 12012 } 12013 handle_2misc_pairwise(s, opcode, u, is_q, size, rn, rd); 12014 return; 12015 case 0x13: /* SHLL, SHLL2 */ 12016 if (u == 0 || size == 3) { 12017 unallocated_encoding(s); 12018 return; 12019 } 12020 if (!fp_access_check(s)) { 12021 return; 12022 } 12023 handle_shll(s, is_q, size, rn, rd); 12024 return; 12025 case 0xa: /* CMLT */ 12026 if (u == 1) { 12027 unallocated_encoding(s); 12028 return; 12029 } 12030 /* fall through */ 12031 case 0x8: /* CMGT, CMGE */ 12032 case 0x9: /* CMEQ, CMLE */ 12033 case 0xb: /* ABS, NEG */ 12034 if (size == 3 && !is_q) { 12035 unallocated_encoding(s); 12036 return; 12037 } 12038 break; 12039 case 0x3: /* SUQADD, USQADD */ 12040 if (size == 3 && !is_q) { 12041 unallocated_encoding(s); 12042 return; 12043 } 12044 if (!fp_access_check(s)) { 12045 return; 12046 } 12047 handle_2misc_satacc(s, false, u, is_q, size, rn, rd); 12048 return; 12049 case 0x7: /* SQABS, SQNEG */ 12050 if (size == 3 && !is_q) { 12051 unallocated_encoding(s); 12052 return; 12053 } 12054 break; 12055 case 0xc ... 0xf: 12056 case 0x16 ... 0x1f: 12057 { 12058 /* Floating point: U, size[1] and opcode indicate operation; 12059 * size[0] indicates single or double precision. 12060 */ 12061 int is_double = extract32(size, 0, 1); 12062 opcode |= (extract32(size, 1, 1) << 5) | (u << 6); 12063 size = is_double ? 3 : 2; 12064 switch (opcode) { 12065 case 0x2f: /* FABS */ 12066 case 0x6f: /* FNEG */ 12067 if (size == 3 && !is_q) { 12068 unallocated_encoding(s); 12069 return; 12070 } 12071 break; 12072 case 0x1d: /* SCVTF */ 12073 case 0x5d: /* UCVTF */ 12074 { 12075 bool is_signed = (opcode == 0x1d) ? true : false; 12076 int elements = is_double ? 2 : is_q ? 4 : 2; 12077 if (is_double && !is_q) { 12078 unallocated_encoding(s); 12079 return; 12080 } 12081 if (!fp_access_check(s)) { 12082 return; 12083 } 12084 handle_simd_intfp_conv(s, rd, rn, elements, is_signed, 0, size); 12085 return; 12086 } 12087 case 0x2c: /* FCMGT (zero) */ 12088 case 0x2d: /* FCMEQ (zero) */ 12089 case 0x2e: /* FCMLT (zero) */ 12090 case 0x6c: /* FCMGE (zero) */ 12091 case 0x6d: /* FCMLE (zero) */ 12092 if (size == 3 && !is_q) { 12093 unallocated_encoding(s); 12094 return; 12095 } 12096 handle_2misc_fcmp_zero(s, opcode, false, u, is_q, size, rn, rd); 12097 return; 12098 case 0x7f: /* FSQRT */ 12099 if (size == 3 && !is_q) { 12100 unallocated_encoding(s); 12101 return; 12102 } 12103 break; 12104 case 0x1a: /* FCVTNS */ 12105 case 0x1b: /* FCVTMS */ 12106 case 0x3a: /* FCVTPS */ 12107 case 0x3b: /* FCVTZS */ 12108 case 0x5a: /* FCVTNU */ 12109 case 0x5b: /* FCVTMU */ 12110 case 0x7a: /* FCVTPU */ 12111 case 0x7b: /* FCVTZU */ 12112 need_fpstatus = true; 12113 rmode = extract32(opcode, 5, 1) | (extract32(opcode, 0, 1) << 1); 12114 if (size == 3 && !is_q) { 12115 unallocated_encoding(s); 12116 return; 12117 } 12118 break; 12119 case 0x5c: /* FCVTAU */ 12120 case 0x1c: /* FCVTAS */ 12121 need_fpstatus = true; 12122 rmode = FPROUNDING_TIEAWAY; 12123 if (size == 3 && !is_q) { 12124 unallocated_encoding(s); 12125 return; 12126 } 12127 break; 12128 case 0x3c: /* URECPE */ 12129 if (size == 3) { 12130 unallocated_encoding(s); 12131 return; 12132 } 12133 /* fall through */ 12134 case 0x3d: /* FRECPE */ 12135 case 0x7d: /* FRSQRTE */ 12136 if (size == 3 && !is_q) { 12137 unallocated_encoding(s); 12138 return; 12139 } 12140 if (!fp_access_check(s)) { 12141 return; 12142 } 12143 handle_2misc_reciprocal(s, opcode, false, u, is_q, size, rn, rd); 12144 return; 12145 case 0x56: /* FCVTXN, FCVTXN2 */ 12146 if (size == 2) { 12147 unallocated_encoding(s); 12148 return; 12149 } 12150 /* fall through */ 12151 case 0x16: /* FCVTN, FCVTN2 */ 12152 /* handle_2misc_narrow does a 2*size -> size operation, but these 12153 * instructions encode the source size rather than dest size. 12154 */ 12155 if (!fp_access_check(s)) { 12156 return; 12157 } 12158 handle_2misc_narrow(s, false, opcode, 0, is_q, size - 1, rn, rd); 12159 return; 12160 case 0x36: /* BFCVTN, BFCVTN2 */ 12161 if (!dc_isar_feature(aa64_bf16, s) || size != 2) { 12162 unallocated_encoding(s); 12163 return; 12164 } 12165 if (!fp_access_check(s)) { 12166 return; 12167 } 12168 handle_2misc_narrow(s, false, opcode, 0, is_q, size - 1, rn, rd); 12169 return; 12170 case 0x17: /* FCVTL, FCVTL2 */ 12171 if (!fp_access_check(s)) { 12172 return; 12173 } 12174 handle_2misc_widening(s, opcode, is_q, size, rn, rd); 12175 return; 12176 case 0x18: /* FRINTN */ 12177 case 0x19: /* FRINTM */ 12178 case 0x38: /* FRINTP */ 12179 case 0x39: /* FRINTZ */ 12180 rmode = extract32(opcode, 5, 1) | (extract32(opcode, 0, 1) << 1); 12181 /* fall through */ 12182 case 0x59: /* FRINTX */ 12183 case 0x79: /* FRINTI */ 12184 need_fpstatus = true; 12185 if (size == 3 && !is_q) { 12186 unallocated_encoding(s); 12187 return; 12188 } 12189 break; 12190 case 0x58: /* FRINTA */ 12191 rmode = FPROUNDING_TIEAWAY; 12192 need_fpstatus = true; 12193 if (size == 3 && !is_q) { 12194 unallocated_encoding(s); 12195 return; 12196 } 12197 break; 12198 case 0x7c: /* URSQRTE */ 12199 if (size == 3) { 12200 unallocated_encoding(s); 12201 return; 12202 } 12203 break; 12204 case 0x1e: /* FRINT32Z */ 12205 case 0x1f: /* FRINT64Z */ 12206 rmode = FPROUNDING_ZERO; 12207 /* fall through */ 12208 case 0x5e: /* FRINT32X */ 12209 case 0x5f: /* FRINT64X */ 12210 need_fpstatus = true; 12211 if ((size == 3 && !is_q) || !dc_isar_feature(aa64_frint, s)) { 12212 unallocated_encoding(s); 12213 return; 12214 } 12215 break; 12216 default: 12217 unallocated_encoding(s); 12218 return; 12219 } 12220 break; 12221 } 12222 default: 12223 unallocated_encoding(s); 12224 return; 12225 } 12226 12227 if (!fp_access_check(s)) { 12228 return; 12229 } 12230 12231 if (need_fpstatus || rmode >= 0) { 12232 tcg_fpstatus = fpstatus_ptr(FPST_FPCR); 12233 } else { 12234 tcg_fpstatus = NULL; 12235 } 12236 if (rmode >= 0) { 12237 tcg_rmode = gen_set_rmode(rmode, tcg_fpstatus); 12238 } else { 12239 tcg_rmode = NULL; 12240 } 12241 12242 switch (opcode) { 12243 case 0x5: 12244 if (u && size == 0) { /* NOT */ 12245 gen_gvec_fn2(s, is_q, rd, rn, tcg_gen_gvec_not, 0); 12246 return; 12247 } 12248 break; 12249 case 0x8: /* CMGT, CMGE */ 12250 if (u) { 12251 gen_gvec_fn2(s, is_q, rd, rn, gen_gvec_cge0, size); 12252 } else { 12253 gen_gvec_fn2(s, is_q, rd, rn, gen_gvec_cgt0, size); 12254 } 12255 return; 12256 case 0x9: /* CMEQ, CMLE */ 12257 if (u) { 12258 gen_gvec_fn2(s, is_q, rd, rn, gen_gvec_cle0, size); 12259 } else { 12260 gen_gvec_fn2(s, is_q, rd, rn, gen_gvec_ceq0, size); 12261 } 12262 return; 12263 case 0xa: /* CMLT */ 12264 gen_gvec_fn2(s, is_q, rd, rn, gen_gvec_clt0, size); 12265 return; 12266 case 0xb: 12267 if (u) { /* ABS, NEG */ 12268 gen_gvec_fn2(s, is_q, rd, rn, tcg_gen_gvec_neg, size); 12269 } else { 12270 gen_gvec_fn2(s, is_q, rd, rn, tcg_gen_gvec_abs, size); 12271 } 12272 return; 12273 } 12274 12275 if (size == 3) { 12276 /* All 64-bit element operations can be shared with scalar 2misc */ 12277 int pass; 12278 12279 /* Coverity claims (size == 3 && !is_q) has been eliminated 12280 * from all paths leading to here. 12281 */ 12282 tcg_debug_assert(is_q); 12283 for (pass = 0; pass < 2; pass++) { 12284 TCGv_i64 tcg_op = tcg_temp_new_i64(); 12285 TCGv_i64 tcg_res = tcg_temp_new_i64(); 12286 12287 read_vec_element(s, tcg_op, rn, pass, MO_64); 12288 12289 handle_2misc_64(s, opcode, u, tcg_res, tcg_op, 12290 tcg_rmode, tcg_fpstatus); 12291 12292 write_vec_element(s, tcg_res, rd, pass, MO_64); 12293 } 12294 } else { 12295 int pass; 12296 12297 for (pass = 0; pass < (is_q ? 4 : 2); pass++) { 12298 TCGv_i32 tcg_op = tcg_temp_new_i32(); 12299 TCGv_i32 tcg_res = tcg_temp_new_i32(); 12300 12301 read_vec_element_i32(s, tcg_op, rn, pass, MO_32); 12302 12303 if (size == 2) { 12304 /* Special cases for 32 bit elements */ 12305 switch (opcode) { 12306 case 0x4: /* CLS */ 12307 if (u) { 12308 tcg_gen_clzi_i32(tcg_res, tcg_op, 32); 12309 } else { 12310 tcg_gen_clrsb_i32(tcg_res, tcg_op); 12311 } 12312 break; 12313 case 0x7: /* SQABS, SQNEG */ 12314 if (u) { 12315 gen_helper_neon_qneg_s32(tcg_res, cpu_env, tcg_op); 12316 } else { 12317 gen_helper_neon_qabs_s32(tcg_res, cpu_env, tcg_op); 12318 } 12319 break; 12320 case 0x2f: /* FABS */ 12321 gen_helper_vfp_abss(tcg_res, tcg_op); 12322 break; 12323 case 0x6f: /* FNEG */ 12324 gen_helper_vfp_negs(tcg_res, tcg_op); 12325 break; 12326 case 0x7f: /* FSQRT */ 12327 gen_helper_vfp_sqrts(tcg_res, tcg_op, cpu_env); 12328 break; 12329 case 0x1a: /* FCVTNS */ 12330 case 0x1b: /* FCVTMS */ 12331 case 0x1c: /* FCVTAS */ 12332 case 0x3a: /* FCVTPS */ 12333 case 0x3b: /* FCVTZS */ 12334 gen_helper_vfp_tosls(tcg_res, tcg_op, 12335 tcg_constant_i32(0), tcg_fpstatus); 12336 break; 12337 case 0x5a: /* FCVTNU */ 12338 case 0x5b: /* FCVTMU */ 12339 case 0x5c: /* FCVTAU */ 12340 case 0x7a: /* FCVTPU */ 12341 case 0x7b: /* FCVTZU */ 12342 gen_helper_vfp_touls(tcg_res, tcg_op, 12343 tcg_constant_i32(0), tcg_fpstatus); 12344 break; 12345 case 0x18: /* FRINTN */ 12346 case 0x19: /* FRINTM */ 12347 case 0x38: /* FRINTP */ 12348 case 0x39: /* FRINTZ */ 12349 case 0x58: /* FRINTA */ 12350 case 0x79: /* FRINTI */ 12351 gen_helper_rints(tcg_res, tcg_op, tcg_fpstatus); 12352 break; 12353 case 0x59: /* FRINTX */ 12354 gen_helper_rints_exact(tcg_res, tcg_op, tcg_fpstatus); 12355 break; 12356 case 0x7c: /* URSQRTE */ 12357 gen_helper_rsqrte_u32(tcg_res, tcg_op); 12358 break; 12359 case 0x1e: /* FRINT32Z */ 12360 case 0x5e: /* FRINT32X */ 12361 gen_helper_frint32_s(tcg_res, tcg_op, tcg_fpstatus); 12362 break; 12363 case 0x1f: /* FRINT64Z */ 12364 case 0x5f: /* FRINT64X */ 12365 gen_helper_frint64_s(tcg_res, tcg_op, tcg_fpstatus); 12366 break; 12367 default: 12368 g_assert_not_reached(); 12369 } 12370 } else { 12371 /* Use helpers for 8 and 16 bit elements */ 12372 switch (opcode) { 12373 case 0x5: /* CNT, RBIT */ 12374 /* For these two insns size is part of the opcode specifier 12375 * (handled earlier); they always operate on byte elements. 12376 */ 12377 if (u) { 12378 gen_helper_neon_rbit_u8(tcg_res, tcg_op); 12379 } else { 12380 gen_helper_neon_cnt_u8(tcg_res, tcg_op); 12381 } 12382 break; 12383 case 0x7: /* SQABS, SQNEG */ 12384 { 12385 NeonGenOneOpEnvFn *genfn; 12386 static NeonGenOneOpEnvFn * const fns[2][2] = { 12387 { gen_helper_neon_qabs_s8, gen_helper_neon_qneg_s8 }, 12388 { gen_helper_neon_qabs_s16, gen_helper_neon_qneg_s16 }, 12389 }; 12390 genfn = fns[size][u]; 12391 genfn(tcg_res, cpu_env, tcg_op); 12392 break; 12393 } 12394 case 0x4: /* CLS, CLZ */ 12395 if (u) { 12396 if (size == 0) { 12397 gen_helper_neon_clz_u8(tcg_res, tcg_op); 12398 } else { 12399 gen_helper_neon_clz_u16(tcg_res, tcg_op); 12400 } 12401 } else { 12402 if (size == 0) { 12403 gen_helper_neon_cls_s8(tcg_res, tcg_op); 12404 } else { 12405 gen_helper_neon_cls_s16(tcg_res, tcg_op); 12406 } 12407 } 12408 break; 12409 default: 12410 g_assert_not_reached(); 12411 } 12412 } 12413 12414 write_vec_element_i32(s, tcg_res, rd, pass, MO_32); 12415 } 12416 } 12417 clear_vec_high(s, is_q, rd); 12418 12419 if (tcg_rmode) { 12420 gen_restore_rmode(tcg_rmode, tcg_fpstatus); 12421 } 12422 } 12423 12424 /* AdvSIMD [scalar] two register miscellaneous (FP16) 12425 * 12426 * 31 30 29 28 27 24 23 22 21 17 16 12 11 10 9 5 4 0 12427 * +---+---+---+---+---------+---+-------------+--------+-----+------+------+ 12428 * | 0 | Q | U | S | 1 1 1 0 | a | 1 1 1 1 0 0 | opcode | 1 0 | Rn | Rd | 12429 * +---+---+---+---+---------+---+-------------+--------+-----+------+------+ 12430 * mask: 1000 1111 0111 1110 0000 1100 0000 0000 0x8f7e 0c00 12431 * val: 0000 1110 0111 1000 0000 1000 0000 0000 0x0e78 0800 12432 * 12433 * This actually covers two groups where scalar access is governed by 12434 * bit 28. A bunch of the instructions (float to integral) only exist 12435 * in the vector form and are un-allocated for the scalar decode. Also 12436 * in the scalar decode Q is always 1. 12437 */ 12438 static void disas_simd_two_reg_misc_fp16(DisasContext *s, uint32_t insn) 12439 { 12440 int fpop, opcode, a, u; 12441 int rn, rd; 12442 bool is_q; 12443 bool is_scalar; 12444 bool only_in_vector = false; 12445 12446 int pass; 12447 TCGv_i32 tcg_rmode = NULL; 12448 TCGv_ptr tcg_fpstatus = NULL; 12449 bool need_fpst = true; 12450 int rmode = -1; 12451 12452 if (!dc_isar_feature(aa64_fp16, s)) { 12453 unallocated_encoding(s); 12454 return; 12455 } 12456 12457 rd = extract32(insn, 0, 5); 12458 rn = extract32(insn, 5, 5); 12459 12460 a = extract32(insn, 23, 1); 12461 u = extract32(insn, 29, 1); 12462 is_scalar = extract32(insn, 28, 1); 12463 is_q = extract32(insn, 30, 1); 12464 12465 opcode = extract32(insn, 12, 5); 12466 fpop = deposit32(opcode, 5, 1, a); 12467 fpop = deposit32(fpop, 6, 1, u); 12468 12469 switch (fpop) { 12470 case 0x1d: /* SCVTF */ 12471 case 0x5d: /* UCVTF */ 12472 { 12473 int elements; 12474 12475 if (is_scalar) { 12476 elements = 1; 12477 } else { 12478 elements = (is_q ? 8 : 4); 12479 } 12480 12481 if (!fp_access_check(s)) { 12482 return; 12483 } 12484 handle_simd_intfp_conv(s, rd, rn, elements, !u, 0, MO_16); 12485 return; 12486 } 12487 break; 12488 case 0x2c: /* FCMGT (zero) */ 12489 case 0x2d: /* FCMEQ (zero) */ 12490 case 0x2e: /* FCMLT (zero) */ 12491 case 0x6c: /* FCMGE (zero) */ 12492 case 0x6d: /* FCMLE (zero) */ 12493 handle_2misc_fcmp_zero(s, fpop, is_scalar, 0, is_q, MO_16, rn, rd); 12494 return; 12495 case 0x3d: /* FRECPE */ 12496 case 0x3f: /* FRECPX */ 12497 break; 12498 case 0x18: /* FRINTN */ 12499 only_in_vector = true; 12500 rmode = FPROUNDING_TIEEVEN; 12501 break; 12502 case 0x19: /* FRINTM */ 12503 only_in_vector = true; 12504 rmode = FPROUNDING_NEGINF; 12505 break; 12506 case 0x38: /* FRINTP */ 12507 only_in_vector = true; 12508 rmode = FPROUNDING_POSINF; 12509 break; 12510 case 0x39: /* FRINTZ */ 12511 only_in_vector = true; 12512 rmode = FPROUNDING_ZERO; 12513 break; 12514 case 0x58: /* FRINTA */ 12515 only_in_vector = true; 12516 rmode = FPROUNDING_TIEAWAY; 12517 break; 12518 case 0x59: /* FRINTX */ 12519 case 0x79: /* FRINTI */ 12520 only_in_vector = true; 12521 /* current rounding mode */ 12522 break; 12523 case 0x1a: /* FCVTNS */ 12524 rmode = FPROUNDING_TIEEVEN; 12525 break; 12526 case 0x1b: /* FCVTMS */ 12527 rmode = FPROUNDING_NEGINF; 12528 break; 12529 case 0x1c: /* FCVTAS */ 12530 rmode = FPROUNDING_TIEAWAY; 12531 break; 12532 case 0x3a: /* FCVTPS */ 12533 rmode = FPROUNDING_POSINF; 12534 break; 12535 case 0x3b: /* FCVTZS */ 12536 rmode = FPROUNDING_ZERO; 12537 break; 12538 case 0x5a: /* FCVTNU */ 12539 rmode = FPROUNDING_TIEEVEN; 12540 break; 12541 case 0x5b: /* FCVTMU */ 12542 rmode = FPROUNDING_NEGINF; 12543 break; 12544 case 0x5c: /* FCVTAU */ 12545 rmode = FPROUNDING_TIEAWAY; 12546 break; 12547 case 0x7a: /* FCVTPU */ 12548 rmode = FPROUNDING_POSINF; 12549 break; 12550 case 0x7b: /* FCVTZU */ 12551 rmode = FPROUNDING_ZERO; 12552 break; 12553 case 0x2f: /* FABS */ 12554 case 0x6f: /* FNEG */ 12555 need_fpst = false; 12556 break; 12557 case 0x7d: /* FRSQRTE */ 12558 case 0x7f: /* FSQRT (vector) */ 12559 break; 12560 default: 12561 unallocated_encoding(s); 12562 return; 12563 } 12564 12565 12566 /* Check additional constraints for the scalar encoding */ 12567 if (is_scalar) { 12568 if (!is_q) { 12569 unallocated_encoding(s); 12570 return; 12571 } 12572 /* FRINTxx is only in the vector form */ 12573 if (only_in_vector) { 12574 unallocated_encoding(s); 12575 return; 12576 } 12577 } 12578 12579 if (!fp_access_check(s)) { 12580 return; 12581 } 12582 12583 if (rmode >= 0 || need_fpst) { 12584 tcg_fpstatus = fpstatus_ptr(FPST_FPCR_F16); 12585 } 12586 12587 if (rmode >= 0) { 12588 tcg_rmode = gen_set_rmode(rmode, tcg_fpstatus); 12589 } 12590 12591 if (is_scalar) { 12592 TCGv_i32 tcg_op = read_fp_hreg(s, rn); 12593 TCGv_i32 tcg_res = tcg_temp_new_i32(); 12594 12595 switch (fpop) { 12596 case 0x1a: /* FCVTNS */ 12597 case 0x1b: /* FCVTMS */ 12598 case 0x1c: /* FCVTAS */ 12599 case 0x3a: /* FCVTPS */ 12600 case 0x3b: /* FCVTZS */ 12601 gen_helper_advsimd_f16tosinth(tcg_res, tcg_op, tcg_fpstatus); 12602 break; 12603 case 0x3d: /* FRECPE */ 12604 gen_helper_recpe_f16(tcg_res, tcg_op, tcg_fpstatus); 12605 break; 12606 case 0x3f: /* FRECPX */ 12607 gen_helper_frecpx_f16(tcg_res, tcg_op, tcg_fpstatus); 12608 break; 12609 case 0x5a: /* FCVTNU */ 12610 case 0x5b: /* FCVTMU */ 12611 case 0x5c: /* FCVTAU */ 12612 case 0x7a: /* FCVTPU */ 12613 case 0x7b: /* FCVTZU */ 12614 gen_helper_advsimd_f16touinth(tcg_res, tcg_op, tcg_fpstatus); 12615 break; 12616 case 0x6f: /* FNEG */ 12617 tcg_gen_xori_i32(tcg_res, tcg_op, 0x8000); 12618 break; 12619 case 0x7d: /* FRSQRTE */ 12620 gen_helper_rsqrte_f16(tcg_res, tcg_op, tcg_fpstatus); 12621 break; 12622 default: 12623 g_assert_not_reached(); 12624 } 12625 12626 /* limit any sign extension going on */ 12627 tcg_gen_andi_i32(tcg_res, tcg_res, 0xffff); 12628 write_fp_sreg(s, rd, tcg_res); 12629 } else { 12630 for (pass = 0; pass < (is_q ? 8 : 4); pass++) { 12631 TCGv_i32 tcg_op = tcg_temp_new_i32(); 12632 TCGv_i32 tcg_res = tcg_temp_new_i32(); 12633 12634 read_vec_element_i32(s, tcg_op, rn, pass, MO_16); 12635 12636 switch (fpop) { 12637 case 0x1a: /* FCVTNS */ 12638 case 0x1b: /* FCVTMS */ 12639 case 0x1c: /* FCVTAS */ 12640 case 0x3a: /* FCVTPS */ 12641 case 0x3b: /* FCVTZS */ 12642 gen_helper_advsimd_f16tosinth(tcg_res, tcg_op, tcg_fpstatus); 12643 break; 12644 case 0x3d: /* FRECPE */ 12645 gen_helper_recpe_f16(tcg_res, tcg_op, tcg_fpstatus); 12646 break; 12647 case 0x5a: /* FCVTNU */ 12648 case 0x5b: /* FCVTMU */ 12649 case 0x5c: /* FCVTAU */ 12650 case 0x7a: /* FCVTPU */ 12651 case 0x7b: /* FCVTZU */ 12652 gen_helper_advsimd_f16touinth(tcg_res, tcg_op, tcg_fpstatus); 12653 break; 12654 case 0x18: /* FRINTN */ 12655 case 0x19: /* FRINTM */ 12656 case 0x38: /* FRINTP */ 12657 case 0x39: /* FRINTZ */ 12658 case 0x58: /* FRINTA */ 12659 case 0x79: /* FRINTI */ 12660 gen_helper_advsimd_rinth(tcg_res, tcg_op, tcg_fpstatus); 12661 break; 12662 case 0x59: /* FRINTX */ 12663 gen_helper_advsimd_rinth_exact(tcg_res, tcg_op, tcg_fpstatus); 12664 break; 12665 case 0x2f: /* FABS */ 12666 tcg_gen_andi_i32(tcg_res, tcg_op, 0x7fff); 12667 break; 12668 case 0x6f: /* FNEG */ 12669 tcg_gen_xori_i32(tcg_res, tcg_op, 0x8000); 12670 break; 12671 case 0x7d: /* FRSQRTE */ 12672 gen_helper_rsqrte_f16(tcg_res, tcg_op, tcg_fpstatus); 12673 break; 12674 case 0x7f: /* FSQRT */ 12675 gen_helper_sqrt_f16(tcg_res, tcg_op, tcg_fpstatus); 12676 break; 12677 default: 12678 g_assert_not_reached(); 12679 } 12680 12681 write_vec_element_i32(s, tcg_res, rd, pass, MO_16); 12682 } 12683 12684 clear_vec_high(s, is_q, rd); 12685 } 12686 12687 if (tcg_rmode) { 12688 gen_restore_rmode(tcg_rmode, tcg_fpstatus); 12689 } 12690 } 12691 12692 /* AdvSIMD scalar x indexed element 12693 * 31 30 29 28 24 23 22 21 20 19 16 15 12 11 10 9 5 4 0 12694 * +-----+---+-----------+------+---+---+------+-----+---+---+------+------+ 12695 * | 0 1 | U | 1 1 1 1 1 | size | L | M | Rm | opc | H | 0 | Rn | Rd | 12696 * +-----+---+-----------+------+---+---+------+-----+---+---+------+------+ 12697 * AdvSIMD vector x indexed element 12698 * 31 30 29 28 24 23 22 21 20 19 16 15 12 11 10 9 5 4 0 12699 * +---+---+---+-----------+------+---+---+------+-----+---+---+------+------+ 12700 * | 0 | Q | U | 0 1 1 1 1 | size | L | M | Rm | opc | H | 0 | Rn | Rd | 12701 * +---+---+---+-----------+------+---+---+------+-----+---+---+------+------+ 12702 */ 12703 static void disas_simd_indexed(DisasContext *s, uint32_t insn) 12704 { 12705 /* This encoding has two kinds of instruction: 12706 * normal, where we perform elt x idxelt => elt for each 12707 * element in the vector 12708 * long, where we perform elt x idxelt and generate a result of 12709 * double the width of the input element 12710 * The long ops have a 'part' specifier (ie come in INSN, INSN2 pairs). 12711 */ 12712 bool is_scalar = extract32(insn, 28, 1); 12713 bool is_q = extract32(insn, 30, 1); 12714 bool u = extract32(insn, 29, 1); 12715 int size = extract32(insn, 22, 2); 12716 int l = extract32(insn, 21, 1); 12717 int m = extract32(insn, 20, 1); 12718 /* Note that the Rm field here is only 4 bits, not 5 as it usually is */ 12719 int rm = extract32(insn, 16, 4); 12720 int opcode = extract32(insn, 12, 4); 12721 int h = extract32(insn, 11, 1); 12722 int rn = extract32(insn, 5, 5); 12723 int rd = extract32(insn, 0, 5); 12724 bool is_long = false; 12725 int is_fp = 0; 12726 bool is_fp16 = false; 12727 int index; 12728 TCGv_ptr fpst; 12729 12730 switch (16 * u + opcode) { 12731 case 0x08: /* MUL */ 12732 case 0x10: /* MLA */ 12733 case 0x14: /* MLS */ 12734 if (is_scalar) { 12735 unallocated_encoding(s); 12736 return; 12737 } 12738 break; 12739 case 0x02: /* SMLAL, SMLAL2 */ 12740 case 0x12: /* UMLAL, UMLAL2 */ 12741 case 0x06: /* SMLSL, SMLSL2 */ 12742 case 0x16: /* UMLSL, UMLSL2 */ 12743 case 0x0a: /* SMULL, SMULL2 */ 12744 case 0x1a: /* UMULL, UMULL2 */ 12745 if (is_scalar) { 12746 unallocated_encoding(s); 12747 return; 12748 } 12749 is_long = true; 12750 break; 12751 case 0x03: /* SQDMLAL, SQDMLAL2 */ 12752 case 0x07: /* SQDMLSL, SQDMLSL2 */ 12753 case 0x0b: /* SQDMULL, SQDMULL2 */ 12754 is_long = true; 12755 break; 12756 case 0x0c: /* SQDMULH */ 12757 case 0x0d: /* SQRDMULH */ 12758 break; 12759 case 0x01: /* FMLA */ 12760 case 0x05: /* FMLS */ 12761 case 0x09: /* FMUL */ 12762 case 0x19: /* FMULX */ 12763 is_fp = 1; 12764 break; 12765 case 0x1d: /* SQRDMLAH */ 12766 case 0x1f: /* SQRDMLSH */ 12767 if (!dc_isar_feature(aa64_rdm, s)) { 12768 unallocated_encoding(s); 12769 return; 12770 } 12771 break; 12772 case 0x0e: /* SDOT */ 12773 case 0x1e: /* UDOT */ 12774 if (is_scalar || size != MO_32 || !dc_isar_feature(aa64_dp, s)) { 12775 unallocated_encoding(s); 12776 return; 12777 } 12778 break; 12779 case 0x0f: 12780 switch (size) { 12781 case 0: /* SUDOT */ 12782 case 2: /* USDOT */ 12783 if (is_scalar || !dc_isar_feature(aa64_i8mm, s)) { 12784 unallocated_encoding(s); 12785 return; 12786 } 12787 size = MO_32; 12788 break; 12789 case 1: /* BFDOT */ 12790 if (is_scalar || !dc_isar_feature(aa64_bf16, s)) { 12791 unallocated_encoding(s); 12792 return; 12793 } 12794 size = MO_32; 12795 break; 12796 case 3: /* BFMLAL{B,T} */ 12797 if (is_scalar || !dc_isar_feature(aa64_bf16, s)) { 12798 unallocated_encoding(s); 12799 return; 12800 } 12801 /* can't set is_fp without other incorrect size checks */ 12802 size = MO_16; 12803 break; 12804 default: 12805 unallocated_encoding(s); 12806 return; 12807 } 12808 break; 12809 case 0x11: /* FCMLA #0 */ 12810 case 0x13: /* FCMLA #90 */ 12811 case 0x15: /* FCMLA #180 */ 12812 case 0x17: /* FCMLA #270 */ 12813 if (is_scalar || !dc_isar_feature(aa64_fcma, s)) { 12814 unallocated_encoding(s); 12815 return; 12816 } 12817 is_fp = 2; 12818 break; 12819 case 0x00: /* FMLAL */ 12820 case 0x04: /* FMLSL */ 12821 case 0x18: /* FMLAL2 */ 12822 case 0x1c: /* FMLSL2 */ 12823 if (is_scalar || size != MO_32 || !dc_isar_feature(aa64_fhm, s)) { 12824 unallocated_encoding(s); 12825 return; 12826 } 12827 size = MO_16; 12828 /* is_fp, but we pass cpu_env not fp_status. */ 12829 break; 12830 default: 12831 unallocated_encoding(s); 12832 return; 12833 } 12834 12835 switch (is_fp) { 12836 case 1: /* normal fp */ 12837 /* convert insn encoded size to MemOp size */ 12838 switch (size) { 12839 case 0: /* half-precision */ 12840 size = MO_16; 12841 is_fp16 = true; 12842 break; 12843 case MO_32: /* single precision */ 12844 case MO_64: /* double precision */ 12845 break; 12846 default: 12847 unallocated_encoding(s); 12848 return; 12849 } 12850 break; 12851 12852 case 2: /* complex fp */ 12853 /* Each indexable element is a complex pair. */ 12854 size += 1; 12855 switch (size) { 12856 case MO_32: 12857 if (h && !is_q) { 12858 unallocated_encoding(s); 12859 return; 12860 } 12861 is_fp16 = true; 12862 break; 12863 case MO_64: 12864 break; 12865 default: 12866 unallocated_encoding(s); 12867 return; 12868 } 12869 break; 12870 12871 default: /* integer */ 12872 switch (size) { 12873 case MO_8: 12874 case MO_64: 12875 unallocated_encoding(s); 12876 return; 12877 } 12878 break; 12879 } 12880 if (is_fp16 && !dc_isar_feature(aa64_fp16, s)) { 12881 unallocated_encoding(s); 12882 return; 12883 } 12884 12885 /* Given MemOp size, adjust register and indexing. */ 12886 switch (size) { 12887 case MO_16: 12888 index = h << 2 | l << 1 | m; 12889 break; 12890 case MO_32: 12891 index = h << 1 | l; 12892 rm |= m << 4; 12893 break; 12894 case MO_64: 12895 if (l || !is_q) { 12896 unallocated_encoding(s); 12897 return; 12898 } 12899 index = h; 12900 rm |= m << 4; 12901 break; 12902 default: 12903 g_assert_not_reached(); 12904 } 12905 12906 if (!fp_access_check(s)) { 12907 return; 12908 } 12909 12910 if (is_fp) { 12911 fpst = fpstatus_ptr(is_fp16 ? FPST_FPCR_F16 : FPST_FPCR); 12912 } else { 12913 fpst = NULL; 12914 } 12915 12916 switch (16 * u + opcode) { 12917 case 0x0e: /* SDOT */ 12918 case 0x1e: /* UDOT */ 12919 gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, index, 12920 u ? gen_helper_gvec_udot_idx_b 12921 : gen_helper_gvec_sdot_idx_b); 12922 return; 12923 case 0x0f: 12924 switch (extract32(insn, 22, 2)) { 12925 case 0: /* SUDOT */ 12926 gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, index, 12927 gen_helper_gvec_sudot_idx_b); 12928 return; 12929 case 1: /* BFDOT */ 12930 gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, index, 12931 gen_helper_gvec_bfdot_idx); 12932 return; 12933 case 2: /* USDOT */ 12934 gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, index, 12935 gen_helper_gvec_usdot_idx_b); 12936 return; 12937 case 3: /* BFMLAL{B,T} */ 12938 gen_gvec_op4_fpst(s, 1, rd, rn, rm, rd, 0, (index << 1) | is_q, 12939 gen_helper_gvec_bfmlal_idx); 12940 return; 12941 } 12942 g_assert_not_reached(); 12943 case 0x11: /* FCMLA #0 */ 12944 case 0x13: /* FCMLA #90 */ 12945 case 0x15: /* FCMLA #180 */ 12946 case 0x17: /* FCMLA #270 */ 12947 { 12948 int rot = extract32(insn, 13, 2); 12949 int data = (index << 2) | rot; 12950 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, rd), 12951 vec_full_reg_offset(s, rn), 12952 vec_full_reg_offset(s, rm), 12953 vec_full_reg_offset(s, rd), fpst, 12954 is_q ? 16 : 8, vec_full_reg_size(s), data, 12955 size == MO_64 12956 ? gen_helper_gvec_fcmlas_idx 12957 : gen_helper_gvec_fcmlah_idx); 12958 } 12959 return; 12960 12961 case 0x00: /* FMLAL */ 12962 case 0x04: /* FMLSL */ 12963 case 0x18: /* FMLAL2 */ 12964 case 0x1c: /* FMLSL2 */ 12965 { 12966 int is_s = extract32(opcode, 2, 1); 12967 int is_2 = u; 12968 int data = (index << 2) | (is_2 << 1) | is_s; 12969 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd), 12970 vec_full_reg_offset(s, rn), 12971 vec_full_reg_offset(s, rm), cpu_env, 12972 is_q ? 16 : 8, vec_full_reg_size(s), 12973 data, gen_helper_gvec_fmlal_idx_a64); 12974 } 12975 return; 12976 12977 case 0x08: /* MUL */ 12978 if (!is_long && !is_scalar) { 12979 static gen_helper_gvec_3 * const fns[3] = { 12980 gen_helper_gvec_mul_idx_h, 12981 gen_helper_gvec_mul_idx_s, 12982 gen_helper_gvec_mul_idx_d, 12983 }; 12984 tcg_gen_gvec_3_ool(vec_full_reg_offset(s, rd), 12985 vec_full_reg_offset(s, rn), 12986 vec_full_reg_offset(s, rm), 12987 is_q ? 16 : 8, vec_full_reg_size(s), 12988 index, fns[size - 1]); 12989 return; 12990 } 12991 break; 12992 12993 case 0x10: /* MLA */ 12994 if (!is_long && !is_scalar) { 12995 static gen_helper_gvec_4 * const fns[3] = { 12996 gen_helper_gvec_mla_idx_h, 12997 gen_helper_gvec_mla_idx_s, 12998 gen_helper_gvec_mla_idx_d, 12999 }; 13000 tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd), 13001 vec_full_reg_offset(s, rn), 13002 vec_full_reg_offset(s, rm), 13003 vec_full_reg_offset(s, rd), 13004 is_q ? 16 : 8, vec_full_reg_size(s), 13005 index, fns[size - 1]); 13006 return; 13007 } 13008 break; 13009 13010 case 0x14: /* MLS */ 13011 if (!is_long && !is_scalar) { 13012 static gen_helper_gvec_4 * const fns[3] = { 13013 gen_helper_gvec_mls_idx_h, 13014 gen_helper_gvec_mls_idx_s, 13015 gen_helper_gvec_mls_idx_d, 13016 }; 13017 tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd), 13018 vec_full_reg_offset(s, rn), 13019 vec_full_reg_offset(s, rm), 13020 vec_full_reg_offset(s, rd), 13021 is_q ? 16 : 8, vec_full_reg_size(s), 13022 index, fns[size - 1]); 13023 return; 13024 } 13025 break; 13026 } 13027 13028 if (size == 3) { 13029 TCGv_i64 tcg_idx = tcg_temp_new_i64(); 13030 int pass; 13031 13032 assert(is_fp && is_q && !is_long); 13033 13034 read_vec_element(s, tcg_idx, rm, index, MO_64); 13035 13036 for (pass = 0; pass < (is_scalar ? 1 : 2); pass++) { 13037 TCGv_i64 tcg_op = tcg_temp_new_i64(); 13038 TCGv_i64 tcg_res = tcg_temp_new_i64(); 13039 13040 read_vec_element(s, tcg_op, rn, pass, MO_64); 13041 13042 switch (16 * u + opcode) { 13043 case 0x05: /* FMLS */ 13044 /* As usual for ARM, separate negation for fused multiply-add */ 13045 gen_helper_vfp_negd(tcg_op, tcg_op); 13046 /* fall through */ 13047 case 0x01: /* FMLA */ 13048 read_vec_element(s, tcg_res, rd, pass, MO_64); 13049 gen_helper_vfp_muladdd(tcg_res, tcg_op, tcg_idx, tcg_res, fpst); 13050 break; 13051 case 0x09: /* FMUL */ 13052 gen_helper_vfp_muld(tcg_res, tcg_op, tcg_idx, fpst); 13053 break; 13054 case 0x19: /* FMULX */ 13055 gen_helper_vfp_mulxd(tcg_res, tcg_op, tcg_idx, fpst); 13056 break; 13057 default: 13058 g_assert_not_reached(); 13059 } 13060 13061 write_vec_element(s, tcg_res, rd, pass, MO_64); 13062 } 13063 13064 clear_vec_high(s, !is_scalar, rd); 13065 } else if (!is_long) { 13066 /* 32 bit floating point, or 16 or 32 bit integer. 13067 * For the 16 bit scalar case we use the usual Neon helpers and 13068 * rely on the fact that 0 op 0 == 0 with no side effects. 13069 */ 13070 TCGv_i32 tcg_idx = tcg_temp_new_i32(); 13071 int pass, maxpasses; 13072 13073 if (is_scalar) { 13074 maxpasses = 1; 13075 } else { 13076 maxpasses = is_q ? 4 : 2; 13077 } 13078 13079 read_vec_element_i32(s, tcg_idx, rm, index, size); 13080 13081 if (size == 1 && !is_scalar) { 13082 /* The simplest way to handle the 16x16 indexed ops is to duplicate 13083 * the index into both halves of the 32 bit tcg_idx and then use 13084 * the usual Neon helpers. 13085 */ 13086 tcg_gen_deposit_i32(tcg_idx, tcg_idx, tcg_idx, 16, 16); 13087 } 13088 13089 for (pass = 0; pass < maxpasses; pass++) { 13090 TCGv_i32 tcg_op = tcg_temp_new_i32(); 13091 TCGv_i32 tcg_res = tcg_temp_new_i32(); 13092 13093 read_vec_element_i32(s, tcg_op, rn, pass, is_scalar ? size : MO_32); 13094 13095 switch (16 * u + opcode) { 13096 case 0x08: /* MUL */ 13097 case 0x10: /* MLA */ 13098 case 0x14: /* MLS */ 13099 { 13100 static NeonGenTwoOpFn * const fns[2][2] = { 13101 { gen_helper_neon_add_u16, gen_helper_neon_sub_u16 }, 13102 { tcg_gen_add_i32, tcg_gen_sub_i32 }, 13103 }; 13104 NeonGenTwoOpFn *genfn; 13105 bool is_sub = opcode == 0x4; 13106 13107 if (size == 1) { 13108 gen_helper_neon_mul_u16(tcg_res, tcg_op, tcg_idx); 13109 } else { 13110 tcg_gen_mul_i32(tcg_res, tcg_op, tcg_idx); 13111 } 13112 if (opcode == 0x8) { 13113 break; 13114 } 13115 read_vec_element_i32(s, tcg_op, rd, pass, MO_32); 13116 genfn = fns[size - 1][is_sub]; 13117 genfn(tcg_res, tcg_op, tcg_res); 13118 break; 13119 } 13120 case 0x05: /* FMLS */ 13121 case 0x01: /* FMLA */ 13122 read_vec_element_i32(s, tcg_res, rd, pass, 13123 is_scalar ? size : MO_32); 13124 switch (size) { 13125 case 1: 13126 if (opcode == 0x5) { 13127 /* As usual for ARM, separate negation for fused 13128 * multiply-add */ 13129 tcg_gen_xori_i32(tcg_op, tcg_op, 0x80008000); 13130 } 13131 if (is_scalar) { 13132 gen_helper_advsimd_muladdh(tcg_res, tcg_op, tcg_idx, 13133 tcg_res, fpst); 13134 } else { 13135 gen_helper_advsimd_muladd2h(tcg_res, tcg_op, tcg_idx, 13136 tcg_res, fpst); 13137 } 13138 break; 13139 case 2: 13140 if (opcode == 0x5) { 13141 /* As usual for ARM, separate negation for 13142 * fused multiply-add */ 13143 tcg_gen_xori_i32(tcg_op, tcg_op, 0x80000000); 13144 } 13145 gen_helper_vfp_muladds(tcg_res, tcg_op, tcg_idx, 13146 tcg_res, fpst); 13147 break; 13148 default: 13149 g_assert_not_reached(); 13150 } 13151 break; 13152 case 0x09: /* FMUL */ 13153 switch (size) { 13154 case 1: 13155 if (is_scalar) { 13156 gen_helper_advsimd_mulh(tcg_res, tcg_op, 13157 tcg_idx, fpst); 13158 } else { 13159 gen_helper_advsimd_mul2h(tcg_res, tcg_op, 13160 tcg_idx, fpst); 13161 } 13162 break; 13163 case 2: 13164 gen_helper_vfp_muls(tcg_res, tcg_op, tcg_idx, fpst); 13165 break; 13166 default: 13167 g_assert_not_reached(); 13168 } 13169 break; 13170 case 0x19: /* FMULX */ 13171 switch (size) { 13172 case 1: 13173 if (is_scalar) { 13174 gen_helper_advsimd_mulxh(tcg_res, tcg_op, 13175 tcg_idx, fpst); 13176 } else { 13177 gen_helper_advsimd_mulx2h(tcg_res, tcg_op, 13178 tcg_idx, fpst); 13179 } 13180 break; 13181 case 2: 13182 gen_helper_vfp_mulxs(tcg_res, tcg_op, tcg_idx, fpst); 13183 break; 13184 default: 13185 g_assert_not_reached(); 13186 } 13187 break; 13188 case 0x0c: /* SQDMULH */ 13189 if (size == 1) { 13190 gen_helper_neon_qdmulh_s16(tcg_res, cpu_env, 13191 tcg_op, tcg_idx); 13192 } else { 13193 gen_helper_neon_qdmulh_s32(tcg_res, cpu_env, 13194 tcg_op, tcg_idx); 13195 } 13196 break; 13197 case 0x0d: /* SQRDMULH */ 13198 if (size == 1) { 13199 gen_helper_neon_qrdmulh_s16(tcg_res, cpu_env, 13200 tcg_op, tcg_idx); 13201 } else { 13202 gen_helper_neon_qrdmulh_s32(tcg_res, cpu_env, 13203 tcg_op, tcg_idx); 13204 } 13205 break; 13206 case 0x1d: /* SQRDMLAH */ 13207 read_vec_element_i32(s, tcg_res, rd, pass, 13208 is_scalar ? size : MO_32); 13209 if (size == 1) { 13210 gen_helper_neon_qrdmlah_s16(tcg_res, cpu_env, 13211 tcg_op, tcg_idx, tcg_res); 13212 } else { 13213 gen_helper_neon_qrdmlah_s32(tcg_res, cpu_env, 13214 tcg_op, tcg_idx, tcg_res); 13215 } 13216 break; 13217 case 0x1f: /* SQRDMLSH */ 13218 read_vec_element_i32(s, tcg_res, rd, pass, 13219 is_scalar ? size : MO_32); 13220 if (size == 1) { 13221 gen_helper_neon_qrdmlsh_s16(tcg_res, cpu_env, 13222 tcg_op, tcg_idx, tcg_res); 13223 } else { 13224 gen_helper_neon_qrdmlsh_s32(tcg_res, cpu_env, 13225 tcg_op, tcg_idx, tcg_res); 13226 } 13227 break; 13228 default: 13229 g_assert_not_reached(); 13230 } 13231 13232 if (is_scalar) { 13233 write_fp_sreg(s, rd, tcg_res); 13234 } else { 13235 write_vec_element_i32(s, tcg_res, rd, pass, MO_32); 13236 } 13237 } 13238 13239 clear_vec_high(s, is_q, rd); 13240 } else { 13241 /* long ops: 16x16->32 or 32x32->64 */ 13242 TCGv_i64 tcg_res[2]; 13243 int pass; 13244 bool satop = extract32(opcode, 0, 1); 13245 MemOp memop = MO_32; 13246 13247 if (satop || !u) { 13248 memop |= MO_SIGN; 13249 } 13250 13251 if (size == 2) { 13252 TCGv_i64 tcg_idx = tcg_temp_new_i64(); 13253 13254 read_vec_element(s, tcg_idx, rm, index, memop); 13255 13256 for (pass = 0; pass < (is_scalar ? 1 : 2); pass++) { 13257 TCGv_i64 tcg_op = tcg_temp_new_i64(); 13258 TCGv_i64 tcg_passres; 13259 int passelt; 13260 13261 if (is_scalar) { 13262 passelt = 0; 13263 } else { 13264 passelt = pass + (is_q * 2); 13265 } 13266 13267 read_vec_element(s, tcg_op, rn, passelt, memop); 13268 13269 tcg_res[pass] = tcg_temp_new_i64(); 13270 13271 if (opcode == 0xa || opcode == 0xb) { 13272 /* Non-accumulating ops */ 13273 tcg_passres = tcg_res[pass]; 13274 } else { 13275 tcg_passres = tcg_temp_new_i64(); 13276 } 13277 13278 tcg_gen_mul_i64(tcg_passres, tcg_op, tcg_idx); 13279 13280 if (satop) { 13281 /* saturating, doubling */ 13282 gen_helper_neon_addl_saturate_s64(tcg_passres, cpu_env, 13283 tcg_passres, tcg_passres); 13284 } 13285 13286 if (opcode == 0xa || opcode == 0xb) { 13287 continue; 13288 } 13289 13290 /* Accumulating op: handle accumulate step */ 13291 read_vec_element(s, tcg_res[pass], rd, pass, MO_64); 13292 13293 switch (opcode) { 13294 case 0x2: /* SMLAL, SMLAL2, UMLAL, UMLAL2 */ 13295 tcg_gen_add_i64(tcg_res[pass], tcg_res[pass], tcg_passres); 13296 break; 13297 case 0x6: /* SMLSL, SMLSL2, UMLSL, UMLSL2 */ 13298 tcg_gen_sub_i64(tcg_res[pass], tcg_res[pass], tcg_passres); 13299 break; 13300 case 0x7: /* SQDMLSL, SQDMLSL2 */ 13301 tcg_gen_neg_i64(tcg_passres, tcg_passres); 13302 /* fall through */ 13303 case 0x3: /* SQDMLAL, SQDMLAL2 */ 13304 gen_helper_neon_addl_saturate_s64(tcg_res[pass], cpu_env, 13305 tcg_res[pass], 13306 tcg_passres); 13307 break; 13308 default: 13309 g_assert_not_reached(); 13310 } 13311 } 13312 13313 clear_vec_high(s, !is_scalar, rd); 13314 } else { 13315 TCGv_i32 tcg_idx = tcg_temp_new_i32(); 13316 13317 assert(size == 1); 13318 read_vec_element_i32(s, tcg_idx, rm, index, size); 13319 13320 if (!is_scalar) { 13321 /* The simplest way to handle the 16x16 indexed ops is to 13322 * duplicate the index into both halves of the 32 bit tcg_idx 13323 * and then use the usual Neon helpers. 13324 */ 13325 tcg_gen_deposit_i32(tcg_idx, tcg_idx, tcg_idx, 16, 16); 13326 } 13327 13328 for (pass = 0; pass < (is_scalar ? 1 : 2); pass++) { 13329 TCGv_i32 tcg_op = tcg_temp_new_i32(); 13330 TCGv_i64 tcg_passres; 13331 13332 if (is_scalar) { 13333 read_vec_element_i32(s, tcg_op, rn, pass, size); 13334 } else { 13335 read_vec_element_i32(s, tcg_op, rn, 13336 pass + (is_q * 2), MO_32); 13337 } 13338 13339 tcg_res[pass] = tcg_temp_new_i64(); 13340 13341 if (opcode == 0xa || opcode == 0xb) { 13342 /* Non-accumulating ops */ 13343 tcg_passres = tcg_res[pass]; 13344 } else { 13345 tcg_passres = tcg_temp_new_i64(); 13346 } 13347 13348 if (memop & MO_SIGN) { 13349 gen_helper_neon_mull_s16(tcg_passres, tcg_op, tcg_idx); 13350 } else { 13351 gen_helper_neon_mull_u16(tcg_passres, tcg_op, tcg_idx); 13352 } 13353 if (satop) { 13354 gen_helper_neon_addl_saturate_s32(tcg_passres, cpu_env, 13355 tcg_passres, tcg_passres); 13356 } 13357 13358 if (opcode == 0xa || opcode == 0xb) { 13359 continue; 13360 } 13361 13362 /* Accumulating op: handle accumulate step */ 13363 read_vec_element(s, tcg_res[pass], rd, pass, MO_64); 13364 13365 switch (opcode) { 13366 case 0x2: /* SMLAL, SMLAL2, UMLAL, UMLAL2 */ 13367 gen_helper_neon_addl_u32(tcg_res[pass], tcg_res[pass], 13368 tcg_passres); 13369 break; 13370 case 0x6: /* SMLSL, SMLSL2, UMLSL, UMLSL2 */ 13371 gen_helper_neon_subl_u32(tcg_res[pass], tcg_res[pass], 13372 tcg_passres); 13373 break; 13374 case 0x7: /* SQDMLSL, SQDMLSL2 */ 13375 gen_helper_neon_negl_u32(tcg_passres, tcg_passres); 13376 /* fall through */ 13377 case 0x3: /* SQDMLAL, SQDMLAL2 */ 13378 gen_helper_neon_addl_saturate_s32(tcg_res[pass], cpu_env, 13379 tcg_res[pass], 13380 tcg_passres); 13381 break; 13382 default: 13383 g_assert_not_reached(); 13384 } 13385 } 13386 13387 if (is_scalar) { 13388 tcg_gen_ext32u_i64(tcg_res[0], tcg_res[0]); 13389 } 13390 } 13391 13392 if (is_scalar) { 13393 tcg_res[1] = tcg_constant_i64(0); 13394 } 13395 13396 for (pass = 0; pass < 2; pass++) { 13397 write_vec_element(s, tcg_res[pass], rd, pass, MO_64); 13398 } 13399 } 13400 } 13401 13402 /* Crypto AES 13403 * 31 24 23 22 21 17 16 12 11 10 9 5 4 0 13404 * +-----------------+------+-----------+--------+-----+------+------+ 13405 * | 0 1 0 0 1 1 1 0 | size | 1 0 1 0 0 | opcode | 1 0 | Rn | Rd | 13406 * +-----------------+------+-----------+--------+-----+------+------+ 13407 */ 13408 static void disas_crypto_aes(DisasContext *s, uint32_t insn) 13409 { 13410 int size = extract32(insn, 22, 2); 13411 int opcode = extract32(insn, 12, 5); 13412 int rn = extract32(insn, 5, 5); 13413 int rd = extract32(insn, 0, 5); 13414 int decrypt; 13415 gen_helper_gvec_2 *genfn2 = NULL; 13416 gen_helper_gvec_3 *genfn3 = NULL; 13417 13418 if (!dc_isar_feature(aa64_aes, s) || size != 0) { 13419 unallocated_encoding(s); 13420 return; 13421 } 13422 13423 switch (opcode) { 13424 case 0x4: /* AESE */ 13425 decrypt = 0; 13426 genfn3 = gen_helper_crypto_aese; 13427 break; 13428 case 0x6: /* AESMC */ 13429 decrypt = 0; 13430 genfn2 = gen_helper_crypto_aesmc; 13431 break; 13432 case 0x5: /* AESD */ 13433 decrypt = 1; 13434 genfn3 = gen_helper_crypto_aese; 13435 break; 13436 case 0x7: /* AESIMC */ 13437 decrypt = 1; 13438 genfn2 = gen_helper_crypto_aesmc; 13439 break; 13440 default: 13441 unallocated_encoding(s); 13442 return; 13443 } 13444 13445 if (!fp_access_check(s)) { 13446 return; 13447 } 13448 if (genfn2) { 13449 gen_gvec_op2_ool(s, true, rd, rn, decrypt, genfn2); 13450 } else { 13451 gen_gvec_op3_ool(s, true, rd, rd, rn, decrypt, genfn3); 13452 } 13453 } 13454 13455 /* Crypto three-reg SHA 13456 * 31 24 23 22 21 20 16 15 14 12 11 10 9 5 4 0 13457 * +-----------------+------+---+------+---+--------+-----+------+------+ 13458 * | 0 1 0 1 1 1 1 0 | size | 0 | Rm | 0 | opcode | 0 0 | Rn | Rd | 13459 * +-----------------+------+---+------+---+--------+-----+------+------+ 13460 */ 13461 static void disas_crypto_three_reg_sha(DisasContext *s, uint32_t insn) 13462 { 13463 int size = extract32(insn, 22, 2); 13464 int opcode = extract32(insn, 12, 3); 13465 int rm = extract32(insn, 16, 5); 13466 int rn = extract32(insn, 5, 5); 13467 int rd = extract32(insn, 0, 5); 13468 gen_helper_gvec_3 *genfn; 13469 bool feature; 13470 13471 if (size != 0) { 13472 unallocated_encoding(s); 13473 return; 13474 } 13475 13476 switch (opcode) { 13477 case 0: /* SHA1C */ 13478 genfn = gen_helper_crypto_sha1c; 13479 feature = dc_isar_feature(aa64_sha1, s); 13480 break; 13481 case 1: /* SHA1P */ 13482 genfn = gen_helper_crypto_sha1p; 13483 feature = dc_isar_feature(aa64_sha1, s); 13484 break; 13485 case 2: /* SHA1M */ 13486 genfn = gen_helper_crypto_sha1m; 13487 feature = dc_isar_feature(aa64_sha1, s); 13488 break; 13489 case 3: /* SHA1SU0 */ 13490 genfn = gen_helper_crypto_sha1su0; 13491 feature = dc_isar_feature(aa64_sha1, s); 13492 break; 13493 case 4: /* SHA256H */ 13494 genfn = gen_helper_crypto_sha256h; 13495 feature = dc_isar_feature(aa64_sha256, s); 13496 break; 13497 case 5: /* SHA256H2 */ 13498 genfn = gen_helper_crypto_sha256h2; 13499 feature = dc_isar_feature(aa64_sha256, s); 13500 break; 13501 case 6: /* SHA256SU1 */ 13502 genfn = gen_helper_crypto_sha256su1; 13503 feature = dc_isar_feature(aa64_sha256, s); 13504 break; 13505 default: 13506 unallocated_encoding(s); 13507 return; 13508 } 13509 13510 if (!feature) { 13511 unallocated_encoding(s); 13512 return; 13513 } 13514 13515 if (!fp_access_check(s)) { 13516 return; 13517 } 13518 gen_gvec_op3_ool(s, true, rd, rn, rm, 0, genfn); 13519 } 13520 13521 /* Crypto two-reg SHA 13522 * 31 24 23 22 21 17 16 12 11 10 9 5 4 0 13523 * +-----------------+------+-----------+--------+-----+------+------+ 13524 * | 0 1 0 1 1 1 1 0 | size | 1 0 1 0 0 | opcode | 1 0 | Rn | Rd | 13525 * +-----------------+------+-----------+--------+-----+------+------+ 13526 */ 13527 static void disas_crypto_two_reg_sha(DisasContext *s, uint32_t insn) 13528 { 13529 int size = extract32(insn, 22, 2); 13530 int opcode = extract32(insn, 12, 5); 13531 int rn = extract32(insn, 5, 5); 13532 int rd = extract32(insn, 0, 5); 13533 gen_helper_gvec_2 *genfn; 13534 bool feature; 13535 13536 if (size != 0) { 13537 unallocated_encoding(s); 13538 return; 13539 } 13540 13541 switch (opcode) { 13542 case 0: /* SHA1H */ 13543 feature = dc_isar_feature(aa64_sha1, s); 13544 genfn = gen_helper_crypto_sha1h; 13545 break; 13546 case 1: /* SHA1SU1 */ 13547 feature = dc_isar_feature(aa64_sha1, s); 13548 genfn = gen_helper_crypto_sha1su1; 13549 break; 13550 case 2: /* SHA256SU0 */ 13551 feature = dc_isar_feature(aa64_sha256, s); 13552 genfn = gen_helper_crypto_sha256su0; 13553 break; 13554 default: 13555 unallocated_encoding(s); 13556 return; 13557 } 13558 13559 if (!feature) { 13560 unallocated_encoding(s); 13561 return; 13562 } 13563 13564 if (!fp_access_check(s)) { 13565 return; 13566 } 13567 gen_gvec_op2_ool(s, true, rd, rn, 0, genfn); 13568 } 13569 13570 static void gen_rax1_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m) 13571 { 13572 tcg_gen_rotli_i64(d, m, 1); 13573 tcg_gen_xor_i64(d, d, n); 13574 } 13575 13576 static void gen_rax1_vec(unsigned vece, TCGv_vec d, TCGv_vec n, TCGv_vec m) 13577 { 13578 tcg_gen_rotli_vec(vece, d, m, 1); 13579 tcg_gen_xor_vec(vece, d, d, n); 13580 } 13581 13582 void gen_gvec_rax1(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs, 13583 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz) 13584 { 13585 static const TCGOpcode vecop_list[] = { INDEX_op_rotli_vec, 0 }; 13586 static const GVecGen3 op = { 13587 .fni8 = gen_rax1_i64, 13588 .fniv = gen_rax1_vec, 13589 .opt_opc = vecop_list, 13590 .fno = gen_helper_crypto_rax1, 13591 .vece = MO_64, 13592 }; 13593 tcg_gen_gvec_3(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, &op); 13594 } 13595 13596 /* Crypto three-reg SHA512 13597 * 31 21 20 16 15 14 13 12 11 10 9 5 4 0 13598 * +-----------------------+------+---+---+-----+--------+------+------+ 13599 * | 1 1 0 0 1 1 1 0 0 1 1 | Rm | 1 | O | 0 0 | opcode | Rn | Rd | 13600 * +-----------------------+------+---+---+-----+--------+------+------+ 13601 */ 13602 static void disas_crypto_three_reg_sha512(DisasContext *s, uint32_t insn) 13603 { 13604 int opcode = extract32(insn, 10, 2); 13605 int o = extract32(insn, 14, 1); 13606 int rm = extract32(insn, 16, 5); 13607 int rn = extract32(insn, 5, 5); 13608 int rd = extract32(insn, 0, 5); 13609 bool feature; 13610 gen_helper_gvec_3 *oolfn = NULL; 13611 GVecGen3Fn *gvecfn = NULL; 13612 13613 if (o == 0) { 13614 switch (opcode) { 13615 case 0: /* SHA512H */ 13616 feature = dc_isar_feature(aa64_sha512, s); 13617 oolfn = gen_helper_crypto_sha512h; 13618 break; 13619 case 1: /* SHA512H2 */ 13620 feature = dc_isar_feature(aa64_sha512, s); 13621 oolfn = gen_helper_crypto_sha512h2; 13622 break; 13623 case 2: /* SHA512SU1 */ 13624 feature = dc_isar_feature(aa64_sha512, s); 13625 oolfn = gen_helper_crypto_sha512su1; 13626 break; 13627 case 3: /* RAX1 */ 13628 feature = dc_isar_feature(aa64_sha3, s); 13629 gvecfn = gen_gvec_rax1; 13630 break; 13631 default: 13632 g_assert_not_reached(); 13633 } 13634 } else { 13635 switch (opcode) { 13636 case 0: /* SM3PARTW1 */ 13637 feature = dc_isar_feature(aa64_sm3, s); 13638 oolfn = gen_helper_crypto_sm3partw1; 13639 break; 13640 case 1: /* SM3PARTW2 */ 13641 feature = dc_isar_feature(aa64_sm3, s); 13642 oolfn = gen_helper_crypto_sm3partw2; 13643 break; 13644 case 2: /* SM4EKEY */ 13645 feature = dc_isar_feature(aa64_sm4, s); 13646 oolfn = gen_helper_crypto_sm4ekey; 13647 break; 13648 default: 13649 unallocated_encoding(s); 13650 return; 13651 } 13652 } 13653 13654 if (!feature) { 13655 unallocated_encoding(s); 13656 return; 13657 } 13658 13659 if (!fp_access_check(s)) { 13660 return; 13661 } 13662 13663 if (oolfn) { 13664 gen_gvec_op3_ool(s, true, rd, rn, rm, 0, oolfn); 13665 } else { 13666 gen_gvec_fn3(s, true, rd, rn, rm, gvecfn, MO_64); 13667 } 13668 } 13669 13670 /* Crypto two-reg SHA512 13671 * 31 12 11 10 9 5 4 0 13672 * +-----------------------------------------+--------+------+------+ 13673 * | 1 1 0 0 1 1 1 0 1 1 0 0 0 0 0 0 1 0 0 0 | opcode | Rn | Rd | 13674 * +-----------------------------------------+--------+------+------+ 13675 */ 13676 static void disas_crypto_two_reg_sha512(DisasContext *s, uint32_t insn) 13677 { 13678 int opcode = extract32(insn, 10, 2); 13679 int rn = extract32(insn, 5, 5); 13680 int rd = extract32(insn, 0, 5); 13681 bool feature; 13682 13683 switch (opcode) { 13684 case 0: /* SHA512SU0 */ 13685 feature = dc_isar_feature(aa64_sha512, s); 13686 break; 13687 case 1: /* SM4E */ 13688 feature = dc_isar_feature(aa64_sm4, s); 13689 break; 13690 default: 13691 unallocated_encoding(s); 13692 return; 13693 } 13694 13695 if (!feature) { 13696 unallocated_encoding(s); 13697 return; 13698 } 13699 13700 if (!fp_access_check(s)) { 13701 return; 13702 } 13703 13704 switch (opcode) { 13705 case 0: /* SHA512SU0 */ 13706 gen_gvec_op2_ool(s, true, rd, rn, 0, gen_helper_crypto_sha512su0); 13707 break; 13708 case 1: /* SM4E */ 13709 gen_gvec_op3_ool(s, true, rd, rd, rn, 0, gen_helper_crypto_sm4e); 13710 break; 13711 default: 13712 g_assert_not_reached(); 13713 } 13714 } 13715 13716 /* Crypto four-register 13717 * 31 23 22 21 20 16 15 14 10 9 5 4 0 13718 * +-------------------+-----+------+---+------+------+------+ 13719 * | 1 1 0 0 1 1 1 0 0 | Op0 | Rm | 0 | Ra | Rn | Rd | 13720 * +-------------------+-----+------+---+------+------+------+ 13721 */ 13722 static void disas_crypto_four_reg(DisasContext *s, uint32_t insn) 13723 { 13724 int op0 = extract32(insn, 21, 2); 13725 int rm = extract32(insn, 16, 5); 13726 int ra = extract32(insn, 10, 5); 13727 int rn = extract32(insn, 5, 5); 13728 int rd = extract32(insn, 0, 5); 13729 bool feature; 13730 13731 switch (op0) { 13732 case 0: /* EOR3 */ 13733 case 1: /* BCAX */ 13734 feature = dc_isar_feature(aa64_sha3, s); 13735 break; 13736 case 2: /* SM3SS1 */ 13737 feature = dc_isar_feature(aa64_sm3, s); 13738 break; 13739 default: 13740 unallocated_encoding(s); 13741 return; 13742 } 13743 13744 if (!feature) { 13745 unallocated_encoding(s); 13746 return; 13747 } 13748 13749 if (!fp_access_check(s)) { 13750 return; 13751 } 13752 13753 if (op0 < 2) { 13754 TCGv_i64 tcg_op1, tcg_op2, tcg_op3, tcg_res[2]; 13755 int pass; 13756 13757 tcg_op1 = tcg_temp_new_i64(); 13758 tcg_op2 = tcg_temp_new_i64(); 13759 tcg_op3 = tcg_temp_new_i64(); 13760 tcg_res[0] = tcg_temp_new_i64(); 13761 tcg_res[1] = tcg_temp_new_i64(); 13762 13763 for (pass = 0; pass < 2; pass++) { 13764 read_vec_element(s, tcg_op1, rn, pass, MO_64); 13765 read_vec_element(s, tcg_op2, rm, pass, MO_64); 13766 read_vec_element(s, tcg_op3, ra, pass, MO_64); 13767 13768 if (op0 == 0) { 13769 /* EOR3 */ 13770 tcg_gen_xor_i64(tcg_res[pass], tcg_op2, tcg_op3); 13771 } else { 13772 /* BCAX */ 13773 tcg_gen_andc_i64(tcg_res[pass], tcg_op2, tcg_op3); 13774 } 13775 tcg_gen_xor_i64(tcg_res[pass], tcg_res[pass], tcg_op1); 13776 } 13777 write_vec_element(s, tcg_res[0], rd, 0, MO_64); 13778 write_vec_element(s, tcg_res[1], rd, 1, MO_64); 13779 } else { 13780 TCGv_i32 tcg_op1, tcg_op2, tcg_op3, tcg_res, tcg_zero; 13781 13782 tcg_op1 = tcg_temp_new_i32(); 13783 tcg_op2 = tcg_temp_new_i32(); 13784 tcg_op3 = tcg_temp_new_i32(); 13785 tcg_res = tcg_temp_new_i32(); 13786 tcg_zero = tcg_constant_i32(0); 13787 13788 read_vec_element_i32(s, tcg_op1, rn, 3, MO_32); 13789 read_vec_element_i32(s, tcg_op2, rm, 3, MO_32); 13790 read_vec_element_i32(s, tcg_op3, ra, 3, MO_32); 13791 13792 tcg_gen_rotri_i32(tcg_res, tcg_op1, 20); 13793 tcg_gen_add_i32(tcg_res, tcg_res, tcg_op2); 13794 tcg_gen_add_i32(tcg_res, tcg_res, tcg_op3); 13795 tcg_gen_rotri_i32(tcg_res, tcg_res, 25); 13796 13797 write_vec_element_i32(s, tcg_zero, rd, 0, MO_32); 13798 write_vec_element_i32(s, tcg_zero, rd, 1, MO_32); 13799 write_vec_element_i32(s, tcg_zero, rd, 2, MO_32); 13800 write_vec_element_i32(s, tcg_res, rd, 3, MO_32); 13801 } 13802 } 13803 13804 /* Crypto XAR 13805 * 31 21 20 16 15 10 9 5 4 0 13806 * +-----------------------+------+--------+------+------+ 13807 * | 1 1 0 0 1 1 1 0 1 0 0 | Rm | imm6 | Rn | Rd | 13808 * +-----------------------+------+--------+------+------+ 13809 */ 13810 static void disas_crypto_xar(DisasContext *s, uint32_t insn) 13811 { 13812 int rm = extract32(insn, 16, 5); 13813 int imm6 = extract32(insn, 10, 6); 13814 int rn = extract32(insn, 5, 5); 13815 int rd = extract32(insn, 0, 5); 13816 13817 if (!dc_isar_feature(aa64_sha3, s)) { 13818 unallocated_encoding(s); 13819 return; 13820 } 13821 13822 if (!fp_access_check(s)) { 13823 return; 13824 } 13825 13826 gen_gvec_xar(MO_64, vec_full_reg_offset(s, rd), 13827 vec_full_reg_offset(s, rn), 13828 vec_full_reg_offset(s, rm), imm6, 16, 13829 vec_full_reg_size(s)); 13830 } 13831 13832 /* Crypto three-reg imm2 13833 * 31 21 20 16 15 14 13 12 11 10 9 5 4 0 13834 * +-----------------------+------+-----+------+--------+------+------+ 13835 * | 1 1 0 0 1 1 1 0 0 1 0 | Rm | 1 0 | imm2 | opcode | Rn | Rd | 13836 * +-----------------------+------+-----+------+--------+------+------+ 13837 */ 13838 static void disas_crypto_three_reg_imm2(DisasContext *s, uint32_t insn) 13839 { 13840 static gen_helper_gvec_3 * const fns[4] = { 13841 gen_helper_crypto_sm3tt1a, gen_helper_crypto_sm3tt1b, 13842 gen_helper_crypto_sm3tt2a, gen_helper_crypto_sm3tt2b, 13843 }; 13844 int opcode = extract32(insn, 10, 2); 13845 int imm2 = extract32(insn, 12, 2); 13846 int rm = extract32(insn, 16, 5); 13847 int rn = extract32(insn, 5, 5); 13848 int rd = extract32(insn, 0, 5); 13849 13850 if (!dc_isar_feature(aa64_sm3, s)) { 13851 unallocated_encoding(s); 13852 return; 13853 } 13854 13855 if (!fp_access_check(s)) { 13856 return; 13857 } 13858 13859 gen_gvec_op3_ool(s, true, rd, rn, rm, imm2, fns[opcode]); 13860 } 13861 13862 /* C3.6 Data processing - SIMD, inc Crypto 13863 * 13864 * As the decode gets a little complex we are using a table based 13865 * approach for this part of the decode. 13866 */ 13867 static const AArch64DecodeTable data_proc_simd[] = { 13868 /* pattern , mask , fn */ 13869 { 0x0e200400, 0x9f200400, disas_simd_three_reg_same }, 13870 { 0x0e008400, 0x9f208400, disas_simd_three_reg_same_extra }, 13871 { 0x0e200000, 0x9f200c00, disas_simd_three_reg_diff }, 13872 { 0x0e200800, 0x9f3e0c00, disas_simd_two_reg_misc }, 13873 { 0x0e300800, 0x9f3e0c00, disas_simd_across_lanes }, 13874 { 0x0e000400, 0x9fe08400, disas_simd_copy }, 13875 { 0x0f000000, 0x9f000400, disas_simd_indexed }, /* vector indexed */ 13876 /* simd_mod_imm decode is a subset of simd_shift_imm, so must precede it */ 13877 { 0x0f000400, 0x9ff80400, disas_simd_mod_imm }, 13878 { 0x0f000400, 0x9f800400, disas_simd_shift_imm }, 13879 { 0x0e000000, 0xbf208c00, disas_simd_tb }, 13880 { 0x0e000800, 0xbf208c00, disas_simd_zip_trn }, 13881 { 0x2e000000, 0xbf208400, disas_simd_ext }, 13882 { 0x5e200400, 0xdf200400, disas_simd_scalar_three_reg_same }, 13883 { 0x5e008400, 0xdf208400, disas_simd_scalar_three_reg_same_extra }, 13884 { 0x5e200000, 0xdf200c00, disas_simd_scalar_three_reg_diff }, 13885 { 0x5e200800, 0xdf3e0c00, disas_simd_scalar_two_reg_misc }, 13886 { 0x5e300800, 0xdf3e0c00, disas_simd_scalar_pairwise }, 13887 { 0x5e000400, 0xdfe08400, disas_simd_scalar_copy }, 13888 { 0x5f000000, 0xdf000400, disas_simd_indexed }, /* scalar indexed */ 13889 { 0x5f000400, 0xdf800400, disas_simd_scalar_shift_imm }, 13890 { 0x4e280800, 0xff3e0c00, disas_crypto_aes }, 13891 { 0x5e000000, 0xff208c00, disas_crypto_three_reg_sha }, 13892 { 0x5e280800, 0xff3e0c00, disas_crypto_two_reg_sha }, 13893 { 0xce608000, 0xffe0b000, disas_crypto_three_reg_sha512 }, 13894 { 0xcec08000, 0xfffff000, disas_crypto_two_reg_sha512 }, 13895 { 0xce000000, 0xff808000, disas_crypto_four_reg }, 13896 { 0xce800000, 0xffe00000, disas_crypto_xar }, 13897 { 0xce408000, 0xffe0c000, disas_crypto_three_reg_imm2 }, 13898 { 0x0e400400, 0x9f60c400, disas_simd_three_reg_same_fp16 }, 13899 { 0x0e780800, 0x8f7e0c00, disas_simd_two_reg_misc_fp16 }, 13900 { 0x5e400400, 0xdf60c400, disas_simd_scalar_three_reg_same_fp16 }, 13901 { 0x00000000, 0x00000000, NULL } 13902 }; 13903 13904 static void disas_data_proc_simd(DisasContext *s, uint32_t insn) 13905 { 13906 /* Note that this is called with all non-FP cases from 13907 * table C3-6 so it must UNDEF for entries not specifically 13908 * allocated to instructions in that table. 13909 */ 13910 AArch64DecodeFn *fn = lookup_disas_fn(&data_proc_simd[0], insn); 13911 if (fn) { 13912 fn(s, insn); 13913 } else { 13914 unallocated_encoding(s); 13915 } 13916 } 13917 13918 /* C3.6 Data processing - SIMD and floating point */ 13919 static void disas_data_proc_simd_fp(DisasContext *s, uint32_t insn) 13920 { 13921 if (extract32(insn, 28, 1) == 1 && extract32(insn, 30, 1) == 0) { 13922 disas_data_proc_fp(s, insn); 13923 } else { 13924 /* SIMD, including crypto */ 13925 disas_data_proc_simd(s, insn); 13926 } 13927 } 13928 13929 static bool trans_OK(DisasContext *s, arg_OK *a) 13930 { 13931 return true; 13932 } 13933 13934 static bool trans_FAIL(DisasContext *s, arg_OK *a) 13935 { 13936 s->is_nonstreaming = true; 13937 return true; 13938 } 13939 13940 /** 13941 * is_guarded_page: 13942 * @env: The cpu environment 13943 * @s: The DisasContext 13944 * 13945 * Return true if the page is guarded. 13946 */ 13947 static bool is_guarded_page(CPUARMState *env, DisasContext *s) 13948 { 13949 uint64_t addr = s->base.pc_first; 13950 #ifdef CONFIG_USER_ONLY 13951 return page_get_flags(addr) & PAGE_BTI; 13952 #else 13953 CPUTLBEntryFull *full; 13954 void *host; 13955 int mmu_idx = arm_to_core_mmu_idx(s->mmu_idx); 13956 int flags; 13957 13958 /* 13959 * We test this immediately after reading an insn, which means 13960 * that the TLB entry must be present and valid, and thus this 13961 * access will never raise an exception. 13962 */ 13963 flags = probe_access_full(env, addr, 0, MMU_INST_FETCH, mmu_idx, 13964 false, &host, &full, 0); 13965 assert(!(flags & TLB_INVALID_MASK)); 13966 13967 return full->guarded; 13968 #endif 13969 } 13970 13971 /** 13972 * btype_destination_ok: 13973 * @insn: The instruction at the branch destination 13974 * @bt: SCTLR_ELx.BT 13975 * @btype: PSTATE.BTYPE, and is non-zero 13976 * 13977 * On a guarded page, there are a limited number of insns 13978 * that may be present at the branch target: 13979 * - branch target identifiers, 13980 * - paciasp, pacibsp, 13981 * - BRK insn 13982 * - HLT insn 13983 * Anything else causes a Branch Target Exception. 13984 * 13985 * Return true if the branch is compatible, false to raise BTITRAP. 13986 */ 13987 static bool btype_destination_ok(uint32_t insn, bool bt, int btype) 13988 { 13989 if ((insn & 0xfffff01fu) == 0xd503201fu) { 13990 /* HINT space */ 13991 switch (extract32(insn, 5, 7)) { 13992 case 0b011001: /* PACIASP */ 13993 case 0b011011: /* PACIBSP */ 13994 /* 13995 * If SCTLR_ELx.BT, then PACI*SP are not compatible 13996 * with btype == 3. Otherwise all btype are ok. 13997 */ 13998 return !bt || btype != 3; 13999 case 0b100000: /* BTI */ 14000 /* Not compatible with any btype. */ 14001 return false; 14002 case 0b100010: /* BTI c */ 14003 /* Not compatible with btype == 3 */ 14004 return btype != 3; 14005 case 0b100100: /* BTI j */ 14006 /* Not compatible with btype == 2 */ 14007 return btype != 2; 14008 case 0b100110: /* BTI jc */ 14009 /* Compatible with any btype. */ 14010 return true; 14011 } 14012 } else { 14013 switch (insn & 0xffe0001fu) { 14014 case 0xd4200000u: /* BRK */ 14015 case 0xd4400000u: /* HLT */ 14016 /* Give priority to the breakpoint exception. */ 14017 return true; 14018 } 14019 } 14020 return false; 14021 } 14022 14023 /* C3.1 A64 instruction index by encoding */ 14024 static void disas_a64_legacy(DisasContext *s, uint32_t insn) 14025 { 14026 switch (extract32(insn, 25, 4)) { 14027 case 0xa: case 0xb: /* Branch, exception generation and system insns */ 14028 disas_b_exc_sys(s, insn); 14029 break; 14030 case 0x4: 14031 case 0x6: 14032 case 0xc: 14033 case 0xe: /* Loads and stores */ 14034 disas_ldst(s, insn); 14035 break; 14036 case 0x5: 14037 case 0xd: /* Data processing - register */ 14038 disas_data_proc_reg(s, insn); 14039 break; 14040 case 0x7: 14041 case 0xf: /* Data processing - SIMD and floating point */ 14042 disas_data_proc_simd_fp(s, insn); 14043 break; 14044 default: 14045 unallocated_encoding(s); 14046 break; 14047 } 14048 } 14049 14050 static void aarch64_tr_init_disas_context(DisasContextBase *dcbase, 14051 CPUState *cpu) 14052 { 14053 DisasContext *dc = container_of(dcbase, DisasContext, base); 14054 CPUARMState *env = cpu->env_ptr; 14055 ARMCPU *arm_cpu = env_archcpu(env); 14056 CPUARMTBFlags tb_flags = arm_tbflags_from_tb(dc->base.tb); 14057 int bound, core_mmu_idx; 14058 14059 dc->isar = &arm_cpu->isar; 14060 dc->condjmp = 0; 14061 dc->pc_save = dc->base.pc_first; 14062 dc->aarch64 = true; 14063 dc->thumb = false; 14064 dc->sctlr_b = 0; 14065 dc->be_data = EX_TBFLAG_ANY(tb_flags, BE_DATA) ? MO_BE : MO_LE; 14066 dc->condexec_mask = 0; 14067 dc->condexec_cond = 0; 14068 core_mmu_idx = EX_TBFLAG_ANY(tb_flags, MMUIDX); 14069 dc->mmu_idx = core_to_aa64_mmu_idx(core_mmu_idx); 14070 dc->tbii = EX_TBFLAG_A64(tb_flags, TBII); 14071 dc->tbid = EX_TBFLAG_A64(tb_flags, TBID); 14072 dc->tcma = EX_TBFLAG_A64(tb_flags, TCMA); 14073 dc->current_el = arm_mmu_idx_to_el(dc->mmu_idx); 14074 #if !defined(CONFIG_USER_ONLY) 14075 dc->user = (dc->current_el == 0); 14076 #endif 14077 dc->fp_excp_el = EX_TBFLAG_ANY(tb_flags, FPEXC_EL); 14078 dc->align_mem = EX_TBFLAG_ANY(tb_flags, ALIGN_MEM); 14079 dc->pstate_il = EX_TBFLAG_ANY(tb_flags, PSTATE__IL); 14080 dc->fgt_active = EX_TBFLAG_ANY(tb_flags, FGT_ACTIVE); 14081 dc->fgt_svc = EX_TBFLAG_ANY(tb_flags, FGT_SVC); 14082 dc->fgt_eret = EX_TBFLAG_A64(tb_flags, FGT_ERET); 14083 dc->sve_excp_el = EX_TBFLAG_A64(tb_flags, SVEEXC_EL); 14084 dc->sme_excp_el = EX_TBFLAG_A64(tb_flags, SMEEXC_EL); 14085 dc->vl = (EX_TBFLAG_A64(tb_flags, VL) + 1) * 16; 14086 dc->svl = (EX_TBFLAG_A64(tb_flags, SVL) + 1) * 16; 14087 dc->pauth_active = EX_TBFLAG_A64(tb_flags, PAUTH_ACTIVE); 14088 dc->bt = EX_TBFLAG_A64(tb_flags, BT); 14089 dc->btype = EX_TBFLAG_A64(tb_flags, BTYPE); 14090 dc->unpriv = EX_TBFLAG_A64(tb_flags, UNPRIV); 14091 dc->ata = EX_TBFLAG_A64(tb_flags, ATA); 14092 dc->mte_active[0] = EX_TBFLAG_A64(tb_flags, MTE_ACTIVE); 14093 dc->mte_active[1] = EX_TBFLAG_A64(tb_flags, MTE0_ACTIVE); 14094 dc->pstate_sm = EX_TBFLAG_A64(tb_flags, PSTATE_SM); 14095 dc->pstate_za = EX_TBFLAG_A64(tb_flags, PSTATE_ZA); 14096 dc->sme_trap_nonstreaming = EX_TBFLAG_A64(tb_flags, SME_TRAP_NONSTREAMING); 14097 dc->vec_len = 0; 14098 dc->vec_stride = 0; 14099 dc->cp_regs = arm_cpu->cp_regs; 14100 dc->features = env->features; 14101 dc->dcz_blocksize = arm_cpu->dcz_blocksize; 14102 14103 #ifdef CONFIG_USER_ONLY 14104 /* In sve_probe_page, we assume TBI is enabled. */ 14105 tcg_debug_assert(dc->tbid & 1); 14106 #endif 14107 14108 /* Single step state. The code-generation logic here is: 14109 * SS_ACTIVE == 0: 14110 * generate code with no special handling for single-stepping (except 14111 * that anything that can make us go to SS_ACTIVE == 1 must end the TB; 14112 * this happens anyway because those changes are all system register or 14113 * PSTATE writes). 14114 * SS_ACTIVE == 1, PSTATE.SS == 1: (active-not-pending) 14115 * emit code for one insn 14116 * emit code to clear PSTATE.SS 14117 * emit code to generate software step exception for completed step 14118 * end TB (as usual for having generated an exception) 14119 * SS_ACTIVE == 1, PSTATE.SS == 0: (active-pending) 14120 * emit code to generate a software step exception 14121 * end the TB 14122 */ 14123 dc->ss_active = EX_TBFLAG_ANY(tb_flags, SS_ACTIVE); 14124 dc->pstate_ss = EX_TBFLAG_ANY(tb_flags, PSTATE__SS); 14125 dc->is_ldex = false; 14126 14127 /* Bound the number of insns to execute to those left on the page. */ 14128 bound = -(dc->base.pc_first | TARGET_PAGE_MASK) / 4; 14129 14130 /* If architectural single step active, limit to 1. */ 14131 if (dc->ss_active) { 14132 bound = 1; 14133 } 14134 dc->base.max_insns = MIN(dc->base.max_insns, bound); 14135 } 14136 14137 static void aarch64_tr_tb_start(DisasContextBase *db, CPUState *cpu) 14138 { 14139 } 14140 14141 static void aarch64_tr_insn_start(DisasContextBase *dcbase, CPUState *cpu) 14142 { 14143 DisasContext *dc = container_of(dcbase, DisasContext, base); 14144 target_ulong pc_arg = dc->base.pc_next; 14145 14146 if (tb_cflags(dcbase->tb) & CF_PCREL) { 14147 pc_arg &= ~TARGET_PAGE_MASK; 14148 } 14149 tcg_gen_insn_start(pc_arg, 0, 0); 14150 dc->insn_start = tcg_last_op(); 14151 } 14152 14153 static void aarch64_tr_translate_insn(DisasContextBase *dcbase, CPUState *cpu) 14154 { 14155 DisasContext *s = container_of(dcbase, DisasContext, base); 14156 CPUARMState *env = cpu->env_ptr; 14157 uint64_t pc = s->base.pc_next; 14158 uint32_t insn; 14159 14160 /* Singlestep exceptions have the highest priority. */ 14161 if (s->ss_active && !s->pstate_ss) { 14162 /* Singlestep state is Active-pending. 14163 * If we're in this state at the start of a TB then either 14164 * a) we just took an exception to an EL which is being debugged 14165 * and this is the first insn in the exception handler 14166 * b) debug exceptions were masked and we just unmasked them 14167 * without changing EL (eg by clearing PSTATE.D) 14168 * In either case we're going to take a swstep exception in the 14169 * "did not step an insn" case, and so the syndrome ISV and EX 14170 * bits should be zero. 14171 */ 14172 assert(s->base.num_insns == 1); 14173 gen_swstep_exception(s, 0, 0); 14174 s->base.is_jmp = DISAS_NORETURN; 14175 s->base.pc_next = pc + 4; 14176 return; 14177 } 14178 14179 if (pc & 3) { 14180 /* 14181 * PC alignment fault. This has priority over the instruction abort 14182 * that we would receive from a translation fault via arm_ldl_code. 14183 * This should only be possible after an indirect branch, at the 14184 * start of the TB. 14185 */ 14186 assert(s->base.num_insns == 1); 14187 gen_helper_exception_pc_alignment(cpu_env, tcg_constant_tl(pc)); 14188 s->base.is_jmp = DISAS_NORETURN; 14189 s->base.pc_next = QEMU_ALIGN_UP(pc, 4); 14190 return; 14191 } 14192 14193 s->pc_curr = pc; 14194 insn = arm_ldl_code(env, &s->base, pc, s->sctlr_b); 14195 s->insn = insn; 14196 s->base.pc_next = pc + 4; 14197 14198 s->fp_access_checked = false; 14199 s->sve_access_checked = false; 14200 14201 if (s->pstate_il) { 14202 /* 14203 * Illegal execution state. This has priority over BTI 14204 * exceptions, but comes after instruction abort exceptions. 14205 */ 14206 gen_exception_insn(s, 0, EXCP_UDEF, syn_illegalstate()); 14207 return; 14208 } 14209 14210 if (dc_isar_feature(aa64_bti, s)) { 14211 if (s->base.num_insns == 1) { 14212 /* 14213 * At the first insn of the TB, compute s->guarded_page. 14214 * We delayed computing this until successfully reading 14215 * the first insn of the TB, above. This (mostly) ensures 14216 * that the softmmu tlb entry has been populated, and the 14217 * page table GP bit is available. 14218 * 14219 * Note that we need to compute this even if btype == 0, 14220 * because this value is used for BR instructions later 14221 * where ENV is not available. 14222 */ 14223 s->guarded_page = is_guarded_page(env, s); 14224 14225 /* First insn can have btype set to non-zero. */ 14226 tcg_debug_assert(s->btype >= 0); 14227 14228 /* 14229 * Note that the Branch Target Exception has fairly high 14230 * priority -- below debugging exceptions but above most 14231 * everything else. This allows us to handle this now 14232 * instead of waiting until the insn is otherwise decoded. 14233 */ 14234 if (s->btype != 0 14235 && s->guarded_page 14236 && !btype_destination_ok(insn, s->bt, s->btype)) { 14237 gen_exception_insn(s, 0, EXCP_UDEF, syn_btitrap(s->btype)); 14238 return; 14239 } 14240 } else { 14241 /* Not the first insn: btype must be 0. */ 14242 tcg_debug_assert(s->btype == 0); 14243 } 14244 } 14245 14246 s->is_nonstreaming = false; 14247 if (s->sme_trap_nonstreaming) { 14248 disas_sme_fa64(s, insn); 14249 } 14250 14251 if (!disas_a64(s, insn) && 14252 !disas_sme(s, insn) && 14253 !disas_sve(s, insn)) { 14254 disas_a64_legacy(s, insn); 14255 } 14256 14257 /* 14258 * After execution of most insns, btype is reset to 0. 14259 * Note that we set btype == -1 when the insn sets btype. 14260 */ 14261 if (s->btype > 0 && s->base.is_jmp != DISAS_NORETURN) { 14262 reset_btype(s); 14263 } 14264 } 14265 14266 static void aarch64_tr_tb_stop(DisasContextBase *dcbase, CPUState *cpu) 14267 { 14268 DisasContext *dc = container_of(dcbase, DisasContext, base); 14269 14270 if (unlikely(dc->ss_active)) { 14271 /* Note that this means single stepping WFI doesn't halt the CPU. 14272 * For conditional branch insns this is harmless unreachable code as 14273 * gen_goto_tb() has already handled emitting the debug exception 14274 * (and thus a tb-jump is not possible when singlestepping). 14275 */ 14276 switch (dc->base.is_jmp) { 14277 default: 14278 gen_a64_update_pc(dc, 4); 14279 /* fall through */ 14280 case DISAS_EXIT: 14281 case DISAS_JUMP: 14282 gen_step_complete_exception(dc); 14283 break; 14284 case DISAS_NORETURN: 14285 break; 14286 } 14287 } else { 14288 switch (dc->base.is_jmp) { 14289 case DISAS_NEXT: 14290 case DISAS_TOO_MANY: 14291 gen_goto_tb(dc, 1, 4); 14292 break; 14293 default: 14294 case DISAS_UPDATE_EXIT: 14295 gen_a64_update_pc(dc, 4); 14296 /* fall through */ 14297 case DISAS_EXIT: 14298 tcg_gen_exit_tb(NULL, 0); 14299 break; 14300 case DISAS_UPDATE_NOCHAIN: 14301 gen_a64_update_pc(dc, 4); 14302 /* fall through */ 14303 case DISAS_JUMP: 14304 tcg_gen_lookup_and_goto_ptr(); 14305 break; 14306 case DISAS_NORETURN: 14307 case DISAS_SWI: 14308 break; 14309 case DISAS_WFE: 14310 gen_a64_update_pc(dc, 4); 14311 gen_helper_wfe(cpu_env); 14312 break; 14313 case DISAS_YIELD: 14314 gen_a64_update_pc(dc, 4); 14315 gen_helper_yield(cpu_env); 14316 break; 14317 case DISAS_WFI: 14318 /* 14319 * This is a special case because we don't want to just halt 14320 * the CPU if trying to debug across a WFI. 14321 */ 14322 gen_a64_update_pc(dc, 4); 14323 gen_helper_wfi(cpu_env, tcg_constant_i32(4)); 14324 /* 14325 * The helper doesn't necessarily throw an exception, but we 14326 * must go back to the main loop to check for interrupts anyway. 14327 */ 14328 tcg_gen_exit_tb(NULL, 0); 14329 break; 14330 } 14331 } 14332 } 14333 14334 static void aarch64_tr_disas_log(const DisasContextBase *dcbase, 14335 CPUState *cpu, FILE *logfile) 14336 { 14337 DisasContext *dc = container_of(dcbase, DisasContext, base); 14338 14339 fprintf(logfile, "IN: %s\n", lookup_symbol(dc->base.pc_first)); 14340 target_disas(logfile, cpu, dc->base.pc_first, dc->base.tb->size); 14341 } 14342 14343 const TranslatorOps aarch64_translator_ops = { 14344 .init_disas_context = aarch64_tr_init_disas_context, 14345 .tb_start = aarch64_tr_tb_start, 14346 .insn_start = aarch64_tr_insn_start, 14347 .translate_insn = aarch64_tr_translate_insn, 14348 .tb_stop = aarch64_tr_tb_stop, 14349 .disas_log = aarch64_tr_disas_log, 14350 }; 14351