1 /*
2 * AArch64 translation
3 *
4 * Copyright (c) 2013 Alexander Graf <agraf@suse.de>
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "qemu/osdep.h"
20
21 #include "translate.h"
22 #include "translate-a64.h"
23 #include "qemu/log.h"
24 #include "disas/disas.h"
25 #include "arm_ldst.h"
26 #include "semihosting/semihost.h"
27 #include "cpregs.h"
28
29 static TCGv_i64 cpu_X[32];
30 static TCGv_i64 cpu_pc;
31
32 /* Load/store exclusive handling */
33 static TCGv_i64 cpu_exclusive_high;
34
35 static const char *regnames[] = {
36 "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7",
37 "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15",
38 "x16", "x17", "x18", "x19", "x20", "x21", "x22", "x23",
39 "x24", "x25", "x26", "x27", "x28", "x29", "lr", "sp"
40 };
41
42 enum a64_shift_type {
43 A64_SHIFT_TYPE_LSL = 0,
44 A64_SHIFT_TYPE_LSR = 1,
45 A64_SHIFT_TYPE_ASR = 2,
46 A64_SHIFT_TYPE_ROR = 3
47 };
48
49 /*
50 * Helpers for extracting complex instruction fields
51 */
52
53 /*
54 * For load/store with an unsigned 12 bit immediate scaled by the element
55 * size. The input has the immediate field in bits [14:3] and the element
56 * size in [2:0].
57 */
uimm_scaled(DisasContext * s,int x)58 static int uimm_scaled(DisasContext *s, int x)
59 {
60 unsigned imm = x >> 3;
61 unsigned scale = extract32(x, 0, 3);
62 return imm << scale;
63 }
64
65 /* For load/store memory tags: scale offset by LOG2_TAG_GRANULE */
scale_by_log2_tag_granule(DisasContext * s,int x)66 static int scale_by_log2_tag_granule(DisasContext *s, int x)
67 {
68 return x << LOG2_TAG_GRANULE;
69 }
70
71 /*
72 * Include the generated decoders.
73 */
74
75 #include "decode-sme-fa64.c.inc"
76 #include "decode-a64.c.inc"
77
78 /* Table based decoder typedefs - used when the relevant bits for decode
79 * are too awkwardly scattered across the instruction (eg SIMD).
80 */
81 typedef void AArch64DecodeFn(DisasContext *s, uint32_t insn);
82
83 typedef struct AArch64DecodeTable {
84 uint32_t pattern;
85 uint32_t mask;
86 AArch64DecodeFn *disas_fn;
87 } AArch64DecodeTable;
88
89 /* initialize TCG globals. */
a64_translate_init(void)90 void a64_translate_init(void)
91 {
92 int i;
93
94 cpu_pc = tcg_global_mem_new_i64(tcg_env,
95 offsetof(CPUARMState, pc),
96 "pc");
97 for (i = 0; i < 32; i++) {
98 cpu_X[i] = tcg_global_mem_new_i64(tcg_env,
99 offsetof(CPUARMState, xregs[i]),
100 regnames[i]);
101 }
102
103 cpu_exclusive_high = tcg_global_mem_new_i64(tcg_env,
104 offsetof(CPUARMState, exclusive_high), "exclusive_high");
105 }
106
107 /*
108 * Return the core mmu_idx to use for A64 load/store insns which
109 * have a "unprivileged load/store" variant. Those insns access
110 * EL0 if executed from an EL which has control over EL0 (usually
111 * EL1) but behave like normal loads and stores if executed from
112 * elsewhere (eg EL3).
113 *
114 * @unpriv : true for the unprivileged encoding; false for the
115 * normal encoding (in which case we will return the same
116 * thing as get_mem_index().
117 */
get_a64_user_mem_index(DisasContext * s,bool unpriv)118 static int get_a64_user_mem_index(DisasContext *s, bool unpriv)
119 {
120 /*
121 * If AccType_UNPRIV is not used, the insn uses AccType_NORMAL,
122 * which is the usual mmu_idx for this cpu state.
123 */
124 ARMMMUIdx useridx = s->mmu_idx;
125
126 if (unpriv && s->unpriv) {
127 /*
128 * We have pre-computed the condition for AccType_UNPRIV.
129 * Therefore we should never get here with a mmu_idx for
130 * which we do not know the corresponding user mmu_idx.
131 */
132 switch (useridx) {
133 case ARMMMUIdx_E10_1:
134 case ARMMMUIdx_E10_1_PAN:
135 useridx = ARMMMUIdx_E10_0;
136 break;
137 case ARMMMUIdx_E20_2:
138 case ARMMMUIdx_E20_2_PAN:
139 useridx = ARMMMUIdx_E20_0;
140 break;
141 default:
142 g_assert_not_reached();
143 }
144 }
145 return arm_to_core_mmu_idx(useridx);
146 }
147
set_btype_raw(int val)148 static void set_btype_raw(int val)
149 {
150 tcg_gen_st_i32(tcg_constant_i32(val), tcg_env,
151 offsetof(CPUARMState, btype));
152 }
153
set_btype(DisasContext * s,int val)154 static void set_btype(DisasContext *s, int val)
155 {
156 /* BTYPE is a 2-bit field, and 0 should be done with reset_btype. */
157 tcg_debug_assert(val >= 1 && val <= 3);
158 set_btype_raw(val);
159 s->btype = -1;
160 }
161
reset_btype(DisasContext * s)162 static void reset_btype(DisasContext *s)
163 {
164 if (s->btype != 0) {
165 set_btype_raw(0);
166 s->btype = 0;
167 }
168 }
169
gen_pc_plus_diff(DisasContext * s,TCGv_i64 dest,target_long diff)170 static void gen_pc_plus_diff(DisasContext *s, TCGv_i64 dest, target_long diff)
171 {
172 assert(s->pc_save != -1);
173 if (tb_cflags(s->base.tb) & CF_PCREL) {
174 tcg_gen_addi_i64(dest, cpu_pc, (s->pc_curr - s->pc_save) + diff);
175 } else {
176 tcg_gen_movi_i64(dest, s->pc_curr + diff);
177 }
178 }
179
gen_a64_update_pc(DisasContext * s,target_long diff)180 void gen_a64_update_pc(DisasContext *s, target_long diff)
181 {
182 gen_pc_plus_diff(s, cpu_pc, diff);
183 s->pc_save = s->pc_curr + diff;
184 }
185
186 /*
187 * Handle Top Byte Ignore (TBI) bits.
188 *
189 * If address tagging is enabled via the TCR TBI bits:
190 * + for EL2 and EL3 there is only one TBI bit, and if it is set
191 * then the address is zero-extended, clearing bits [63:56]
192 * + for EL0 and EL1, TBI0 controls addresses with bit 55 == 0
193 * and TBI1 controls addresses with bit 55 == 1.
194 * If the appropriate TBI bit is set for the address then
195 * the address is sign-extended from bit 55 into bits [63:56]
196 *
197 * Here We have concatenated TBI{1,0} into tbi.
198 */
gen_top_byte_ignore(DisasContext * s,TCGv_i64 dst,TCGv_i64 src,int tbi)199 static void gen_top_byte_ignore(DisasContext *s, TCGv_i64 dst,
200 TCGv_i64 src, int tbi)
201 {
202 if (tbi == 0) {
203 /* Load unmodified address */
204 tcg_gen_mov_i64(dst, src);
205 } else if (!regime_has_2_ranges(s->mmu_idx)) {
206 /* Force tag byte to all zero */
207 tcg_gen_extract_i64(dst, src, 0, 56);
208 } else {
209 /* Sign-extend from bit 55. */
210 tcg_gen_sextract_i64(dst, src, 0, 56);
211
212 switch (tbi) {
213 case 1:
214 /* tbi0 but !tbi1: only use the extension if positive */
215 tcg_gen_and_i64(dst, dst, src);
216 break;
217 case 2:
218 /* !tbi0 but tbi1: only use the extension if negative */
219 tcg_gen_or_i64(dst, dst, src);
220 break;
221 case 3:
222 /* tbi0 and tbi1: always use the extension */
223 break;
224 default:
225 g_assert_not_reached();
226 }
227 }
228 }
229
gen_a64_set_pc(DisasContext * s,TCGv_i64 src)230 static void gen_a64_set_pc(DisasContext *s, TCGv_i64 src)
231 {
232 /*
233 * If address tagging is enabled for instructions via the TCR TBI bits,
234 * then loading an address into the PC will clear out any tag.
235 */
236 gen_top_byte_ignore(s, cpu_pc, src, s->tbii);
237 s->pc_save = -1;
238 }
239
240 /*
241 * Handle MTE and/or TBI.
242 *
243 * For TBI, ideally, we would do nothing. Proper behaviour on fault is
244 * for the tag to be present in the FAR_ELx register. But for user-only
245 * mode we do not have a TLB with which to implement this, so we must
246 * remove the top byte now.
247 *
248 * Always return a fresh temporary that we can increment independently
249 * of the write-back address.
250 */
251
clean_data_tbi(DisasContext * s,TCGv_i64 addr)252 TCGv_i64 clean_data_tbi(DisasContext *s, TCGv_i64 addr)
253 {
254 TCGv_i64 clean = tcg_temp_new_i64();
255 #ifdef CONFIG_USER_ONLY
256 gen_top_byte_ignore(s, clean, addr, s->tbid);
257 #else
258 tcg_gen_mov_i64(clean, addr);
259 #endif
260 return clean;
261 }
262
263 /* Insert a zero tag into src, with the result at dst. */
gen_address_with_allocation_tag0(TCGv_i64 dst,TCGv_i64 src)264 static void gen_address_with_allocation_tag0(TCGv_i64 dst, TCGv_i64 src)
265 {
266 tcg_gen_andi_i64(dst, src, ~MAKE_64BIT_MASK(56, 4));
267 }
268
gen_probe_access(DisasContext * s,TCGv_i64 ptr,MMUAccessType acc,int log2_size)269 static void gen_probe_access(DisasContext *s, TCGv_i64 ptr,
270 MMUAccessType acc, int log2_size)
271 {
272 gen_helper_probe_access(tcg_env, ptr,
273 tcg_constant_i32(acc),
274 tcg_constant_i32(get_mem_index(s)),
275 tcg_constant_i32(1 << log2_size));
276 }
277
278 /*
279 * For MTE, check a single logical or atomic access. This probes a single
280 * address, the exact one specified. The size and alignment of the access
281 * is not relevant to MTE, per se, but watchpoints do require the size,
282 * and we want to recognize those before making any other changes to state.
283 */
gen_mte_check1_mmuidx(DisasContext * s,TCGv_i64 addr,bool is_write,bool tag_checked,MemOp memop,bool is_unpriv,int core_idx)284 static TCGv_i64 gen_mte_check1_mmuidx(DisasContext *s, TCGv_i64 addr,
285 bool is_write, bool tag_checked,
286 MemOp memop, bool is_unpriv,
287 int core_idx)
288 {
289 if (tag_checked && s->mte_active[is_unpriv]) {
290 TCGv_i64 ret;
291 int desc = 0;
292
293 desc = FIELD_DP32(desc, MTEDESC, MIDX, core_idx);
294 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid);
295 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma);
296 desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write);
297 desc = FIELD_DP32(desc, MTEDESC, ALIGN, get_alignment_bits(memop));
298 desc = FIELD_DP32(desc, MTEDESC, SIZEM1, memop_size(memop) - 1);
299
300 ret = tcg_temp_new_i64();
301 gen_helper_mte_check(ret, tcg_env, tcg_constant_i32(desc), addr);
302
303 return ret;
304 }
305 return clean_data_tbi(s, addr);
306 }
307
gen_mte_check1(DisasContext * s,TCGv_i64 addr,bool is_write,bool tag_checked,MemOp memop)308 TCGv_i64 gen_mte_check1(DisasContext *s, TCGv_i64 addr, bool is_write,
309 bool tag_checked, MemOp memop)
310 {
311 return gen_mte_check1_mmuidx(s, addr, is_write, tag_checked, memop,
312 false, get_mem_index(s));
313 }
314
315 /*
316 * For MTE, check multiple logical sequential accesses.
317 */
gen_mte_checkN(DisasContext * s,TCGv_i64 addr,bool is_write,bool tag_checked,int total_size,MemOp single_mop)318 TCGv_i64 gen_mte_checkN(DisasContext *s, TCGv_i64 addr, bool is_write,
319 bool tag_checked, int total_size, MemOp single_mop)
320 {
321 if (tag_checked && s->mte_active[0]) {
322 TCGv_i64 ret;
323 int desc = 0;
324
325 desc = FIELD_DP32(desc, MTEDESC, MIDX, get_mem_index(s));
326 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid);
327 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma);
328 desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write);
329 desc = FIELD_DP32(desc, MTEDESC, ALIGN, get_alignment_bits(single_mop));
330 desc = FIELD_DP32(desc, MTEDESC, SIZEM1, total_size - 1);
331
332 ret = tcg_temp_new_i64();
333 gen_helper_mte_check(ret, tcg_env, tcg_constant_i32(desc), addr);
334
335 return ret;
336 }
337 return clean_data_tbi(s, addr);
338 }
339
340 /*
341 * Generate the special alignment check that applies to AccType_ATOMIC
342 * and AccType_ORDERED insns under FEAT_LSE2: the access need not be
343 * naturally aligned, but it must not cross a 16-byte boundary.
344 * See AArch64.CheckAlignment().
345 */
check_lse2_align(DisasContext * s,int rn,int imm,bool is_write,MemOp mop)346 static void check_lse2_align(DisasContext *s, int rn, int imm,
347 bool is_write, MemOp mop)
348 {
349 TCGv_i32 tmp;
350 TCGv_i64 addr;
351 TCGLabel *over_label;
352 MMUAccessType type;
353 int mmu_idx;
354
355 tmp = tcg_temp_new_i32();
356 tcg_gen_extrl_i64_i32(tmp, cpu_reg_sp(s, rn));
357 tcg_gen_addi_i32(tmp, tmp, imm & 15);
358 tcg_gen_andi_i32(tmp, tmp, 15);
359 tcg_gen_addi_i32(tmp, tmp, memop_size(mop));
360
361 over_label = gen_new_label();
362 tcg_gen_brcondi_i32(TCG_COND_LEU, tmp, 16, over_label);
363
364 addr = tcg_temp_new_i64();
365 tcg_gen_addi_i64(addr, cpu_reg_sp(s, rn), imm);
366
367 type = is_write ? MMU_DATA_STORE : MMU_DATA_LOAD,
368 mmu_idx = get_mem_index(s);
369 gen_helper_unaligned_access(tcg_env, addr, tcg_constant_i32(type),
370 tcg_constant_i32(mmu_idx));
371
372 gen_set_label(over_label);
373
374 }
375
376 /* Handle the alignment check for AccType_ATOMIC instructions. */
check_atomic_align(DisasContext * s,int rn,MemOp mop)377 static MemOp check_atomic_align(DisasContext *s, int rn, MemOp mop)
378 {
379 MemOp size = mop & MO_SIZE;
380
381 if (size == MO_8) {
382 return mop;
383 }
384
385 /*
386 * If size == MO_128, this is a LDXP, and the operation is single-copy
387 * atomic for each doubleword, not the entire quadword; it still must
388 * be quadword aligned.
389 */
390 if (size == MO_128) {
391 return finalize_memop_atom(s, MO_128 | MO_ALIGN,
392 MO_ATOM_IFALIGN_PAIR);
393 }
394 if (dc_isar_feature(aa64_lse2, s)) {
395 check_lse2_align(s, rn, 0, true, mop);
396 } else {
397 mop |= MO_ALIGN;
398 }
399 return finalize_memop(s, mop);
400 }
401
402 /* Handle the alignment check for AccType_ORDERED instructions. */
check_ordered_align(DisasContext * s,int rn,int imm,bool is_write,MemOp mop)403 static MemOp check_ordered_align(DisasContext *s, int rn, int imm,
404 bool is_write, MemOp mop)
405 {
406 MemOp size = mop & MO_SIZE;
407
408 if (size == MO_8) {
409 return mop;
410 }
411 if (size == MO_128) {
412 return finalize_memop_atom(s, MO_128 | MO_ALIGN,
413 MO_ATOM_IFALIGN_PAIR);
414 }
415 if (!dc_isar_feature(aa64_lse2, s)) {
416 mop |= MO_ALIGN;
417 } else if (!s->naa) {
418 check_lse2_align(s, rn, imm, is_write, mop);
419 }
420 return finalize_memop(s, mop);
421 }
422
423 typedef struct DisasCompare64 {
424 TCGCond cond;
425 TCGv_i64 value;
426 } DisasCompare64;
427
a64_test_cc(DisasCompare64 * c64,int cc)428 static void a64_test_cc(DisasCompare64 *c64, int cc)
429 {
430 DisasCompare c32;
431
432 arm_test_cc(&c32, cc);
433
434 /*
435 * Sign-extend the 32-bit value so that the GE/LT comparisons work
436 * properly. The NE/EQ comparisons are also fine with this choice.
437 */
438 c64->cond = c32.cond;
439 c64->value = tcg_temp_new_i64();
440 tcg_gen_ext_i32_i64(c64->value, c32.value);
441 }
442
gen_rebuild_hflags(DisasContext * s)443 static void gen_rebuild_hflags(DisasContext *s)
444 {
445 gen_helper_rebuild_hflags_a64(tcg_env, tcg_constant_i32(s->current_el));
446 }
447
gen_exception_internal(int excp)448 static void gen_exception_internal(int excp)
449 {
450 assert(excp_is_internal(excp));
451 gen_helper_exception_internal(tcg_env, tcg_constant_i32(excp));
452 }
453
gen_exception_internal_insn(DisasContext * s,int excp)454 static void gen_exception_internal_insn(DisasContext *s, int excp)
455 {
456 gen_a64_update_pc(s, 0);
457 gen_exception_internal(excp);
458 s->base.is_jmp = DISAS_NORETURN;
459 }
460
gen_exception_bkpt_insn(DisasContext * s,uint32_t syndrome)461 static void gen_exception_bkpt_insn(DisasContext *s, uint32_t syndrome)
462 {
463 gen_a64_update_pc(s, 0);
464 gen_helper_exception_bkpt_insn(tcg_env, tcg_constant_i32(syndrome));
465 s->base.is_jmp = DISAS_NORETURN;
466 }
467
gen_step_complete_exception(DisasContext * s)468 static void gen_step_complete_exception(DisasContext *s)
469 {
470 /* We just completed step of an insn. Move from Active-not-pending
471 * to Active-pending, and then also take the swstep exception.
472 * This corresponds to making the (IMPDEF) choice to prioritize
473 * swstep exceptions over asynchronous exceptions taken to an exception
474 * level where debug is disabled. This choice has the advantage that
475 * we do not need to maintain internal state corresponding to the
476 * ISV/EX syndrome bits between completion of the step and generation
477 * of the exception, and our syndrome information is always correct.
478 */
479 gen_ss_advance(s);
480 gen_swstep_exception(s, 1, s->is_ldex);
481 s->base.is_jmp = DISAS_NORETURN;
482 }
483
use_goto_tb(DisasContext * s,uint64_t dest)484 static inline bool use_goto_tb(DisasContext *s, uint64_t dest)
485 {
486 if (s->ss_active) {
487 return false;
488 }
489 return translator_use_goto_tb(&s->base, dest);
490 }
491
gen_goto_tb(DisasContext * s,int n,int64_t diff)492 static void gen_goto_tb(DisasContext *s, int n, int64_t diff)
493 {
494 if (use_goto_tb(s, s->pc_curr + diff)) {
495 /*
496 * For pcrel, the pc must always be up-to-date on entry to
497 * the linked TB, so that it can use simple additions for all
498 * further adjustments. For !pcrel, the linked TB is compiled
499 * to know its full virtual address, so we can delay the
500 * update to pc to the unlinked path. A long chain of links
501 * can thus avoid many updates to the PC.
502 */
503 if (tb_cflags(s->base.tb) & CF_PCREL) {
504 gen_a64_update_pc(s, diff);
505 tcg_gen_goto_tb(n);
506 } else {
507 tcg_gen_goto_tb(n);
508 gen_a64_update_pc(s, diff);
509 }
510 tcg_gen_exit_tb(s->base.tb, n);
511 s->base.is_jmp = DISAS_NORETURN;
512 } else {
513 gen_a64_update_pc(s, diff);
514 if (s->ss_active) {
515 gen_step_complete_exception(s);
516 } else {
517 tcg_gen_lookup_and_goto_ptr();
518 s->base.is_jmp = DISAS_NORETURN;
519 }
520 }
521 }
522
523 /*
524 * Register access functions
525 *
526 * These functions are used for directly accessing a register in where
527 * changes to the final register value are likely to be made. If you
528 * need to use a register for temporary calculation (e.g. index type
529 * operations) use the read_* form.
530 *
531 * B1.2.1 Register mappings
532 *
533 * In instruction register encoding 31 can refer to ZR (zero register) or
534 * the SP (stack pointer) depending on context. In QEMU's case we map SP
535 * to cpu_X[31] and ZR accesses to a temporary which can be discarded.
536 * This is the point of the _sp forms.
537 */
cpu_reg(DisasContext * s,int reg)538 TCGv_i64 cpu_reg(DisasContext *s, int reg)
539 {
540 if (reg == 31) {
541 TCGv_i64 t = tcg_temp_new_i64();
542 tcg_gen_movi_i64(t, 0);
543 return t;
544 } else {
545 return cpu_X[reg];
546 }
547 }
548
549 /* register access for when 31 == SP */
cpu_reg_sp(DisasContext * s,int reg)550 TCGv_i64 cpu_reg_sp(DisasContext *s, int reg)
551 {
552 return cpu_X[reg];
553 }
554
555 /* read a cpu register in 32bit/64bit mode. Returns a TCGv_i64
556 * representing the register contents. This TCGv is an auto-freed
557 * temporary so it need not be explicitly freed, and may be modified.
558 */
read_cpu_reg(DisasContext * s,int reg,int sf)559 TCGv_i64 read_cpu_reg(DisasContext *s, int reg, int sf)
560 {
561 TCGv_i64 v = tcg_temp_new_i64();
562 if (reg != 31) {
563 if (sf) {
564 tcg_gen_mov_i64(v, cpu_X[reg]);
565 } else {
566 tcg_gen_ext32u_i64(v, cpu_X[reg]);
567 }
568 } else {
569 tcg_gen_movi_i64(v, 0);
570 }
571 return v;
572 }
573
read_cpu_reg_sp(DisasContext * s,int reg,int sf)574 TCGv_i64 read_cpu_reg_sp(DisasContext *s, int reg, int sf)
575 {
576 TCGv_i64 v = tcg_temp_new_i64();
577 if (sf) {
578 tcg_gen_mov_i64(v, cpu_X[reg]);
579 } else {
580 tcg_gen_ext32u_i64(v, cpu_X[reg]);
581 }
582 return v;
583 }
584
585 /* Return the offset into CPUARMState of a slice (from
586 * the least significant end) of FP register Qn (ie
587 * Dn, Sn, Hn or Bn).
588 * (Note that this is not the same mapping as for A32; see cpu.h)
589 */
fp_reg_offset(DisasContext * s,int regno,MemOp size)590 static inline int fp_reg_offset(DisasContext *s, int regno, MemOp size)
591 {
592 return vec_reg_offset(s, regno, 0, size);
593 }
594
595 /* Offset of the high half of the 128 bit vector Qn */
fp_reg_hi_offset(DisasContext * s,int regno)596 static inline int fp_reg_hi_offset(DisasContext *s, int regno)
597 {
598 return vec_reg_offset(s, regno, 1, MO_64);
599 }
600
601 /* Convenience accessors for reading and writing single and double
602 * FP registers. Writing clears the upper parts of the associated
603 * 128 bit vector register, as required by the architecture.
604 * Note that unlike the GP register accessors, the values returned
605 * by the read functions must be manually freed.
606 */
read_fp_dreg(DisasContext * s,int reg)607 static TCGv_i64 read_fp_dreg(DisasContext *s, int reg)
608 {
609 TCGv_i64 v = tcg_temp_new_i64();
610
611 tcg_gen_ld_i64(v, tcg_env, fp_reg_offset(s, reg, MO_64));
612 return v;
613 }
614
read_fp_sreg(DisasContext * s,int reg)615 static TCGv_i32 read_fp_sreg(DisasContext *s, int reg)
616 {
617 TCGv_i32 v = tcg_temp_new_i32();
618
619 tcg_gen_ld_i32(v, tcg_env, fp_reg_offset(s, reg, MO_32));
620 return v;
621 }
622
read_fp_hreg(DisasContext * s,int reg)623 static TCGv_i32 read_fp_hreg(DisasContext *s, int reg)
624 {
625 TCGv_i32 v = tcg_temp_new_i32();
626
627 tcg_gen_ld16u_i32(v, tcg_env, fp_reg_offset(s, reg, MO_16));
628 return v;
629 }
630
631 /* Clear the bits above an N-bit vector, for N = (is_q ? 128 : 64).
632 * If SVE is not enabled, then there are only 128 bits in the vector.
633 */
clear_vec_high(DisasContext * s,bool is_q,int rd)634 static void clear_vec_high(DisasContext *s, bool is_q, int rd)
635 {
636 unsigned ofs = fp_reg_offset(s, rd, MO_64);
637 unsigned vsz = vec_full_reg_size(s);
638
639 /* Nop move, with side effect of clearing the tail. */
640 tcg_gen_gvec_mov(MO_64, ofs, ofs, is_q ? 16 : 8, vsz);
641 }
642
write_fp_dreg(DisasContext * s,int reg,TCGv_i64 v)643 void write_fp_dreg(DisasContext *s, int reg, TCGv_i64 v)
644 {
645 unsigned ofs = fp_reg_offset(s, reg, MO_64);
646
647 tcg_gen_st_i64(v, tcg_env, ofs);
648 clear_vec_high(s, false, reg);
649 }
650
write_fp_sreg(DisasContext * s,int reg,TCGv_i32 v)651 static void write_fp_sreg(DisasContext *s, int reg, TCGv_i32 v)
652 {
653 TCGv_i64 tmp = tcg_temp_new_i64();
654
655 tcg_gen_extu_i32_i64(tmp, v);
656 write_fp_dreg(s, reg, tmp);
657 }
658
659 /* Expand a 2-operand AdvSIMD vector operation using an expander function. */
gen_gvec_fn2(DisasContext * s,bool is_q,int rd,int rn,GVecGen2Fn * gvec_fn,int vece)660 static void gen_gvec_fn2(DisasContext *s, bool is_q, int rd, int rn,
661 GVecGen2Fn *gvec_fn, int vece)
662 {
663 gvec_fn(vece, vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn),
664 is_q ? 16 : 8, vec_full_reg_size(s));
665 }
666
667 /* Expand a 2-operand + immediate AdvSIMD vector operation using
668 * an expander function.
669 */
gen_gvec_fn2i(DisasContext * s,bool is_q,int rd,int rn,int64_t imm,GVecGen2iFn * gvec_fn,int vece)670 static void gen_gvec_fn2i(DisasContext *s, bool is_q, int rd, int rn,
671 int64_t imm, GVecGen2iFn *gvec_fn, int vece)
672 {
673 gvec_fn(vece, vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn),
674 imm, is_q ? 16 : 8, vec_full_reg_size(s));
675 }
676
677 /* Expand a 3-operand AdvSIMD vector operation using an expander function. */
gen_gvec_fn3(DisasContext * s,bool is_q,int rd,int rn,int rm,GVecGen3Fn * gvec_fn,int vece)678 static void gen_gvec_fn3(DisasContext *s, bool is_q, int rd, int rn, int rm,
679 GVecGen3Fn *gvec_fn, int vece)
680 {
681 gvec_fn(vece, vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn),
682 vec_full_reg_offset(s, rm), is_q ? 16 : 8, vec_full_reg_size(s));
683 }
684
685 /* Expand a 4-operand AdvSIMD vector operation using an expander function. */
gen_gvec_fn4(DisasContext * s,bool is_q,int rd,int rn,int rm,int rx,GVecGen4Fn * gvec_fn,int vece)686 static void gen_gvec_fn4(DisasContext *s, bool is_q, int rd, int rn, int rm,
687 int rx, GVecGen4Fn *gvec_fn, int vece)
688 {
689 gvec_fn(vece, vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn),
690 vec_full_reg_offset(s, rm), vec_full_reg_offset(s, rx),
691 is_q ? 16 : 8, vec_full_reg_size(s));
692 }
693
694 /* Expand a 2-operand operation using an out-of-line helper. */
gen_gvec_op2_ool(DisasContext * s,bool is_q,int rd,int rn,int data,gen_helper_gvec_2 * fn)695 static void gen_gvec_op2_ool(DisasContext *s, bool is_q, int rd,
696 int rn, int data, gen_helper_gvec_2 *fn)
697 {
698 tcg_gen_gvec_2_ool(vec_full_reg_offset(s, rd),
699 vec_full_reg_offset(s, rn),
700 is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
701 }
702
703 /* Expand a 3-operand operation using an out-of-line helper. */
gen_gvec_op3_ool(DisasContext * s,bool is_q,int rd,int rn,int rm,int data,gen_helper_gvec_3 * fn)704 static void gen_gvec_op3_ool(DisasContext *s, bool is_q, int rd,
705 int rn, int rm, int data, gen_helper_gvec_3 *fn)
706 {
707 tcg_gen_gvec_3_ool(vec_full_reg_offset(s, rd),
708 vec_full_reg_offset(s, rn),
709 vec_full_reg_offset(s, rm),
710 is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
711 }
712
713 /* Expand a 3-operand + fpstatus pointer + simd data value operation using
714 * an out-of-line helper.
715 */
gen_gvec_op3_fpst(DisasContext * s,bool is_q,int rd,int rn,int rm,bool is_fp16,int data,gen_helper_gvec_3_ptr * fn)716 static void gen_gvec_op3_fpst(DisasContext *s, bool is_q, int rd, int rn,
717 int rm, bool is_fp16, int data,
718 gen_helper_gvec_3_ptr *fn)
719 {
720 TCGv_ptr fpst = fpstatus_ptr(is_fp16 ? FPST_FPCR_F16 : FPST_FPCR);
721 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd),
722 vec_full_reg_offset(s, rn),
723 vec_full_reg_offset(s, rm), fpst,
724 is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
725 }
726
727 /* Expand a 3-operand + qc + operation using an out-of-line helper. */
gen_gvec_op3_qc(DisasContext * s,bool is_q,int rd,int rn,int rm,gen_helper_gvec_3_ptr * fn)728 static void gen_gvec_op3_qc(DisasContext *s, bool is_q, int rd, int rn,
729 int rm, gen_helper_gvec_3_ptr *fn)
730 {
731 TCGv_ptr qc_ptr = tcg_temp_new_ptr();
732
733 tcg_gen_addi_ptr(qc_ptr, tcg_env, offsetof(CPUARMState, vfp.qc));
734 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd),
735 vec_full_reg_offset(s, rn),
736 vec_full_reg_offset(s, rm), qc_ptr,
737 is_q ? 16 : 8, vec_full_reg_size(s), 0, fn);
738 }
739
740 /* Expand a 4-operand operation using an out-of-line helper. */
gen_gvec_op4_ool(DisasContext * s,bool is_q,int rd,int rn,int rm,int ra,int data,gen_helper_gvec_4 * fn)741 static void gen_gvec_op4_ool(DisasContext *s, bool is_q, int rd, int rn,
742 int rm, int ra, int data, gen_helper_gvec_4 *fn)
743 {
744 tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd),
745 vec_full_reg_offset(s, rn),
746 vec_full_reg_offset(s, rm),
747 vec_full_reg_offset(s, ra),
748 is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
749 }
750
751 /*
752 * Expand a 4-operand + fpstatus pointer + simd data value operation using
753 * an out-of-line helper.
754 */
gen_gvec_op4_fpst(DisasContext * s,bool is_q,int rd,int rn,int rm,int ra,bool is_fp16,int data,gen_helper_gvec_4_ptr * fn)755 static void gen_gvec_op4_fpst(DisasContext *s, bool is_q, int rd, int rn,
756 int rm, int ra, bool is_fp16, int data,
757 gen_helper_gvec_4_ptr *fn)
758 {
759 TCGv_ptr fpst = fpstatus_ptr(is_fp16 ? FPST_FPCR_F16 : FPST_FPCR);
760 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, rd),
761 vec_full_reg_offset(s, rn),
762 vec_full_reg_offset(s, rm),
763 vec_full_reg_offset(s, ra), fpst,
764 is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
765 }
766
767 /* Set ZF and NF based on a 64 bit result. This is alas fiddlier
768 * than the 32 bit equivalent.
769 */
gen_set_NZ64(TCGv_i64 result)770 static inline void gen_set_NZ64(TCGv_i64 result)
771 {
772 tcg_gen_extr_i64_i32(cpu_ZF, cpu_NF, result);
773 tcg_gen_or_i32(cpu_ZF, cpu_ZF, cpu_NF);
774 }
775
776 /* Set NZCV as for a logical operation: NZ as per result, CV cleared. */
gen_logic_CC(int sf,TCGv_i64 result)777 static inline void gen_logic_CC(int sf, TCGv_i64 result)
778 {
779 if (sf) {
780 gen_set_NZ64(result);
781 } else {
782 tcg_gen_extrl_i64_i32(cpu_ZF, result);
783 tcg_gen_mov_i32(cpu_NF, cpu_ZF);
784 }
785 tcg_gen_movi_i32(cpu_CF, 0);
786 tcg_gen_movi_i32(cpu_VF, 0);
787 }
788
789 /* dest = T0 + T1; compute C, N, V and Z flags */
gen_add64_CC(TCGv_i64 dest,TCGv_i64 t0,TCGv_i64 t1)790 static void gen_add64_CC(TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
791 {
792 TCGv_i64 result, flag, tmp;
793 result = tcg_temp_new_i64();
794 flag = tcg_temp_new_i64();
795 tmp = tcg_temp_new_i64();
796
797 tcg_gen_movi_i64(tmp, 0);
798 tcg_gen_add2_i64(result, flag, t0, tmp, t1, tmp);
799
800 tcg_gen_extrl_i64_i32(cpu_CF, flag);
801
802 gen_set_NZ64(result);
803
804 tcg_gen_xor_i64(flag, result, t0);
805 tcg_gen_xor_i64(tmp, t0, t1);
806 tcg_gen_andc_i64(flag, flag, tmp);
807 tcg_gen_extrh_i64_i32(cpu_VF, flag);
808
809 tcg_gen_mov_i64(dest, result);
810 }
811
gen_add32_CC(TCGv_i64 dest,TCGv_i64 t0,TCGv_i64 t1)812 static void gen_add32_CC(TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
813 {
814 TCGv_i32 t0_32 = tcg_temp_new_i32();
815 TCGv_i32 t1_32 = tcg_temp_new_i32();
816 TCGv_i32 tmp = tcg_temp_new_i32();
817
818 tcg_gen_movi_i32(tmp, 0);
819 tcg_gen_extrl_i64_i32(t0_32, t0);
820 tcg_gen_extrl_i64_i32(t1_32, t1);
821 tcg_gen_add2_i32(cpu_NF, cpu_CF, t0_32, tmp, t1_32, tmp);
822 tcg_gen_mov_i32(cpu_ZF, cpu_NF);
823 tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32);
824 tcg_gen_xor_i32(tmp, t0_32, t1_32);
825 tcg_gen_andc_i32(cpu_VF, cpu_VF, tmp);
826 tcg_gen_extu_i32_i64(dest, cpu_NF);
827 }
828
gen_add_CC(int sf,TCGv_i64 dest,TCGv_i64 t0,TCGv_i64 t1)829 static void gen_add_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
830 {
831 if (sf) {
832 gen_add64_CC(dest, t0, t1);
833 } else {
834 gen_add32_CC(dest, t0, t1);
835 }
836 }
837
838 /* dest = T0 - T1; compute C, N, V and Z flags */
gen_sub64_CC(TCGv_i64 dest,TCGv_i64 t0,TCGv_i64 t1)839 static void gen_sub64_CC(TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
840 {
841 /* 64 bit arithmetic */
842 TCGv_i64 result, flag, tmp;
843
844 result = tcg_temp_new_i64();
845 flag = tcg_temp_new_i64();
846 tcg_gen_sub_i64(result, t0, t1);
847
848 gen_set_NZ64(result);
849
850 tcg_gen_setcond_i64(TCG_COND_GEU, flag, t0, t1);
851 tcg_gen_extrl_i64_i32(cpu_CF, flag);
852
853 tcg_gen_xor_i64(flag, result, t0);
854 tmp = tcg_temp_new_i64();
855 tcg_gen_xor_i64(tmp, t0, t1);
856 tcg_gen_and_i64(flag, flag, tmp);
857 tcg_gen_extrh_i64_i32(cpu_VF, flag);
858 tcg_gen_mov_i64(dest, result);
859 }
860
gen_sub32_CC(TCGv_i64 dest,TCGv_i64 t0,TCGv_i64 t1)861 static void gen_sub32_CC(TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
862 {
863 /* 32 bit arithmetic */
864 TCGv_i32 t0_32 = tcg_temp_new_i32();
865 TCGv_i32 t1_32 = tcg_temp_new_i32();
866 TCGv_i32 tmp;
867
868 tcg_gen_extrl_i64_i32(t0_32, t0);
869 tcg_gen_extrl_i64_i32(t1_32, t1);
870 tcg_gen_sub_i32(cpu_NF, t0_32, t1_32);
871 tcg_gen_mov_i32(cpu_ZF, cpu_NF);
872 tcg_gen_setcond_i32(TCG_COND_GEU, cpu_CF, t0_32, t1_32);
873 tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32);
874 tmp = tcg_temp_new_i32();
875 tcg_gen_xor_i32(tmp, t0_32, t1_32);
876 tcg_gen_and_i32(cpu_VF, cpu_VF, tmp);
877 tcg_gen_extu_i32_i64(dest, cpu_NF);
878 }
879
gen_sub_CC(int sf,TCGv_i64 dest,TCGv_i64 t0,TCGv_i64 t1)880 static void gen_sub_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
881 {
882 if (sf) {
883 gen_sub64_CC(dest, t0, t1);
884 } else {
885 gen_sub32_CC(dest, t0, t1);
886 }
887 }
888
889 /* dest = T0 + T1 + CF; do not compute flags. */
gen_adc(int sf,TCGv_i64 dest,TCGv_i64 t0,TCGv_i64 t1)890 static void gen_adc(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
891 {
892 TCGv_i64 flag = tcg_temp_new_i64();
893 tcg_gen_extu_i32_i64(flag, cpu_CF);
894 tcg_gen_add_i64(dest, t0, t1);
895 tcg_gen_add_i64(dest, dest, flag);
896
897 if (!sf) {
898 tcg_gen_ext32u_i64(dest, dest);
899 }
900 }
901
902 /* dest = T0 + T1 + CF; compute C, N, V and Z flags. */
gen_adc_CC(int sf,TCGv_i64 dest,TCGv_i64 t0,TCGv_i64 t1)903 static void gen_adc_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
904 {
905 if (sf) {
906 TCGv_i64 result = tcg_temp_new_i64();
907 TCGv_i64 cf_64 = tcg_temp_new_i64();
908 TCGv_i64 vf_64 = tcg_temp_new_i64();
909 TCGv_i64 tmp = tcg_temp_new_i64();
910 TCGv_i64 zero = tcg_constant_i64(0);
911
912 tcg_gen_extu_i32_i64(cf_64, cpu_CF);
913 tcg_gen_add2_i64(result, cf_64, t0, zero, cf_64, zero);
914 tcg_gen_add2_i64(result, cf_64, result, cf_64, t1, zero);
915 tcg_gen_extrl_i64_i32(cpu_CF, cf_64);
916 gen_set_NZ64(result);
917
918 tcg_gen_xor_i64(vf_64, result, t0);
919 tcg_gen_xor_i64(tmp, t0, t1);
920 tcg_gen_andc_i64(vf_64, vf_64, tmp);
921 tcg_gen_extrh_i64_i32(cpu_VF, vf_64);
922
923 tcg_gen_mov_i64(dest, result);
924 } else {
925 TCGv_i32 t0_32 = tcg_temp_new_i32();
926 TCGv_i32 t1_32 = tcg_temp_new_i32();
927 TCGv_i32 tmp = tcg_temp_new_i32();
928 TCGv_i32 zero = tcg_constant_i32(0);
929
930 tcg_gen_extrl_i64_i32(t0_32, t0);
931 tcg_gen_extrl_i64_i32(t1_32, t1);
932 tcg_gen_add2_i32(cpu_NF, cpu_CF, t0_32, zero, cpu_CF, zero);
933 tcg_gen_add2_i32(cpu_NF, cpu_CF, cpu_NF, cpu_CF, t1_32, zero);
934
935 tcg_gen_mov_i32(cpu_ZF, cpu_NF);
936 tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32);
937 tcg_gen_xor_i32(tmp, t0_32, t1_32);
938 tcg_gen_andc_i32(cpu_VF, cpu_VF, tmp);
939 tcg_gen_extu_i32_i64(dest, cpu_NF);
940 }
941 }
942
943 /*
944 * Load/Store generators
945 */
946
947 /*
948 * Store from GPR register to memory.
949 */
do_gpr_st_memidx(DisasContext * s,TCGv_i64 source,TCGv_i64 tcg_addr,MemOp memop,int memidx,bool iss_valid,unsigned int iss_srt,bool iss_sf,bool iss_ar)950 static void do_gpr_st_memidx(DisasContext *s, TCGv_i64 source,
951 TCGv_i64 tcg_addr, MemOp memop, int memidx,
952 bool iss_valid,
953 unsigned int iss_srt,
954 bool iss_sf, bool iss_ar)
955 {
956 tcg_gen_qemu_st_i64(source, tcg_addr, memidx, memop);
957
958 if (iss_valid) {
959 uint32_t syn;
960
961 syn = syn_data_abort_with_iss(0,
962 (memop & MO_SIZE),
963 false,
964 iss_srt,
965 iss_sf,
966 iss_ar,
967 0, 0, 0, 0, 0, false);
968 disas_set_insn_syndrome(s, syn);
969 }
970 }
971
do_gpr_st(DisasContext * s,TCGv_i64 source,TCGv_i64 tcg_addr,MemOp memop,bool iss_valid,unsigned int iss_srt,bool iss_sf,bool iss_ar)972 static void do_gpr_st(DisasContext *s, TCGv_i64 source,
973 TCGv_i64 tcg_addr, MemOp memop,
974 bool iss_valid,
975 unsigned int iss_srt,
976 bool iss_sf, bool iss_ar)
977 {
978 do_gpr_st_memidx(s, source, tcg_addr, memop, get_mem_index(s),
979 iss_valid, iss_srt, iss_sf, iss_ar);
980 }
981
982 /*
983 * Load from memory to GPR register
984 */
do_gpr_ld_memidx(DisasContext * s,TCGv_i64 dest,TCGv_i64 tcg_addr,MemOp memop,bool extend,int memidx,bool iss_valid,unsigned int iss_srt,bool iss_sf,bool iss_ar)985 static void do_gpr_ld_memidx(DisasContext *s, TCGv_i64 dest, TCGv_i64 tcg_addr,
986 MemOp memop, bool extend, int memidx,
987 bool iss_valid, unsigned int iss_srt,
988 bool iss_sf, bool iss_ar)
989 {
990 tcg_gen_qemu_ld_i64(dest, tcg_addr, memidx, memop);
991
992 if (extend && (memop & MO_SIGN)) {
993 g_assert((memop & MO_SIZE) <= MO_32);
994 tcg_gen_ext32u_i64(dest, dest);
995 }
996
997 if (iss_valid) {
998 uint32_t syn;
999
1000 syn = syn_data_abort_with_iss(0,
1001 (memop & MO_SIZE),
1002 (memop & MO_SIGN) != 0,
1003 iss_srt,
1004 iss_sf,
1005 iss_ar,
1006 0, 0, 0, 0, 0, false);
1007 disas_set_insn_syndrome(s, syn);
1008 }
1009 }
1010
do_gpr_ld(DisasContext * s,TCGv_i64 dest,TCGv_i64 tcg_addr,MemOp memop,bool extend,bool iss_valid,unsigned int iss_srt,bool iss_sf,bool iss_ar)1011 static void do_gpr_ld(DisasContext *s, TCGv_i64 dest, TCGv_i64 tcg_addr,
1012 MemOp memop, bool extend,
1013 bool iss_valid, unsigned int iss_srt,
1014 bool iss_sf, bool iss_ar)
1015 {
1016 do_gpr_ld_memidx(s, dest, tcg_addr, memop, extend, get_mem_index(s),
1017 iss_valid, iss_srt, iss_sf, iss_ar);
1018 }
1019
1020 /*
1021 * Store from FP register to memory
1022 */
do_fp_st(DisasContext * s,int srcidx,TCGv_i64 tcg_addr,MemOp mop)1023 static void do_fp_st(DisasContext *s, int srcidx, TCGv_i64 tcg_addr, MemOp mop)
1024 {
1025 /* This writes the bottom N bits of a 128 bit wide vector to memory */
1026 TCGv_i64 tmplo = tcg_temp_new_i64();
1027
1028 tcg_gen_ld_i64(tmplo, tcg_env, fp_reg_offset(s, srcidx, MO_64));
1029
1030 if ((mop & MO_SIZE) < MO_128) {
1031 tcg_gen_qemu_st_i64(tmplo, tcg_addr, get_mem_index(s), mop);
1032 } else {
1033 TCGv_i64 tmphi = tcg_temp_new_i64();
1034 TCGv_i128 t16 = tcg_temp_new_i128();
1035
1036 tcg_gen_ld_i64(tmphi, tcg_env, fp_reg_hi_offset(s, srcidx));
1037 tcg_gen_concat_i64_i128(t16, tmplo, tmphi);
1038
1039 tcg_gen_qemu_st_i128(t16, tcg_addr, get_mem_index(s), mop);
1040 }
1041 }
1042
1043 /*
1044 * Load from memory to FP register
1045 */
do_fp_ld(DisasContext * s,int destidx,TCGv_i64 tcg_addr,MemOp mop)1046 static void do_fp_ld(DisasContext *s, int destidx, TCGv_i64 tcg_addr, MemOp mop)
1047 {
1048 /* This always zero-extends and writes to a full 128 bit wide vector */
1049 TCGv_i64 tmplo = tcg_temp_new_i64();
1050 TCGv_i64 tmphi = NULL;
1051
1052 if ((mop & MO_SIZE) < MO_128) {
1053 tcg_gen_qemu_ld_i64(tmplo, tcg_addr, get_mem_index(s), mop);
1054 } else {
1055 TCGv_i128 t16 = tcg_temp_new_i128();
1056
1057 tcg_gen_qemu_ld_i128(t16, tcg_addr, get_mem_index(s), mop);
1058
1059 tmphi = tcg_temp_new_i64();
1060 tcg_gen_extr_i128_i64(tmplo, tmphi, t16);
1061 }
1062
1063 tcg_gen_st_i64(tmplo, tcg_env, fp_reg_offset(s, destidx, MO_64));
1064
1065 if (tmphi) {
1066 tcg_gen_st_i64(tmphi, tcg_env, fp_reg_hi_offset(s, destidx));
1067 }
1068 clear_vec_high(s, tmphi != NULL, destidx);
1069 }
1070
1071 /*
1072 * Vector load/store helpers.
1073 *
1074 * The principal difference between this and a FP load is that we don't
1075 * zero extend as we are filling a partial chunk of the vector register.
1076 * These functions don't support 128 bit loads/stores, which would be
1077 * normal load/store operations.
1078 *
1079 * The _i32 versions are useful when operating on 32 bit quantities
1080 * (eg for floating point single or using Neon helper functions).
1081 */
1082
1083 /* Get value of an element within a vector register */
read_vec_element(DisasContext * s,TCGv_i64 tcg_dest,int srcidx,int element,MemOp memop)1084 static void read_vec_element(DisasContext *s, TCGv_i64 tcg_dest, int srcidx,
1085 int element, MemOp memop)
1086 {
1087 int vect_off = vec_reg_offset(s, srcidx, element, memop & MO_SIZE);
1088 switch ((unsigned)memop) {
1089 case MO_8:
1090 tcg_gen_ld8u_i64(tcg_dest, tcg_env, vect_off);
1091 break;
1092 case MO_16:
1093 tcg_gen_ld16u_i64(tcg_dest, tcg_env, vect_off);
1094 break;
1095 case MO_32:
1096 tcg_gen_ld32u_i64(tcg_dest, tcg_env, vect_off);
1097 break;
1098 case MO_8|MO_SIGN:
1099 tcg_gen_ld8s_i64(tcg_dest, tcg_env, vect_off);
1100 break;
1101 case MO_16|MO_SIGN:
1102 tcg_gen_ld16s_i64(tcg_dest, tcg_env, vect_off);
1103 break;
1104 case MO_32|MO_SIGN:
1105 tcg_gen_ld32s_i64(tcg_dest, tcg_env, vect_off);
1106 break;
1107 case MO_64:
1108 case MO_64|MO_SIGN:
1109 tcg_gen_ld_i64(tcg_dest, tcg_env, vect_off);
1110 break;
1111 default:
1112 g_assert_not_reached();
1113 }
1114 }
1115
read_vec_element_i32(DisasContext * s,TCGv_i32 tcg_dest,int srcidx,int element,MemOp memop)1116 static void read_vec_element_i32(DisasContext *s, TCGv_i32 tcg_dest, int srcidx,
1117 int element, MemOp memop)
1118 {
1119 int vect_off = vec_reg_offset(s, srcidx, element, memop & MO_SIZE);
1120 switch (memop) {
1121 case MO_8:
1122 tcg_gen_ld8u_i32(tcg_dest, tcg_env, vect_off);
1123 break;
1124 case MO_16:
1125 tcg_gen_ld16u_i32(tcg_dest, tcg_env, vect_off);
1126 break;
1127 case MO_8|MO_SIGN:
1128 tcg_gen_ld8s_i32(tcg_dest, tcg_env, vect_off);
1129 break;
1130 case MO_16|MO_SIGN:
1131 tcg_gen_ld16s_i32(tcg_dest, tcg_env, vect_off);
1132 break;
1133 case MO_32:
1134 case MO_32|MO_SIGN:
1135 tcg_gen_ld_i32(tcg_dest, tcg_env, vect_off);
1136 break;
1137 default:
1138 g_assert_not_reached();
1139 }
1140 }
1141
1142 /* Set value of an element within a vector register */
write_vec_element(DisasContext * s,TCGv_i64 tcg_src,int destidx,int element,MemOp memop)1143 static void write_vec_element(DisasContext *s, TCGv_i64 tcg_src, int destidx,
1144 int element, MemOp memop)
1145 {
1146 int vect_off = vec_reg_offset(s, destidx, element, memop & MO_SIZE);
1147 switch (memop) {
1148 case MO_8:
1149 tcg_gen_st8_i64(tcg_src, tcg_env, vect_off);
1150 break;
1151 case MO_16:
1152 tcg_gen_st16_i64(tcg_src, tcg_env, vect_off);
1153 break;
1154 case MO_32:
1155 tcg_gen_st32_i64(tcg_src, tcg_env, vect_off);
1156 break;
1157 case MO_64:
1158 tcg_gen_st_i64(tcg_src, tcg_env, vect_off);
1159 break;
1160 default:
1161 g_assert_not_reached();
1162 }
1163 }
1164
write_vec_element_i32(DisasContext * s,TCGv_i32 tcg_src,int destidx,int element,MemOp memop)1165 static void write_vec_element_i32(DisasContext *s, TCGv_i32 tcg_src,
1166 int destidx, int element, MemOp memop)
1167 {
1168 int vect_off = vec_reg_offset(s, destidx, element, memop & MO_SIZE);
1169 switch (memop) {
1170 case MO_8:
1171 tcg_gen_st8_i32(tcg_src, tcg_env, vect_off);
1172 break;
1173 case MO_16:
1174 tcg_gen_st16_i32(tcg_src, tcg_env, vect_off);
1175 break;
1176 case MO_32:
1177 tcg_gen_st_i32(tcg_src, tcg_env, vect_off);
1178 break;
1179 default:
1180 g_assert_not_reached();
1181 }
1182 }
1183
1184 /* Store from vector register to memory */
do_vec_st(DisasContext * s,int srcidx,int element,TCGv_i64 tcg_addr,MemOp mop)1185 static void do_vec_st(DisasContext *s, int srcidx, int element,
1186 TCGv_i64 tcg_addr, MemOp mop)
1187 {
1188 TCGv_i64 tcg_tmp = tcg_temp_new_i64();
1189
1190 read_vec_element(s, tcg_tmp, srcidx, element, mop & MO_SIZE);
1191 tcg_gen_qemu_st_i64(tcg_tmp, tcg_addr, get_mem_index(s), mop);
1192 }
1193
1194 /* Load from memory to vector register */
do_vec_ld(DisasContext * s,int destidx,int element,TCGv_i64 tcg_addr,MemOp mop)1195 static void do_vec_ld(DisasContext *s, int destidx, int element,
1196 TCGv_i64 tcg_addr, MemOp mop)
1197 {
1198 TCGv_i64 tcg_tmp = tcg_temp_new_i64();
1199
1200 tcg_gen_qemu_ld_i64(tcg_tmp, tcg_addr, get_mem_index(s), mop);
1201 write_vec_element(s, tcg_tmp, destidx, element, mop & MO_SIZE);
1202 }
1203
1204 /* Check that FP/Neon access is enabled. If it is, return
1205 * true. If not, emit code to generate an appropriate exception,
1206 * and return false; the caller should not emit any code for
1207 * the instruction. Note that this check must happen after all
1208 * unallocated-encoding checks (otherwise the syndrome information
1209 * for the resulting exception will be incorrect).
1210 */
fp_access_check_only(DisasContext * s)1211 static bool fp_access_check_only(DisasContext *s)
1212 {
1213 if (s->fp_excp_el) {
1214 assert(!s->fp_access_checked);
1215 s->fp_access_checked = true;
1216
1217 gen_exception_insn_el(s, 0, EXCP_UDEF,
1218 syn_fp_access_trap(1, 0xe, false, 0),
1219 s->fp_excp_el);
1220 return false;
1221 }
1222 s->fp_access_checked = true;
1223 return true;
1224 }
1225
fp_access_check(DisasContext * s)1226 static bool fp_access_check(DisasContext *s)
1227 {
1228 if (!fp_access_check_only(s)) {
1229 return false;
1230 }
1231 if (s->sme_trap_nonstreaming && s->is_nonstreaming) {
1232 gen_exception_insn(s, 0, EXCP_UDEF,
1233 syn_smetrap(SME_ET_Streaming, false));
1234 return false;
1235 }
1236 return true;
1237 }
1238
1239 /*
1240 * Check that SVE access is enabled. If it is, return true.
1241 * If not, emit code to generate an appropriate exception and return false.
1242 * This function corresponds to CheckSVEEnabled().
1243 */
sve_access_check(DisasContext * s)1244 bool sve_access_check(DisasContext *s)
1245 {
1246 if (s->pstate_sm || !dc_isar_feature(aa64_sve, s)) {
1247 assert(dc_isar_feature(aa64_sme, s));
1248 if (!sme_sm_enabled_check(s)) {
1249 goto fail_exit;
1250 }
1251 } else if (s->sve_excp_el) {
1252 gen_exception_insn_el(s, 0, EXCP_UDEF,
1253 syn_sve_access_trap(), s->sve_excp_el);
1254 goto fail_exit;
1255 }
1256 s->sve_access_checked = true;
1257 return fp_access_check(s);
1258
1259 fail_exit:
1260 /* Assert that we only raise one exception per instruction. */
1261 assert(!s->sve_access_checked);
1262 s->sve_access_checked = true;
1263 return false;
1264 }
1265
1266 /*
1267 * Check that SME access is enabled, raise an exception if not.
1268 * Note that this function corresponds to CheckSMEAccess and is
1269 * only used directly for cpregs.
1270 */
sme_access_check(DisasContext * s)1271 static bool sme_access_check(DisasContext *s)
1272 {
1273 if (s->sme_excp_el) {
1274 gen_exception_insn_el(s, 0, EXCP_UDEF,
1275 syn_smetrap(SME_ET_AccessTrap, false),
1276 s->sme_excp_el);
1277 return false;
1278 }
1279 return true;
1280 }
1281
1282 /* This function corresponds to CheckSMEEnabled. */
sme_enabled_check(DisasContext * s)1283 bool sme_enabled_check(DisasContext *s)
1284 {
1285 /*
1286 * Note that unlike sve_excp_el, we have not constrained sme_excp_el
1287 * to be zero when fp_excp_el has priority. This is because we need
1288 * sme_excp_el by itself for cpregs access checks.
1289 */
1290 if (!s->fp_excp_el || s->sme_excp_el < s->fp_excp_el) {
1291 s->fp_access_checked = true;
1292 return sme_access_check(s);
1293 }
1294 return fp_access_check_only(s);
1295 }
1296
1297 /* Common subroutine for CheckSMEAnd*Enabled. */
sme_enabled_check_with_svcr(DisasContext * s,unsigned req)1298 bool sme_enabled_check_with_svcr(DisasContext *s, unsigned req)
1299 {
1300 if (!sme_enabled_check(s)) {
1301 return false;
1302 }
1303 if (FIELD_EX64(req, SVCR, SM) && !s->pstate_sm) {
1304 gen_exception_insn(s, 0, EXCP_UDEF,
1305 syn_smetrap(SME_ET_NotStreaming, false));
1306 return false;
1307 }
1308 if (FIELD_EX64(req, SVCR, ZA) && !s->pstate_za) {
1309 gen_exception_insn(s, 0, EXCP_UDEF,
1310 syn_smetrap(SME_ET_InactiveZA, false));
1311 return false;
1312 }
1313 return true;
1314 }
1315
1316 /*
1317 * This utility function is for doing register extension with an
1318 * optional shift. You will likely want to pass a temporary for the
1319 * destination register. See DecodeRegExtend() in the ARM ARM.
1320 */
ext_and_shift_reg(TCGv_i64 tcg_out,TCGv_i64 tcg_in,int option,unsigned int shift)1321 static void ext_and_shift_reg(TCGv_i64 tcg_out, TCGv_i64 tcg_in,
1322 int option, unsigned int shift)
1323 {
1324 int extsize = extract32(option, 0, 2);
1325 bool is_signed = extract32(option, 2, 1);
1326
1327 tcg_gen_ext_i64(tcg_out, tcg_in, extsize | (is_signed ? MO_SIGN : 0));
1328 tcg_gen_shli_i64(tcg_out, tcg_out, shift);
1329 }
1330
gen_check_sp_alignment(DisasContext * s)1331 static inline void gen_check_sp_alignment(DisasContext *s)
1332 {
1333 /* The AArch64 architecture mandates that (if enabled via PSTATE
1334 * or SCTLR bits) there is a check that SP is 16-aligned on every
1335 * SP-relative load or store (with an exception generated if it is not).
1336 * In line with general QEMU practice regarding misaligned accesses,
1337 * we omit these checks for the sake of guest program performance.
1338 * This function is provided as a hook so we can more easily add these
1339 * checks in future (possibly as a "favour catching guest program bugs
1340 * over speed" user selectable option).
1341 */
1342 }
1343
1344 /*
1345 * This provides a simple table based table lookup decoder. It is
1346 * intended to be used when the relevant bits for decode are too
1347 * awkwardly placed and switch/if based logic would be confusing and
1348 * deeply nested. Since it's a linear search through the table, tables
1349 * should be kept small.
1350 *
1351 * It returns the first handler where insn & mask == pattern, or
1352 * NULL if there is no match.
1353 * The table is terminated by an empty mask (i.e. 0)
1354 */
lookup_disas_fn(const AArch64DecodeTable * table,uint32_t insn)1355 static inline AArch64DecodeFn *lookup_disas_fn(const AArch64DecodeTable *table,
1356 uint32_t insn)
1357 {
1358 const AArch64DecodeTable *tptr = table;
1359
1360 while (tptr->mask) {
1361 if ((insn & tptr->mask) == tptr->pattern) {
1362 return tptr->disas_fn;
1363 }
1364 tptr++;
1365 }
1366 return NULL;
1367 }
1368
1369 /*
1370 * The instruction disassembly implemented here matches
1371 * the instruction encoding classifications in chapter C4
1372 * of the ARM Architecture Reference Manual (DDI0487B_a);
1373 * classification names and decode diagrams here should generally
1374 * match up with those in the manual.
1375 */
1376
trans_B(DisasContext * s,arg_i * a)1377 static bool trans_B(DisasContext *s, arg_i *a)
1378 {
1379 reset_btype(s);
1380 gen_goto_tb(s, 0, a->imm);
1381 return true;
1382 }
1383
trans_BL(DisasContext * s,arg_i * a)1384 static bool trans_BL(DisasContext *s, arg_i *a)
1385 {
1386 gen_pc_plus_diff(s, cpu_reg(s, 30), curr_insn_len(s));
1387 reset_btype(s);
1388 gen_goto_tb(s, 0, a->imm);
1389 return true;
1390 }
1391
1392
trans_CBZ(DisasContext * s,arg_cbz * a)1393 static bool trans_CBZ(DisasContext *s, arg_cbz *a)
1394 {
1395 DisasLabel match;
1396 TCGv_i64 tcg_cmp;
1397
1398 tcg_cmp = read_cpu_reg(s, a->rt, a->sf);
1399 reset_btype(s);
1400
1401 match = gen_disas_label(s);
1402 tcg_gen_brcondi_i64(a->nz ? TCG_COND_NE : TCG_COND_EQ,
1403 tcg_cmp, 0, match.label);
1404 gen_goto_tb(s, 0, 4);
1405 set_disas_label(s, match);
1406 gen_goto_tb(s, 1, a->imm);
1407 return true;
1408 }
1409
trans_TBZ(DisasContext * s,arg_tbz * a)1410 static bool trans_TBZ(DisasContext *s, arg_tbz *a)
1411 {
1412 DisasLabel match;
1413 TCGv_i64 tcg_cmp;
1414
1415 tcg_cmp = tcg_temp_new_i64();
1416 tcg_gen_andi_i64(tcg_cmp, cpu_reg(s, a->rt), 1ULL << a->bitpos);
1417
1418 reset_btype(s);
1419
1420 match = gen_disas_label(s);
1421 tcg_gen_brcondi_i64(a->nz ? TCG_COND_NE : TCG_COND_EQ,
1422 tcg_cmp, 0, match.label);
1423 gen_goto_tb(s, 0, 4);
1424 set_disas_label(s, match);
1425 gen_goto_tb(s, 1, a->imm);
1426 return true;
1427 }
1428
trans_B_cond(DisasContext * s,arg_B_cond * a)1429 static bool trans_B_cond(DisasContext *s, arg_B_cond *a)
1430 {
1431 /* BC.cond is only present with FEAT_HBC */
1432 if (a->c && !dc_isar_feature(aa64_hbc, s)) {
1433 return false;
1434 }
1435 reset_btype(s);
1436 if (a->cond < 0x0e) {
1437 /* genuinely conditional branches */
1438 DisasLabel match = gen_disas_label(s);
1439 arm_gen_test_cc(a->cond, match.label);
1440 gen_goto_tb(s, 0, 4);
1441 set_disas_label(s, match);
1442 gen_goto_tb(s, 1, a->imm);
1443 } else {
1444 /* 0xe and 0xf are both "always" conditions */
1445 gen_goto_tb(s, 0, a->imm);
1446 }
1447 return true;
1448 }
1449
set_btype_for_br(DisasContext * s,int rn)1450 static void set_btype_for_br(DisasContext *s, int rn)
1451 {
1452 if (dc_isar_feature(aa64_bti, s)) {
1453 /* BR to {x16,x17} or !guard -> 1, else 3. */
1454 set_btype(s, rn == 16 || rn == 17 || !s->guarded_page ? 1 : 3);
1455 }
1456 }
1457
set_btype_for_blr(DisasContext * s)1458 static void set_btype_for_blr(DisasContext *s)
1459 {
1460 if (dc_isar_feature(aa64_bti, s)) {
1461 /* BLR sets BTYPE to 2, regardless of source guarded page. */
1462 set_btype(s, 2);
1463 }
1464 }
1465
trans_BR(DisasContext * s,arg_r * a)1466 static bool trans_BR(DisasContext *s, arg_r *a)
1467 {
1468 gen_a64_set_pc(s, cpu_reg(s, a->rn));
1469 set_btype_for_br(s, a->rn);
1470 s->base.is_jmp = DISAS_JUMP;
1471 return true;
1472 }
1473
trans_BLR(DisasContext * s,arg_r * a)1474 static bool trans_BLR(DisasContext *s, arg_r *a)
1475 {
1476 TCGv_i64 dst = cpu_reg(s, a->rn);
1477 TCGv_i64 lr = cpu_reg(s, 30);
1478 if (dst == lr) {
1479 TCGv_i64 tmp = tcg_temp_new_i64();
1480 tcg_gen_mov_i64(tmp, dst);
1481 dst = tmp;
1482 }
1483 gen_pc_plus_diff(s, lr, curr_insn_len(s));
1484 gen_a64_set_pc(s, dst);
1485 set_btype_for_blr(s);
1486 s->base.is_jmp = DISAS_JUMP;
1487 return true;
1488 }
1489
trans_RET(DisasContext * s,arg_r * a)1490 static bool trans_RET(DisasContext *s, arg_r *a)
1491 {
1492 gen_a64_set_pc(s, cpu_reg(s, a->rn));
1493 s->base.is_jmp = DISAS_JUMP;
1494 return true;
1495 }
1496
auth_branch_target(DisasContext * s,TCGv_i64 dst,TCGv_i64 modifier,bool use_key_a)1497 static TCGv_i64 auth_branch_target(DisasContext *s, TCGv_i64 dst,
1498 TCGv_i64 modifier, bool use_key_a)
1499 {
1500 TCGv_i64 truedst;
1501 /*
1502 * Return the branch target for a BRAA/RETA/etc, which is either
1503 * just the destination dst, or that value with the pauth check
1504 * done and the code removed from the high bits.
1505 */
1506 if (!s->pauth_active) {
1507 return dst;
1508 }
1509
1510 truedst = tcg_temp_new_i64();
1511 if (use_key_a) {
1512 gen_helper_autia_combined(truedst, tcg_env, dst, modifier);
1513 } else {
1514 gen_helper_autib_combined(truedst, tcg_env, dst, modifier);
1515 }
1516 return truedst;
1517 }
1518
trans_BRAZ(DisasContext * s,arg_braz * a)1519 static bool trans_BRAZ(DisasContext *s, arg_braz *a)
1520 {
1521 TCGv_i64 dst;
1522
1523 if (!dc_isar_feature(aa64_pauth, s)) {
1524 return false;
1525 }
1526
1527 dst = auth_branch_target(s, cpu_reg(s, a->rn), tcg_constant_i64(0), !a->m);
1528 gen_a64_set_pc(s, dst);
1529 set_btype_for_br(s, a->rn);
1530 s->base.is_jmp = DISAS_JUMP;
1531 return true;
1532 }
1533
trans_BLRAZ(DisasContext * s,arg_braz * a)1534 static bool trans_BLRAZ(DisasContext *s, arg_braz *a)
1535 {
1536 TCGv_i64 dst, lr;
1537
1538 if (!dc_isar_feature(aa64_pauth, s)) {
1539 return false;
1540 }
1541
1542 dst = auth_branch_target(s, cpu_reg(s, a->rn), tcg_constant_i64(0), !a->m);
1543 lr = cpu_reg(s, 30);
1544 if (dst == lr) {
1545 TCGv_i64 tmp = tcg_temp_new_i64();
1546 tcg_gen_mov_i64(tmp, dst);
1547 dst = tmp;
1548 }
1549 gen_pc_plus_diff(s, lr, curr_insn_len(s));
1550 gen_a64_set_pc(s, dst);
1551 set_btype_for_blr(s);
1552 s->base.is_jmp = DISAS_JUMP;
1553 return true;
1554 }
1555
trans_RETA(DisasContext * s,arg_reta * a)1556 static bool trans_RETA(DisasContext *s, arg_reta *a)
1557 {
1558 TCGv_i64 dst;
1559
1560 dst = auth_branch_target(s, cpu_reg(s, 30), cpu_X[31], !a->m);
1561 gen_a64_set_pc(s, dst);
1562 s->base.is_jmp = DISAS_JUMP;
1563 return true;
1564 }
1565
trans_BRA(DisasContext * s,arg_bra * a)1566 static bool trans_BRA(DisasContext *s, arg_bra *a)
1567 {
1568 TCGv_i64 dst;
1569
1570 if (!dc_isar_feature(aa64_pauth, s)) {
1571 return false;
1572 }
1573 dst = auth_branch_target(s, cpu_reg(s,a->rn), cpu_reg_sp(s, a->rm), !a->m);
1574 gen_a64_set_pc(s, dst);
1575 set_btype_for_br(s, a->rn);
1576 s->base.is_jmp = DISAS_JUMP;
1577 return true;
1578 }
1579
trans_BLRA(DisasContext * s,arg_bra * a)1580 static bool trans_BLRA(DisasContext *s, arg_bra *a)
1581 {
1582 TCGv_i64 dst, lr;
1583
1584 if (!dc_isar_feature(aa64_pauth, s)) {
1585 return false;
1586 }
1587 dst = auth_branch_target(s, cpu_reg(s, a->rn), cpu_reg_sp(s, a->rm), !a->m);
1588 lr = cpu_reg(s, 30);
1589 if (dst == lr) {
1590 TCGv_i64 tmp = tcg_temp_new_i64();
1591 tcg_gen_mov_i64(tmp, dst);
1592 dst = tmp;
1593 }
1594 gen_pc_plus_diff(s, lr, curr_insn_len(s));
1595 gen_a64_set_pc(s, dst);
1596 set_btype_for_blr(s);
1597 s->base.is_jmp = DISAS_JUMP;
1598 return true;
1599 }
1600
trans_ERET(DisasContext * s,arg_ERET * a)1601 static bool trans_ERET(DisasContext *s, arg_ERET *a)
1602 {
1603 TCGv_i64 dst;
1604
1605 if (s->current_el == 0) {
1606 return false;
1607 }
1608 if (s->fgt_eret) {
1609 gen_exception_insn_el(s, 0, EXCP_UDEF, syn_erettrap(0), 2);
1610 return true;
1611 }
1612 dst = tcg_temp_new_i64();
1613 tcg_gen_ld_i64(dst, tcg_env,
1614 offsetof(CPUARMState, elr_el[s->current_el]));
1615
1616 translator_io_start(&s->base);
1617
1618 gen_helper_exception_return(tcg_env, dst);
1619 /* Must exit loop to check un-masked IRQs */
1620 s->base.is_jmp = DISAS_EXIT;
1621 return true;
1622 }
1623
trans_ERETA(DisasContext * s,arg_reta * a)1624 static bool trans_ERETA(DisasContext *s, arg_reta *a)
1625 {
1626 TCGv_i64 dst;
1627
1628 if (!dc_isar_feature(aa64_pauth, s)) {
1629 return false;
1630 }
1631 if (s->current_el == 0) {
1632 return false;
1633 }
1634 /* The FGT trap takes precedence over an auth trap. */
1635 if (s->fgt_eret) {
1636 gen_exception_insn_el(s, 0, EXCP_UDEF, syn_erettrap(a->m ? 3 : 2), 2);
1637 return true;
1638 }
1639 dst = tcg_temp_new_i64();
1640 tcg_gen_ld_i64(dst, tcg_env,
1641 offsetof(CPUARMState, elr_el[s->current_el]));
1642
1643 dst = auth_branch_target(s, dst, cpu_X[31], !a->m);
1644
1645 translator_io_start(&s->base);
1646
1647 gen_helper_exception_return(tcg_env, dst);
1648 /* Must exit loop to check un-masked IRQs */
1649 s->base.is_jmp = DISAS_EXIT;
1650 return true;
1651 }
1652
trans_NOP(DisasContext * s,arg_NOP * a)1653 static bool trans_NOP(DisasContext *s, arg_NOP *a)
1654 {
1655 return true;
1656 }
1657
trans_YIELD(DisasContext * s,arg_YIELD * a)1658 static bool trans_YIELD(DisasContext *s, arg_YIELD *a)
1659 {
1660 /*
1661 * When running in MTTCG we don't generate jumps to the yield and
1662 * WFE helpers as it won't affect the scheduling of other vCPUs.
1663 * If we wanted to more completely model WFE/SEV so we don't busy
1664 * spin unnecessarily we would need to do something more involved.
1665 */
1666 if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) {
1667 s->base.is_jmp = DISAS_YIELD;
1668 }
1669 return true;
1670 }
1671
trans_WFI(DisasContext * s,arg_WFI * a)1672 static bool trans_WFI(DisasContext *s, arg_WFI *a)
1673 {
1674 s->base.is_jmp = DISAS_WFI;
1675 return true;
1676 }
1677
trans_WFE(DisasContext * s,arg_WFI * a)1678 static bool trans_WFE(DisasContext *s, arg_WFI *a)
1679 {
1680 /*
1681 * When running in MTTCG we don't generate jumps to the yield and
1682 * WFE helpers as it won't affect the scheduling of other vCPUs.
1683 * If we wanted to more completely model WFE/SEV so we don't busy
1684 * spin unnecessarily we would need to do something more involved.
1685 */
1686 if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) {
1687 s->base.is_jmp = DISAS_WFE;
1688 }
1689 return true;
1690 }
1691
trans_XPACLRI(DisasContext * s,arg_XPACLRI * a)1692 static bool trans_XPACLRI(DisasContext *s, arg_XPACLRI *a)
1693 {
1694 if (s->pauth_active) {
1695 gen_helper_xpaci(cpu_X[30], tcg_env, cpu_X[30]);
1696 }
1697 return true;
1698 }
1699
trans_PACIA1716(DisasContext * s,arg_PACIA1716 * a)1700 static bool trans_PACIA1716(DisasContext *s, arg_PACIA1716 *a)
1701 {
1702 if (s->pauth_active) {
1703 gen_helper_pacia(cpu_X[17], tcg_env, cpu_X[17], cpu_X[16]);
1704 }
1705 return true;
1706 }
1707
trans_PACIB1716(DisasContext * s,arg_PACIB1716 * a)1708 static bool trans_PACIB1716(DisasContext *s, arg_PACIB1716 *a)
1709 {
1710 if (s->pauth_active) {
1711 gen_helper_pacib(cpu_X[17], tcg_env, cpu_X[17], cpu_X[16]);
1712 }
1713 return true;
1714 }
1715
trans_AUTIA1716(DisasContext * s,arg_AUTIA1716 * a)1716 static bool trans_AUTIA1716(DisasContext *s, arg_AUTIA1716 *a)
1717 {
1718 if (s->pauth_active) {
1719 gen_helper_autia(cpu_X[17], tcg_env, cpu_X[17], cpu_X[16]);
1720 }
1721 return true;
1722 }
1723
trans_AUTIB1716(DisasContext * s,arg_AUTIB1716 * a)1724 static bool trans_AUTIB1716(DisasContext *s, arg_AUTIB1716 *a)
1725 {
1726 if (s->pauth_active) {
1727 gen_helper_autib(cpu_X[17], tcg_env, cpu_X[17], cpu_X[16]);
1728 }
1729 return true;
1730 }
1731
trans_ESB(DisasContext * s,arg_ESB * a)1732 static bool trans_ESB(DisasContext *s, arg_ESB *a)
1733 {
1734 /* Without RAS, we must implement this as NOP. */
1735 if (dc_isar_feature(aa64_ras, s)) {
1736 /*
1737 * QEMU does not have a source of physical SErrors,
1738 * so we are only concerned with virtual SErrors.
1739 * The pseudocode in the ARM for this case is
1740 * if PSTATE.EL IN {EL0, EL1} && EL2Enabled() then
1741 * AArch64.vESBOperation();
1742 * Most of the condition can be evaluated at translation time.
1743 * Test for EL2 present, and defer test for SEL2 to runtime.
1744 */
1745 if (s->current_el <= 1 && arm_dc_feature(s, ARM_FEATURE_EL2)) {
1746 gen_helper_vesb(tcg_env);
1747 }
1748 }
1749 return true;
1750 }
1751
trans_PACIAZ(DisasContext * s,arg_PACIAZ * a)1752 static bool trans_PACIAZ(DisasContext *s, arg_PACIAZ *a)
1753 {
1754 if (s->pauth_active) {
1755 gen_helper_pacia(cpu_X[30], tcg_env, cpu_X[30], tcg_constant_i64(0));
1756 }
1757 return true;
1758 }
1759
trans_PACIASP(DisasContext * s,arg_PACIASP * a)1760 static bool trans_PACIASP(DisasContext *s, arg_PACIASP *a)
1761 {
1762 if (s->pauth_active) {
1763 gen_helper_pacia(cpu_X[30], tcg_env, cpu_X[30], cpu_X[31]);
1764 }
1765 return true;
1766 }
1767
trans_PACIBZ(DisasContext * s,arg_PACIBZ * a)1768 static bool trans_PACIBZ(DisasContext *s, arg_PACIBZ *a)
1769 {
1770 if (s->pauth_active) {
1771 gen_helper_pacib(cpu_X[30], tcg_env, cpu_X[30], tcg_constant_i64(0));
1772 }
1773 return true;
1774 }
1775
trans_PACIBSP(DisasContext * s,arg_PACIBSP * a)1776 static bool trans_PACIBSP(DisasContext *s, arg_PACIBSP *a)
1777 {
1778 if (s->pauth_active) {
1779 gen_helper_pacib(cpu_X[30], tcg_env, cpu_X[30], cpu_X[31]);
1780 }
1781 return true;
1782 }
1783
trans_AUTIAZ(DisasContext * s,arg_AUTIAZ * a)1784 static bool trans_AUTIAZ(DisasContext *s, arg_AUTIAZ *a)
1785 {
1786 if (s->pauth_active) {
1787 gen_helper_autia(cpu_X[30], tcg_env, cpu_X[30], tcg_constant_i64(0));
1788 }
1789 return true;
1790 }
1791
trans_AUTIASP(DisasContext * s,arg_AUTIASP * a)1792 static bool trans_AUTIASP(DisasContext *s, arg_AUTIASP *a)
1793 {
1794 if (s->pauth_active) {
1795 gen_helper_autia(cpu_X[30], tcg_env, cpu_X[30], cpu_X[31]);
1796 }
1797 return true;
1798 }
1799
trans_AUTIBZ(DisasContext * s,arg_AUTIBZ * a)1800 static bool trans_AUTIBZ(DisasContext *s, arg_AUTIBZ *a)
1801 {
1802 if (s->pauth_active) {
1803 gen_helper_autib(cpu_X[30], tcg_env, cpu_X[30], tcg_constant_i64(0));
1804 }
1805 return true;
1806 }
1807
trans_AUTIBSP(DisasContext * s,arg_AUTIBSP * a)1808 static bool trans_AUTIBSP(DisasContext *s, arg_AUTIBSP *a)
1809 {
1810 if (s->pauth_active) {
1811 gen_helper_autib(cpu_X[30], tcg_env, cpu_X[30], cpu_X[31]);
1812 }
1813 return true;
1814 }
1815
trans_CLREX(DisasContext * s,arg_CLREX * a)1816 static bool trans_CLREX(DisasContext *s, arg_CLREX *a)
1817 {
1818 tcg_gen_movi_i64(cpu_exclusive_addr, -1);
1819 return true;
1820 }
1821
trans_DSB_DMB(DisasContext * s,arg_DSB_DMB * a)1822 static bool trans_DSB_DMB(DisasContext *s, arg_DSB_DMB *a)
1823 {
1824 /* We handle DSB and DMB the same way */
1825 TCGBar bar;
1826
1827 switch (a->types) {
1828 case 1: /* MBReqTypes_Reads */
1829 bar = TCG_BAR_SC | TCG_MO_LD_LD | TCG_MO_LD_ST;
1830 break;
1831 case 2: /* MBReqTypes_Writes */
1832 bar = TCG_BAR_SC | TCG_MO_ST_ST;
1833 break;
1834 default: /* MBReqTypes_All */
1835 bar = TCG_BAR_SC | TCG_MO_ALL;
1836 break;
1837 }
1838 tcg_gen_mb(bar);
1839 return true;
1840 }
1841
trans_ISB(DisasContext * s,arg_ISB * a)1842 static bool trans_ISB(DisasContext *s, arg_ISB *a)
1843 {
1844 /*
1845 * We need to break the TB after this insn to execute
1846 * self-modifying code correctly and also to take
1847 * any pending interrupts immediately.
1848 */
1849 reset_btype(s);
1850 gen_goto_tb(s, 0, 4);
1851 return true;
1852 }
1853
trans_SB(DisasContext * s,arg_SB * a)1854 static bool trans_SB(DisasContext *s, arg_SB *a)
1855 {
1856 if (!dc_isar_feature(aa64_sb, s)) {
1857 return false;
1858 }
1859 /*
1860 * TODO: There is no speculation barrier opcode for TCG;
1861 * MB and end the TB instead.
1862 */
1863 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_SC);
1864 gen_goto_tb(s, 0, 4);
1865 return true;
1866 }
1867
trans_CFINV(DisasContext * s,arg_CFINV * a)1868 static bool trans_CFINV(DisasContext *s, arg_CFINV *a)
1869 {
1870 if (!dc_isar_feature(aa64_condm_4, s)) {
1871 return false;
1872 }
1873 tcg_gen_xori_i32(cpu_CF, cpu_CF, 1);
1874 return true;
1875 }
1876
trans_XAFLAG(DisasContext * s,arg_XAFLAG * a)1877 static bool trans_XAFLAG(DisasContext *s, arg_XAFLAG *a)
1878 {
1879 TCGv_i32 z;
1880
1881 if (!dc_isar_feature(aa64_condm_5, s)) {
1882 return false;
1883 }
1884
1885 z = tcg_temp_new_i32();
1886
1887 tcg_gen_setcondi_i32(TCG_COND_EQ, z, cpu_ZF, 0);
1888
1889 /*
1890 * (!C & !Z) << 31
1891 * (!(C | Z)) << 31
1892 * ~((C | Z) << 31)
1893 * ~-(C | Z)
1894 * (C | Z) - 1
1895 */
1896 tcg_gen_or_i32(cpu_NF, cpu_CF, z);
1897 tcg_gen_subi_i32(cpu_NF, cpu_NF, 1);
1898
1899 /* !(Z & C) */
1900 tcg_gen_and_i32(cpu_ZF, z, cpu_CF);
1901 tcg_gen_xori_i32(cpu_ZF, cpu_ZF, 1);
1902
1903 /* (!C & Z) << 31 -> -(Z & ~C) */
1904 tcg_gen_andc_i32(cpu_VF, z, cpu_CF);
1905 tcg_gen_neg_i32(cpu_VF, cpu_VF);
1906
1907 /* C | Z */
1908 tcg_gen_or_i32(cpu_CF, cpu_CF, z);
1909
1910 return true;
1911 }
1912
trans_AXFLAG(DisasContext * s,arg_AXFLAG * a)1913 static bool trans_AXFLAG(DisasContext *s, arg_AXFLAG *a)
1914 {
1915 if (!dc_isar_feature(aa64_condm_5, s)) {
1916 return false;
1917 }
1918
1919 tcg_gen_sari_i32(cpu_VF, cpu_VF, 31); /* V ? -1 : 0 */
1920 tcg_gen_andc_i32(cpu_CF, cpu_CF, cpu_VF); /* C & !V */
1921
1922 /* !(Z | V) -> !(!ZF | V) -> ZF & !V -> ZF & ~VF */
1923 tcg_gen_andc_i32(cpu_ZF, cpu_ZF, cpu_VF);
1924
1925 tcg_gen_movi_i32(cpu_NF, 0);
1926 tcg_gen_movi_i32(cpu_VF, 0);
1927
1928 return true;
1929 }
1930
trans_MSR_i_UAO(DisasContext * s,arg_i * a)1931 static bool trans_MSR_i_UAO(DisasContext *s, arg_i *a)
1932 {
1933 if (!dc_isar_feature(aa64_uao, s) || s->current_el == 0) {
1934 return false;
1935 }
1936 if (a->imm & 1) {
1937 set_pstate_bits(PSTATE_UAO);
1938 } else {
1939 clear_pstate_bits(PSTATE_UAO);
1940 }
1941 gen_rebuild_hflags(s);
1942 s->base.is_jmp = DISAS_TOO_MANY;
1943 return true;
1944 }
1945
trans_MSR_i_PAN(DisasContext * s,arg_i * a)1946 static bool trans_MSR_i_PAN(DisasContext *s, arg_i *a)
1947 {
1948 if (!dc_isar_feature(aa64_pan, s) || s->current_el == 0) {
1949 return false;
1950 }
1951 if (a->imm & 1) {
1952 set_pstate_bits(PSTATE_PAN);
1953 } else {
1954 clear_pstate_bits(PSTATE_PAN);
1955 }
1956 gen_rebuild_hflags(s);
1957 s->base.is_jmp = DISAS_TOO_MANY;
1958 return true;
1959 }
1960
trans_MSR_i_SPSEL(DisasContext * s,arg_i * a)1961 static bool trans_MSR_i_SPSEL(DisasContext *s, arg_i *a)
1962 {
1963 if (s->current_el == 0) {
1964 return false;
1965 }
1966 gen_helper_msr_i_spsel(tcg_env, tcg_constant_i32(a->imm & PSTATE_SP));
1967 s->base.is_jmp = DISAS_TOO_MANY;
1968 return true;
1969 }
1970
trans_MSR_i_SBSS(DisasContext * s,arg_i * a)1971 static bool trans_MSR_i_SBSS(DisasContext *s, arg_i *a)
1972 {
1973 if (!dc_isar_feature(aa64_ssbs, s)) {
1974 return false;
1975 }
1976 if (a->imm & 1) {
1977 set_pstate_bits(PSTATE_SSBS);
1978 } else {
1979 clear_pstate_bits(PSTATE_SSBS);
1980 }
1981 /* Don't need to rebuild hflags since SSBS is a nop */
1982 s->base.is_jmp = DISAS_TOO_MANY;
1983 return true;
1984 }
1985
trans_MSR_i_DIT(DisasContext * s,arg_i * a)1986 static bool trans_MSR_i_DIT(DisasContext *s, arg_i *a)
1987 {
1988 if (!dc_isar_feature(aa64_dit, s)) {
1989 return false;
1990 }
1991 if (a->imm & 1) {
1992 set_pstate_bits(PSTATE_DIT);
1993 } else {
1994 clear_pstate_bits(PSTATE_DIT);
1995 }
1996 /* There's no need to rebuild hflags because DIT is a nop */
1997 s->base.is_jmp = DISAS_TOO_MANY;
1998 return true;
1999 }
2000
trans_MSR_i_TCO(DisasContext * s,arg_i * a)2001 static bool trans_MSR_i_TCO(DisasContext *s, arg_i *a)
2002 {
2003 if (dc_isar_feature(aa64_mte, s)) {
2004 /* Full MTE is enabled -- set the TCO bit as directed. */
2005 if (a->imm & 1) {
2006 set_pstate_bits(PSTATE_TCO);
2007 } else {
2008 clear_pstate_bits(PSTATE_TCO);
2009 }
2010 gen_rebuild_hflags(s);
2011 /* Many factors, including TCO, go into MTE_ACTIVE. */
2012 s->base.is_jmp = DISAS_UPDATE_NOCHAIN;
2013 return true;
2014 } else if (dc_isar_feature(aa64_mte_insn_reg, s)) {
2015 /* Only "instructions accessible at EL0" -- PSTATE.TCO is WI. */
2016 return true;
2017 } else {
2018 /* Insn not present */
2019 return false;
2020 }
2021 }
2022
trans_MSR_i_DAIFSET(DisasContext * s,arg_i * a)2023 static bool trans_MSR_i_DAIFSET(DisasContext *s, arg_i *a)
2024 {
2025 gen_helper_msr_i_daifset(tcg_env, tcg_constant_i32(a->imm));
2026 s->base.is_jmp = DISAS_TOO_MANY;
2027 return true;
2028 }
2029
trans_MSR_i_DAIFCLEAR(DisasContext * s,arg_i * a)2030 static bool trans_MSR_i_DAIFCLEAR(DisasContext *s, arg_i *a)
2031 {
2032 gen_helper_msr_i_daifclear(tcg_env, tcg_constant_i32(a->imm));
2033 /* Exit the cpu loop to re-evaluate pending IRQs. */
2034 s->base.is_jmp = DISAS_UPDATE_EXIT;
2035 return true;
2036 }
2037
trans_MSR_i_SVCR(DisasContext * s,arg_MSR_i_SVCR * a)2038 static bool trans_MSR_i_SVCR(DisasContext *s, arg_MSR_i_SVCR *a)
2039 {
2040 if (!dc_isar_feature(aa64_sme, s) || a->mask == 0) {
2041 return false;
2042 }
2043 if (sme_access_check(s)) {
2044 int old = s->pstate_sm | (s->pstate_za << 1);
2045 int new = a->imm * 3;
2046
2047 if ((old ^ new) & a->mask) {
2048 /* At least one bit changes. */
2049 gen_helper_set_svcr(tcg_env, tcg_constant_i32(new),
2050 tcg_constant_i32(a->mask));
2051 s->base.is_jmp = DISAS_TOO_MANY;
2052 }
2053 }
2054 return true;
2055 }
2056
gen_get_nzcv(TCGv_i64 tcg_rt)2057 static void gen_get_nzcv(TCGv_i64 tcg_rt)
2058 {
2059 TCGv_i32 tmp = tcg_temp_new_i32();
2060 TCGv_i32 nzcv = tcg_temp_new_i32();
2061
2062 /* build bit 31, N */
2063 tcg_gen_andi_i32(nzcv, cpu_NF, (1U << 31));
2064 /* build bit 30, Z */
2065 tcg_gen_setcondi_i32(TCG_COND_EQ, tmp, cpu_ZF, 0);
2066 tcg_gen_deposit_i32(nzcv, nzcv, tmp, 30, 1);
2067 /* build bit 29, C */
2068 tcg_gen_deposit_i32(nzcv, nzcv, cpu_CF, 29, 1);
2069 /* build bit 28, V */
2070 tcg_gen_shri_i32(tmp, cpu_VF, 31);
2071 tcg_gen_deposit_i32(nzcv, nzcv, tmp, 28, 1);
2072 /* generate result */
2073 tcg_gen_extu_i32_i64(tcg_rt, nzcv);
2074 }
2075
gen_set_nzcv(TCGv_i64 tcg_rt)2076 static void gen_set_nzcv(TCGv_i64 tcg_rt)
2077 {
2078 TCGv_i32 nzcv = tcg_temp_new_i32();
2079
2080 /* take NZCV from R[t] */
2081 tcg_gen_extrl_i64_i32(nzcv, tcg_rt);
2082
2083 /* bit 31, N */
2084 tcg_gen_andi_i32(cpu_NF, nzcv, (1U << 31));
2085 /* bit 30, Z */
2086 tcg_gen_andi_i32(cpu_ZF, nzcv, (1 << 30));
2087 tcg_gen_setcondi_i32(TCG_COND_EQ, cpu_ZF, cpu_ZF, 0);
2088 /* bit 29, C */
2089 tcg_gen_andi_i32(cpu_CF, nzcv, (1 << 29));
2090 tcg_gen_shri_i32(cpu_CF, cpu_CF, 29);
2091 /* bit 28, V */
2092 tcg_gen_andi_i32(cpu_VF, nzcv, (1 << 28));
2093 tcg_gen_shli_i32(cpu_VF, cpu_VF, 3);
2094 }
2095
gen_sysreg_undef(DisasContext * s,bool isread,uint8_t op0,uint8_t op1,uint8_t op2,uint8_t crn,uint8_t crm,uint8_t rt)2096 static void gen_sysreg_undef(DisasContext *s, bool isread,
2097 uint8_t op0, uint8_t op1, uint8_t op2,
2098 uint8_t crn, uint8_t crm, uint8_t rt)
2099 {
2100 /*
2101 * Generate code to emit an UNDEF with correct syndrome
2102 * information for a failed system register access.
2103 * This is EC_UNCATEGORIZED (ie a standard UNDEF) in most cases,
2104 * but if FEAT_IDST is implemented then read accesses to registers
2105 * in the feature ID space are reported with the EC_SYSTEMREGISTERTRAP
2106 * syndrome.
2107 */
2108 uint32_t syndrome;
2109
2110 if (isread && dc_isar_feature(aa64_ids, s) &&
2111 arm_cpreg_encoding_in_idspace(op0, op1, op2, crn, crm)) {
2112 syndrome = syn_aa64_sysregtrap(op0, op1, op2, crn, crm, rt, isread);
2113 } else {
2114 syndrome = syn_uncategorized();
2115 }
2116 gen_exception_insn(s, 0, EXCP_UDEF, syndrome);
2117 }
2118
2119 /* MRS - move from system register
2120 * MSR (register) - move to system register
2121 * SYS
2122 * SYSL
2123 * These are all essentially the same insn in 'read' and 'write'
2124 * versions, with varying op0 fields.
2125 */
handle_sys(DisasContext * s,bool isread,unsigned int op0,unsigned int op1,unsigned int op2,unsigned int crn,unsigned int crm,unsigned int rt)2126 static void handle_sys(DisasContext *s, bool isread,
2127 unsigned int op0, unsigned int op1, unsigned int op2,
2128 unsigned int crn, unsigned int crm, unsigned int rt)
2129 {
2130 uint32_t key = ENCODE_AA64_CP_REG(CP_REG_ARM64_SYSREG_CP,
2131 crn, crm, op0, op1, op2);
2132 const ARMCPRegInfo *ri = get_arm_cp_reginfo(s->cp_regs, key);
2133 bool need_exit_tb = false;
2134 TCGv_ptr tcg_ri = NULL;
2135 TCGv_i64 tcg_rt;
2136 uint32_t syndrome;
2137
2138 if (crn == 11 || crn == 15) {
2139 /*
2140 * Check for TIDCP trap, which must take precedence over
2141 * the UNDEF for "no such register" etc.
2142 */
2143 syndrome = syn_aa64_sysregtrap(op0, op1, op2, crn, crm, rt, isread);
2144 switch (s->current_el) {
2145 case 0:
2146 if (dc_isar_feature(aa64_tidcp1, s)) {
2147 gen_helper_tidcp_el0(tcg_env, tcg_constant_i32(syndrome));
2148 }
2149 break;
2150 case 1:
2151 gen_helper_tidcp_el1(tcg_env, tcg_constant_i32(syndrome));
2152 break;
2153 }
2154 }
2155
2156 if (!ri) {
2157 /* Unknown register; this might be a guest error or a QEMU
2158 * unimplemented feature.
2159 */
2160 qemu_log_mask(LOG_UNIMP, "%s access to unsupported AArch64 "
2161 "system register op0:%d op1:%d crn:%d crm:%d op2:%d\n",
2162 isread ? "read" : "write", op0, op1, crn, crm, op2);
2163 gen_sysreg_undef(s, isread, op0, op1, op2, crn, crm, rt);
2164 return;
2165 }
2166
2167 /* Check access permissions */
2168 if (!cp_access_ok(s->current_el, ri, isread)) {
2169 gen_sysreg_undef(s, isread, op0, op1, op2, crn, crm, rt);
2170 return;
2171 }
2172
2173 if (ri->accessfn || (ri->fgt && s->fgt_active)) {
2174 /* Emit code to perform further access permissions checks at
2175 * runtime; this may result in an exception.
2176 */
2177 syndrome = syn_aa64_sysregtrap(op0, op1, op2, crn, crm, rt, isread);
2178 gen_a64_update_pc(s, 0);
2179 tcg_ri = tcg_temp_new_ptr();
2180 gen_helper_access_check_cp_reg(tcg_ri, tcg_env,
2181 tcg_constant_i32(key),
2182 tcg_constant_i32(syndrome),
2183 tcg_constant_i32(isread));
2184 } else if (ri->type & ARM_CP_RAISES_EXC) {
2185 /*
2186 * The readfn or writefn might raise an exception;
2187 * synchronize the CPU state in case it does.
2188 */
2189 gen_a64_update_pc(s, 0);
2190 }
2191
2192 /* Handle special cases first */
2193 switch (ri->type & ARM_CP_SPECIAL_MASK) {
2194 case 0:
2195 break;
2196 case ARM_CP_NOP:
2197 return;
2198 case ARM_CP_NZCV:
2199 tcg_rt = cpu_reg(s, rt);
2200 if (isread) {
2201 gen_get_nzcv(tcg_rt);
2202 } else {
2203 gen_set_nzcv(tcg_rt);
2204 }
2205 return;
2206 case ARM_CP_CURRENTEL:
2207 /* Reads as current EL value from pstate, which is
2208 * guaranteed to be constant by the tb flags.
2209 */
2210 tcg_rt = cpu_reg(s, rt);
2211 tcg_gen_movi_i64(tcg_rt, s->current_el << 2);
2212 return;
2213 case ARM_CP_DC_ZVA:
2214 /* Writes clear the aligned block of memory which rt points into. */
2215 if (s->mte_active[0]) {
2216 int desc = 0;
2217
2218 desc = FIELD_DP32(desc, MTEDESC, MIDX, get_mem_index(s));
2219 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid);
2220 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma);
2221
2222 tcg_rt = tcg_temp_new_i64();
2223 gen_helper_mte_check_zva(tcg_rt, tcg_env,
2224 tcg_constant_i32(desc), cpu_reg(s, rt));
2225 } else {
2226 tcg_rt = clean_data_tbi(s, cpu_reg(s, rt));
2227 }
2228 gen_helper_dc_zva(tcg_env, tcg_rt);
2229 return;
2230 case ARM_CP_DC_GVA:
2231 {
2232 TCGv_i64 clean_addr, tag;
2233
2234 /*
2235 * DC_GVA, like DC_ZVA, requires that we supply the original
2236 * pointer for an invalid page. Probe that address first.
2237 */
2238 tcg_rt = cpu_reg(s, rt);
2239 clean_addr = clean_data_tbi(s, tcg_rt);
2240 gen_probe_access(s, clean_addr, MMU_DATA_STORE, MO_8);
2241
2242 if (s->ata[0]) {
2243 /* Extract the tag from the register to match STZGM. */
2244 tag = tcg_temp_new_i64();
2245 tcg_gen_shri_i64(tag, tcg_rt, 56);
2246 gen_helper_stzgm_tags(tcg_env, clean_addr, tag);
2247 }
2248 }
2249 return;
2250 case ARM_CP_DC_GZVA:
2251 {
2252 TCGv_i64 clean_addr, tag;
2253
2254 /* For DC_GZVA, we can rely on DC_ZVA for the proper fault. */
2255 tcg_rt = cpu_reg(s, rt);
2256 clean_addr = clean_data_tbi(s, tcg_rt);
2257 gen_helper_dc_zva(tcg_env, clean_addr);
2258
2259 if (s->ata[0]) {
2260 /* Extract the tag from the register to match STZGM. */
2261 tag = tcg_temp_new_i64();
2262 tcg_gen_shri_i64(tag, tcg_rt, 56);
2263 gen_helper_stzgm_tags(tcg_env, clean_addr, tag);
2264 }
2265 }
2266 return;
2267 default:
2268 g_assert_not_reached();
2269 }
2270 if ((ri->type & ARM_CP_FPU) && !fp_access_check_only(s)) {
2271 return;
2272 } else if ((ri->type & ARM_CP_SVE) && !sve_access_check(s)) {
2273 return;
2274 } else if ((ri->type & ARM_CP_SME) && !sme_access_check(s)) {
2275 return;
2276 }
2277
2278 if (ri->type & ARM_CP_IO) {
2279 /* I/O operations must end the TB here (whether read or write) */
2280 need_exit_tb = translator_io_start(&s->base);
2281 }
2282
2283 tcg_rt = cpu_reg(s, rt);
2284
2285 if (isread) {
2286 if (ri->type & ARM_CP_CONST) {
2287 tcg_gen_movi_i64(tcg_rt, ri->resetvalue);
2288 } else if (ri->readfn) {
2289 if (!tcg_ri) {
2290 tcg_ri = gen_lookup_cp_reg(key);
2291 }
2292 gen_helper_get_cp_reg64(tcg_rt, tcg_env, tcg_ri);
2293 } else {
2294 tcg_gen_ld_i64(tcg_rt, tcg_env, ri->fieldoffset);
2295 }
2296 } else {
2297 if (ri->type & ARM_CP_CONST) {
2298 /* If not forbidden by access permissions, treat as WI */
2299 return;
2300 } else if (ri->writefn) {
2301 if (!tcg_ri) {
2302 tcg_ri = gen_lookup_cp_reg(key);
2303 }
2304 gen_helper_set_cp_reg64(tcg_env, tcg_ri, tcg_rt);
2305 } else {
2306 tcg_gen_st_i64(tcg_rt, tcg_env, ri->fieldoffset);
2307 }
2308 }
2309
2310 if (!isread && !(ri->type & ARM_CP_SUPPRESS_TB_END)) {
2311 /*
2312 * A write to any coprocessor register that ends a TB
2313 * must rebuild the hflags for the next TB.
2314 */
2315 gen_rebuild_hflags(s);
2316 /*
2317 * We default to ending the TB on a coprocessor register write,
2318 * but allow this to be suppressed by the register definition
2319 * (usually only necessary to work around guest bugs).
2320 */
2321 need_exit_tb = true;
2322 }
2323 if (need_exit_tb) {
2324 s->base.is_jmp = DISAS_UPDATE_EXIT;
2325 }
2326 }
2327
trans_SYS(DisasContext * s,arg_SYS * a)2328 static bool trans_SYS(DisasContext *s, arg_SYS *a)
2329 {
2330 handle_sys(s, a->l, a->op0, a->op1, a->op2, a->crn, a->crm, a->rt);
2331 return true;
2332 }
2333
trans_SVC(DisasContext * s,arg_i * a)2334 static bool trans_SVC(DisasContext *s, arg_i *a)
2335 {
2336 /*
2337 * For SVC, HVC and SMC we advance the single-step state
2338 * machine before taking the exception. This is architecturally
2339 * mandated, to ensure that single-stepping a system call
2340 * instruction works properly.
2341 */
2342 uint32_t syndrome = syn_aa64_svc(a->imm);
2343 if (s->fgt_svc) {
2344 gen_exception_insn_el(s, 0, EXCP_UDEF, syndrome, 2);
2345 return true;
2346 }
2347 gen_ss_advance(s);
2348 gen_exception_insn(s, 4, EXCP_SWI, syndrome);
2349 return true;
2350 }
2351
trans_HVC(DisasContext * s,arg_i * a)2352 static bool trans_HVC(DisasContext *s, arg_i *a)
2353 {
2354 int target_el = s->current_el == 3 ? 3 : 2;
2355
2356 if (s->current_el == 0) {
2357 unallocated_encoding(s);
2358 return true;
2359 }
2360 /*
2361 * The pre HVC helper handles cases when HVC gets trapped
2362 * as an undefined insn by runtime configuration.
2363 */
2364 gen_a64_update_pc(s, 0);
2365 gen_helper_pre_hvc(tcg_env);
2366 /* Architecture requires ss advance before we do the actual work */
2367 gen_ss_advance(s);
2368 gen_exception_insn_el(s, 4, EXCP_HVC, syn_aa64_hvc(a->imm), target_el);
2369 return true;
2370 }
2371
trans_SMC(DisasContext * s,arg_i * a)2372 static bool trans_SMC(DisasContext *s, arg_i *a)
2373 {
2374 if (s->current_el == 0) {
2375 unallocated_encoding(s);
2376 return true;
2377 }
2378 gen_a64_update_pc(s, 0);
2379 gen_helper_pre_smc(tcg_env, tcg_constant_i32(syn_aa64_smc(a->imm)));
2380 /* Architecture requires ss advance before we do the actual work */
2381 gen_ss_advance(s);
2382 gen_exception_insn_el(s, 4, EXCP_SMC, syn_aa64_smc(a->imm), 3);
2383 return true;
2384 }
2385
trans_BRK(DisasContext * s,arg_i * a)2386 static bool trans_BRK(DisasContext *s, arg_i *a)
2387 {
2388 gen_exception_bkpt_insn(s, syn_aa64_bkpt(a->imm));
2389 return true;
2390 }
2391
trans_HLT(DisasContext * s,arg_i * a)2392 static bool trans_HLT(DisasContext *s, arg_i *a)
2393 {
2394 /*
2395 * HLT. This has two purposes.
2396 * Architecturally, it is an external halting debug instruction.
2397 * Since QEMU doesn't implement external debug, we treat this as
2398 * it is required for halting debug disabled: it will UNDEF.
2399 * Secondly, "HLT 0xf000" is the A64 semihosting syscall instruction.
2400 */
2401 if (semihosting_enabled(s->current_el == 0) && a->imm == 0xf000) {
2402 gen_exception_internal_insn(s, EXCP_SEMIHOST);
2403 } else {
2404 unallocated_encoding(s);
2405 }
2406 return true;
2407 }
2408
2409 /*
2410 * Load/Store exclusive instructions are implemented by remembering
2411 * the value/address loaded, and seeing if these are the same
2412 * when the store is performed. This is not actually the architecturally
2413 * mandated semantics, but it works for typical guest code sequences
2414 * and avoids having to monitor regular stores.
2415 *
2416 * The store exclusive uses the atomic cmpxchg primitives to avoid
2417 * races in multi-threaded linux-user and when MTTCG softmmu is
2418 * enabled.
2419 */
gen_load_exclusive(DisasContext * s,int rt,int rt2,int rn,int size,bool is_pair)2420 static void gen_load_exclusive(DisasContext *s, int rt, int rt2, int rn,
2421 int size, bool is_pair)
2422 {
2423 int idx = get_mem_index(s);
2424 TCGv_i64 dirty_addr, clean_addr;
2425 MemOp memop = check_atomic_align(s, rn, size + is_pair);
2426
2427 s->is_ldex = true;
2428 dirty_addr = cpu_reg_sp(s, rn);
2429 clean_addr = gen_mte_check1(s, dirty_addr, false, rn != 31, memop);
2430
2431 g_assert(size <= 3);
2432 if (is_pair) {
2433 g_assert(size >= 2);
2434 if (size == 2) {
2435 tcg_gen_qemu_ld_i64(cpu_exclusive_val, clean_addr, idx, memop);
2436 if (s->be_data == MO_LE) {
2437 tcg_gen_extract_i64(cpu_reg(s, rt), cpu_exclusive_val, 0, 32);
2438 tcg_gen_extract_i64(cpu_reg(s, rt2), cpu_exclusive_val, 32, 32);
2439 } else {
2440 tcg_gen_extract_i64(cpu_reg(s, rt), cpu_exclusive_val, 32, 32);
2441 tcg_gen_extract_i64(cpu_reg(s, rt2), cpu_exclusive_val, 0, 32);
2442 }
2443 } else {
2444 TCGv_i128 t16 = tcg_temp_new_i128();
2445
2446 tcg_gen_qemu_ld_i128(t16, clean_addr, idx, memop);
2447
2448 if (s->be_data == MO_LE) {
2449 tcg_gen_extr_i128_i64(cpu_exclusive_val,
2450 cpu_exclusive_high, t16);
2451 } else {
2452 tcg_gen_extr_i128_i64(cpu_exclusive_high,
2453 cpu_exclusive_val, t16);
2454 }
2455 tcg_gen_mov_i64(cpu_reg(s, rt), cpu_exclusive_val);
2456 tcg_gen_mov_i64(cpu_reg(s, rt2), cpu_exclusive_high);
2457 }
2458 } else {
2459 tcg_gen_qemu_ld_i64(cpu_exclusive_val, clean_addr, idx, memop);
2460 tcg_gen_mov_i64(cpu_reg(s, rt), cpu_exclusive_val);
2461 }
2462 tcg_gen_mov_i64(cpu_exclusive_addr, clean_addr);
2463 }
2464
gen_store_exclusive(DisasContext * s,int rd,int rt,int rt2,int rn,int size,int is_pair)2465 static void gen_store_exclusive(DisasContext *s, int rd, int rt, int rt2,
2466 int rn, int size, int is_pair)
2467 {
2468 /* if (env->exclusive_addr == addr && env->exclusive_val == [addr]
2469 * && (!is_pair || env->exclusive_high == [addr + datasize])) {
2470 * [addr] = {Rt};
2471 * if (is_pair) {
2472 * [addr + datasize] = {Rt2};
2473 * }
2474 * {Rd} = 0;
2475 * } else {
2476 * {Rd} = 1;
2477 * }
2478 * env->exclusive_addr = -1;
2479 */
2480 TCGLabel *fail_label = gen_new_label();
2481 TCGLabel *done_label = gen_new_label();
2482 TCGv_i64 tmp, clean_addr;
2483 MemOp memop;
2484
2485 /*
2486 * FIXME: We are out of spec here. We have recorded only the address
2487 * from load_exclusive, not the entire range, and we assume that the
2488 * size of the access on both sides match. The architecture allows the
2489 * store to be smaller than the load, so long as the stored bytes are
2490 * within the range recorded by the load.
2491 */
2492
2493 /* See AArch64.ExclusiveMonitorsPass() and AArch64.IsExclusiveVA(). */
2494 clean_addr = clean_data_tbi(s, cpu_reg_sp(s, rn));
2495 tcg_gen_brcond_i64(TCG_COND_NE, clean_addr, cpu_exclusive_addr, fail_label);
2496
2497 /*
2498 * The write, and any associated faults, only happen if the virtual
2499 * and physical addresses pass the exclusive monitor check. These
2500 * faults are exceedingly unlikely, because normally the guest uses
2501 * the exact same address register for the load_exclusive, and we
2502 * would have recognized these faults there.
2503 *
2504 * It is possible to trigger an alignment fault pre-LSE2, e.g. with an
2505 * unaligned 4-byte write within the range of an aligned 8-byte load.
2506 * With LSE2, the store would need to cross a 16-byte boundary when the
2507 * load did not, which would mean the store is outside the range
2508 * recorded for the monitor, which would have failed a corrected monitor
2509 * check above. For now, we assume no size change and retain the
2510 * MO_ALIGN to let tcg know what we checked in the load_exclusive.
2511 *
2512 * It is possible to trigger an MTE fault, by performing the load with
2513 * a virtual address with a valid tag and performing the store with the
2514 * same virtual address and a different invalid tag.
2515 */
2516 memop = size + is_pair;
2517 if (memop == MO_128 || !dc_isar_feature(aa64_lse2, s)) {
2518 memop |= MO_ALIGN;
2519 }
2520 memop = finalize_memop(s, memop);
2521 gen_mte_check1(s, cpu_reg_sp(s, rn), true, rn != 31, memop);
2522
2523 tmp = tcg_temp_new_i64();
2524 if (is_pair) {
2525 if (size == 2) {
2526 if (s->be_data == MO_LE) {
2527 tcg_gen_concat32_i64(tmp, cpu_reg(s, rt), cpu_reg(s, rt2));
2528 } else {
2529 tcg_gen_concat32_i64(tmp, cpu_reg(s, rt2), cpu_reg(s, rt));
2530 }
2531 tcg_gen_atomic_cmpxchg_i64(tmp, cpu_exclusive_addr,
2532 cpu_exclusive_val, tmp,
2533 get_mem_index(s), memop);
2534 tcg_gen_setcond_i64(TCG_COND_NE, tmp, tmp, cpu_exclusive_val);
2535 } else {
2536 TCGv_i128 t16 = tcg_temp_new_i128();
2537 TCGv_i128 c16 = tcg_temp_new_i128();
2538 TCGv_i64 a, b;
2539
2540 if (s->be_data == MO_LE) {
2541 tcg_gen_concat_i64_i128(t16, cpu_reg(s, rt), cpu_reg(s, rt2));
2542 tcg_gen_concat_i64_i128(c16, cpu_exclusive_val,
2543 cpu_exclusive_high);
2544 } else {
2545 tcg_gen_concat_i64_i128(t16, cpu_reg(s, rt2), cpu_reg(s, rt));
2546 tcg_gen_concat_i64_i128(c16, cpu_exclusive_high,
2547 cpu_exclusive_val);
2548 }
2549
2550 tcg_gen_atomic_cmpxchg_i128(t16, cpu_exclusive_addr, c16, t16,
2551 get_mem_index(s), memop);
2552
2553 a = tcg_temp_new_i64();
2554 b = tcg_temp_new_i64();
2555 if (s->be_data == MO_LE) {
2556 tcg_gen_extr_i128_i64(a, b, t16);
2557 } else {
2558 tcg_gen_extr_i128_i64(b, a, t16);
2559 }
2560
2561 tcg_gen_xor_i64(a, a, cpu_exclusive_val);
2562 tcg_gen_xor_i64(b, b, cpu_exclusive_high);
2563 tcg_gen_or_i64(tmp, a, b);
2564
2565 tcg_gen_setcondi_i64(TCG_COND_NE, tmp, tmp, 0);
2566 }
2567 } else {
2568 tcg_gen_atomic_cmpxchg_i64(tmp, cpu_exclusive_addr, cpu_exclusive_val,
2569 cpu_reg(s, rt), get_mem_index(s), memop);
2570 tcg_gen_setcond_i64(TCG_COND_NE, tmp, tmp, cpu_exclusive_val);
2571 }
2572 tcg_gen_mov_i64(cpu_reg(s, rd), tmp);
2573 tcg_gen_br(done_label);
2574
2575 gen_set_label(fail_label);
2576 tcg_gen_movi_i64(cpu_reg(s, rd), 1);
2577 gen_set_label(done_label);
2578 tcg_gen_movi_i64(cpu_exclusive_addr, -1);
2579 }
2580
gen_compare_and_swap(DisasContext * s,int rs,int rt,int rn,int size)2581 static void gen_compare_and_swap(DisasContext *s, int rs, int rt,
2582 int rn, int size)
2583 {
2584 TCGv_i64 tcg_rs = cpu_reg(s, rs);
2585 TCGv_i64 tcg_rt = cpu_reg(s, rt);
2586 int memidx = get_mem_index(s);
2587 TCGv_i64 clean_addr;
2588 MemOp memop;
2589
2590 if (rn == 31) {
2591 gen_check_sp_alignment(s);
2592 }
2593 memop = check_atomic_align(s, rn, size);
2594 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn), true, rn != 31, memop);
2595 tcg_gen_atomic_cmpxchg_i64(tcg_rs, clean_addr, tcg_rs, tcg_rt,
2596 memidx, memop);
2597 }
2598
gen_compare_and_swap_pair(DisasContext * s,int rs,int rt,int rn,int size)2599 static void gen_compare_and_swap_pair(DisasContext *s, int rs, int rt,
2600 int rn, int size)
2601 {
2602 TCGv_i64 s1 = cpu_reg(s, rs);
2603 TCGv_i64 s2 = cpu_reg(s, rs + 1);
2604 TCGv_i64 t1 = cpu_reg(s, rt);
2605 TCGv_i64 t2 = cpu_reg(s, rt + 1);
2606 TCGv_i64 clean_addr;
2607 int memidx = get_mem_index(s);
2608 MemOp memop;
2609
2610 if (rn == 31) {
2611 gen_check_sp_alignment(s);
2612 }
2613
2614 /* This is a single atomic access, despite the "pair". */
2615 memop = check_atomic_align(s, rn, size + 1);
2616 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn), true, rn != 31, memop);
2617
2618 if (size == 2) {
2619 TCGv_i64 cmp = tcg_temp_new_i64();
2620 TCGv_i64 val = tcg_temp_new_i64();
2621
2622 if (s->be_data == MO_LE) {
2623 tcg_gen_concat32_i64(val, t1, t2);
2624 tcg_gen_concat32_i64(cmp, s1, s2);
2625 } else {
2626 tcg_gen_concat32_i64(val, t2, t1);
2627 tcg_gen_concat32_i64(cmp, s2, s1);
2628 }
2629
2630 tcg_gen_atomic_cmpxchg_i64(cmp, clean_addr, cmp, val, memidx, memop);
2631
2632 if (s->be_data == MO_LE) {
2633 tcg_gen_extr32_i64(s1, s2, cmp);
2634 } else {
2635 tcg_gen_extr32_i64(s2, s1, cmp);
2636 }
2637 } else {
2638 TCGv_i128 cmp = tcg_temp_new_i128();
2639 TCGv_i128 val = tcg_temp_new_i128();
2640
2641 if (s->be_data == MO_LE) {
2642 tcg_gen_concat_i64_i128(val, t1, t2);
2643 tcg_gen_concat_i64_i128(cmp, s1, s2);
2644 } else {
2645 tcg_gen_concat_i64_i128(val, t2, t1);
2646 tcg_gen_concat_i64_i128(cmp, s2, s1);
2647 }
2648
2649 tcg_gen_atomic_cmpxchg_i128(cmp, clean_addr, cmp, val, memidx, memop);
2650
2651 if (s->be_data == MO_LE) {
2652 tcg_gen_extr_i128_i64(s1, s2, cmp);
2653 } else {
2654 tcg_gen_extr_i128_i64(s2, s1, cmp);
2655 }
2656 }
2657 }
2658
2659 /*
2660 * Compute the ISS.SF bit for syndrome information if an exception
2661 * is taken on a load or store. This indicates whether the instruction
2662 * is accessing a 32-bit or 64-bit register. This logic is derived
2663 * from the ARMv8 specs for LDR (Shared decode for all encodings).
2664 */
ldst_iss_sf(int size,bool sign,bool ext)2665 static bool ldst_iss_sf(int size, bool sign, bool ext)
2666 {
2667
2668 if (sign) {
2669 /*
2670 * Signed loads are 64 bit results if we are not going to
2671 * do a zero-extend from 32 to 64 after the load.
2672 * (For a store, sign and ext are always false.)
2673 */
2674 return !ext;
2675 } else {
2676 /* Unsigned loads/stores work at the specified size */
2677 return size == MO_64;
2678 }
2679 }
2680
trans_STXR(DisasContext * s,arg_stxr * a)2681 static bool trans_STXR(DisasContext *s, arg_stxr *a)
2682 {
2683 if (a->rn == 31) {
2684 gen_check_sp_alignment(s);
2685 }
2686 if (a->lasr) {
2687 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_STRL);
2688 }
2689 gen_store_exclusive(s, a->rs, a->rt, a->rt2, a->rn, a->sz, false);
2690 return true;
2691 }
2692
trans_LDXR(DisasContext * s,arg_stxr * a)2693 static bool trans_LDXR(DisasContext *s, arg_stxr *a)
2694 {
2695 if (a->rn == 31) {
2696 gen_check_sp_alignment(s);
2697 }
2698 gen_load_exclusive(s, a->rt, a->rt2, a->rn, a->sz, false);
2699 if (a->lasr) {
2700 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ);
2701 }
2702 return true;
2703 }
2704
trans_STLR(DisasContext * s,arg_stlr * a)2705 static bool trans_STLR(DisasContext *s, arg_stlr *a)
2706 {
2707 TCGv_i64 clean_addr;
2708 MemOp memop;
2709 bool iss_sf = ldst_iss_sf(a->sz, false, false);
2710
2711 /*
2712 * StoreLORelease is the same as Store-Release for QEMU, but
2713 * needs the feature-test.
2714 */
2715 if (!a->lasr && !dc_isar_feature(aa64_lor, s)) {
2716 return false;
2717 }
2718 /* Generate ISS for non-exclusive accesses including LASR. */
2719 if (a->rn == 31) {
2720 gen_check_sp_alignment(s);
2721 }
2722 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_STRL);
2723 memop = check_ordered_align(s, a->rn, 0, true, a->sz);
2724 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn),
2725 true, a->rn != 31, memop);
2726 do_gpr_st(s, cpu_reg(s, a->rt), clean_addr, memop, true, a->rt,
2727 iss_sf, a->lasr);
2728 return true;
2729 }
2730
trans_LDAR(DisasContext * s,arg_stlr * a)2731 static bool trans_LDAR(DisasContext *s, arg_stlr *a)
2732 {
2733 TCGv_i64 clean_addr;
2734 MemOp memop;
2735 bool iss_sf = ldst_iss_sf(a->sz, false, false);
2736
2737 /* LoadLOAcquire is the same as Load-Acquire for QEMU. */
2738 if (!a->lasr && !dc_isar_feature(aa64_lor, s)) {
2739 return false;
2740 }
2741 /* Generate ISS for non-exclusive accesses including LASR. */
2742 if (a->rn == 31) {
2743 gen_check_sp_alignment(s);
2744 }
2745 memop = check_ordered_align(s, a->rn, 0, false, a->sz);
2746 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn),
2747 false, a->rn != 31, memop);
2748 do_gpr_ld(s, cpu_reg(s, a->rt), clean_addr, memop, false, true,
2749 a->rt, iss_sf, a->lasr);
2750 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ);
2751 return true;
2752 }
2753
trans_STXP(DisasContext * s,arg_stxr * a)2754 static bool trans_STXP(DisasContext *s, arg_stxr *a)
2755 {
2756 if (a->rn == 31) {
2757 gen_check_sp_alignment(s);
2758 }
2759 if (a->lasr) {
2760 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_STRL);
2761 }
2762 gen_store_exclusive(s, a->rs, a->rt, a->rt2, a->rn, a->sz, true);
2763 return true;
2764 }
2765
trans_LDXP(DisasContext * s,arg_stxr * a)2766 static bool trans_LDXP(DisasContext *s, arg_stxr *a)
2767 {
2768 if (a->rn == 31) {
2769 gen_check_sp_alignment(s);
2770 }
2771 gen_load_exclusive(s, a->rt, a->rt2, a->rn, a->sz, true);
2772 if (a->lasr) {
2773 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ);
2774 }
2775 return true;
2776 }
2777
trans_CASP(DisasContext * s,arg_CASP * a)2778 static bool trans_CASP(DisasContext *s, arg_CASP *a)
2779 {
2780 if (!dc_isar_feature(aa64_atomics, s)) {
2781 return false;
2782 }
2783 if (((a->rt | a->rs) & 1) != 0) {
2784 return false;
2785 }
2786
2787 gen_compare_and_swap_pair(s, a->rs, a->rt, a->rn, a->sz);
2788 return true;
2789 }
2790
trans_CAS(DisasContext * s,arg_CAS * a)2791 static bool trans_CAS(DisasContext *s, arg_CAS *a)
2792 {
2793 if (!dc_isar_feature(aa64_atomics, s)) {
2794 return false;
2795 }
2796 gen_compare_and_swap(s, a->rs, a->rt, a->rn, a->sz);
2797 return true;
2798 }
2799
trans_LD_lit(DisasContext * s,arg_ldlit * a)2800 static bool trans_LD_lit(DisasContext *s, arg_ldlit *a)
2801 {
2802 bool iss_sf = ldst_iss_sf(a->sz, a->sign, false);
2803 TCGv_i64 tcg_rt = cpu_reg(s, a->rt);
2804 TCGv_i64 clean_addr = tcg_temp_new_i64();
2805 MemOp memop = finalize_memop(s, a->sz + a->sign * MO_SIGN);
2806
2807 gen_pc_plus_diff(s, clean_addr, a->imm);
2808 do_gpr_ld(s, tcg_rt, clean_addr, memop,
2809 false, true, a->rt, iss_sf, false);
2810 return true;
2811 }
2812
trans_LD_lit_v(DisasContext * s,arg_ldlit * a)2813 static bool trans_LD_lit_v(DisasContext *s, arg_ldlit *a)
2814 {
2815 /* Load register (literal), vector version */
2816 TCGv_i64 clean_addr;
2817 MemOp memop;
2818
2819 if (!fp_access_check(s)) {
2820 return true;
2821 }
2822 memop = finalize_memop_asimd(s, a->sz);
2823 clean_addr = tcg_temp_new_i64();
2824 gen_pc_plus_diff(s, clean_addr, a->imm);
2825 do_fp_ld(s, a->rt, clean_addr, memop);
2826 return true;
2827 }
2828
op_addr_ldstpair_pre(DisasContext * s,arg_ldstpair * a,TCGv_i64 * clean_addr,TCGv_i64 * dirty_addr,uint64_t offset,bool is_store,MemOp mop)2829 static void op_addr_ldstpair_pre(DisasContext *s, arg_ldstpair *a,
2830 TCGv_i64 *clean_addr, TCGv_i64 *dirty_addr,
2831 uint64_t offset, bool is_store, MemOp mop)
2832 {
2833 if (a->rn == 31) {
2834 gen_check_sp_alignment(s);
2835 }
2836
2837 *dirty_addr = read_cpu_reg_sp(s, a->rn, 1);
2838 if (!a->p) {
2839 tcg_gen_addi_i64(*dirty_addr, *dirty_addr, offset);
2840 }
2841
2842 *clean_addr = gen_mte_checkN(s, *dirty_addr, is_store,
2843 (a->w || a->rn != 31), 2 << a->sz, mop);
2844 }
2845
op_addr_ldstpair_post(DisasContext * s,arg_ldstpair * a,TCGv_i64 dirty_addr,uint64_t offset)2846 static void op_addr_ldstpair_post(DisasContext *s, arg_ldstpair *a,
2847 TCGv_i64 dirty_addr, uint64_t offset)
2848 {
2849 if (a->w) {
2850 if (a->p) {
2851 tcg_gen_addi_i64(dirty_addr, dirty_addr, offset);
2852 }
2853 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), dirty_addr);
2854 }
2855 }
2856
trans_STP(DisasContext * s,arg_ldstpair * a)2857 static bool trans_STP(DisasContext *s, arg_ldstpair *a)
2858 {
2859 uint64_t offset = a->imm << a->sz;
2860 TCGv_i64 clean_addr, dirty_addr, tcg_rt, tcg_rt2;
2861 MemOp mop = finalize_memop(s, a->sz);
2862
2863 op_addr_ldstpair_pre(s, a, &clean_addr, &dirty_addr, offset, true, mop);
2864 tcg_rt = cpu_reg(s, a->rt);
2865 tcg_rt2 = cpu_reg(s, a->rt2);
2866 /*
2867 * We built mop above for the single logical access -- rebuild it
2868 * now for the paired operation.
2869 *
2870 * With LSE2, non-sign-extending pairs are treated atomically if
2871 * aligned, and if unaligned one of the pair will be completely
2872 * within a 16-byte block and that element will be atomic.
2873 * Otherwise each element is separately atomic.
2874 * In all cases, issue one operation with the correct atomicity.
2875 */
2876 mop = a->sz + 1;
2877 if (s->align_mem) {
2878 mop |= (a->sz == 2 ? MO_ALIGN_4 : MO_ALIGN_8);
2879 }
2880 mop = finalize_memop_pair(s, mop);
2881 if (a->sz == 2) {
2882 TCGv_i64 tmp = tcg_temp_new_i64();
2883
2884 if (s->be_data == MO_LE) {
2885 tcg_gen_concat32_i64(tmp, tcg_rt, tcg_rt2);
2886 } else {
2887 tcg_gen_concat32_i64(tmp, tcg_rt2, tcg_rt);
2888 }
2889 tcg_gen_qemu_st_i64(tmp, clean_addr, get_mem_index(s), mop);
2890 } else {
2891 TCGv_i128 tmp = tcg_temp_new_i128();
2892
2893 if (s->be_data == MO_LE) {
2894 tcg_gen_concat_i64_i128(tmp, tcg_rt, tcg_rt2);
2895 } else {
2896 tcg_gen_concat_i64_i128(tmp, tcg_rt2, tcg_rt);
2897 }
2898 tcg_gen_qemu_st_i128(tmp, clean_addr, get_mem_index(s), mop);
2899 }
2900 op_addr_ldstpair_post(s, a, dirty_addr, offset);
2901 return true;
2902 }
2903
trans_LDP(DisasContext * s,arg_ldstpair * a)2904 static bool trans_LDP(DisasContext *s, arg_ldstpair *a)
2905 {
2906 uint64_t offset = a->imm << a->sz;
2907 TCGv_i64 clean_addr, dirty_addr, tcg_rt, tcg_rt2;
2908 MemOp mop = finalize_memop(s, a->sz);
2909
2910 op_addr_ldstpair_pre(s, a, &clean_addr, &dirty_addr, offset, false, mop);
2911 tcg_rt = cpu_reg(s, a->rt);
2912 tcg_rt2 = cpu_reg(s, a->rt2);
2913
2914 /*
2915 * We built mop above for the single logical access -- rebuild it
2916 * now for the paired operation.
2917 *
2918 * With LSE2, non-sign-extending pairs are treated atomically if
2919 * aligned, and if unaligned one of the pair will be completely
2920 * within a 16-byte block and that element will be atomic.
2921 * Otherwise each element is separately atomic.
2922 * In all cases, issue one operation with the correct atomicity.
2923 *
2924 * This treats sign-extending loads like zero-extending loads,
2925 * since that reuses the most code below.
2926 */
2927 mop = a->sz + 1;
2928 if (s->align_mem) {
2929 mop |= (a->sz == 2 ? MO_ALIGN_4 : MO_ALIGN_8);
2930 }
2931 mop = finalize_memop_pair(s, mop);
2932 if (a->sz == 2) {
2933 int o2 = s->be_data == MO_LE ? 32 : 0;
2934 int o1 = o2 ^ 32;
2935
2936 tcg_gen_qemu_ld_i64(tcg_rt, clean_addr, get_mem_index(s), mop);
2937 if (a->sign) {
2938 tcg_gen_sextract_i64(tcg_rt2, tcg_rt, o2, 32);
2939 tcg_gen_sextract_i64(tcg_rt, tcg_rt, o1, 32);
2940 } else {
2941 tcg_gen_extract_i64(tcg_rt2, tcg_rt, o2, 32);
2942 tcg_gen_extract_i64(tcg_rt, tcg_rt, o1, 32);
2943 }
2944 } else {
2945 TCGv_i128 tmp = tcg_temp_new_i128();
2946
2947 tcg_gen_qemu_ld_i128(tmp, clean_addr, get_mem_index(s), mop);
2948 if (s->be_data == MO_LE) {
2949 tcg_gen_extr_i128_i64(tcg_rt, tcg_rt2, tmp);
2950 } else {
2951 tcg_gen_extr_i128_i64(tcg_rt2, tcg_rt, tmp);
2952 }
2953 }
2954 op_addr_ldstpair_post(s, a, dirty_addr, offset);
2955 return true;
2956 }
2957
trans_STP_v(DisasContext * s,arg_ldstpair * a)2958 static bool trans_STP_v(DisasContext *s, arg_ldstpair *a)
2959 {
2960 uint64_t offset = a->imm << a->sz;
2961 TCGv_i64 clean_addr, dirty_addr;
2962 MemOp mop;
2963
2964 if (!fp_access_check(s)) {
2965 return true;
2966 }
2967
2968 /* LSE2 does not merge FP pairs; leave these as separate operations. */
2969 mop = finalize_memop_asimd(s, a->sz);
2970 op_addr_ldstpair_pre(s, a, &clean_addr, &dirty_addr, offset, true, mop);
2971 do_fp_st(s, a->rt, clean_addr, mop);
2972 tcg_gen_addi_i64(clean_addr, clean_addr, 1 << a->sz);
2973 do_fp_st(s, a->rt2, clean_addr, mop);
2974 op_addr_ldstpair_post(s, a, dirty_addr, offset);
2975 return true;
2976 }
2977
trans_LDP_v(DisasContext * s,arg_ldstpair * a)2978 static bool trans_LDP_v(DisasContext *s, arg_ldstpair *a)
2979 {
2980 uint64_t offset = a->imm << a->sz;
2981 TCGv_i64 clean_addr, dirty_addr;
2982 MemOp mop;
2983
2984 if (!fp_access_check(s)) {
2985 return true;
2986 }
2987
2988 /* LSE2 does not merge FP pairs; leave these as separate operations. */
2989 mop = finalize_memop_asimd(s, a->sz);
2990 op_addr_ldstpair_pre(s, a, &clean_addr, &dirty_addr, offset, false, mop);
2991 do_fp_ld(s, a->rt, clean_addr, mop);
2992 tcg_gen_addi_i64(clean_addr, clean_addr, 1 << a->sz);
2993 do_fp_ld(s, a->rt2, clean_addr, mop);
2994 op_addr_ldstpair_post(s, a, dirty_addr, offset);
2995 return true;
2996 }
2997
trans_STGP(DisasContext * s,arg_ldstpair * a)2998 static bool trans_STGP(DisasContext *s, arg_ldstpair *a)
2999 {
3000 TCGv_i64 clean_addr, dirty_addr, tcg_rt, tcg_rt2;
3001 uint64_t offset = a->imm << LOG2_TAG_GRANULE;
3002 MemOp mop;
3003 TCGv_i128 tmp;
3004
3005 /* STGP only comes in one size. */
3006 tcg_debug_assert(a->sz == MO_64);
3007
3008 if (!dc_isar_feature(aa64_mte_insn_reg, s)) {
3009 return false;
3010 }
3011
3012 if (a->rn == 31) {
3013 gen_check_sp_alignment(s);
3014 }
3015
3016 dirty_addr = read_cpu_reg_sp(s, a->rn, 1);
3017 if (!a->p) {
3018 tcg_gen_addi_i64(dirty_addr, dirty_addr, offset);
3019 }
3020
3021 clean_addr = clean_data_tbi(s, dirty_addr);
3022 tcg_rt = cpu_reg(s, a->rt);
3023 tcg_rt2 = cpu_reg(s, a->rt2);
3024
3025 /*
3026 * STGP is defined as two 8-byte memory operations, aligned to TAG_GRANULE,
3027 * and one tag operation. We implement it as one single aligned 16-byte
3028 * memory operation for convenience. Note that the alignment ensures
3029 * MO_ATOM_IFALIGN_PAIR produces 8-byte atomicity for the memory store.
3030 */
3031 mop = finalize_memop_atom(s, MO_128 | MO_ALIGN, MO_ATOM_IFALIGN_PAIR);
3032
3033 tmp = tcg_temp_new_i128();
3034 if (s->be_data == MO_LE) {
3035 tcg_gen_concat_i64_i128(tmp, tcg_rt, tcg_rt2);
3036 } else {
3037 tcg_gen_concat_i64_i128(tmp, tcg_rt2, tcg_rt);
3038 }
3039 tcg_gen_qemu_st_i128(tmp, clean_addr, get_mem_index(s), mop);
3040
3041 /* Perform the tag store, if tag access enabled. */
3042 if (s->ata[0]) {
3043 if (tb_cflags(s->base.tb) & CF_PARALLEL) {
3044 gen_helper_stg_parallel(tcg_env, dirty_addr, dirty_addr);
3045 } else {
3046 gen_helper_stg(tcg_env, dirty_addr, dirty_addr);
3047 }
3048 }
3049
3050 op_addr_ldstpair_post(s, a, dirty_addr, offset);
3051 return true;
3052 }
3053
op_addr_ldst_imm_pre(DisasContext * s,arg_ldst_imm * a,TCGv_i64 * clean_addr,TCGv_i64 * dirty_addr,uint64_t offset,bool is_store,MemOp mop)3054 static void op_addr_ldst_imm_pre(DisasContext *s, arg_ldst_imm *a,
3055 TCGv_i64 *clean_addr, TCGv_i64 *dirty_addr,
3056 uint64_t offset, bool is_store, MemOp mop)
3057 {
3058 int memidx;
3059
3060 if (a->rn == 31) {
3061 gen_check_sp_alignment(s);
3062 }
3063
3064 *dirty_addr = read_cpu_reg_sp(s, a->rn, 1);
3065 if (!a->p) {
3066 tcg_gen_addi_i64(*dirty_addr, *dirty_addr, offset);
3067 }
3068 memidx = get_a64_user_mem_index(s, a->unpriv);
3069 *clean_addr = gen_mte_check1_mmuidx(s, *dirty_addr, is_store,
3070 a->w || a->rn != 31,
3071 mop, a->unpriv, memidx);
3072 }
3073
op_addr_ldst_imm_post(DisasContext * s,arg_ldst_imm * a,TCGv_i64 dirty_addr,uint64_t offset)3074 static void op_addr_ldst_imm_post(DisasContext *s, arg_ldst_imm *a,
3075 TCGv_i64 dirty_addr, uint64_t offset)
3076 {
3077 if (a->w) {
3078 if (a->p) {
3079 tcg_gen_addi_i64(dirty_addr, dirty_addr, offset);
3080 }
3081 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), dirty_addr);
3082 }
3083 }
3084
trans_STR_i(DisasContext * s,arg_ldst_imm * a)3085 static bool trans_STR_i(DisasContext *s, arg_ldst_imm *a)
3086 {
3087 bool iss_sf, iss_valid = !a->w;
3088 TCGv_i64 clean_addr, dirty_addr, tcg_rt;
3089 int memidx = get_a64_user_mem_index(s, a->unpriv);
3090 MemOp mop = finalize_memop(s, a->sz + a->sign * MO_SIGN);
3091
3092 op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, true, mop);
3093
3094 tcg_rt = cpu_reg(s, a->rt);
3095 iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext);
3096
3097 do_gpr_st_memidx(s, tcg_rt, clean_addr, mop, memidx,
3098 iss_valid, a->rt, iss_sf, false);
3099 op_addr_ldst_imm_post(s, a, dirty_addr, a->imm);
3100 return true;
3101 }
3102
trans_LDR_i(DisasContext * s,arg_ldst_imm * a)3103 static bool trans_LDR_i(DisasContext *s, arg_ldst_imm *a)
3104 {
3105 bool iss_sf, iss_valid = !a->w;
3106 TCGv_i64 clean_addr, dirty_addr, tcg_rt;
3107 int memidx = get_a64_user_mem_index(s, a->unpriv);
3108 MemOp mop = finalize_memop(s, a->sz + a->sign * MO_SIGN);
3109
3110 op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, false, mop);
3111
3112 tcg_rt = cpu_reg(s, a->rt);
3113 iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext);
3114
3115 do_gpr_ld_memidx(s, tcg_rt, clean_addr, mop,
3116 a->ext, memidx, iss_valid, a->rt, iss_sf, false);
3117 op_addr_ldst_imm_post(s, a, dirty_addr, a->imm);
3118 return true;
3119 }
3120
trans_STR_v_i(DisasContext * s,arg_ldst_imm * a)3121 static bool trans_STR_v_i(DisasContext *s, arg_ldst_imm *a)
3122 {
3123 TCGv_i64 clean_addr, dirty_addr;
3124 MemOp mop;
3125
3126 if (!fp_access_check(s)) {
3127 return true;
3128 }
3129 mop = finalize_memop_asimd(s, a->sz);
3130 op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, true, mop);
3131 do_fp_st(s, a->rt, clean_addr, mop);
3132 op_addr_ldst_imm_post(s, a, dirty_addr, a->imm);
3133 return true;
3134 }
3135
trans_LDR_v_i(DisasContext * s,arg_ldst_imm * a)3136 static bool trans_LDR_v_i(DisasContext *s, arg_ldst_imm *a)
3137 {
3138 TCGv_i64 clean_addr, dirty_addr;
3139 MemOp mop;
3140
3141 if (!fp_access_check(s)) {
3142 return true;
3143 }
3144 mop = finalize_memop_asimd(s, a->sz);
3145 op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, false, mop);
3146 do_fp_ld(s, a->rt, clean_addr, mop);
3147 op_addr_ldst_imm_post(s, a, dirty_addr, a->imm);
3148 return true;
3149 }
3150
op_addr_ldst_pre(DisasContext * s,arg_ldst * a,TCGv_i64 * clean_addr,TCGv_i64 * dirty_addr,bool is_store,MemOp memop)3151 static void op_addr_ldst_pre(DisasContext *s, arg_ldst *a,
3152 TCGv_i64 *clean_addr, TCGv_i64 *dirty_addr,
3153 bool is_store, MemOp memop)
3154 {
3155 TCGv_i64 tcg_rm;
3156
3157 if (a->rn == 31) {
3158 gen_check_sp_alignment(s);
3159 }
3160 *dirty_addr = read_cpu_reg_sp(s, a->rn, 1);
3161
3162 tcg_rm = read_cpu_reg(s, a->rm, 1);
3163 ext_and_shift_reg(tcg_rm, tcg_rm, a->opt, a->s ? a->sz : 0);
3164
3165 tcg_gen_add_i64(*dirty_addr, *dirty_addr, tcg_rm);
3166 *clean_addr = gen_mte_check1(s, *dirty_addr, is_store, true, memop);
3167 }
3168
trans_LDR(DisasContext * s,arg_ldst * a)3169 static bool trans_LDR(DisasContext *s, arg_ldst *a)
3170 {
3171 TCGv_i64 clean_addr, dirty_addr, tcg_rt;
3172 bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext);
3173 MemOp memop;
3174
3175 if (extract32(a->opt, 1, 1) == 0) {
3176 return false;
3177 }
3178
3179 memop = finalize_memop(s, a->sz + a->sign * MO_SIGN);
3180 op_addr_ldst_pre(s, a, &clean_addr, &dirty_addr, false, memop);
3181 tcg_rt = cpu_reg(s, a->rt);
3182 do_gpr_ld(s, tcg_rt, clean_addr, memop,
3183 a->ext, true, a->rt, iss_sf, false);
3184 return true;
3185 }
3186
trans_STR(DisasContext * s,arg_ldst * a)3187 static bool trans_STR(DisasContext *s, arg_ldst *a)
3188 {
3189 TCGv_i64 clean_addr, dirty_addr, tcg_rt;
3190 bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext);
3191 MemOp memop;
3192
3193 if (extract32(a->opt, 1, 1) == 0) {
3194 return false;
3195 }
3196
3197 memop = finalize_memop(s, a->sz);
3198 op_addr_ldst_pre(s, a, &clean_addr, &dirty_addr, true, memop);
3199 tcg_rt = cpu_reg(s, a->rt);
3200 do_gpr_st(s, tcg_rt, clean_addr, memop, true, a->rt, iss_sf, false);
3201 return true;
3202 }
3203
trans_LDR_v(DisasContext * s,arg_ldst * a)3204 static bool trans_LDR_v(DisasContext *s, arg_ldst *a)
3205 {
3206 TCGv_i64 clean_addr, dirty_addr;
3207 MemOp memop;
3208
3209 if (extract32(a->opt, 1, 1) == 0) {
3210 return false;
3211 }
3212
3213 if (!fp_access_check(s)) {
3214 return true;
3215 }
3216
3217 memop = finalize_memop_asimd(s, a->sz);
3218 op_addr_ldst_pre(s, a, &clean_addr, &dirty_addr, false, memop);
3219 do_fp_ld(s, a->rt, clean_addr, memop);
3220 return true;
3221 }
3222
trans_STR_v(DisasContext * s,arg_ldst * a)3223 static bool trans_STR_v(DisasContext *s, arg_ldst *a)
3224 {
3225 TCGv_i64 clean_addr, dirty_addr;
3226 MemOp memop;
3227
3228 if (extract32(a->opt, 1, 1) == 0) {
3229 return false;
3230 }
3231
3232 if (!fp_access_check(s)) {
3233 return true;
3234 }
3235
3236 memop = finalize_memop_asimd(s, a->sz);
3237 op_addr_ldst_pre(s, a, &clean_addr, &dirty_addr, true, memop);
3238 do_fp_st(s, a->rt, clean_addr, memop);
3239 return true;
3240 }
3241
3242
do_atomic_ld(DisasContext * s,arg_atomic * a,AtomicThreeOpFn * fn,int sign,bool invert)3243 static bool do_atomic_ld(DisasContext *s, arg_atomic *a, AtomicThreeOpFn *fn,
3244 int sign, bool invert)
3245 {
3246 MemOp mop = a->sz | sign;
3247 TCGv_i64 clean_addr, tcg_rs, tcg_rt;
3248
3249 if (a->rn == 31) {
3250 gen_check_sp_alignment(s);
3251 }
3252 mop = check_atomic_align(s, a->rn, mop);
3253 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn), false,
3254 a->rn != 31, mop);
3255 tcg_rs = read_cpu_reg(s, a->rs, true);
3256 tcg_rt = cpu_reg(s, a->rt);
3257 if (invert) {
3258 tcg_gen_not_i64(tcg_rs, tcg_rs);
3259 }
3260 /*
3261 * The tcg atomic primitives are all full barriers. Therefore we
3262 * can ignore the Acquire and Release bits of this instruction.
3263 */
3264 fn(tcg_rt, clean_addr, tcg_rs, get_mem_index(s), mop);
3265
3266 if (mop & MO_SIGN) {
3267 switch (a->sz) {
3268 case MO_8:
3269 tcg_gen_ext8u_i64(tcg_rt, tcg_rt);
3270 break;
3271 case MO_16:
3272 tcg_gen_ext16u_i64(tcg_rt, tcg_rt);
3273 break;
3274 case MO_32:
3275 tcg_gen_ext32u_i64(tcg_rt, tcg_rt);
3276 break;
3277 case MO_64:
3278 break;
3279 default:
3280 g_assert_not_reached();
3281 }
3282 }
3283 return true;
3284 }
3285
3286 TRANS_FEAT(LDADD, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_add_i64, 0, false)
3287 TRANS_FEAT(LDCLR, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_and_i64, 0, true)
3288 TRANS_FEAT(LDEOR, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_xor_i64, 0, false)
3289 TRANS_FEAT(LDSET, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_or_i64, 0, false)
TRANS_FEAT(LDSMAX,aa64_atomics,do_atomic_ld,a,tcg_gen_atomic_fetch_smax_i64,MO_SIGN,false)3290 TRANS_FEAT(LDSMAX, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_smax_i64, MO_SIGN, false)
3291 TRANS_FEAT(LDSMIN, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_smin_i64, MO_SIGN, false)
3292 TRANS_FEAT(LDUMAX, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_umax_i64, 0, false)
3293 TRANS_FEAT(LDUMIN, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_umin_i64, 0, false)
3294 TRANS_FEAT(SWP, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_xchg_i64, 0, false)
3295
3296 static bool trans_LDAPR(DisasContext *s, arg_LDAPR *a)
3297 {
3298 bool iss_sf = ldst_iss_sf(a->sz, false, false);
3299 TCGv_i64 clean_addr;
3300 MemOp mop;
3301
3302 if (!dc_isar_feature(aa64_atomics, s) ||
3303 !dc_isar_feature(aa64_rcpc_8_3, s)) {
3304 return false;
3305 }
3306 if (a->rn == 31) {
3307 gen_check_sp_alignment(s);
3308 }
3309 mop = check_atomic_align(s, a->rn, a->sz);
3310 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn), false,
3311 a->rn != 31, mop);
3312 /*
3313 * LDAPR* are a special case because they are a simple load, not a
3314 * fetch-and-do-something op.
3315 * The architectural consistency requirements here are weaker than
3316 * full load-acquire (we only need "load-acquire processor consistent"),
3317 * but we choose to implement them as full LDAQ.
3318 */
3319 do_gpr_ld(s, cpu_reg(s, a->rt), clean_addr, mop, false,
3320 true, a->rt, iss_sf, true);
3321 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ);
3322 return true;
3323 }
3324
trans_LDRA(DisasContext * s,arg_LDRA * a)3325 static bool trans_LDRA(DisasContext *s, arg_LDRA *a)
3326 {
3327 TCGv_i64 clean_addr, dirty_addr, tcg_rt;
3328 MemOp memop;
3329
3330 /* Load with pointer authentication */
3331 if (!dc_isar_feature(aa64_pauth, s)) {
3332 return false;
3333 }
3334
3335 if (a->rn == 31) {
3336 gen_check_sp_alignment(s);
3337 }
3338 dirty_addr = read_cpu_reg_sp(s, a->rn, 1);
3339
3340 if (s->pauth_active) {
3341 if (!a->m) {
3342 gen_helper_autda_combined(dirty_addr, tcg_env, dirty_addr,
3343 tcg_constant_i64(0));
3344 } else {
3345 gen_helper_autdb_combined(dirty_addr, tcg_env, dirty_addr,
3346 tcg_constant_i64(0));
3347 }
3348 }
3349
3350 tcg_gen_addi_i64(dirty_addr, dirty_addr, a->imm);
3351
3352 memop = finalize_memop(s, MO_64);
3353
3354 /* Note that "clean" and "dirty" here refer to TBI not PAC. */
3355 clean_addr = gen_mte_check1(s, dirty_addr, false,
3356 a->w || a->rn != 31, memop);
3357
3358 tcg_rt = cpu_reg(s, a->rt);
3359 do_gpr_ld(s, tcg_rt, clean_addr, memop,
3360 /* extend */ false, /* iss_valid */ !a->w,
3361 /* iss_srt */ a->rt, /* iss_sf */ true, /* iss_ar */ false);
3362
3363 if (a->w) {
3364 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), dirty_addr);
3365 }
3366 return true;
3367 }
3368
trans_LDAPR_i(DisasContext * s,arg_ldapr_stlr_i * a)3369 static bool trans_LDAPR_i(DisasContext *s, arg_ldapr_stlr_i *a)
3370 {
3371 TCGv_i64 clean_addr, dirty_addr;
3372 MemOp mop = a->sz | (a->sign ? MO_SIGN : 0);
3373 bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext);
3374
3375 if (!dc_isar_feature(aa64_rcpc_8_4, s)) {
3376 return false;
3377 }
3378
3379 if (a->rn == 31) {
3380 gen_check_sp_alignment(s);
3381 }
3382
3383 mop = check_ordered_align(s, a->rn, a->imm, false, mop);
3384 dirty_addr = read_cpu_reg_sp(s, a->rn, 1);
3385 tcg_gen_addi_i64(dirty_addr, dirty_addr, a->imm);
3386 clean_addr = clean_data_tbi(s, dirty_addr);
3387
3388 /*
3389 * Load-AcquirePC semantics; we implement as the slightly more
3390 * restrictive Load-Acquire.
3391 */
3392 do_gpr_ld(s, cpu_reg(s, a->rt), clean_addr, mop, a->ext, true,
3393 a->rt, iss_sf, true);
3394 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ);
3395 return true;
3396 }
3397
trans_STLR_i(DisasContext * s,arg_ldapr_stlr_i * a)3398 static bool trans_STLR_i(DisasContext *s, arg_ldapr_stlr_i *a)
3399 {
3400 TCGv_i64 clean_addr, dirty_addr;
3401 MemOp mop = a->sz;
3402 bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext);
3403
3404 if (!dc_isar_feature(aa64_rcpc_8_4, s)) {
3405 return false;
3406 }
3407
3408 /* TODO: ARMv8.4-LSE SCTLR.nAA */
3409
3410 if (a->rn == 31) {
3411 gen_check_sp_alignment(s);
3412 }
3413
3414 mop = check_ordered_align(s, a->rn, a->imm, true, mop);
3415 dirty_addr = read_cpu_reg_sp(s, a->rn, 1);
3416 tcg_gen_addi_i64(dirty_addr, dirty_addr, a->imm);
3417 clean_addr = clean_data_tbi(s, dirty_addr);
3418
3419 /* Store-Release semantics */
3420 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_STRL);
3421 do_gpr_st(s, cpu_reg(s, a->rt), clean_addr, mop, true, a->rt, iss_sf, true);
3422 return true;
3423 }
3424
trans_LD_mult(DisasContext * s,arg_ldst_mult * a)3425 static bool trans_LD_mult(DisasContext *s, arg_ldst_mult *a)
3426 {
3427 TCGv_i64 clean_addr, tcg_rn, tcg_ebytes;
3428 MemOp endian, align, mop;
3429
3430 int total; /* total bytes */
3431 int elements; /* elements per vector */
3432 int r;
3433 int size = a->sz;
3434
3435 if (!a->p && a->rm != 0) {
3436 /* For non-postindexed accesses the Rm field must be 0 */
3437 return false;
3438 }
3439 if (size == 3 && !a->q && a->selem != 1) {
3440 return false;
3441 }
3442 if (!fp_access_check(s)) {
3443 return true;
3444 }
3445
3446 if (a->rn == 31) {
3447 gen_check_sp_alignment(s);
3448 }
3449
3450 /* For our purposes, bytes are always little-endian. */
3451 endian = s->be_data;
3452 if (size == 0) {
3453 endian = MO_LE;
3454 }
3455
3456 total = a->rpt * a->selem * (a->q ? 16 : 8);
3457 tcg_rn = cpu_reg_sp(s, a->rn);
3458
3459 /*
3460 * Issue the MTE check vs the logical repeat count, before we
3461 * promote consecutive little-endian elements below.
3462 */
3463 clean_addr = gen_mte_checkN(s, tcg_rn, false, a->p || a->rn != 31, total,
3464 finalize_memop_asimd(s, size));
3465
3466 /*
3467 * Consecutive little-endian elements from a single register
3468 * can be promoted to a larger little-endian operation.
3469 */
3470 align = MO_ALIGN;
3471 if (a->selem == 1 && endian == MO_LE) {
3472 align = pow2_align(size);
3473 size = 3;
3474 }
3475 if (!s->align_mem) {
3476 align = 0;
3477 }
3478 mop = endian | size | align;
3479
3480 elements = (a->q ? 16 : 8) >> size;
3481 tcg_ebytes = tcg_constant_i64(1 << size);
3482 for (r = 0; r < a->rpt; r++) {
3483 int e;
3484 for (e = 0; e < elements; e++) {
3485 int xs;
3486 for (xs = 0; xs < a->selem; xs++) {
3487 int tt = (a->rt + r + xs) % 32;
3488 do_vec_ld(s, tt, e, clean_addr, mop);
3489 tcg_gen_add_i64(clean_addr, clean_addr, tcg_ebytes);
3490 }
3491 }
3492 }
3493
3494 /*
3495 * For non-quad operations, setting a slice of the low 64 bits of
3496 * the register clears the high 64 bits (in the ARM ARM pseudocode
3497 * this is implicit in the fact that 'rval' is a 64 bit wide
3498 * variable). For quad operations, we might still need to zero
3499 * the high bits of SVE.
3500 */
3501 for (r = 0; r < a->rpt * a->selem; r++) {
3502 int tt = (a->rt + r) % 32;
3503 clear_vec_high(s, a->q, tt);
3504 }
3505
3506 if (a->p) {
3507 if (a->rm == 31) {
3508 tcg_gen_addi_i64(tcg_rn, tcg_rn, total);
3509 } else {
3510 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm));
3511 }
3512 }
3513 return true;
3514 }
3515
trans_ST_mult(DisasContext * s,arg_ldst_mult * a)3516 static bool trans_ST_mult(DisasContext *s, arg_ldst_mult *a)
3517 {
3518 TCGv_i64 clean_addr, tcg_rn, tcg_ebytes;
3519 MemOp endian, align, mop;
3520
3521 int total; /* total bytes */
3522 int elements; /* elements per vector */
3523 int r;
3524 int size = a->sz;
3525
3526 if (!a->p && a->rm != 0) {
3527 /* For non-postindexed accesses the Rm field must be 0 */
3528 return false;
3529 }
3530 if (size == 3 && !a->q && a->selem != 1) {
3531 return false;
3532 }
3533 if (!fp_access_check(s)) {
3534 return true;
3535 }
3536
3537 if (a->rn == 31) {
3538 gen_check_sp_alignment(s);
3539 }
3540
3541 /* For our purposes, bytes are always little-endian. */
3542 endian = s->be_data;
3543 if (size == 0) {
3544 endian = MO_LE;
3545 }
3546
3547 total = a->rpt * a->selem * (a->q ? 16 : 8);
3548 tcg_rn = cpu_reg_sp(s, a->rn);
3549
3550 /*
3551 * Issue the MTE check vs the logical repeat count, before we
3552 * promote consecutive little-endian elements below.
3553 */
3554 clean_addr = gen_mte_checkN(s, tcg_rn, true, a->p || a->rn != 31, total,
3555 finalize_memop_asimd(s, size));
3556
3557 /*
3558 * Consecutive little-endian elements from a single register
3559 * can be promoted to a larger little-endian operation.
3560 */
3561 align = MO_ALIGN;
3562 if (a->selem == 1 && endian == MO_LE) {
3563 align = pow2_align(size);
3564 size = 3;
3565 }
3566 if (!s->align_mem) {
3567 align = 0;
3568 }
3569 mop = endian | size | align;
3570
3571 elements = (a->q ? 16 : 8) >> size;
3572 tcg_ebytes = tcg_constant_i64(1 << size);
3573 for (r = 0; r < a->rpt; r++) {
3574 int e;
3575 for (e = 0; e < elements; e++) {
3576 int xs;
3577 for (xs = 0; xs < a->selem; xs++) {
3578 int tt = (a->rt + r + xs) % 32;
3579 do_vec_st(s, tt, e, clean_addr, mop);
3580 tcg_gen_add_i64(clean_addr, clean_addr, tcg_ebytes);
3581 }
3582 }
3583 }
3584
3585 if (a->p) {
3586 if (a->rm == 31) {
3587 tcg_gen_addi_i64(tcg_rn, tcg_rn, total);
3588 } else {
3589 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm));
3590 }
3591 }
3592 return true;
3593 }
3594
trans_ST_single(DisasContext * s,arg_ldst_single * a)3595 static bool trans_ST_single(DisasContext *s, arg_ldst_single *a)
3596 {
3597 int xs, total, rt;
3598 TCGv_i64 clean_addr, tcg_rn, tcg_ebytes;
3599 MemOp mop;
3600
3601 if (!a->p && a->rm != 0) {
3602 return false;
3603 }
3604 if (!fp_access_check(s)) {
3605 return true;
3606 }
3607
3608 if (a->rn == 31) {
3609 gen_check_sp_alignment(s);
3610 }
3611
3612 total = a->selem << a->scale;
3613 tcg_rn = cpu_reg_sp(s, a->rn);
3614
3615 mop = finalize_memop_asimd(s, a->scale);
3616 clean_addr = gen_mte_checkN(s, tcg_rn, true, a->p || a->rn != 31,
3617 total, mop);
3618
3619 tcg_ebytes = tcg_constant_i64(1 << a->scale);
3620 for (xs = 0, rt = a->rt; xs < a->selem; xs++, rt = (rt + 1) % 32) {
3621 do_vec_st(s, rt, a->index, clean_addr, mop);
3622 tcg_gen_add_i64(clean_addr, clean_addr, tcg_ebytes);
3623 }
3624
3625 if (a->p) {
3626 if (a->rm == 31) {
3627 tcg_gen_addi_i64(tcg_rn, tcg_rn, total);
3628 } else {
3629 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm));
3630 }
3631 }
3632 return true;
3633 }
3634
trans_LD_single(DisasContext * s,arg_ldst_single * a)3635 static bool trans_LD_single(DisasContext *s, arg_ldst_single *a)
3636 {
3637 int xs, total, rt;
3638 TCGv_i64 clean_addr, tcg_rn, tcg_ebytes;
3639 MemOp mop;
3640
3641 if (!a->p && a->rm != 0) {
3642 return false;
3643 }
3644 if (!fp_access_check(s)) {
3645 return true;
3646 }
3647
3648 if (a->rn == 31) {
3649 gen_check_sp_alignment(s);
3650 }
3651
3652 total = a->selem << a->scale;
3653 tcg_rn = cpu_reg_sp(s, a->rn);
3654
3655 mop = finalize_memop_asimd(s, a->scale);
3656 clean_addr = gen_mte_checkN(s, tcg_rn, false, a->p || a->rn != 31,
3657 total, mop);
3658
3659 tcg_ebytes = tcg_constant_i64(1 << a->scale);
3660 for (xs = 0, rt = a->rt; xs < a->selem; xs++, rt = (rt + 1) % 32) {
3661 do_vec_ld(s, rt, a->index, clean_addr, mop);
3662 tcg_gen_add_i64(clean_addr, clean_addr, tcg_ebytes);
3663 }
3664
3665 if (a->p) {
3666 if (a->rm == 31) {
3667 tcg_gen_addi_i64(tcg_rn, tcg_rn, total);
3668 } else {
3669 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm));
3670 }
3671 }
3672 return true;
3673 }
3674
trans_LD_single_repl(DisasContext * s,arg_LD_single_repl * a)3675 static bool trans_LD_single_repl(DisasContext *s, arg_LD_single_repl *a)
3676 {
3677 int xs, total, rt;
3678 TCGv_i64 clean_addr, tcg_rn, tcg_ebytes;
3679 MemOp mop;
3680
3681 if (!a->p && a->rm != 0) {
3682 return false;
3683 }
3684 if (!fp_access_check(s)) {
3685 return true;
3686 }
3687
3688 if (a->rn == 31) {
3689 gen_check_sp_alignment(s);
3690 }
3691
3692 total = a->selem << a->scale;
3693 tcg_rn = cpu_reg_sp(s, a->rn);
3694
3695 mop = finalize_memop_asimd(s, a->scale);
3696 clean_addr = gen_mte_checkN(s, tcg_rn, false, a->p || a->rn != 31,
3697 total, mop);
3698
3699 tcg_ebytes = tcg_constant_i64(1 << a->scale);
3700 for (xs = 0, rt = a->rt; xs < a->selem; xs++, rt = (rt + 1) % 32) {
3701 /* Load and replicate to all elements */
3702 TCGv_i64 tcg_tmp = tcg_temp_new_i64();
3703
3704 tcg_gen_qemu_ld_i64(tcg_tmp, clean_addr, get_mem_index(s), mop);
3705 tcg_gen_gvec_dup_i64(a->scale, vec_full_reg_offset(s, rt),
3706 (a->q + 1) * 8, vec_full_reg_size(s), tcg_tmp);
3707 tcg_gen_add_i64(clean_addr, clean_addr, tcg_ebytes);
3708 }
3709
3710 if (a->p) {
3711 if (a->rm == 31) {
3712 tcg_gen_addi_i64(tcg_rn, tcg_rn, total);
3713 } else {
3714 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm));
3715 }
3716 }
3717 return true;
3718 }
3719
trans_STZGM(DisasContext * s,arg_ldst_tag * a)3720 static bool trans_STZGM(DisasContext *s, arg_ldst_tag *a)
3721 {
3722 TCGv_i64 addr, clean_addr, tcg_rt;
3723 int size = 4 << s->dcz_blocksize;
3724
3725 if (!dc_isar_feature(aa64_mte, s)) {
3726 return false;
3727 }
3728 if (s->current_el == 0) {
3729 return false;
3730 }
3731
3732 if (a->rn == 31) {
3733 gen_check_sp_alignment(s);
3734 }
3735
3736 addr = read_cpu_reg_sp(s, a->rn, true);
3737 tcg_gen_addi_i64(addr, addr, a->imm);
3738 tcg_rt = cpu_reg(s, a->rt);
3739
3740 if (s->ata[0]) {
3741 gen_helper_stzgm_tags(tcg_env, addr, tcg_rt);
3742 }
3743 /*
3744 * The non-tags portion of STZGM is mostly like DC_ZVA,
3745 * except the alignment happens before the access.
3746 */
3747 clean_addr = clean_data_tbi(s, addr);
3748 tcg_gen_andi_i64(clean_addr, clean_addr, -size);
3749 gen_helper_dc_zva(tcg_env, clean_addr);
3750 return true;
3751 }
3752
trans_STGM(DisasContext * s,arg_ldst_tag * a)3753 static bool trans_STGM(DisasContext *s, arg_ldst_tag *a)
3754 {
3755 TCGv_i64 addr, clean_addr, tcg_rt;
3756
3757 if (!dc_isar_feature(aa64_mte, s)) {
3758 return false;
3759 }
3760 if (s->current_el == 0) {
3761 return false;
3762 }
3763
3764 if (a->rn == 31) {
3765 gen_check_sp_alignment(s);
3766 }
3767
3768 addr = read_cpu_reg_sp(s, a->rn, true);
3769 tcg_gen_addi_i64(addr, addr, a->imm);
3770 tcg_rt = cpu_reg(s, a->rt);
3771
3772 if (s->ata[0]) {
3773 gen_helper_stgm(tcg_env, addr, tcg_rt);
3774 } else {
3775 MMUAccessType acc = MMU_DATA_STORE;
3776 int size = 4 << s->gm_blocksize;
3777
3778 clean_addr = clean_data_tbi(s, addr);
3779 tcg_gen_andi_i64(clean_addr, clean_addr, -size);
3780 gen_probe_access(s, clean_addr, acc, size);
3781 }
3782 return true;
3783 }
3784
trans_LDGM(DisasContext * s,arg_ldst_tag * a)3785 static bool trans_LDGM(DisasContext *s, arg_ldst_tag *a)
3786 {
3787 TCGv_i64 addr, clean_addr, tcg_rt;
3788
3789 if (!dc_isar_feature(aa64_mte, s)) {
3790 return false;
3791 }
3792 if (s->current_el == 0) {
3793 return false;
3794 }
3795
3796 if (a->rn == 31) {
3797 gen_check_sp_alignment(s);
3798 }
3799
3800 addr = read_cpu_reg_sp(s, a->rn, true);
3801 tcg_gen_addi_i64(addr, addr, a->imm);
3802 tcg_rt = cpu_reg(s, a->rt);
3803
3804 if (s->ata[0]) {
3805 gen_helper_ldgm(tcg_rt, tcg_env, addr);
3806 } else {
3807 MMUAccessType acc = MMU_DATA_LOAD;
3808 int size = 4 << s->gm_blocksize;
3809
3810 clean_addr = clean_data_tbi(s, addr);
3811 tcg_gen_andi_i64(clean_addr, clean_addr, -size);
3812 gen_probe_access(s, clean_addr, acc, size);
3813 /* The result tags are zeros. */
3814 tcg_gen_movi_i64(tcg_rt, 0);
3815 }
3816 return true;
3817 }
3818
trans_LDG(DisasContext * s,arg_ldst_tag * a)3819 static bool trans_LDG(DisasContext *s, arg_ldst_tag *a)
3820 {
3821 TCGv_i64 addr, clean_addr, tcg_rt;
3822
3823 if (!dc_isar_feature(aa64_mte_insn_reg, s)) {
3824 return false;
3825 }
3826
3827 if (a->rn == 31) {
3828 gen_check_sp_alignment(s);
3829 }
3830
3831 addr = read_cpu_reg_sp(s, a->rn, true);
3832 if (!a->p) {
3833 /* pre-index or signed offset */
3834 tcg_gen_addi_i64(addr, addr, a->imm);
3835 }
3836
3837 tcg_gen_andi_i64(addr, addr, -TAG_GRANULE);
3838 tcg_rt = cpu_reg(s, a->rt);
3839 if (s->ata[0]) {
3840 gen_helper_ldg(tcg_rt, tcg_env, addr, tcg_rt);
3841 } else {
3842 /*
3843 * Tag access disabled: we must check for aborts on the load
3844 * load from [rn+offset], and then insert a 0 tag into rt.
3845 */
3846 clean_addr = clean_data_tbi(s, addr);
3847 gen_probe_access(s, clean_addr, MMU_DATA_LOAD, MO_8);
3848 gen_address_with_allocation_tag0(tcg_rt, tcg_rt);
3849 }
3850
3851 if (a->w) {
3852 /* pre-index or post-index */
3853 if (a->p) {
3854 /* post-index */
3855 tcg_gen_addi_i64(addr, addr, a->imm);
3856 }
3857 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), addr);
3858 }
3859 return true;
3860 }
3861
do_STG(DisasContext * s,arg_ldst_tag * a,bool is_zero,bool is_pair)3862 static bool do_STG(DisasContext *s, arg_ldst_tag *a, bool is_zero, bool is_pair)
3863 {
3864 TCGv_i64 addr, tcg_rt;
3865
3866 if (a->rn == 31) {
3867 gen_check_sp_alignment(s);
3868 }
3869
3870 addr = read_cpu_reg_sp(s, a->rn, true);
3871 if (!a->p) {
3872 /* pre-index or signed offset */
3873 tcg_gen_addi_i64(addr, addr, a->imm);
3874 }
3875 tcg_rt = cpu_reg_sp(s, a->rt);
3876 if (!s->ata[0]) {
3877 /*
3878 * For STG and ST2G, we need to check alignment and probe memory.
3879 * TODO: For STZG and STZ2G, we could rely on the stores below,
3880 * at least for system mode; user-only won't enforce alignment.
3881 */
3882 if (is_pair) {
3883 gen_helper_st2g_stub(tcg_env, addr);
3884 } else {
3885 gen_helper_stg_stub(tcg_env, addr);
3886 }
3887 } else if (tb_cflags(s->base.tb) & CF_PARALLEL) {
3888 if (is_pair) {
3889 gen_helper_st2g_parallel(tcg_env, addr, tcg_rt);
3890 } else {
3891 gen_helper_stg_parallel(tcg_env, addr, tcg_rt);
3892 }
3893 } else {
3894 if (is_pair) {
3895 gen_helper_st2g(tcg_env, addr, tcg_rt);
3896 } else {
3897 gen_helper_stg(tcg_env, addr, tcg_rt);
3898 }
3899 }
3900
3901 if (is_zero) {
3902 TCGv_i64 clean_addr = clean_data_tbi(s, addr);
3903 TCGv_i64 zero64 = tcg_constant_i64(0);
3904 TCGv_i128 zero128 = tcg_temp_new_i128();
3905 int mem_index = get_mem_index(s);
3906 MemOp mop = finalize_memop(s, MO_128 | MO_ALIGN);
3907
3908 tcg_gen_concat_i64_i128(zero128, zero64, zero64);
3909
3910 /* This is 1 or 2 atomic 16-byte operations. */
3911 tcg_gen_qemu_st_i128(zero128, clean_addr, mem_index, mop);
3912 if (is_pair) {
3913 tcg_gen_addi_i64(clean_addr, clean_addr, 16);
3914 tcg_gen_qemu_st_i128(zero128, clean_addr, mem_index, mop);
3915 }
3916 }
3917
3918 if (a->w) {
3919 /* pre-index or post-index */
3920 if (a->p) {
3921 /* post-index */
3922 tcg_gen_addi_i64(addr, addr, a->imm);
3923 }
3924 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), addr);
3925 }
3926 return true;
3927 }
3928
3929 TRANS_FEAT(STG, aa64_mte_insn_reg, do_STG, a, false, false)
3930 TRANS_FEAT(STZG, aa64_mte_insn_reg, do_STG, a, true, false)
3931 TRANS_FEAT(ST2G, aa64_mte_insn_reg, do_STG, a, false, true)
3932 TRANS_FEAT(STZ2G, aa64_mte_insn_reg, do_STG, a, true, true)
3933
3934 typedef void SetFn(TCGv_env, TCGv_i32, TCGv_i32);
3935
do_SET(DisasContext * s,arg_set * a,bool is_epilogue,bool is_setg,SetFn fn)3936 static bool do_SET(DisasContext *s, arg_set *a, bool is_epilogue,
3937 bool is_setg, SetFn fn)
3938 {
3939 int memidx;
3940 uint32_t syndrome, desc = 0;
3941
3942 if (is_setg && !dc_isar_feature(aa64_mte, s)) {
3943 return false;
3944 }
3945
3946 /*
3947 * UNPREDICTABLE cases: we choose to UNDEF, which allows
3948 * us to pull this check before the CheckMOPSEnabled() test
3949 * (which we do in the helper function)
3950 */
3951 if (a->rs == a->rn || a->rs == a->rd || a->rn == a->rd ||
3952 a->rd == 31 || a->rn == 31) {
3953 return false;
3954 }
3955
3956 memidx = get_a64_user_mem_index(s, a->unpriv);
3957
3958 /*
3959 * We pass option_a == true, matching our implementation;
3960 * we pass wrong_option == false: helper function may set that bit.
3961 */
3962 syndrome = syn_mop(true, is_setg, (a->nontemp << 1) | a->unpriv,
3963 is_epilogue, false, true, a->rd, a->rs, a->rn);
3964
3965 if (is_setg ? s->ata[a->unpriv] : s->mte_active[a->unpriv]) {
3966 /* We may need to do MTE tag checking, so assemble the descriptor */
3967 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid);
3968 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma);
3969 desc = FIELD_DP32(desc, MTEDESC, WRITE, true);
3970 /* SIZEM1 and ALIGN we leave 0 (byte write) */
3971 }
3972 /* The helper function always needs the memidx even with MTE disabled */
3973 desc = FIELD_DP32(desc, MTEDESC, MIDX, memidx);
3974
3975 /*
3976 * The helper needs the register numbers, but since they're in
3977 * the syndrome anyway, we let it extract them from there rather
3978 * than passing in an extra three integer arguments.
3979 */
3980 fn(tcg_env, tcg_constant_i32(syndrome), tcg_constant_i32(desc));
3981 return true;
3982 }
3983
3984 TRANS_FEAT(SETP, aa64_mops, do_SET, a, false, false, gen_helper_setp)
3985 TRANS_FEAT(SETM, aa64_mops, do_SET, a, false, false, gen_helper_setm)
3986 TRANS_FEAT(SETE, aa64_mops, do_SET, a, true, false, gen_helper_sete)
3987 TRANS_FEAT(SETGP, aa64_mops, do_SET, a, false, true, gen_helper_setgp)
3988 TRANS_FEAT(SETGM, aa64_mops, do_SET, a, false, true, gen_helper_setgm)
3989 TRANS_FEAT(SETGE, aa64_mops, do_SET, a, true, true, gen_helper_setge)
3990
3991 typedef void CpyFn(TCGv_env, TCGv_i32, TCGv_i32, TCGv_i32);
3992
do_CPY(DisasContext * s,arg_cpy * a,bool is_epilogue,CpyFn fn)3993 static bool do_CPY(DisasContext *s, arg_cpy *a, bool is_epilogue, CpyFn fn)
3994 {
3995 int rmemidx, wmemidx;
3996 uint32_t syndrome, rdesc = 0, wdesc = 0;
3997 bool wunpriv = extract32(a->options, 0, 1);
3998 bool runpriv = extract32(a->options, 1, 1);
3999
4000 /*
4001 * UNPREDICTABLE cases: we choose to UNDEF, which allows
4002 * us to pull this check before the CheckMOPSEnabled() test
4003 * (which we do in the helper function)
4004 */
4005 if (a->rs == a->rn || a->rs == a->rd || a->rn == a->rd ||
4006 a->rd == 31 || a->rs == 31 || a->rn == 31) {
4007 return false;
4008 }
4009
4010 rmemidx = get_a64_user_mem_index(s, runpriv);
4011 wmemidx = get_a64_user_mem_index(s, wunpriv);
4012
4013 /*
4014 * We pass option_a == true, matching our implementation;
4015 * we pass wrong_option == false: helper function may set that bit.
4016 */
4017 syndrome = syn_mop(false, false, a->options, is_epilogue,
4018 false, true, a->rd, a->rs, a->rn);
4019
4020 /* If we need to do MTE tag checking, assemble the descriptors */
4021 if (s->mte_active[runpriv]) {
4022 rdesc = FIELD_DP32(rdesc, MTEDESC, TBI, s->tbid);
4023 rdesc = FIELD_DP32(rdesc, MTEDESC, TCMA, s->tcma);
4024 }
4025 if (s->mte_active[wunpriv]) {
4026 wdesc = FIELD_DP32(wdesc, MTEDESC, TBI, s->tbid);
4027 wdesc = FIELD_DP32(wdesc, MTEDESC, TCMA, s->tcma);
4028 wdesc = FIELD_DP32(wdesc, MTEDESC, WRITE, true);
4029 }
4030 /* The helper function needs these parts of the descriptor regardless */
4031 rdesc = FIELD_DP32(rdesc, MTEDESC, MIDX, rmemidx);
4032 wdesc = FIELD_DP32(wdesc, MTEDESC, MIDX, wmemidx);
4033
4034 /*
4035 * The helper needs the register numbers, but since they're in
4036 * the syndrome anyway, we let it extract them from there rather
4037 * than passing in an extra three integer arguments.
4038 */
4039 fn(tcg_env, tcg_constant_i32(syndrome), tcg_constant_i32(wdesc),
4040 tcg_constant_i32(rdesc));
4041 return true;
4042 }
4043
4044 TRANS_FEAT(CPYP, aa64_mops, do_CPY, a, false, gen_helper_cpyp)
4045 TRANS_FEAT(CPYM, aa64_mops, do_CPY, a, false, gen_helper_cpym)
4046 TRANS_FEAT(CPYE, aa64_mops, do_CPY, a, true, gen_helper_cpye)
4047 TRANS_FEAT(CPYFP, aa64_mops, do_CPY, a, false, gen_helper_cpyfp)
4048 TRANS_FEAT(CPYFM, aa64_mops, do_CPY, a, false, gen_helper_cpyfm)
4049 TRANS_FEAT(CPYFE, aa64_mops, do_CPY, a, true, gen_helper_cpyfe)
4050
4051 typedef void ArithTwoOp(TCGv_i64, TCGv_i64, TCGv_i64);
4052
gen_rri(DisasContext * s,arg_rri_sf * a,bool rd_sp,bool rn_sp,ArithTwoOp * fn)4053 static bool gen_rri(DisasContext *s, arg_rri_sf *a,
4054 bool rd_sp, bool rn_sp, ArithTwoOp *fn)
4055 {
4056 TCGv_i64 tcg_rn = rn_sp ? cpu_reg_sp(s, a->rn) : cpu_reg(s, a->rn);
4057 TCGv_i64 tcg_rd = rd_sp ? cpu_reg_sp(s, a->rd) : cpu_reg(s, a->rd);
4058 TCGv_i64 tcg_imm = tcg_constant_i64(a->imm);
4059
4060 fn(tcg_rd, tcg_rn, tcg_imm);
4061 if (!a->sf) {
4062 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
4063 }
4064 return true;
4065 }
4066
4067 /*
4068 * PC-rel. addressing
4069 */
4070
trans_ADR(DisasContext * s,arg_ri * a)4071 static bool trans_ADR(DisasContext *s, arg_ri *a)
4072 {
4073 gen_pc_plus_diff(s, cpu_reg(s, a->rd), a->imm);
4074 return true;
4075 }
4076
trans_ADRP(DisasContext * s,arg_ri * a)4077 static bool trans_ADRP(DisasContext *s, arg_ri *a)
4078 {
4079 int64_t offset = (int64_t)a->imm << 12;
4080
4081 /* The page offset is ok for CF_PCREL. */
4082 offset -= s->pc_curr & 0xfff;
4083 gen_pc_plus_diff(s, cpu_reg(s, a->rd), offset);
4084 return true;
4085 }
4086
4087 /*
4088 * Add/subtract (immediate)
4089 */
4090 TRANS(ADD_i, gen_rri, a, 1, 1, tcg_gen_add_i64)
4091 TRANS(SUB_i, gen_rri, a, 1, 1, tcg_gen_sub_i64)
4092 TRANS(ADDS_i, gen_rri, a, 0, 1, a->sf ? gen_add64_CC : gen_add32_CC)
4093 TRANS(SUBS_i, gen_rri, a, 0, 1, a->sf ? gen_sub64_CC : gen_sub32_CC)
4094
4095 /*
4096 * Add/subtract (immediate, with tags)
4097 */
4098
gen_add_sub_imm_with_tags(DisasContext * s,arg_rri_tag * a,bool sub_op)4099 static bool gen_add_sub_imm_with_tags(DisasContext *s, arg_rri_tag *a,
4100 bool sub_op)
4101 {
4102 TCGv_i64 tcg_rn, tcg_rd;
4103 int imm;
4104
4105 imm = a->uimm6 << LOG2_TAG_GRANULE;
4106 if (sub_op) {
4107 imm = -imm;
4108 }
4109
4110 tcg_rn = cpu_reg_sp(s, a->rn);
4111 tcg_rd = cpu_reg_sp(s, a->rd);
4112
4113 if (s->ata[0]) {
4114 gen_helper_addsubg(tcg_rd, tcg_env, tcg_rn,
4115 tcg_constant_i32(imm),
4116 tcg_constant_i32(a->uimm4));
4117 } else {
4118 tcg_gen_addi_i64(tcg_rd, tcg_rn, imm);
4119 gen_address_with_allocation_tag0(tcg_rd, tcg_rd);
4120 }
4121 return true;
4122 }
4123
TRANS_FEAT(ADDG_i,aa64_mte_insn_reg,gen_add_sub_imm_with_tags,a,false)4124 TRANS_FEAT(ADDG_i, aa64_mte_insn_reg, gen_add_sub_imm_with_tags, a, false)
4125 TRANS_FEAT(SUBG_i, aa64_mte_insn_reg, gen_add_sub_imm_with_tags, a, true)
4126
4127 /* The input should be a value in the bottom e bits (with higher
4128 * bits zero); returns that value replicated into every element
4129 * of size e in a 64 bit integer.
4130 */
4131 static uint64_t bitfield_replicate(uint64_t mask, unsigned int e)
4132 {
4133 assert(e != 0);
4134 while (e < 64) {
4135 mask |= mask << e;
4136 e *= 2;
4137 }
4138 return mask;
4139 }
4140
4141 /*
4142 * Logical (immediate)
4143 */
4144
4145 /*
4146 * Simplified variant of pseudocode DecodeBitMasks() for the case where we
4147 * only require the wmask. Returns false if the imms/immr/immn are a reserved
4148 * value (ie should cause a guest UNDEF exception), and true if they are
4149 * valid, in which case the decoded bit pattern is written to result.
4150 */
logic_imm_decode_wmask(uint64_t * result,unsigned int immn,unsigned int imms,unsigned int immr)4151 bool logic_imm_decode_wmask(uint64_t *result, unsigned int immn,
4152 unsigned int imms, unsigned int immr)
4153 {
4154 uint64_t mask;
4155 unsigned e, levels, s, r;
4156 int len;
4157
4158 assert(immn < 2 && imms < 64 && immr < 64);
4159
4160 /* The bit patterns we create here are 64 bit patterns which
4161 * are vectors of identical elements of size e = 2, 4, 8, 16, 32 or
4162 * 64 bits each. Each element contains the same value: a run
4163 * of between 1 and e-1 non-zero bits, rotated within the
4164 * element by between 0 and e-1 bits.
4165 *
4166 * The element size and run length are encoded into immn (1 bit)
4167 * and imms (6 bits) as follows:
4168 * 64 bit elements: immn = 1, imms = <length of run - 1>
4169 * 32 bit elements: immn = 0, imms = 0 : <length of run - 1>
4170 * 16 bit elements: immn = 0, imms = 10 : <length of run - 1>
4171 * 8 bit elements: immn = 0, imms = 110 : <length of run - 1>
4172 * 4 bit elements: immn = 0, imms = 1110 : <length of run - 1>
4173 * 2 bit elements: immn = 0, imms = 11110 : <length of run - 1>
4174 * Notice that immn = 0, imms = 11111x is the only combination
4175 * not covered by one of the above options; this is reserved.
4176 * Further, <length of run - 1> all-ones is a reserved pattern.
4177 *
4178 * In all cases the rotation is by immr % e (and immr is 6 bits).
4179 */
4180
4181 /* First determine the element size */
4182 len = 31 - clz32((immn << 6) | (~imms & 0x3f));
4183 if (len < 1) {
4184 /* This is the immn == 0, imms == 0x11111x case */
4185 return false;
4186 }
4187 e = 1 << len;
4188
4189 levels = e - 1;
4190 s = imms & levels;
4191 r = immr & levels;
4192
4193 if (s == levels) {
4194 /* <length of run - 1> mustn't be all-ones. */
4195 return false;
4196 }
4197
4198 /* Create the value of one element: s+1 set bits rotated
4199 * by r within the element (which is e bits wide)...
4200 */
4201 mask = MAKE_64BIT_MASK(0, s + 1);
4202 if (r) {
4203 mask = (mask >> r) | (mask << (e - r));
4204 mask &= MAKE_64BIT_MASK(0, e);
4205 }
4206 /* ...then replicate the element over the whole 64 bit value */
4207 mask = bitfield_replicate(mask, e);
4208 *result = mask;
4209 return true;
4210 }
4211
gen_rri_log(DisasContext * s,arg_rri_log * a,bool set_cc,void (* fn)(TCGv_i64,TCGv_i64,int64_t))4212 static bool gen_rri_log(DisasContext *s, arg_rri_log *a, bool set_cc,
4213 void (*fn)(TCGv_i64, TCGv_i64, int64_t))
4214 {
4215 TCGv_i64 tcg_rd, tcg_rn;
4216 uint64_t imm;
4217
4218 /* Some immediate field values are reserved. */
4219 if (!logic_imm_decode_wmask(&imm, extract32(a->dbm, 12, 1),
4220 extract32(a->dbm, 0, 6),
4221 extract32(a->dbm, 6, 6))) {
4222 return false;
4223 }
4224 if (!a->sf) {
4225 imm &= 0xffffffffull;
4226 }
4227
4228 tcg_rd = set_cc ? cpu_reg(s, a->rd) : cpu_reg_sp(s, a->rd);
4229 tcg_rn = cpu_reg(s, a->rn);
4230
4231 fn(tcg_rd, tcg_rn, imm);
4232 if (set_cc) {
4233 gen_logic_CC(a->sf, tcg_rd);
4234 }
4235 if (!a->sf) {
4236 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
4237 }
4238 return true;
4239 }
4240
TRANS(AND_i,gen_rri_log,a,false,tcg_gen_andi_i64)4241 TRANS(AND_i, gen_rri_log, a, false, tcg_gen_andi_i64)
4242 TRANS(ORR_i, gen_rri_log, a, false, tcg_gen_ori_i64)
4243 TRANS(EOR_i, gen_rri_log, a, false, tcg_gen_xori_i64)
4244 TRANS(ANDS_i, gen_rri_log, a, true, tcg_gen_andi_i64)
4245
4246 /*
4247 * Move wide (immediate)
4248 */
4249
4250 static bool trans_MOVZ(DisasContext *s, arg_movw *a)
4251 {
4252 int pos = a->hw << 4;
4253 tcg_gen_movi_i64(cpu_reg(s, a->rd), (uint64_t)a->imm << pos);
4254 return true;
4255 }
4256
trans_MOVN(DisasContext * s,arg_movw * a)4257 static bool trans_MOVN(DisasContext *s, arg_movw *a)
4258 {
4259 int pos = a->hw << 4;
4260 uint64_t imm = a->imm;
4261
4262 imm = ~(imm << pos);
4263 if (!a->sf) {
4264 imm = (uint32_t)imm;
4265 }
4266 tcg_gen_movi_i64(cpu_reg(s, a->rd), imm);
4267 return true;
4268 }
4269
trans_MOVK(DisasContext * s,arg_movw * a)4270 static bool trans_MOVK(DisasContext *s, arg_movw *a)
4271 {
4272 int pos = a->hw << 4;
4273 TCGv_i64 tcg_rd, tcg_im;
4274
4275 tcg_rd = cpu_reg(s, a->rd);
4276 tcg_im = tcg_constant_i64(a->imm);
4277 tcg_gen_deposit_i64(tcg_rd, tcg_rd, tcg_im, pos, 16);
4278 if (!a->sf) {
4279 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
4280 }
4281 return true;
4282 }
4283
4284 /*
4285 * Bitfield
4286 */
4287
trans_SBFM(DisasContext * s,arg_SBFM * a)4288 static bool trans_SBFM(DisasContext *s, arg_SBFM *a)
4289 {
4290 TCGv_i64 tcg_rd = cpu_reg(s, a->rd);
4291 TCGv_i64 tcg_tmp = read_cpu_reg(s, a->rn, 1);
4292 unsigned int bitsize = a->sf ? 64 : 32;
4293 unsigned int ri = a->immr;
4294 unsigned int si = a->imms;
4295 unsigned int pos, len;
4296
4297 if (si >= ri) {
4298 /* Wd<s-r:0> = Wn<s:r> */
4299 len = (si - ri) + 1;
4300 tcg_gen_sextract_i64(tcg_rd, tcg_tmp, ri, len);
4301 if (!a->sf) {
4302 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
4303 }
4304 } else {
4305 /* Wd<32+s-r,32-r> = Wn<s:0> */
4306 len = si + 1;
4307 pos = (bitsize - ri) & (bitsize - 1);
4308
4309 if (len < ri) {
4310 /*
4311 * Sign extend the destination field from len to fill the
4312 * balance of the word. Let the deposit below insert all
4313 * of those sign bits.
4314 */
4315 tcg_gen_sextract_i64(tcg_tmp, tcg_tmp, 0, len);
4316 len = ri;
4317 }
4318
4319 /*
4320 * We start with zero, and we haven't modified any bits outside
4321 * bitsize, therefore no final zero-extension is unneeded for !sf.
4322 */
4323 tcg_gen_deposit_z_i64(tcg_rd, tcg_tmp, pos, len);
4324 }
4325 return true;
4326 }
4327
trans_UBFM(DisasContext * s,arg_UBFM * a)4328 static bool trans_UBFM(DisasContext *s, arg_UBFM *a)
4329 {
4330 TCGv_i64 tcg_rd = cpu_reg(s, a->rd);
4331 TCGv_i64 tcg_tmp = read_cpu_reg(s, a->rn, 1);
4332 unsigned int bitsize = a->sf ? 64 : 32;
4333 unsigned int ri = a->immr;
4334 unsigned int si = a->imms;
4335 unsigned int pos, len;
4336
4337 tcg_rd = cpu_reg(s, a->rd);
4338 tcg_tmp = read_cpu_reg(s, a->rn, 1);
4339
4340 if (si >= ri) {
4341 /* Wd<s-r:0> = Wn<s:r> */
4342 len = (si - ri) + 1;
4343 tcg_gen_extract_i64(tcg_rd, tcg_tmp, ri, len);
4344 } else {
4345 /* Wd<32+s-r,32-r> = Wn<s:0> */
4346 len = si + 1;
4347 pos = (bitsize - ri) & (bitsize - 1);
4348 tcg_gen_deposit_z_i64(tcg_rd, tcg_tmp, pos, len);
4349 }
4350 return true;
4351 }
4352
trans_BFM(DisasContext * s,arg_BFM * a)4353 static bool trans_BFM(DisasContext *s, arg_BFM *a)
4354 {
4355 TCGv_i64 tcg_rd = cpu_reg(s, a->rd);
4356 TCGv_i64 tcg_tmp = read_cpu_reg(s, a->rn, 1);
4357 unsigned int bitsize = a->sf ? 64 : 32;
4358 unsigned int ri = a->immr;
4359 unsigned int si = a->imms;
4360 unsigned int pos, len;
4361
4362 tcg_rd = cpu_reg(s, a->rd);
4363 tcg_tmp = read_cpu_reg(s, a->rn, 1);
4364
4365 if (si >= ri) {
4366 /* Wd<s-r:0> = Wn<s:r> */
4367 tcg_gen_shri_i64(tcg_tmp, tcg_tmp, ri);
4368 len = (si - ri) + 1;
4369 pos = 0;
4370 } else {
4371 /* Wd<32+s-r,32-r> = Wn<s:0> */
4372 len = si + 1;
4373 pos = (bitsize - ri) & (bitsize - 1);
4374 }
4375
4376 tcg_gen_deposit_i64(tcg_rd, tcg_rd, tcg_tmp, pos, len);
4377 if (!a->sf) {
4378 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
4379 }
4380 return true;
4381 }
4382
trans_EXTR(DisasContext * s,arg_extract * a)4383 static bool trans_EXTR(DisasContext *s, arg_extract *a)
4384 {
4385 TCGv_i64 tcg_rd, tcg_rm, tcg_rn;
4386
4387 tcg_rd = cpu_reg(s, a->rd);
4388
4389 if (unlikely(a->imm == 0)) {
4390 /*
4391 * tcg shl_i32/shl_i64 is undefined for 32/64 bit shifts,
4392 * so an extract from bit 0 is a special case.
4393 */
4394 if (a->sf) {
4395 tcg_gen_mov_i64(tcg_rd, cpu_reg(s, a->rm));
4396 } else {
4397 tcg_gen_ext32u_i64(tcg_rd, cpu_reg(s, a->rm));
4398 }
4399 } else {
4400 tcg_rm = cpu_reg(s, a->rm);
4401 tcg_rn = cpu_reg(s, a->rn);
4402
4403 if (a->sf) {
4404 /* Specialization to ROR happens in EXTRACT2. */
4405 tcg_gen_extract2_i64(tcg_rd, tcg_rm, tcg_rn, a->imm);
4406 } else {
4407 TCGv_i32 t0 = tcg_temp_new_i32();
4408
4409 tcg_gen_extrl_i64_i32(t0, tcg_rm);
4410 if (a->rm == a->rn) {
4411 tcg_gen_rotri_i32(t0, t0, a->imm);
4412 } else {
4413 TCGv_i32 t1 = tcg_temp_new_i32();
4414 tcg_gen_extrl_i64_i32(t1, tcg_rn);
4415 tcg_gen_extract2_i32(t0, t0, t1, a->imm);
4416 }
4417 tcg_gen_extu_i32_i64(tcg_rd, t0);
4418 }
4419 }
4420 return true;
4421 }
4422
4423 /* Shift a TCGv src by TCGv shift_amount, put result in dst.
4424 * Note that it is the caller's responsibility to ensure that the
4425 * shift amount is in range (ie 0..31 or 0..63) and provide the ARM
4426 * mandated semantics for out of range shifts.
4427 */
shift_reg(TCGv_i64 dst,TCGv_i64 src,int sf,enum a64_shift_type shift_type,TCGv_i64 shift_amount)4428 static void shift_reg(TCGv_i64 dst, TCGv_i64 src, int sf,
4429 enum a64_shift_type shift_type, TCGv_i64 shift_amount)
4430 {
4431 switch (shift_type) {
4432 case A64_SHIFT_TYPE_LSL:
4433 tcg_gen_shl_i64(dst, src, shift_amount);
4434 break;
4435 case A64_SHIFT_TYPE_LSR:
4436 tcg_gen_shr_i64(dst, src, shift_amount);
4437 break;
4438 case A64_SHIFT_TYPE_ASR:
4439 if (!sf) {
4440 tcg_gen_ext32s_i64(dst, src);
4441 }
4442 tcg_gen_sar_i64(dst, sf ? src : dst, shift_amount);
4443 break;
4444 case A64_SHIFT_TYPE_ROR:
4445 if (sf) {
4446 tcg_gen_rotr_i64(dst, src, shift_amount);
4447 } else {
4448 TCGv_i32 t0, t1;
4449 t0 = tcg_temp_new_i32();
4450 t1 = tcg_temp_new_i32();
4451 tcg_gen_extrl_i64_i32(t0, src);
4452 tcg_gen_extrl_i64_i32(t1, shift_amount);
4453 tcg_gen_rotr_i32(t0, t0, t1);
4454 tcg_gen_extu_i32_i64(dst, t0);
4455 }
4456 break;
4457 default:
4458 assert(FALSE); /* all shift types should be handled */
4459 break;
4460 }
4461
4462 if (!sf) { /* zero extend final result */
4463 tcg_gen_ext32u_i64(dst, dst);
4464 }
4465 }
4466
4467 /* Shift a TCGv src by immediate, put result in dst.
4468 * The shift amount must be in range (this should always be true as the
4469 * relevant instructions will UNDEF on bad shift immediates).
4470 */
shift_reg_imm(TCGv_i64 dst,TCGv_i64 src,int sf,enum a64_shift_type shift_type,unsigned int shift_i)4471 static void shift_reg_imm(TCGv_i64 dst, TCGv_i64 src, int sf,
4472 enum a64_shift_type shift_type, unsigned int shift_i)
4473 {
4474 assert(shift_i < (sf ? 64 : 32));
4475
4476 if (shift_i == 0) {
4477 tcg_gen_mov_i64(dst, src);
4478 } else {
4479 shift_reg(dst, src, sf, shift_type, tcg_constant_i64(shift_i));
4480 }
4481 }
4482
4483 /* Logical (shifted register)
4484 * 31 30 29 28 24 23 22 21 20 16 15 10 9 5 4 0
4485 * +----+-----+-----------+-------+---+------+--------+------+------+
4486 * | sf | opc | 0 1 0 1 0 | shift | N | Rm | imm6 | Rn | Rd |
4487 * +----+-----+-----------+-------+---+------+--------+------+------+
4488 */
disas_logic_reg(DisasContext * s,uint32_t insn)4489 static void disas_logic_reg(DisasContext *s, uint32_t insn)
4490 {
4491 TCGv_i64 tcg_rd, tcg_rn, tcg_rm;
4492 unsigned int sf, opc, shift_type, invert, rm, shift_amount, rn, rd;
4493
4494 sf = extract32(insn, 31, 1);
4495 opc = extract32(insn, 29, 2);
4496 shift_type = extract32(insn, 22, 2);
4497 invert = extract32(insn, 21, 1);
4498 rm = extract32(insn, 16, 5);
4499 shift_amount = extract32(insn, 10, 6);
4500 rn = extract32(insn, 5, 5);
4501 rd = extract32(insn, 0, 5);
4502
4503 if (!sf && (shift_amount & (1 << 5))) {
4504 unallocated_encoding(s);
4505 return;
4506 }
4507
4508 tcg_rd = cpu_reg(s, rd);
4509
4510 if (opc == 1 && shift_amount == 0 && shift_type == 0 && rn == 31) {
4511 /* Unshifted ORR and ORN with WZR/XZR is the standard encoding for
4512 * register-register MOV and MVN, so it is worth special casing.
4513 */
4514 tcg_rm = cpu_reg(s, rm);
4515 if (invert) {
4516 tcg_gen_not_i64(tcg_rd, tcg_rm);
4517 if (!sf) {
4518 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
4519 }
4520 } else {
4521 if (sf) {
4522 tcg_gen_mov_i64(tcg_rd, tcg_rm);
4523 } else {
4524 tcg_gen_ext32u_i64(tcg_rd, tcg_rm);
4525 }
4526 }
4527 return;
4528 }
4529
4530 tcg_rm = read_cpu_reg(s, rm, sf);
4531
4532 if (shift_amount) {
4533 shift_reg_imm(tcg_rm, tcg_rm, sf, shift_type, shift_amount);
4534 }
4535
4536 tcg_rn = cpu_reg(s, rn);
4537
4538 switch (opc | (invert << 2)) {
4539 case 0: /* AND */
4540 case 3: /* ANDS */
4541 tcg_gen_and_i64(tcg_rd, tcg_rn, tcg_rm);
4542 break;
4543 case 1: /* ORR */
4544 tcg_gen_or_i64(tcg_rd, tcg_rn, tcg_rm);
4545 break;
4546 case 2: /* EOR */
4547 tcg_gen_xor_i64(tcg_rd, tcg_rn, tcg_rm);
4548 break;
4549 case 4: /* BIC */
4550 case 7: /* BICS */
4551 tcg_gen_andc_i64(tcg_rd, tcg_rn, tcg_rm);
4552 break;
4553 case 5: /* ORN */
4554 tcg_gen_orc_i64(tcg_rd, tcg_rn, tcg_rm);
4555 break;
4556 case 6: /* EON */
4557 tcg_gen_eqv_i64(tcg_rd, tcg_rn, tcg_rm);
4558 break;
4559 default:
4560 assert(FALSE);
4561 break;
4562 }
4563
4564 if (!sf) {
4565 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
4566 }
4567
4568 if (opc == 3) {
4569 gen_logic_CC(sf, tcg_rd);
4570 }
4571 }
4572
4573 /*
4574 * Add/subtract (extended register)
4575 *
4576 * 31|30|29|28 24|23 22|21|20 16|15 13|12 10|9 5|4 0|
4577 * +--+--+--+-----------+-----+--+-------+------+------+----+----+
4578 * |sf|op| S| 0 1 0 1 1 | opt | 1| Rm |option| imm3 | Rn | Rd |
4579 * +--+--+--+-----------+-----+--+-------+------+------+----+----+
4580 *
4581 * sf: 0 -> 32bit, 1 -> 64bit
4582 * op: 0 -> add , 1 -> sub
4583 * S: 1 -> set flags
4584 * opt: 00
4585 * option: extension type (see DecodeRegExtend)
4586 * imm3: optional shift to Rm
4587 *
4588 * Rd = Rn + LSL(extend(Rm), amount)
4589 */
disas_add_sub_ext_reg(DisasContext * s,uint32_t insn)4590 static void disas_add_sub_ext_reg(DisasContext *s, uint32_t insn)
4591 {
4592 int rd = extract32(insn, 0, 5);
4593 int rn = extract32(insn, 5, 5);
4594 int imm3 = extract32(insn, 10, 3);
4595 int option = extract32(insn, 13, 3);
4596 int rm = extract32(insn, 16, 5);
4597 int opt = extract32(insn, 22, 2);
4598 bool setflags = extract32(insn, 29, 1);
4599 bool sub_op = extract32(insn, 30, 1);
4600 bool sf = extract32(insn, 31, 1);
4601
4602 TCGv_i64 tcg_rm, tcg_rn; /* temps */
4603 TCGv_i64 tcg_rd;
4604 TCGv_i64 tcg_result;
4605
4606 if (imm3 > 4 || opt != 0) {
4607 unallocated_encoding(s);
4608 return;
4609 }
4610
4611 /* non-flag setting ops may use SP */
4612 if (!setflags) {
4613 tcg_rd = cpu_reg_sp(s, rd);
4614 } else {
4615 tcg_rd = cpu_reg(s, rd);
4616 }
4617 tcg_rn = read_cpu_reg_sp(s, rn, sf);
4618
4619 tcg_rm = read_cpu_reg(s, rm, sf);
4620 ext_and_shift_reg(tcg_rm, tcg_rm, option, imm3);
4621
4622 tcg_result = tcg_temp_new_i64();
4623
4624 if (!setflags) {
4625 if (sub_op) {
4626 tcg_gen_sub_i64(tcg_result, tcg_rn, tcg_rm);
4627 } else {
4628 tcg_gen_add_i64(tcg_result, tcg_rn, tcg_rm);
4629 }
4630 } else {
4631 if (sub_op) {
4632 gen_sub_CC(sf, tcg_result, tcg_rn, tcg_rm);
4633 } else {
4634 gen_add_CC(sf, tcg_result, tcg_rn, tcg_rm);
4635 }
4636 }
4637
4638 if (sf) {
4639 tcg_gen_mov_i64(tcg_rd, tcg_result);
4640 } else {
4641 tcg_gen_ext32u_i64(tcg_rd, tcg_result);
4642 }
4643 }
4644
4645 /*
4646 * Add/subtract (shifted register)
4647 *
4648 * 31 30 29 28 24 23 22 21 20 16 15 10 9 5 4 0
4649 * +--+--+--+-----------+-----+--+-------+---------+------+------+
4650 * |sf|op| S| 0 1 0 1 1 |shift| 0| Rm | imm6 | Rn | Rd |
4651 * +--+--+--+-----------+-----+--+-------+---------+------+------+
4652 *
4653 * sf: 0 -> 32bit, 1 -> 64bit
4654 * op: 0 -> add , 1 -> sub
4655 * S: 1 -> set flags
4656 * shift: 00 -> LSL, 01 -> LSR, 10 -> ASR, 11 -> RESERVED
4657 * imm6: Shift amount to apply to Rm before the add/sub
4658 */
disas_add_sub_reg(DisasContext * s,uint32_t insn)4659 static void disas_add_sub_reg(DisasContext *s, uint32_t insn)
4660 {
4661 int rd = extract32(insn, 0, 5);
4662 int rn = extract32(insn, 5, 5);
4663 int imm6 = extract32(insn, 10, 6);
4664 int rm = extract32(insn, 16, 5);
4665 int shift_type = extract32(insn, 22, 2);
4666 bool setflags = extract32(insn, 29, 1);
4667 bool sub_op = extract32(insn, 30, 1);
4668 bool sf = extract32(insn, 31, 1);
4669
4670 TCGv_i64 tcg_rd = cpu_reg(s, rd);
4671 TCGv_i64 tcg_rn, tcg_rm;
4672 TCGv_i64 tcg_result;
4673
4674 if ((shift_type == 3) || (!sf && (imm6 > 31))) {
4675 unallocated_encoding(s);
4676 return;
4677 }
4678
4679 tcg_rn = read_cpu_reg(s, rn, sf);
4680 tcg_rm = read_cpu_reg(s, rm, sf);
4681
4682 shift_reg_imm(tcg_rm, tcg_rm, sf, shift_type, imm6);
4683
4684 tcg_result = tcg_temp_new_i64();
4685
4686 if (!setflags) {
4687 if (sub_op) {
4688 tcg_gen_sub_i64(tcg_result, tcg_rn, tcg_rm);
4689 } else {
4690 tcg_gen_add_i64(tcg_result, tcg_rn, tcg_rm);
4691 }
4692 } else {
4693 if (sub_op) {
4694 gen_sub_CC(sf, tcg_result, tcg_rn, tcg_rm);
4695 } else {
4696 gen_add_CC(sf, tcg_result, tcg_rn, tcg_rm);
4697 }
4698 }
4699
4700 if (sf) {
4701 tcg_gen_mov_i64(tcg_rd, tcg_result);
4702 } else {
4703 tcg_gen_ext32u_i64(tcg_rd, tcg_result);
4704 }
4705 }
4706
4707 /* Data-processing (3 source)
4708 *
4709 * 31 30 29 28 24 23 21 20 16 15 14 10 9 5 4 0
4710 * +--+------+-----------+------+------+----+------+------+------+
4711 * |sf| op54 | 1 1 0 1 1 | op31 | Rm | o0 | Ra | Rn | Rd |
4712 * +--+------+-----------+------+------+----+------+------+------+
4713 */
disas_data_proc_3src(DisasContext * s,uint32_t insn)4714 static void disas_data_proc_3src(DisasContext *s, uint32_t insn)
4715 {
4716 int rd = extract32(insn, 0, 5);
4717 int rn = extract32(insn, 5, 5);
4718 int ra = extract32(insn, 10, 5);
4719 int rm = extract32(insn, 16, 5);
4720 int op_id = (extract32(insn, 29, 3) << 4) |
4721 (extract32(insn, 21, 3) << 1) |
4722 extract32(insn, 15, 1);
4723 bool sf = extract32(insn, 31, 1);
4724 bool is_sub = extract32(op_id, 0, 1);
4725 bool is_high = extract32(op_id, 2, 1);
4726 bool is_signed = false;
4727 TCGv_i64 tcg_op1;
4728 TCGv_i64 tcg_op2;
4729 TCGv_i64 tcg_tmp;
4730
4731 /* Note that op_id is sf:op54:op31:o0 so it includes the 32/64 size flag */
4732 switch (op_id) {
4733 case 0x42: /* SMADDL */
4734 case 0x43: /* SMSUBL */
4735 case 0x44: /* SMULH */
4736 is_signed = true;
4737 break;
4738 case 0x0: /* MADD (32bit) */
4739 case 0x1: /* MSUB (32bit) */
4740 case 0x40: /* MADD (64bit) */
4741 case 0x41: /* MSUB (64bit) */
4742 case 0x4a: /* UMADDL */
4743 case 0x4b: /* UMSUBL */
4744 case 0x4c: /* UMULH */
4745 break;
4746 default:
4747 unallocated_encoding(s);
4748 return;
4749 }
4750
4751 if (is_high) {
4752 TCGv_i64 low_bits = tcg_temp_new_i64(); /* low bits discarded */
4753 TCGv_i64 tcg_rd = cpu_reg(s, rd);
4754 TCGv_i64 tcg_rn = cpu_reg(s, rn);
4755 TCGv_i64 tcg_rm = cpu_reg(s, rm);
4756
4757 if (is_signed) {
4758 tcg_gen_muls2_i64(low_bits, tcg_rd, tcg_rn, tcg_rm);
4759 } else {
4760 tcg_gen_mulu2_i64(low_bits, tcg_rd, tcg_rn, tcg_rm);
4761 }
4762 return;
4763 }
4764
4765 tcg_op1 = tcg_temp_new_i64();
4766 tcg_op2 = tcg_temp_new_i64();
4767 tcg_tmp = tcg_temp_new_i64();
4768
4769 if (op_id < 0x42) {
4770 tcg_gen_mov_i64(tcg_op1, cpu_reg(s, rn));
4771 tcg_gen_mov_i64(tcg_op2, cpu_reg(s, rm));
4772 } else {
4773 if (is_signed) {
4774 tcg_gen_ext32s_i64(tcg_op1, cpu_reg(s, rn));
4775 tcg_gen_ext32s_i64(tcg_op2, cpu_reg(s, rm));
4776 } else {
4777 tcg_gen_ext32u_i64(tcg_op1, cpu_reg(s, rn));
4778 tcg_gen_ext32u_i64(tcg_op2, cpu_reg(s, rm));
4779 }
4780 }
4781
4782 if (ra == 31 && !is_sub) {
4783 /* Special-case MADD with rA == XZR; it is the standard MUL alias */
4784 tcg_gen_mul_i64(cpu_reg(s, rd), tcg_op1, tcg_op2);
4785 } else {
4786 tcg_gen_mul_i64(tcg_tmp, tcg_op1, tcg_op2);
4787 if (is_sub) {
4788 tcg_gen_sub_i64(cpu_reg(s, rd), cpu_reg(s, ra), tcg_tmp);
4789 } else {
4790 tcg_gen_add_i64(cpu_reg(s, rd), cpu_reg(s, ra), tcg_tmp);
4791 }
4792 }
4793
4794 if (!sf) {
4795 tcg_gen_ext32u_i64(cpu_reg(s, rd), cpu_reg(s, rd));
4796 }
4797 }
4798
4799 /* Add/subtract (with carry)
4800 * 31 30 29 28 27 26 25 24 23 22 21 20 16 15 10 9 5 4 0
4801 * +--+--+--+------------------------+------+-------------+------+-----+
4802 * |sf|op| S| 1 1 0 1 0 0 0 0 | rm | 0 0 0 0 0 0 | Rn | Rd |
4803 * +--+--+--+------------------------+------+-------------+------+-----+
4804 */
4805
disas_adc_sbc(DisasContext * s,uint32_t insn)4806 static void disas_adc_sbc(DisasContext *s, uint32_t insn)
4807 {
4808 unsigned int sf, op, setflags, rm, rn, rd;
4809 TCGv_i64 tcg_y, tcg_rn, tcg_rd;
4810
4811 sf = extract32(insn, 31, 1);
4812 op = extract32(insn, 30, 1);
4813 setflags = extract32(insn, 29, 1);
4814 rm = extract32(insn, 16, 5);
4815 rn = extract32(insn, 5, 5);
4816 rd = extract32(insn, 0, 5);
4817
4818 tcg_rd = cpu_reg(s, rd);
4819 tcg_rn = cpu_reg(s, rn);
4820
4821 if (op) {
4822 tcg_y = tcg_temp_new_i64();
4823 tcg_gen_not_i64(tcg_y, cpu_reg(s, rm));
4824 } else {
4825 tcg_y = cpu_reg(s, rm);
4826 }
4827
4828 if (setflags) {
4829 gen_adc_CC(sf, tcg_rd, tcg_rn, tcg_y);
4830 } else {
4831 gen_adc(sf, tcg_rd, tcg_rn, tcg_y);
4832 }
4833 }
4834
4835 /*
4836 * Rotate right into flags
4837 * 31 30 29 21 15 10 5 4 0
4838 * +--+--+--+-----------------+--------+-----------+------+--+------+
4839 * |sf|op| S| 1 1 0 1 0 0 0 0 | imm6 | 0 0 0 0 1 | Rn |o2| mask |
4840 * +--+--+--+-----------------+--------+-----------+------+--+------+
4841 */
disas_rotate_right_into_flags(DisasContext * s,uint32_t insn)4842 static void disas_rotate_right_into_flags(DisasContext *s, uint32_t insn)
4843 {
4844 int mask = extract32(insn, 0, 4);
4845 int o2 = extract32(insn, 4, 1);
4846 int rn = extract32(insn, 5, 5);
4847 int imm6 = extract32(insn, 15, 6);
4848 int sf_op_s = extract32(insn, 29, 3);
4849 TCGv_i64 tcg_rn;
4850 TCGv_i32 nzcv;
4851
4852 if (sf_op_s != 5 || o2 != 0 || !dc_isar_feature(aa64_condm_4, s)) {
4853 unallocated_encoding(s);
4854 return;
4855 }
4856
4857 tcg_rn = read_cpu_reg(s, rn, 1);
4858 tcg_gen_rotri_i64(tcg_rn, tcg_rn, imm6);
4859
4860 nzcv = tcg_temp_new_i32();
4861 tcg_gen_extrl_i64_i32(nzcv, tcg_rn);
4862
4863 if (mask & 8) { /* N */
4864 tcg_gen_shli_i32(cpu_NF, nzcv, 31 - 3);
4865 }
4866 if (mask & 4) { /* Z */
4867 tcg_gen_not_i32(cpu_ZF, nzcv);
4868 tcg_gen_andi_i32(cpu_ZF, cpu_ZF, 4);
4869 }
4870 if (mask & 2) { /* C */
4871 tcg_gen_extract_i32(cpu_CF, nzcv, 1, 1);
4872 }
4873 if (mask & 1) { /* V */
4874 tcg_gen_shli_i32(cpu_VF, nzcv, 31 - 0);
4875 }
4876 }
4877
4878 /*
4879 * Evaluate into flags
4880 * 31 30 29 21 15 14 10 5 4 0
4881 * +--+--+--+-----------------+---------+----+---------+------+--+------+
4882 * |sf|op| S| 1 1 0 1 0 0 0 0 | opcode2 | sz | 0 0 1 0 | Rn |o3| mask |
4883 * +--+--+--+-----------------+---------+----+---------+------+--+------+
4884 */
disas_evaluate_into_flags(DisasContext * s,uint32_t insn)4885 static void disas_evaluate_into_flags(DisasContext *s, uint32_t insn)
4886 {
4887 int o3_mask = extract32(insn, 0, 5);
4888 int rn = extract32(insn, 5, 5);
4889 int o2 = extract32(insn, 15, 6);
4890 int sz = extract32(insn, 14, 1);
4891 int sf_op_s = extract32(insn, 29, 3);
4892 TCGv_i32 tmp;
4893 int shift;
4894
4895 if (sf_op_s != 1 || o2 != 0 || o3_mask != 0xd ||
4896 !dc_isar_feature(aa64_condm_4, s)) {
4897 unallocated_encoding(s);
4898 return;
4899 }
4900 shift = sz ? 16 : 24; /* SETF16 or SETF8 */
4901
4902 tmp = tcg_temp_new_i32();
4903 tcg_gen_extrl_i64_i32(tmp, cpu_reg(s, rn));
4904 tcg_gen_shli_i32(cpu_NF, tmp, shift);
4905 tcg_gen_shli_i32(cpu_VF, tmp, shift - 1);
4906 tcg_gen_mov_i32(cpu_ZF, cpu_NF);
4907 tcg_gen_xor_i32(cpu_VF, cpu_VF, cpu_NF);
4908 }
4909
4910 /* Conditional compare (immediate / register)
4911 * 31 30 29 28 27 26 25 24 23 22 21 20 16 15 12 11 10 9 5 4 3 0
4912 * +--+--+--+------------------------+--------+------+----+--+------+--+-----+
4913 * |sf|op| S| 1 1 0 1 0 0 1 0 |imm5/rm | cond |i/r |o2| Rn |o3|nzcv |
4914 * +--+--+--+------------------------+--------+------+----+--+------+--+-----+
4915 * [1] y [0] [0]
4916 */
disas_cc(DisasContext * s,uint32_t insn)4917 static void disas_cc(DisasContext *s, uint32_t insn)
4918 {
4919 unsigned int sf, op, y, cond, rn, nzcv, is_imm;
4920 TCGv_i32 tcg_t0, tcg_t1, tcg_t2;
4921 TCGv_i64 tcg_tmp, tcg_y, tcg_rn;
4922 DisasCompare c;
4923
4924 if (!extract32(insn, 29, 1)) {
4925 unallocated_encoding(s);
4926 return;
4927 }
4928 if (insn & (1 << 10 | 1 << 4)) {
4929 unallocated_encoding(s);
4930 return;
4931 }
4932 sf = extract32(insn, 31, 1);
4933 op = extract32(insn, 30, 1);
4934 is_imm = extract32(insn, 11, 1);
4935 y = extract32(insn, 16, 5); /* y = rm (reg) or imm5 (imm) */
4936 cond = extract32(insn, 12, 4);
4937 rn = extract32(insn, 5, 5);
4938 nzcv = extract32(insn, 0, 4);
4939
4940 /* Set T0 = !COND. */
4941 tcg_t0 = tcg_temp_new_i32();
4942 arm_test_cc(&c, cond);
4943 tcg_gen_setcondi_i32(tcg_invert_cond(c.cond), tcg_t0, c.value, 0);
4944
4945 /* Load the arguments for the new comparison. */
4946 if (is_imm) {
4947 tcg_y = tcg_temp_new_i64();
4948 tcg_gen_movi_i64(tcg_y, y);
4949 } else {
4950 tcg_y = cpu_reg(s, y);
4951 }
4952 tcg_rn = cpu_reg(s, rn);
4953
4954 /* Set the flags for the new comparison. */
4955 tcg_tmp = tcg_temp_new_i64();
4956 if (op) {
4957 gen_sub_CC(sf, tcg_tmp, tcg_rn, tcg_y);
4958 } else {
4959 gen_add_CC(sf, tcg_tmp, tcg_rn, tcg_y);
4960 }
4961
4962 /* If COND was false, force the flags to #nzcv. Compute two masks
4963 * to help with this: T1 = (COND ? 0 : -1), T2 = (COND ? -1 : 0).
4964 * For tcg hosts that support ANDC, we can make do with just T1.
4965 * In either case, allow the tcg optimizer to delete any unused mask.
4966 */
4967 tcg_t1 = tcg_temp_new_i32();
4968 tcg_t2 = tcg_temp_new_i32();
4969 tcg_gen_neg_i32(tcg_t1, tcg_t0);
4970 tcg_gen_subi_i32(tcg_t2, tcg_t0, 1);
4971
4972 if (nzcv & 8) { /* N */
4973 tcg_gen_or_i32(cpu_NF, cpu_NF, tcg_t1);
4974 } else {
4975 if (TCG_TARGET_HAS_andc_i32) {
4976 tcg_gen_andc_i32(cpu_NF, cpu_NF, tcg_t1);
4977 } else {
4978 tcg_gen_and_i32(cpu_NF, cpu_NF, tcg_t2);
4979 }
4980 }
4981 if (nzcv & 4) { /* Z */
4982 if (TCG_TARGET_HAS_andc_i32) {
4983 tcg_gen_andc_i32(cpu_ZF, cpu_ZF, tcg_t1);
4984 } else {
4985 tcg_gen_and_i32(cpu_ZF, cpu_ZF, tcg_t2);
4986 }
4987 } else {
4988 tcg_gen_or_i32(cpu_ZF, cpu_ZF, tcg_t0);
4989 }
4990 if (nzcv & 2) { /* C */
4991 tcg_gen_or_i32(cpu_CF, cpu_CF, tcg_t0);
4992 } else {
4993 if (TCG_TARGET_HAS_andc_i32) {
4994 tcg_gen_andc_i32(cpu_CF, cpu_CF, tcg_t1);
4995 } else {
4996 tcg_gen_and_i32(cpu_CF, cpu_CF, tcg_t2);
4997 }
4998 }
4999 if (nzcv & 1) { /* V */
5000 tcg_gen_or_i32(cpu_VF, cpu_VF, tcg_t1);
5001 } else {
5002 if (TCG_TARGET_HAS_andc_i32) {
5003 tcg_gen_andc_i32(cpu_VF, cpu_VF, tcg_t1);
5004 } else {
5005 tcg_gen_and_i32(cpu_VF, cpu_VF, tcg_t2);
5006 }
5007 }
5008 }
5009
5010 /* Conditional select
5011 * 31 30 29 28 21 20 16 15 12 11 10 9 5 4 0
5012 * +----+----+---+-----------------+------+------+-----+------+------+
5013 * | sf | op | S | 1 1 0 1 0 1 0 0 | Rm | cond | op2 | Rn | Rd |
5014 * +----+----+---+-----------------+------+------+-----+------+------+
5015 */
disas_cond_select(DisasContext * s,uint32_t insn)5016 static void disas_cond_select(DisasContext *s, uint32_t insn)
5017 {
5018 unsigned int sf, else_inv, rm, cond, else_inc, rn, rd;
5019 TCGv_i64 tcg_rd, zero;
5020 DisasCompare64 c;
5021
5022 if (extract32(insn, 29, 1) || extract32(insn, 11, 1)) {
5023 /* S == 1 or op2<1> == 1 */
5024 unallocated_encoding(s);
5025 return;
5026 }
5027 sf = extract32(insn, 31, 1);
5028 else_inv = extract32(insn, 30, 1);
5029 rm = extract32(insn, 16, 5);
5030 cond = extract32(insn, 12, 4);
5031 else_inc = extract32(insn, 10, 1);
5032 rn = extract32(insn, 5, 5);
5033 rd = extract32(insn, 0, 5);
5034
5035 tcg_rd = cpu_reg(s, rd);
5036
5037 a64_test_cc(&c, cond);
5038 zero = tcg_constant_i64(0);
5039
5040 if (rn == 31 && rm == 31 && (else_inc ^ else_inv)) {
5041 /* CSET & CSETM. */
5042 if (else_inv) {
5043 tcg_gen_negsetcond_i64(tcg_invert_cond(c.cond),
5044 tcg_rd, c.value, zero);
5045 } else {
5046 tcg_gen_setcond_i64(tcg_invert_cond(c.cond),
5047 tcg_rd, c.value, zero);
5048 }
5049 } else {
5050 TCGv_i64 t_true = cpu_reg(s, rn);
5051 TCGv_i64 t_false = read_cpu_reg(s, rm, 1);
5052 if (else_inv && else_inc) {
5053 tcg_gen_neg_i64(t_false, t_false);
5054 } else if (else_inv) {
5055 tcg_gen_not_i64(t_false, t_false);
5056 } else if (else_inc) {
5057 tcg_gen_addi_i64(t_false, t_false, 1);
5058 }
5059 tcg_gen_movcond_i64(c.cond, tcg_rd, c.value, zero, t_true, t_false);
5060 }
5061
5062 if (!sf) {
5063 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
5064 }
5065 }
5066
handle_clz(DisasContext * s,unsigned int sf,unsigned int rn,unsigned int rd)5067 static void handle_clz(DisasContext *s, unsigned int sf,
5068 unsigned int rn, unsigned int rd)
5069 {
5070 TCGv_i64 tcg_rd, tcg_rn;
5071 tcg_rd = cpu_reg(s, rd);
5072 tcg_rn = cpu_reg(s, rn);
5073
5074 if (sf) {
5075 tcg_gen_clzi_i64(tcg_rd, tcg_rn, 64);
5076 } else {
5077 TCGv_i32 tcg_tmp32 = tcg_temp_new_i32();
5078 tcg_gen_extrl_i64_i32(tcg_tmp32, tcg_rn);
5079 tcg_gen_clzi_i32(tcg_tmp32, tcg_tmp32, 32);
5080 tcg_gen_extu_i32_i64(tcg_rd, tcg_tmp32);
5081 }
5082 }
5083
handle_cls(DisasContext * s,unsigned int sf,unsigned int rn,unsigned int rd)5084 static void handle_cls(DisasContext *s, unsigned int sf,
5085 unsigned int rn, unsigned int rd)
5086 {
5087 TCGv_i64 tcg_rd, tcg_rn;
5088 tcg_rd = cpu_reg(s, rd);
5089 tcg_rn = cpu_reg(s, rn);
5090
5091 if (sf) {
5092 tcg_gen_clrsb_i64(tcg_rd, tcg_rn);
5093 } else {
5094 TCGv_i32 tcg_tmp32 = tcg_temp_new_i32();
5095 tcg_gen_extrl_i64_i32(tcg_tmp32, tcg_rn);
5096 tcg_gen_clrsb_i32(tcg_tmp32, tcg_tmp32);
5097 tcg_gen_extu_i32_i64(tcg_rd, tcg_tmp32);
5098 }
5099 }
5100
handle_rbit(DisasContext * s,unsigned int sf,unsigned int rn,unsigned int rd)5101 static void handle_rbit(DisasContext *s, unsigned int sf,
5102 unsigned int rn, unsigned int rd)
5103 {
5104 TCGv_i64 tcg_rd, tcg_rn;
5105 tcg_rd = cpu_reg(s, rd);
5106 tcg_rn = cpu_reg(s, rn);
5107
5108 if (sf) {
5109 gen_helper_rbit64(tcg_rd, tcg_rn);
5110 } else {
5111 TCGv_i32 tcg_tmp32 = tcg_temp_new_i32();
5112 tcg_gen_extrl_i64_i32(tcg_tmp32, tcg_rn);
5113 gen_helper_rbit(tcg_tmp32, tcg_tmp32);
5114 tcg_gen_extu_i32_i64(tcg_rd, tcg_tmp32);
5115 }
5116 }
5117
5118 /* REV with sf==1, opcode==3 ("REV64") */
handle_rev64(DisasContext * s,unsigned int sf,unsigned int rn,unsigned int rd)5119 static void handle_rev64(DisasContext *s, unsigned int sf,
5120 unsigned int rn, unsigned int rd)
5121 {
5122 if (!sf) {
5123 unallocated_encoding(s);
5124 return;
5125 }
5126 tcg_gen_bswap64_i64(cpu_reg(s, rd), cpu_reg(s, rn));
5127 }
5128
5129 /* REV with sf==0, opcode==2
5130 * REV32 (sf==1, opcode==2)
5131 */
handle_rev32(DisasContext * s,unsigned int sf,unsigned int rn,unsigned int rd)5132 static void handle_rev32(DisasContext *s, unsigned int sf,
5133 unsigned int rn, unsigned int rd)
5134 {
5135 TCGv_i64 tcg_rd = cpu_reg(s, rd);
5136 TCGv_i64 tcg_rn = cpu_reg(s, rn);
5137
5138 if (sf) {
5139 tcg_gen_bswap64_i64(tcg_rd, tcg_rn);
5140 tcg_gen_rotri_i64(tcg_rd, tcg_rd, 32);
5141 } else {
5142 tcg_gen_bswap32_i64(tcg_rd, tcg_rn, TCG_BSWAP_OZ);
5143 }
5144 }
5145
5146 /* REV16 (opcode==1) */
handle_rev16(DisasContext * s,unsigned int sf,unsigned int rn,unsigned int rd)5147 static void handle_rev16(DisasContext *s, unsigned int sf,
5148 unsigned int rn, unsigned int rd)
5149 {
5150 TCGv_i64 tcg_rd = cpu_reg(s, rd);
5151 TCGv_i64 tcg_tmp = tcg_temp_new_i64();
5152 TCGv_i64 tcg_rn = read_cpu_reg(s, rn, sf);
5153 TCGv_i64 mask = tcg_constant_i64(sf ? 0x00ff00ff00ff00ffull : 0x00ff00ff);
5154
5155 tcg_gen_shri_i64(tcg_tmp, tcg_rn, 8);
5156 tcg_gen_and_i64(tcg_rd, tcg_rn, mask);
5157 tcg_gen_and_i64(tcg_tmp, tcg_tmp, mask);
5158 tcg_gen_shli_i64(tcg_rd, tcg_rd, 8);
5159 tcg_gen_or_i64(tcg_rd, tcg_rd, tcg_tmp);
5160 }
5161
5162 /* Data-processing (1 source)
5163 * 31 30 29 28 21 20 16 15 10 9 5 4 0
5164 * +----+---+---+-----------------+---------+--------+------+------+
5165 * | sf | 1 | S | 1 1 0 1 0 1 1 0 | opcode2 | opcode | Rn | Rd |
5166 * +----+---+---+-----------------+---------+--------+------+------+
5167 */
disas_data_proc_1src(DisasContext * s,uint32_t insn)5168 static void disas_data_proc_1src(DisasContext *s, uint32_t insn)
5169 {
5170 unsigned int sf, opcode, opcode2, rn, rd;
5171 TCGv_i64 tcg_rd;
5172
5173 if (extract32(insn, 29, 1)) {
5174 unallocated_encoding(s);
5175 return;
5176 }
5177
5178 sf = extract32(insn, 31, 1);
5179 opcode = extract32(insn, 10, 6);
5180 opcode2 = extract32(insn, 16, 5);
5181 rn = extract32(insn, 5, 5);
5182 rd = extract32(insn, 0, 5);
5183
5184 #define MAP(SF, O2, O1) ((SF) | (O1 << 1) | (O2 << 7))
5185
5186 switch (MAP(sf, opcode2, opcode)) {
5187 case MAP(0, 0x00, 0x00): /* RBIT */
5188 case MAP(1, 0x00, 0x00):
5189 handle_rbit(s, sf, rn, rd);
5190 break;
5191 case MAP(0, 0x00, 0x01): /* REV16 */
5192 case MAP(1, 0x00, 0x01):
5193 handle_rev16(s, sf, rn, rd);
5194 break;
5195 case MAP(0, 0x00, 0x02): /* REV/REV32 */
5196 case MAP(1, 0x00, 0x02):
5197 handle_rev32(s, sf, rn, rd);
5198 break;
5199 case MAP(1, 0x00, 0x03): /* REV64 */
5200 handle_rev64(s, sf, rn, rd);
5201 break;
5202 case MAP(0, 0x00, 0x04): /* CLZ */
5203 case MAP(1, 0x00, 0x04):
5204 handle_clz(s, sf, rn, rd);
5205 break;
5206 case MAP(0, 0x00, 0x05): /* CLS */
5207 case MAP(1, 0x00, 0x05):
5208 handle_cls(s, sf, rn, rd);
5209 break;
5210 case MAP(1, 0x01, 0x00): /* PACIA */
5211 if (s->pauth_active) {
5212 tcg_rd = cpu_reg(s, rd);
5213 gen_helper_pacia(tcg_rd, tcg_env, tcg_rd, cpu_reg_sp(s, rn));
5214 } else if (!dc_isar_feature(aa64_pauth, s)) {
5215 goto do_unallocated;
5216 }
5217 break;
5218 case MAP(1, 0x01, 0x01): /* PACIB */
5219 if (s->pauth_active) {
5220 tcg_rd = cpu_reg(s, rd);
5221 gen_helper_pacib(tcg_rd, tcg_env, tcg_rd, cpu_reg_sp(s, rn));
5222 } else if (!dc_isar_feature(aa64_pauth, s)) {
5223 goto do_unallocated;
5224 }
5225 break;
5226 case MAP(1, 0x01, 0x02): /* PACDA */
5227 if (s->pauth_active) {
5228 tcg_rd = cpu_reg(s, rd);
5229 gen_helper_pacda(tcg_rd, tcg_env, tcg_rd, cpu_reg_sp(s, rn));
5230 } else if (!dc_isar_feature(aa64_pauth, s)) {
5231 goto do_unallocated;
5232 }
5233 break;
5234 case MAP(1, 0x01, 0x03): /* PACDB */
5235 if (s->pauth_active) {
5236 tcg_rd = cpu_reg(s, rd);
5237 gen_helper_pacdb(tcg_rd, tcg_env, tcg_rd, cpu_reg_sp(s, rn));
5238 } else if (!dc_isar_feature(aa64_pauth, s)) {
5239 goto do_unallocated;
5240 }
5241 break;
5242 case MAP(1, 0x01, 0x04): /* AUTIA */
5243 if (s->pauth_active) {
5244 tcg_rd = cpu_reg(s, rd);
5245 gen_helper_autia(tcg_rd, tcg_env, tcg_rd, cpu_reg_sp(s, rn));
5246 } else if (!dc_isar_feature(aa64_pauth, s)) {
5247 goto do_unallocated;
5248 }
5249 break;
5250 case MAP(1, 0x01, 0x05): /* AUTIB */
5251 if (s->pauth_active) {
5252 tcg_rd = cpu_reg(s, rd);
5253 gen_helper_autib(tcg_rd, tcg_env, tcg_rd, cpu_reg_sp(s, rn));
5254 } else if (!dc_isar_feature(aa64_pauth, s)) {
5255 goto do_unallocated;
5256 }
5257 break;
5258 case MAP(1, 0x01, 0x06): /* AUTDA */
5259 if (s->pauth_active) {
5260 tcg_rd = cpu_reg(s, rd);
5261 gen_helper_autda(tcg_rd, tcg_env, tcg_rd, cpu_reg_sp(s, rn));
5262 } else if (!dc_isar_feature(aa64_pauth, s)) {
5263 goto do_unallocated;
5264 }
5265 break;
5266 case MAP(1, 0x01, 0x07): /* AUTDB */
5267 if (s->pauth_active) {
5268 tcg_rd = cpu_reg(s, rd);
5269 gen_helper_autdb(tcg_rd, tcg_env, tcg_rd, cpu_reg_sp(s, rn));
5270 } else if (!dc_isar_feature(aa64_pauth, s)) {
5271 goto do_unallocated;
5272 }
5273 break;
5274 case MAP(1, 0x01, 0x08): /* PACIZA */
5275 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) {
5276 goto do_unallocated;
5277 } else if (s->pauth_active) {
5278 tcg_rd = cpu_reg(s, rd);
5279 gen_helper_pacia(tcg_rd, tcg_env, tcg_rd, tcg_constant_i64(0));
5280 }
5281 break;
5282 case MAP(1, 0x01, 0x09): /* PACIZB */
5283 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) {
5284 goto do_unallocated;
5285 } else if (s->pauth_active) {
5286 tcg_rd = cpu_reg(s, rd);
5287 gen_helper_pacib(tcg_rd, tcg_env, tcg_rd, tcg_constant_i64(0));
5288 }
5289 break;
5290 case MAP(1, 0x01, 0x0a): /* PACDZA */
5291 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) {
5292 goto do_unallocated;
5293 } else if (s->pauth_active) {
5294 tcg_rd = cpu_reg(s, rd);
5295 gen_helper_pacda(tcg_rd, tcg_env, tcg_rd, tcg_constant_i64(0));
5296 }
5297 break;
5298 case MAP(1, 0x01, 0x0b): /* PACDZB */
5299 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) {
5300 goto do_unallocated;
5301 } else if (s->pauth_active) {
5302 tcg_rd = cpu_reg(s, rd);
5303 gen_helper_pacdb(tcg_rd, tcg_env, tcg_rd, tcg_constant_i64(0));
5304 }
5305 break;
5306 case MAP(1, 0x01, 0x0c): /* AUTIZA */
5307 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) {
5308 goto do_unallocated;
5309 } else if (s->pauth_active) {
5310 tcg_rd = cpu_reg(s, rd);
5311 gen_helper_autia(tcg_rd, tcg_env, tcg_rd, tcg_constant_i64(0));
5312 }
5313 break;
5314 case MAP(1, 0x01, 0x0d): /* AUTIZB */
5315 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) {
5316 goto do_unallocated;
5317 } else if (s->pauth_active) {
5318 tcg_rd = cpu_reg(s, rd);
5319 gen_helper_autib(tcg_rd, tcg_env, tcg_rd, tcg_constant_i64(0));
5320 }
5321 break;
5322 case MAP(1, 0x01, 0x0e): /* AUTDZA */
5323 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) {
5324 goto do_unallocated;
5325 } else if (s->pauth_active) {
5326 tcg_rd = cpu_reg(s, rd);
5327 gen_helper_autda(tcg_rd, tcg_env, tcg_rd, tcg_constant_i64(0));
5328 }
5329 break;
5330 case MAP(1, 0x01, 0x0f): /* AUTDZB */
5331 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) {
5332 goto do_unallocated;
5333 } else if (s->pauth_active) {
5334 tcg_rd = cpu_reg(s, rd);
5335 gen_helper_autdb(tcg_rd, tcg_env, tcg_rd, tcg_constant_i64(0));
5336 }
5337 break;
5338 case MAP(1, 0x01, 0x10): /* XPACI */
5339 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) {
5340 goto do_unallocated;
5341 } else if (s->pauth_active) {
5342 tcg_rd = cpu_reg(s, rd);
5343 gen_helper_xpaci(tcg_rd, tcg_env, tcg_rd);
5344 }
5345 break;
5346 case MAP(1, 0x01, 0x11): /* XPACD */
5347 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) {
5348 goto do_unallocated;
5349 } else if (s->pauth_active) {
5350 tcg_rd = cpu_reg(s, rd);
5351 gen_helper_xpacd(tcg_rd, tcg_env, tcg_rd);
5352 }
5353 break;
5354 default:
5355 do_unallocated:
5356 unallocated_encoding(s);
5357 break;
5358 }
5359
5360 #undef MAP
5361 }
5362
handle_div(DisasContext * s,bool is_signed,unsigned int sf,unsigned int rm,unsigned int rn,unsigned int rd)5363 static void handle_div(DisasContext *s, bool is_signed, unsigned int sf,
5364 unsigned int rm, unsigned int rn, unsigned int rd)
5365 {
5366 TCGv_i64 tcg_n, tcg_m, tcg_rd;
5367 tcg_rd = cpu_reg(s, rd);
5368
5369 if (!sf && is_signed) {
5370 tcg_n = tcg_temp_new_i64();
5371 tcg_m = tcg_temp_new_i64();
5372 tcg_gen_ext32s_i64(tcg_n, cpu_reg(s, rn));
5373 tcg_gen_ext32s_i64(tcg_m, cpu_reg(s, rm));
5374 } else {
5375 tcg_n = read_cpu_reg(s, rn, sf);
5376 tcg_m = read_cpu_reg(s, rm, sf);
5377 }
5378
5379 if (is_signed) {
5380 gen_helper_sdiv64(tcg_rd, tcg_n, tcg_m);
5381 } else {
5382 gen_helper_udiv64(tcg_rd, tcg_n, tcg_m);
5383 }
5384
5385 if (!sf) { /* zero extend final result */
5386 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
5387 }
5388 }
5389
5390 /* LSLV, LSRV, ASRV, RORV */
handle_shift_reg(DisasContext * s,enum a64_shift_type shift_type,unsigned int sf,unsigned int rm,unsigned int rn,unsigned int rd)5391 static void handle_shift_reg(DisasContext *s,
5392 enum a64_shift_type shift_type, unsigned int sf,
5393 unsigned int rm, unsigned int rn, unsigned int rd)
5394 {
5395 TCGv_i64 tcg_shift = tcg_temp_new_i64();
5396 TCGv_i64 tcg_rd = cpu_reg(s, rd);
5397 TCGv_i64 tcg_rn = read_cpu_reg(s, rn, sf);
5398
5399 tcg_gen_andi_i64(tcg_shift, cpu_reg(s, rm), sf ? 63 : 31);
5400 shift_reg(tcg_rd, tcg_rn, sf, shift_type, tcg_shift);
5401 }
5402
5403 /* CRC32[BHWX], CRC32C[BHWX] */
handle_crc32(DisasContext * s,unsigned int sf,unsigned int sz,bool crc32c,unsigned int rm,unsigned int rn,unsigned int rd)5404 static void handle_crc32(DisasContext *s,
5405 unsigned int sf, unsigned int sz, bool crc32c,
5406 unsigned int rm, unsigned int rn, unsigned int rd)
5407 {
5408 TCGv_i64 tcg_acc, tcg_val;
5409 TCGv_i32 tcg_bytes;
5410
5411 if (!dc_isar_feature(aa64_crc32, s)
5412 || (sf == 1 && sz != 3)
5413 || (sf == 0 && sz == 3)) {
5414 unallocated_encoding(s);
5415 return;
5416 }
5417
5418 if (sz == 3) {
5419 tcg_val = cpu_reg(s, rm);
5420 } else {
5421 uint64_t mask;
5422 switch (sz) {
5423 case 0:
5424 mask = 0xFF;
5425 break;
5426 case 1:
5427 mask = 0xFFFF;
5428 break;
5429 case 2:
5430 mask = 0xFFFFFFFF;
5431 break;
5432 default:
5433 g_assert_not_reached();
5434 }
5435 tcg_val = tcg_temp_new_i64();
5436 tcg_gen_andi_i64(tcg_val, cpu_reg(s, rm), mask);
5437 }
5438
5439 tcg_acc = cpu_reg(s, rn);
5440 tcg_bytes = tcg_constant_i32(1 << sz);
5441
5442 if (crc32c) {
5443 gen_helper_crc32c_64(cpu_reg(s, rd), tcg_acc, tcg_val, tcg_bytes);
5444 } else {
5445 gen_helper_crc32_64(cpu_reg(s, rd), tcg_acc, tcg_val, tcg_bytes);
5446 }
5447 }
5448
5449 /* Data-processing (2 source)
5450 * 31 30 29 28 21 20 16 15 10 9 5 4 0
5451 * +----+---+---+-----------------+------+--------+------+------+
5452 * | sf | 0 | S | 1 1 0 1 0 1 1 0 | Rm | opcode | Rn | Rd |
5453 * +----+---+---+-----------------+------+--------+------+------+
5454 */
disas_data_proc_2src(DisasContext * s,uint32_t insn)5455 static void disas_data_proc_2src(DisasContext *s, uint32_t insn)
5456 {
5457 unsigned int sf, rm, opcode, rn, rd, setflag;
5458 sf = extract32(insn, 31, 1);
5459 setflag = extract32(insn, 29, 1);
5460 rm = extract32(insn, 16, 5);
5461 opcode = extract32(insn, 10, 6);
5462 rn = extract32(insn, 5, 5);
5463 rd = extract32(insn, 0, 5);
5464
5465 if (setflag && opcode != 0) {
5466 unallocated_encoding(s);
5467 return;
5468 }
5469
5470 switch (opcode) {
5471 case 0: /* SUBP(S) */
5472 if (sf == 0 || !dc_isar_feature(aa64_mte_insn_reg, s)) {
5473 goto do_unallocated;
5474 } else {
5475 TCGv_i64 tcg_n, tcg_m, tcg_d;
5476
5477 tcg_n = read_cpu_reg_sp(s, rn, true);
5478 tcg_m = read_cpu_reg_sp(s, rm, true);
5479 tcg_gen_sextract_i64(tcg_n, tcg_n, 0, 56);
5480 tcg_gen_sextract_i64(tcg_m, tcg_m, 0, 56);
5481 tcg_d = cpu_reg(s, rd);
5482
5483 if (setflag) {
5484 gen_sub_CC(true, tcg_d, tcg_n, tcg_m);
5485 } else {
5486 tcg_gen_sub_i64(tcg_d, tcg_n, tcg_m);
5487 }
5488 }
5489 break;
5490 case 2: /* UDIV */
5491 handle_div(s, false, sf, rm, rn, rd);
5492 break;
5493 case 3: /* SDIV */
5494 handle_div(s, true, sf, rm, rn, rd);
5495 break;
5496 case 4: /* IRG */
5497 if (sf == 0 || !dc_isar_feature(aa64_mte_insn_reg, s)) {
5498 goto do_unallocated;
5499 }
5500 if (s->ata[0]) {
5501 gen_helper_irg(cpu_reg_sp(s, rd), tcg_env,
5502 cpu_reg_sp(s, rn), cpu_reg(s, rm));
5503 } else {
5504 gen_address_with_allocation_tag0(cpu_reg_sp(s, rd),
5505 cpu_reg_sp(s, rn));
5506 }
5507 break;
5508 case 5: /* GMI */
5509 if (sf == 0 || !dc_isar_feature(aa64_mte_insn_reg, s)) {
5510 goto do_unallocated;
5511 } else {
5512 TCGv_i64 t = tcg_temp_new_i64();
5513
5514 tcg_gen_extract_i64(t, cpu_reg_sp(s, rn), 56, 4);
5515 tcg_gen_shl_i64(t, tcg_constant_i64(1), t);
5516 tcg_gen_or_i64(cpu_reg(s, rd), cpu_reg(s, rm), t);
5517 }
5518 break;
5519 case 8: /* LSLV */
5520 handle_shift_reg(s, A64_SHIFT_TYPE_LSL, sf, rm, rn, rd);
5521 break;
5522 case 9: /* LSRV */
5523 handle_shift_reg(s, A64_SHIFT_TYPE_LSR, sf, rm, rn, rd);
5524 break;
5525 case 10: /* ASRV */
5526 handle_shift_reg(s, A64_SHIFT_TYPE_ASR, sf, rm, rn, rd);
5527 break;
5528 case 11: /* RORV */
5529 handle_shift_reg(s, A64_SHIFT_TYPE_ROR, sf, rm, rn, rd);
5530 break;
5531 case 12: /* PACGA */
5532 if (sf == 0 || !dc_isar_feature(aa64_pauth, s)) {
5533 goto do_unallocated;
5534 }
5535 gen_helper_pacga(cpu_reg(s, rd), tcg_env,
5536 cpu_reg(s, rn), cpu_reg_sp(s, rm));
5537 break;
5538 case 16:
5539 case 17:
5540 case 18:
5541 case 19:
5542 case 20:
5543 case 21:
5544 case 22:
5545 case 23: /* CRC32 */
5546 {
5547 int sz = extract32(opcode, 0, 2);
5548 bool crc32c = extract32(opcode, 2, 1);
5549 handle_crc32(s, sf, sz, crc32c, rm, rn, rd);
5550 break;
5551 }
5552 default:
5553 do_unallocated:
5554 unallocated_encoding(s);
5555 break;
5556 }
5557 }
5558
5559 /*
5560 * Data processing - register
5561 * 31 30 29 28 25 21 20 16 10 0
5562 * +--+---+--+---+-------+-----+-------+-------+---------+
5563 * | |op0| |op1| 1 0 1 | op2 | | op3 | |
5564 * +--+---+--+---+-------+-----+-------+-------+---------+
5565 */
disas_data_proc_reg(DisasContext * s,uint32_t insn)5566 static void disas_data_proc_reg(DisasContext *s, uint32_t insn)
5567 {
5568 int op0 = extract32(insn, 30, 1);
5569 int op1 = extract32(insn, 28, 1);
5570 int op2 = extract32(insn, 21, 4);
5571 int op3 = extract32(insn, 10, 6);
5572
5573 if (!op1) {
5574 if (op2 & 8) {
5575 if (op2 & 1) {
5576 /* Add/sub (extended register) */
5577 disas_add_sub_ext_reg(s, insn);
5578 } else {
5579 /* Add/sub (shifted register) */
5580 disas_add_sub_reg(s, insn);
5581 }
5582 } else {
5583 /* Logical (shifted register) */
5584 disas_logic_reg(s, insn);
5585 }
5586 return;
5587 }
5588
5589 switch (op2) {
5590 case 0x0:
5591 switch (op3) {
5592 case 0x00: /* Add/subtract (with carry) */
5593 disas_adc_sbc(s, insn);
5594 break;
5595
5596 case 0x01: /* Rotate right into flags */
5597 case 0x21:
5598 disas_rotate_right_into_flags(s, insn);
5599 break;
5600
5601 case 0x02: /* Evaluate into flags */
5602 case 0x12:
5603 case 0x22:
5604 case 0x32:
5605 disas_evaluate_into_flags(s, insn);
5606 break;
5607
5608 default:
5609 goto do_unallocated;
5610 }
5611 break;
5612
5613 case 0x2: /* Conditional compare */
5614 disas_cc(s, insn); /* both imm and reg forms */
5615 break;
5616
5617 case 0x4: /* Conditional select */
5618 disas_cond_select(s, insn);
5619 break;
5620
5621 case 0x6: /* Data-processing */
5622 if (op0) { /* (1 source) */
5623 disas_data_proc_1src(s, insn);
5624 } else { /* (2 source) */
5625 disas_data_proc_2src(s, insn);
5626 }
5627 break;
5628 case 0x8 ... 0xf: /* (3 source) */
5629 disas_data_proc_3src(s, insn);
5630 break;
5631
5632 default:
5633 do_unallocated:
5634 unallocated_encoding(s);
5635 break;
5636 }
5637 }
5638
handle_fp_compare(DisasContext * s,int size,unsigned int rn,unsigned int rm,bool cmp_with_zero,bool signal_all_nans)5639 static void handle_fp_compare(DisasContext *s, int size,
5640 unsigned int rn, unsigned int rm,
5641 bool cmp_with_zero, bool signal_all_nans)
5642 {
5643 TCGv_i64 tcg_flags = tcg_temp_new_i64();
5644 TCGv_ptr fpst = fpstatus_ptr(size == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
5645
5646 if (size == MO_64) {
5647 TCGv_i64 tcg_vn, tcg_vm;
5648
5649 tcg_vn = read_fp_dreg(s, rn);
5650 if (cmp_with_zero) {
5651 tcg_vm = tcg_constant_i64(0);
5652 } else {
5653 tcg_vm = read_fp_dreg(s, rm);
5654 }
5655 if (signal_all_nans) {
5656 gen_helper_vfp_cmped_a64(tcg_flags, tcg_vn, tcg_vm, fpst);
5657 } else {
5658 gen_helper_vfp_cmpd_a64(tcg_flags, tcg_vn, tcg_vm, fpst);
5659 }
5660 } else {
5661 TCGv_i32 tcg_vn = tcg_temp_new_i32();
5662 TCGv_i32 tcg_vm = tcg_temp_new_i32();
5663
5664 read_vec_element_i32(s, tcg_vn, rn, 0, size);
5665 if (cmp_with_zero) {
5666 tcg_gen_movi_i32(tcg_vm, 0);
5667 } else {
5668 read_vec_element_i32(s, tcg_vm, rm, 0, size);
5669 }
5670
5671 switch (size) {
5672 case MO_32:
5673 if (signal_all_nans) {
5674 gen_helper_vfp_cmpes_a64(tcg_flags, tcg_vn, tcg_vm, fpst);
5675 } else {
5676 gen_helper_vfp_cmps_a64(tcg_flags, tcg_vn, tcg_vm, fpst);
5677 }
5678 break;
5679 case MO_16:
5680 if (signal_all_nans) {
5681 gen_helper_vfp_cmpeh_a64(tcg_flags, tcg_vn, tcg_vm, fpst);
5682 } else {
5683 gen_helper_vfp_cmph_a64(tcg_flags, tcg_vn, tcg_vm, fpst);
5684 }
5685 break;
5686 default:
5687 g_assert_not_reached();
5688 }
5689 }
5690
5691 gen_set_nzcv(tcg_flags);
5692 }
5693
5694 /* Floating point compare
5695 * 31 30 29 28 24 23 22 21 20 16 15 14 13 10 9 5 4 0
5696 * +---+---+---+-----------+------+---+------+-----+---------+------+-------+
5697 * | M | 0 | S | 1 1 1 1 0 | type | 1 | Rm | op | 1 0 0 0 | Rn | op2 |
5698 * +---+---+---+-----------+------+---+------+-----+---------+------+-------+
5699 */
disas_fp_compare(DisasContext * s,uint32_t insn)5700 static void disas_fp_compare(DisasContext *s, uint32_t insn)
5701 {
5702 unsigned int mos, type, rm, op, rn, opc, op2r;
5703 int size;
5704
5705 mos = extract32(insn, 29, 3);
5706 type = extract32(insn, 22, 2);
5707 rm = extract32(insn, 16, 5);
5708 op = extract32(insn, 14, 2);
5709 rn = extract32(insn, 5, 5);
5710 opc = extract32(insn, 3, 2);
5711 op2r = extract32(insn, 0, 3);
5712
5713 if (mos || op || op2r) {
5714 unallocated_encoding(s);
5715 return;
5716 }
5717
5718 switch (type) {
5719 case 0:
5720 size = MO_32;
5721 break;
5722 case 1:
5723 size = MO_64;
5724 break;
5725 case 3:
5726 size = MO_16;
5727 if (dc_isar_feature(aa64_fp16, s)) {
5728 break;
5729 }
5730 /* fallthru */
5731 default:
5732 unallocated_encoding(s);
5733 return;
5734 }
5735
5736 if (!fp_access_check(s)) {
5737 return;
5738 }
5739
5740 handle_fp_compare(s, size, rn, rm, opc & 1, opc & 2);
5741 }
5742
5743 /* Floating point conditional compare
5744 * 31 30 29 28 24 23 22 21 20 16 15 12 11 10 9 5 4 3 0
5745 * +---+---+---+-----------+------+---+------+------+-----+------+----+------+
5746 * | M | 0 | S | 1 1 1 1 0 | type | 1 | Rm | cond | 0 1 | Rn | op | nzcv |
5747 * +---+---+---+-----------+------+---+------+------+-----+------+----+------+
5748 */
disas_fp_ccomp(DisasContext * s,uint32_t insn)5749 static void disas_fp_ccomp(DisasContext *s, uint32_t insn)
5750 {
5751 unsigned int mos, type, rm, cond, rn, op, nzcv;
5752 TCGLabel *label_continue = NULL;
5753 int size;
5754
5755 mos = extract32(insn, 29, 3);
5756 type = extract32(insn, 22, 2);
5757 rm = extract32(insn, 16, 5);
5758 cond = extract32(insn, 12, 4);
5759 rn = extract32(insn, 5, 5);
5760 op = extract32(insn, 4, 1);
5761 nzcv = extract32(insn, 0, 4);
5762
5763 if (mos) {
5764 unallocated_encoding(s);
5765 return;
5766 }
5767
5768 switch (type) {
5769 case 0:
5770 size = MO_32;
5771 break;
5772 case 1:
5773 size = MO_64;
5774 break;
5775 case 3:
5776 size = MO_16;
5777 if (dc_isar_feature(aa64_fp16, s)) {
5778 break;
5779 }
5780 /* fallthru */
5781 default:
5782 unallocated_encoding(s);
5783 return;
5784 }
5785
5786 if (!fp_access_check(s)) {
5787 return;
5788 }
5789
5790 if (cond < 0x0e) { /* not always */
5791 TCGLabel *label_match = gen_new_label();
5792 label_continue = gen_new_label();
5793 arm_gen_test_cc(cond, label_match);
5794 /* nomatch: */
5795 gen_set_nzcv(tcg_constant_i64(nzcv << 28));
5796 tcg_gen_br(label_continue);
5797 gen_set_label(label_match);
5798 }
5799
5800 handle_fp_compare(s, size, rn, rm, false, op);
5801
5802 if (cond < 0x0e) {
5803 gen_set_label(label_continue);
5804 }
5805 }
5806
5807 /* Floating point conditional select
5808 * 31 30 29 28 24 23 22 21 20 16 15 12 11 10 9 5 4 0
5809 * +---+---+---+-----------+------+---+------+------+-----+------+------+
5810 * | M | 0 | S | 1 1 1 1 0 | type | 1 | Rm | cond | 1 1 | Rn | Rd |
5811 * +---+---+---+-----------+------+---+------+------+-----+------+------+
5812 */
disas_fp_csel(DisasContext * s,uint32_t insn)5813 static void disas_fp_csel(DisasContext *s, uint32_t insn)
5814 {
5815 unsigned int mos, type, rm, cond, rn, rd;
5816 TCGv_i64 t_true, t_false;
5817 DisasCompare64 c;
5818 MemOp sz;
5819
5820 mos = extract32(insn, 29, 3);
5821 type = extract32(insn, 22, 2);
5822 rm = extract32(insn, 16, 5);
5823 cond = extract32(insn, 12, 4);
5824 rn = extract32(insn, 5, 5);
5825 rd = extract32(insn, 0, 5);
5826
5827 if (mos) {
5828 unallocated_encoding(s);
5829 return;
5830 }
5831
5832 switch (type) {
5833 case 0:
5834 sz = MO_32;
5835 break;
5836 case 1:
5837 sz = MO_64;
5838 break;
5839 case 3:
5840 sz = MO_16;
5841 if (dc_isar_feature(aa64_fp16, s)) {
5842 break;
5843 }
5844 /* fallthru */
5845 default:
5846 unallocated_encoding(s);
5847 return;
5848 }
5849
5850 if (!fp_access_check(s)) {
5851 return;
5852 }
5853
5854 /* Zero extend sreg & hreg inputs to 64 bits now. */
5855 t_true = tcg_temp_new_i64();
5856 t_false = tcg_temp_new_i64();
5857 read_vec_element(s, t_true, rn, 0, sz);
5858 read_vec_element(s, t_false, rm, 0, sz);
5859
5860 a64_test_cc(&c, cond);
5861 tcg_gen_movcond_i64(c.cond, t_true, c.value, tcg_constant_i64(0),
5862 t_true, t_false);
5863
5864 /* Note that sregs & hregs write back zeros to the high bits,
5865 and we've already done the zero-extension. */
5866 write_fp_dreg(s, rd, t_true);
5867 }
5868
5869 /* Floating-point data-processing (1 source) - half precision */
handle_fp_1src_half(DisasContext * s,int opcode,int rd,int rn)5870 static void handle_fp_1src_half(DisasContext *s, int opcode, int rd, int rn)
5871 {
5872 TCGv_ptr fpst = NULL;
5873 TCGv_i32 tcg_op = read_fp_hreg(s, rn);
5874 TCGv_i32 tcg_res = tcg_temp_new_i32();
5875
5876 switch (opcode) {
5877 case 0x0: /* FMOV */
5878 tcg_gen_mov_i32(tcg_res, tcg_op);
5879 break;
5880 case 0x1: /* FABS */
5881 tcg_gen_andi_i32(tcg_res, tcg_op, 0x7fff);
5882 break;
5883 case 0x2: /* FNEG */
5884 tcg_gen_xori_i32(tcg_res, tcg_op, 0x8000);
5885 break;
5886 case 0x3: /* FSQRT */
5887 fpst = fpstatus_ptr(FPST_FPCR_F16);
5888 gen_helper_sqrt_f16(tcg_res, tcg_op, fpst);
5889 break;
5890 case 0x8: /* FRINTN */
5891 case 0x9: /* FRINTP */
5892 case 0xa: /* FRINTM */
5893 case 0xb: /* FRINTZ */
5894 case 0xc: /* FRINTA */
5895 {
5896 TCGv_i32 tcg_rmode;
5897
5898 fpst = fpstatus_ptr(FPST_FPCR_F16);
5899 tcg_rmode = gen_set_rmode(opcode & 7, fpst);
5900 gen_helper_advsimd_rinth(tcg_res, tcg_op, fpst);
5901 gen_restore_rmode(tcg_rmode, fpst);
5902 break;
5903 }
5904 case 0xe: /* FRINTX */
5905 fpst = fpstatus_ptr(FPST_FPCR_F16);
5906 gen_helper_advsimd_rinth_exact(tcg_res, tcg_op, fpst);
5907 break;
5908 case 0xf: /* FRINTI */
5909 fpst = fpstatus_ptr(FPST_FPCR_F16);
5910 gen_helper_advsimd_rinth(tcg_res, tcg_op, fpst);
5911 break;
5912 default:
5913 g_assert_not_reached();
5914 }
5915
5916 write_fp_sreg(s, rd, tcg_res);
5917 }
5918
5919 /* Floating-point data-processing (1 source) - single precision */
handle_fp_1src_single(DisasContext * s,int opcode,int rd,int rn)5920 static void handle_fp_1src_single(DisasContext *s, int opcode, int rd, int rn)
5921 {
5922 void (*gen_fpst)(TCGv_i32, TCGv_i32, TCGv_ptr);
5923 TCGv_i32 tcg_op, tcg_res;
5924 TCGv_ptr fpst;
5925 int rmode = -1;
5926
5927 tcg_op = read_fp_sreg(s, rn);
5928 tcg_res = tcg_temp_new_i32();
5929
5930 switch (opcode) {
5931 case 0x0: /* FMOV */
5932 tcg_gen_mov_i32(tcg_res, tcg_op);
5933 goto done;
5934 case 0x1: /* FABS */
5935 gen_helper_vfp_abss(tcg_res, tcg_op);
5936 goto done;
5937 case 0x2: /* FNEG */
5938 gen_helper_vfp_negs(tcg_res, tcg_op);
5939 goto done;
5940 case 0x3: /* FSQRT */
5941 gen_helper_vfp_sqrts(tcg_res, tcg_op, tcg_env);
5942 goto done;
5943 case 0x6: /* BFCVT */
5944 gen_fpst = gen_helper_bfcvt;
5945 break;
5946 case 0x8: /* FRINTN */
5947 case 0x9: /* FRINTP */
5948 case 0xa: /* FRINTM */
5949 case 0xb: /* FRINTZ */
5950 case 0xc: /* FRINTA */
5951 rmode = opcode & 7;
5952 gen_fpst = gen_helper_rints;
5953 break;
5954 case 0xe: /* FRINTX */
5955 gen_fpst = gen_helper_rints_exact;
5956 break;
5957 case 0xf: /* FRINTI */
5958 gen_fpst = gen_helper_rints;
5959 break;
5960 case 0x10: /* FRINT32Z */
5961 rmode = FPROUNDING_ZERO;
5962 gen_fpst = gen_helper_frint32_s;
5963 break;
5964 case 0x11: /* FRINT32X */
5965 gen_fpst = gen_helper_frint32_s;
5966 break;
5967 case 0x12: /* FRINT64Z */
5968 rmode = FPROUNDING_ZERO;
5969 gen_fpst = gen_helper_frint64_s;
5970 break;
5971 case 0x13: /* FRINT64X */
5972 gen_fpst = gen_helper_frint64_s;
5973 break;
5974 default:
5975 g_assert_not_reached();
5976 }
5977
5978 fpst = fpstatus_ptr(FPST_FPCR);
5979 if (rmode >= 0) {
5980 TCGv_i32 tcg_rmode = gen_set_rmode(rmode, fpst);
5981 gen_fpst(tcg_res, tcg_op, fpst);
5982 gen_restore_rmode(tcg_rmode, fpst);
5983 } else {
5984 gen_fpst(tcg_res, tcg_op, fpst);
5985 }
5986
5987 done:
5988 write_fp_sreg(s, rd, tcg_res);
5989 }
5990
5991 /* Floating-point data-processing (1 source) - double precision */
handle_fp_1src_double(DisasContext * s,int opcode,int rd,int rn)5992 static void handle_fp_1src_double(DisasContext *s, int opcode, int rd, int rn)
5993 {
5994 void (*gen_fpst)(TCGv_i64, TCGv_i64, TCGv_ptr);
5995 TCGv_i64 tcg_op, tcg_res;
5996 TCGv_ptr fpst;
5997 int rmode = -1;
5998
5999 switch (opcode) {
6000 case 0x0: /* FMOV */
6001 gen_gvec_fn2(s, false, rd, rn, tcg_gen_gvec_mov, 0);
6002 return;
6003 }
6004
6005 tcg_op = read_fp_dreg(s, rn);
6006 tcg_res = tcg_temp_new_i64();
6007
6008 switch (opcode) {
6009 case 0x1: /* FABS */
6010 gen_helper_vfp_absd(tcg_res, tcg_op);
6011 goto done;
6012 case 0x2: /* FNEG */
6013 gen_helper_vfp_negd(tcg_res, tcg_op);
6014 goto done;
6015 case 0x3: /* FSQRT */
6016 gen_helper_vfp_sqrtd(tcg_res, tcg_op, tcg_env);
6017 goto done;
6018 case 0x8: /* FRINTN */
6019 case 0x9: /* FRINTP */
6020 case 0xa: /* FRINTM */
6021 case 0xb: /* FRINTZ */
6022 case 0xc: /* FRINTA */
6023 rmode = opcode & 7;
6024 gen_fpst = gen_helper_rintd;
6025 break;
6026 case 0xe: /* FRINTX */
6027 gen_fpst = gen_helper_rintd_exact;
6028 break;
6029 case 0xf: /* FRINTI */
6030 gen_fpst = gen_helper_rintd;
6031 break;
6032 case 0x10: /* FRINT32Z */
6033 rmode = FPROUNDING_ZERO;
6034 gen_fpst = gen_helper_frint32_d;
6035 break;
6036 case 0x11: /* FRINT32X */
6037 gen_fpst = gen_helper_frint32_d;
6038 break;
6039 case 0x12: /* FRINT64Z */
6040 rmode = FPROUNDING_ZERO;
6041 gen_fpst = gen_helper_frint64_d;
6042 break;
6043 case 0x13: /* FRINT64X */
6044 gen_fpst = gen_helper_frint64_d;
6045 break;
6046 default:
6047 g_assert_not_reached();
6048 }
6049
6050 fpst = fpstatus_ptr(FPST_FPCR);
6051 if (rmode >= 0) {
6052 TCGv_i32 tcg_rmode = gen_set_rmode(rmode, fpst);
6053 gen_fpst(tcg_res, tcg_op, fpst);
6054 gen_restore_rmode(tcg_rmode, fpst);
6055 } else {
6056 gen_fpst(tcg_res, tcg_op, fpst);
6057 }
6058
6059 done:
6060 write_fp_dreg(s, rd, tcg_res);
6061 }
6062
handle_fp_fcvt(DisasContext * s,int opcode,int rd,int rn,int dtype,int ntype)6063 static void handle_fp_fcvt(DisasContext *s, int opcode,
6064 int rd, int rn, int dtype, int ntype)
6065 {
6066 switch (ntype) {
6067 case 0x0:
6068 {
6069 TCGv_i32 tcg_rn = read_fp_sreg(s, rn);
6070 if (dtype == 1) {
6071 /* Single to double */
6072 TCGv_i64 tcg_rd = tcg_temp_new_i64();
6073 gen_helper_vfp_fcvtds(tcg_rd, tcg_rn, tcg_env);
6074 write_fp_dreg(s, rd, tcg_rd);
6075 } else {
6076 /* Single to half */
6077 TCGv_i32 tcg_rd = tcg_temp_new_i32();
6078 TCGv_i32 ahp = get_ahp_flag();
6079 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR);
6080
6081 gen_helper_vfp_fcvt_f32_to_f16(tcg_rd, tcg_rn, fpst, ahp);
6082 /* write_fp_sreg is OK here because top half of tcg_rd is zero */
6083 write_fp_sreg(s, rd, tcg_rd);
6084 }
6085 break;
6086 }
6087 case 0x1:
6088 {
6089 TCGv_i64 tcg_rn = read_fp_dreg(s, rn);
6090 TCGv_i32 tcg_rd = tcg_temp_new_i32();
6091 if (dtype == 0) {
6092 /* Double to single */
6093 gen_helper_vfp_fcvtsd(tcg_rd, tcg_rn, tcg_env);
6094 } else {
6095 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR);
6096 TCGv_i32 ahp = get_ahp_flag();
6097 /* Double to half */
6098 gen_helper_vfp_fcvt_f64_to_f16(tcg_rd, tcg_rn, fpst, ahp);
6099 /* write_fp_sreg is OK here because top half of tcg_rd is zero */
6100 }
6101 write_fp_sreg(s, rd, tcg_rd);
6102 break;
6103 }
6104 case 0x3:
6105 {
6106 TCGv_i32 tcg_rn = read_fp_sreg(s, rn);
6107 TCGv_ptr tcg_fpst = fpstatus_ptr(FPST_FPCR);
6108 TCGv_i32 tcg_ahp = get_ahp_flag();
6109 tcg_gen_ext16u_i32(tcg_rn, tcg_rn);
6110 if (dtype == 0) {
6111 /* Half to single */
6112 TCGv_i32 tcg_rd = tcg_temp_new_i32();
6113 gen_helper_vfp_fcvt_f16_to_f32(tcg_rd, tcg_rn, tcg_fpst, tcg_ahp);
6114 write_fp_sreg(s, rd, tcg_rd);
6115 } else {
6116 /* Half to double */
6117 TCGv_i64 tcg_rd = tcg_temp_new_i64();
6118 gen_helper_vfp_fcvt_f16_to_f64(tcg_rd, tcg_rn, tcg_fpst, tcg_ahp);
6119 write_fp_dreg(s, rd, tcg_rd);
6120 }
6121 break;
6122 }
6123 default:
6124 g_assert_not_reached();
6125 }
6126 }
6127
6128 /* Floating point data-processing (1 source)
6129 * 31 30 29 28 24 23 22 21 20 15 14 10 9 5 4 0
6130 * +---+---+---+-----------+------+---+--------+-----------+------+------+
6131 * | M | 0 | S | 1 1 1 1 0 | type | 1 | opcode | 1 0 0 0 0 | Rn | Rd |
6132 * +---+---+---+-----------+------+---+--------+-----------+------+------+
6133 */
disas_fp_1src(DisasContext * s,uint32_t insn)6134 static void disas_fp_1src(DisasContext *s, uint32_t insn)
6135 {
6136 int mos = extract32(insn, 29, 3);
6137 int type = extract32(insn, 22, 2);
6138 int opcode = extract32(insn, 15, 6);
6139 int rn = extract32(insn, 5, 5);
6140 int rd = extract32(insn, 0, 5);
6141
6142 if (mos) {
6143 goto do_unallocated;
6144 }
6145
6146 switch (opcode) {
6147 case 0x4: case 0x5: case 0x7:
6148 {
6149 /* FCVT between half, single and double precision */
6150 int dtype = extract32(opcode, 0, 2);
6151 if (type == 2 || dtype == type) {
6152 goto do_unallocated;
6153 }
6154 if (!fp_access_check(s)) {
6155 return;
6156 }
6157
6158 handle_fp_fcvt(s, opcode, rd, rn, dtype, type);
6159 break;
6160 }
6161
6162 case 0x10 ... 0x13: /* FRINT{32,64}{X,Z} */
6163 if (type > 1 || !dc_isar_feature(aa64_frint, s)) {
6164 goto do_unallocated;
6165 }
6166 /* fall through */
6167 case 0x0 ... 0x3:
6168 case 0x8 ... 0xc:
6169 case 0xe ... 0xf:
6170 /* 32-to-32 and 64-to-64 ops */
6171 switch (type) {
6172 case 0:
6173 if (!fp_access_check(s)) {
6174 return;
6175 }
6176 handle_fp_1src_single(s, opcode, rd, rn);
6177 break;
6178 case 1:
6179 if (!fp_access_check(s)) {
6180 return;
6181 }
6182 handle_fp_1src_double(s, opcode, rd, rn);
6183 break;
6184 case 3:
6185 if (!dc_isar_feature(aa64_fp16, s)) {
6186 goto do_unallocated;
6187 }
6188
6189 if (!fp_access_check(s)) {
6190 return;
6191 }
6192 handle_fp_1src_half(s, opcode, rd, rn);
6193 break;
6194 default:
6195 goto do_unallocated;
6196 }
6197 break;
6198
6199 case 0x6:
6200 switch (type) {
6201 case 1: /* BFCVT */
6202 if (!dc_isar_feature(aa64_bf16, s)) {
6203 goto do_unallocated;
6204 }
6205 if (!fp_access_check(s)) {
6206 return;
6207 }
6208 handle_fp_1src_single(s, opcode, rd, rn);
6209 break;
6210 default:
6211 goto do_unallocated;
6212 }
6213 break;
6214
6215 default:
6216 do_unallocated:
6217 unallocated_encoding(s);
6218 break;
6219 }
6220 }
6221
6222 /* Floating-point data-processing (2 source) - single precision */
handle_fp_2src_single(DisasContext * s,int opcode,int rd,int rn,int rm)6223 static void handle_fp_2src_single(DisasContext *s, int opcode,
6224 int rd, int rn, int rm)
6225 {
6226 TCGv_i32 tcg_op1;
6227 TCGv_i32 tcg_op2;
6228 TCGv_i32 tcg_res;
6229 TCGv_ptr fpst;
6230
6231 tcg_res = tcg_temp_new_i32();
6232 fpst = fpstatus_ptr(FPST_FPCR);
6233 tcg_op1 = read_fp_sreg(s, rn);
6234 tcg_op2 = read_fp_sreg(s, rm);
6235
6236 switch (opcode) {
6237 case 0x0: /* FMUL */
6238 gen_helper_vfp_muls(tcg_res, tcg_op1, tcg_op2, fpst);
6239 break;
6240 case 0x1: /* FDIV */
6241 gen_helper_vfp_divs(tcg_res, tcg_op1, tcg_op2, fpst);
6242 break;
6243 case 0x2: /* FADD */
6244 gen_helper_vfp_adds(tcg_res, tcg_op1, tcg_op2, fpst);
6245 break;
6246 case 0x3: /* FSUB */
6247 gen_helper_vfp_subs(tcg_res, tcg_op1, tcg_op2, fpst);
6248 break;
6249 case 0x4: /* FMAX */
6250 gen_helper_vfp_maxs(tcg_res, tcg_op1, tcg_op2, fpst);
6251 break;
6252 case 0x5: /* FMIN */
6253 gen_helper_vfp_mins(tcg_res, tcg_op1, tcg_op2, fpst);
6254 break;
6255 case 0x6: /* FMAXNM */
6256 gen_helper_vfp_maxnums(tcg_res, tcg_op1, tcg_op2, fpst);
6257 break;
6258 case 0x7: /* FMINNM */
6259 gen_helper_vfp_minnums(tcg_res, tcg_op1, tcg_op2, fpst);
6260 break;
6261 case 0x8: /* FNMUL */
6262 gen_helper_vfp_muls(tcg_res, tcg_op1, tcg_op2, fpst);
6263 gen_helper_vfp_negs(tcg_res, tcg_res);
6264 break;
6265 }
6266
6267 write_fp_sreg(s, rd, tcg_res);
6268 }
6269
6270 /* Floating-point data-processing (2 source) - double precision */
handle_fp_2src_double(DisasContext * s,int opcode,int rd,int rn,int rm)6271 static void handle_fp_2src_double(DisasContext *s, int opcode,
6272 int rd, int rn, int rm)
6273 {
6274 TCGv_i64 tcg_op1;
6275 TCGv_i64 tcg_op2;
6276 TCGv_i64 tcg_res;
6277 TCGv_ptr fpst;
6278
6279 tcg_res = tcg_temp_new_i64();
6280 fpst = fpstatus_ptr(FPST_FPCR);
6281 tcg_op1 = read_fp_dreg(s, rn);
6282 tcg_op2 = read_fp_dreg(s, rm);
6283
6284 switch (opcode) {
6285 case 0x0: /* FMUL */
6286 gen_helper_vfp_muld(tcg_res, tcg_op1, tcg_op2, fpst);
6287 break;
6288 case 0x1: /* FDIV */
6289 gen_helper_vfp_divd(tcg_res, tcg_op1, tcg_op2, fpst);
6290 break;
6291 case 0x2: /* FADD */
6292 gen_helper_vfp_addd(tcg_res, tcg_op1, tcg_op2, fpst);
6293 break;
6294 case 0x3: /* FSUB */
6295 gen_helper_vfp_subd(tcg_res, tcg_op1, tcg_op2, fpst);
6296 break;
6297 case 0x4: /* FMAX */
6298 gen_helper_vfp_maxd(tcg_res, tcg_op1, tcg_op2, fpst);
6299 break;
6300 case 0x5: /* FMIN */
6301 gen_helper_vfp_mind(tcg_res, tcg_op1, tcg_op2, fpst);
6302 break;
6303 case 0x6: /* FMAXNM */
6304 gen_helper_vfp_maxnumd(tcg_res, tcg_op1, tcg_op2, fpst);
6305 break;
6306 case 0x7: /* FMINNM */
6307 gen_helper_vfp_minnumd(tcg_res, tcg_op1, tcg_op2, fpst);
6308 break;
6309 case 0x8: /* FNMUL */
6310 gen_helper_vfp_muld(tcg_res, tcg_op1, tcg_op2, fpst);
6311 gen_helper_vfp_negd(tcg_res, tcg_res);
6312 break;
6313 }
6314
6315 write_fp_dreg(s, rd, tcg_res);
6316 }
6317
6318 /* Floating-point data-processing (2 source) - half precision */
handle_fp_2src_half(DisasContext * s,int opcode,int rd,int rn,int rm)6319 static void handle_fp_2src_half(DisasContext *s, int opcode,
6320 int rd, int rn, int rm)
6321 {
6322 TCGv_i32 tcg_op1;
6323 TCGv_i32 tcg_op2;
6324 TCGv_i32 tcg_res;
6325 TCGv_ptr fpst;
6326
6327 tcg_res = tcg_temp_new_i32();
6328 fpst = fpstatus_ptr(FPST_FPCR_F16);
6329 tcg_op1 = read_fp_hreg(s, rn);
6330 tcg_op2 = read_fp_hreg(s, rm);
6331
6332 switch (opcode) {
6333 case 0x0: /* FMUL */
6334 gen_helper_advsimd_mulh(tcg_res, tcg_op1, tcg_op2, fpst);
6335 break;
6336 case 0x1: /* FDIV */
6337 gen_helper_advsimd_divh(tcg_res, tcg_op1, tcg_op2, fpst);
6338 break;
6339 case 0x2: /* FADD */
6340 gen_helper_advsimd_addh(tcg_res, tcg_op1, tcg_op2, fpst);
6341 break;
6342 case 0x3: /* FSUB */
6343 gen_helper_advsimd_subh(tcg_res, tcg_op1, tcg_op2, fpst);
6344 break;
6345 case 0x4: /* FMAX */
6346 gen_helper_advsimd_maxh(tcg_res, tcg_op1, tcg_op2, fpst);
6347 break;
6348 case 0x5: /* FMIN */
6349 gen_helper_advsimd_minh(tcg_res, tcg_op1, tcg_op2, fpst);
6350 break;
6351 case 0x6: /* FMAXNM */
6352 gen_helper_advsimd_maxnumh(tcg_res, tcg_op1, tcg_op2, fpst);
6353 break;
6354 case 0x7: /* FMINNM */
6355 gen_helper_advsimd_minnumh(tcg_res, tcg_op1, tcg_op2, fpst);
6356 break;
6357 case 0x8: /* FNMUL */
6358 gen_helper_advsimd_mulh(tcg_res, tcg_op1, tcg_op2, fpst);
6359 tcg_gen_xori_i32(tcg_res, tcg_res, 0x8000);
6360 break;
6361 default:
6362 g_assert_not_reached();
6363 }
6364
6365 write_fp_sreg(s, rd, tcg_res);
6366 }
6367
6368 /* Floating point data-processing (2 source)
6369 * 31 30 29 28 24 23 22 21 20 16 15 12 11 10 9 5 4 0
6370 * +---+---+---+-----------+------+---+------+--------+-----+------+------+
6371 * | M | 0 | S | 1 1 1 1 0 | type | 1 | Rm | opcode | 1 0 | Rn | Rd |
6372 * +---+---+---+-----------+------+---+------+--------+-----+------+------+
6373 */
disas_fp_2src(DisasContext * s,uint32_t insn)6374 static void disas_fp_2src(DisasContext *s, uint32_t insn)
6375 {
6376 int mos = extract32(insn, 29, 3);
6377 int type = extract32(insn, 22, 2);
6378 int rd = extract32(insn, 0, 5);
6379 int rn = extract32(insn, 5, 5);
6380 int rm = extract32(insn, 16, 5);
6381 int opcode = extract32(insn, 12, 4);
6382
6383 if (opcode > 8 || mos) {
6384 unallocated_encoding(s);
6385 return;
6386 }
6387
6388 switch (type) {
6389 case 0:
6390 if (!fp_access_check(s)) {
6391 return;
6392 }
6393 handle_fp_2src_single(s, opcode, rd, rn, rm);
6394 break;
6395 case 1:
6396 if (!fp_access_check(s)) {
6397 return;
6398 }
6399 handle_fp_2src_double(s, opcode, rd, rn, rm);
6400 break;
6401 case 3:
6402 if (!dc_isar_feature(aa64_fp16, s)) {
6403 unallocated_encoding(s);
6404 return;
6405 }
6406 if (!fp_access_check(s)) {
6407 return;
6408 }
6409 handle_fp_2src_half(s, opcode, rd, rn, rm);
6410 break;
6411 default:
6412 unallocated_encoding(s);
6413 }
6414 }
6415
6416 /* Floating-point data-processing (3 source) - single precision */
handle_fp_3src_single(DisasContext * s,bool o0,bool o1,int rd,int rn,int rm,int ra)6417 static void handle_fp_3src_single(DisasContext *s, bool o0, bool o1,
6418 int rd, int rn, int rm, int ra)
6419 {
6420 TCGv_i32 tcg_op1, tcg_op2, tcg_op3;
6421 TCGv_i32 tcg_res = tcg_temp_new_i32();
6422 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR);
6423
6424 tcg_op1 = read_fp_sreg(s, rn);
6425 tcg_op2 = read_fp_sreg(s, rm);
6426 tcg_op3 = read_fp_sreg(s, ra);
6427
6428 /* These are fused multiply-add, and must be done as one
6429 * floating point operation with no rounding between the
6430 * multiplication and addition steps.
6431 * NB that doing the negations here as separate steps is
6432 * correct : an input NaN should come out with its sign bit
6433 * flipped if it is a negated-input.
6434 */
6435 if (o1 == true) {
6436 gen_helper_vfp_negs(tcg_op3, tcg_op3);
6437 }
6438
6439 if (o0 != o1) {
6440 gen_helper_vfp_negs(tcg_op1, tcg_op1);
6441 }
6442
6443 gen_helper_vfp_muladds(tcg_res, tcg_op1, tcg_op2, tcg_op3, fpst);
6444
6445 write_fp_sreg(s, rd, tcg_res);
6446 }
6447
6448 /* Floating-point data-processing (3 source) - double precision */
handle_fp_3src_double(DisasContext * s,bool o0,bool o1,int rd,int rn,int rm,int ra)6449 static void handle_fp_3src_double(DisasContext *s, bool o0, bool o1,
6450 int rd, int rn, int rm, int ra)
6451 {
6452 TCGv_i64 tcg_op1, tcg_op2, tcg_op3;
6453 TCGv_i64 tcg_res = tcg_temp_new_i64();
6454 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR);
6455
6456 tcg_op1 = read_fp_dreg(s, rn);
6457 tcg_op2 = read_fp_dreg(s, rm);
6458 tcg_op3 = read_fp_dreg(s, ra);
6459
6460 /* These are fused multiply-add, and must be done as one
6461 * floating point operation with no rounding between the
6462 * multiplication and addition steps.
6463 * NB that doing the negations here as separate steps is
6464 * correct : an input NaN should come out with its sign bit
6465 * flipped if it is a negated-input.
6466 */
6467 if (o1 == true) {
6468 gen_helper_vfp_negd(tcg_op3, tcg_op3);
6469 }
6470
6471 if (o0 != o1) {
6472 gen_helper_vfp_negd(tcg_op1, tcg_op1);
6473 }
6474
6475 gen_helper_vfp_muladdd(tcg_res, tcg_op1, tcg_op2, tcg_op3, fpst);
6476
6477 write_fp_dreg(s, rd, tcg_res);
6478 }
6479
6480 /* Floating-point data-processing (3 source) - half precision */
handle_fp_3src_half(DisasContext * s,bool o0,bool o1,int rd,int rn,int rm,int ra)6481 static void handle_fp_3src_half(DisasContext *s, bool o0, bool o1,
6482 int rd, int rn, int rm, int ra)
6483 {
6484 TCGv_i32 tcg_op1, tcg_op2, tcg_op3;
6485 TCGv_i32 tcg_res = tcg_temp_new_i32();
6486 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR_F16);
6487
6488 tcg_op1 = read_fp_hreg(s, rn);
6489 tcg_op2 = read_fp_hreg(s, rm);
6490 tcg_op3 = read_fp_hreg(s, ra);
6491
6492 /* These are fused multiply-add, and must be done as one
6493 * floating point operation with no rounding between the
6494 * multiplication and addition steps.
6495 * NB that doing the negations here as separate steps is
6496 * correct : an input NaN should come out with its sign bit
6497 * flipped if it is a negated-input.
6498 */
6499 if (o1 == true) {
6500 tcg_gen_xori_i32(tcg_op3, tcg_op3, 0x8000);
6501 }
6502
6503 if (o0 != o1) {
6504 tcg_gen_xori_i32(tcg_op1, tcg_op1, 0x8000);
6505 }
6506
6507 gen_helper_advsimd_muladdh(tcg_res, tcg_op1, tcg_op2, tcg_op3, fpst);
6508
6509 write_fp_sreg(s, rd, tcg_res);
6510 }
6511
6512 /* Floating point data-processing (3 source)
6513 * 31 30 29 28 24 23 22 21 20 16 15 14 10 9 5 4 0
6514 * +---+---+---+-----------+------+----+------+----+------+------+------+
6515 * | M | 0 | S | 1 1 1 1 1 | type | o1 | Rm | o0 | Ra | Rn | Rd |
6516 * +---+---+---+-----------+------+----+------+----+------+------+------+
6517 */
disas_fp_3src(DisasContext * s,uint32_t insn)6518 static void disas_fp_3src(DisasContext *s, uint32_t insn)
6519 {
6520 int mos = extract32(insn, 29, 3);
6521 int type = extract32(insn, 22, 2);
6522 int rd = extract32(insn, 0, 5);
6523 int rn = extract32(insn, 5, 5);
6524 int ra = extract32(insn, 10, 5);
6525 int rm = extract32(insn, 16, 5);
6526 bool o0 = extract32(insn, 15, 1);
6527 bool o1 = extract32(insn, 21, 1);
6528
6529 if (mos) {
6530 unallocated_encoding(s);
6531 return;
6532 }
6533
6534 switch (type) {
6535 case 0:
6536 if (!fp_access_check(s)) {
6537 return;
6538 }
6539 handle_fp_3src_single(s, o0, o1, rd, rn, rm, ra);
6540 break;
6541 case 1:
6542 if (!fp_access_check(s)) {
6543 return;
6544 }
6545 handle_fp_3src_double(s, o0, o1, rd, rn, rm, ra);
6546 break;
6547 case 3:
6548 if (!dc_isar_feature(aa64_fp16, s)) {
6549 unallocated_encoding(s);
6550 return;
6551 }
6552 if (!fp_access_check(s)) {
6553 return;
6554 }
6555 handle_fp_3src_half(s, o0, o1, rd, rn, rm, ra);
6556 break;
6557 default:
6558 unallocated_encoding(s);
6559 }
6560 }
6561
6562 /* Floating point immediate
6563 * 31 30 29 28 24 23 22 21 20 13 12 10 9 5 4 0
6564 * +---+---+---+-----------+------+---+------------+-------+------+------+
6565 * | M | 0 | S | 1 1 1 1 0 | type | 1 | imm8 | 1 0 0 | imm5 | Rd |
6566 * +---+---+---+-----------+------+---+------------+-------+------+------+
6567 */
disas_fp_imm(DisasContext * s,uint32_t insn)6568 static void disas_fp_imm(DisasContext *s, uint32_t insn)
6569 {
6570 int rd = extract32(insn, 0, 5);
6571 int imm5 = extract32(insn, 5, 5);
6572 int imm8 = extract32(insn, 13, 8);
6573 int type = extract32(insn, 22, 2);
6574 int mos = extract32(insn, 29, 3);
6575 uint64_t imm;
6576 MemOp sz;
6577
6578 if (mos || imm5) {
6579 unallocated_encoding(s);
6580 return;
6581 }
6582
6583 switch (type) {
6584 case 0:
6585 sz = MO_32;
6586 break;
6587 case 1:
6588 sz = MO_64;
6589 break;
6590 case 3:
6591 sz = MO_16;
6592 if (dc_isar_feature(aa64_fp16, s)) {
6593 break;
6594 }
6595 /* fallthru */
6596 default:
6597 unallocated_encoding(s);
6598 return;
6599 }
6600
6601 if (!fp_access_check(s)) {
6602 return;
6603 }
6604
6605 imm = vfp_expand_imm(sz, imm8);
6606 write_fp_dreg(s, rd, tcg_constant_i64(imm));
6607 }
6608
6609 /* Handle floating point <=> fixed point conversions. Note that we can
6610 * also deal with fp <=> integer conversions as a special case (scale == 64)
6611 * OPTME: consider handling that special case specially or at least skipping
6612 * the call to scalbn in the helpers for zero shifts.
6613 */
handle_fpfpcvt(DisasContext * s,int rd,int rn,int opcode,bool itof,int rmode,int scale,int sf,int type)6614 static void handle_fpfpcvt(DisasContext *s, int rd, int rn, int opcode,
6615 bool itof, int rmode, int scale, int sf, int type)
6616 {
6617 bool is_signed = !(opcode & 1);
6618 TCGv_ptr tcg_fpstatus;
6619 TCGv_i32 tcg_shift, tcg_single;
6620 TCGv_i64 tcg_double;
6621
6622 tcg_fpstatus = fpstatus_ptr(type == 3 ? FPST_FPCR_F16 : FPST_FPCR);
6623
6624 tcg_shift = tcg_constant_i32(64 - scale);
6625
6626 if (itof) {
6627 TCGv_i64 tcg_int = cpu_reg(s, rn);
6628 if (!sf) {
6629 TCGv_i64 tcg_extend = tcg_temp_new_i64();
6630
6631 if (is_signed) {
6632 tcg_gen_ext32s_i64(tcg_extend, tcg_int);
6633 } else {
6634 tcg_gen_ext32u_i64(tcg_extend, tcg_int);
6635 }
6636
6637 tcg_int = tcg_extend;
6638 }
6639
6640 switch (type) {
6641 case 1: /* float64 */
6642 tcg_double = tcg_temp_new_i64();
6643 if (is_signed) {
6644 gen_helper_vfp_sqtod(tcg_double, tcg_int,
6645 tcg_shift, tcg_fpstatus);
6646 } else {
6647 gen_helper_vfp_uqtod(tcg_double, tcg_int,
6648 tcg_shift, tcg_fpstatus);
6649 }
6650 write_fp_dreg(s, rd, tcg_double);
6651 break;
6652
6653 case 0: /* float32 */
6654 tcg_single = tcg_temp_new_i32();
6655 if (is_signed) {
6656 gen_helper_vfp_sqtos(tcg_single, tcg_int,
6657 tcg_shift, tcg_fpstatus);
6658 } else {
6659 gen_helper_vfp_uqtos(tcg_single, tcg_int,
6660 tcg_shift, tcg_fpstatus);
6661 }
6662 write_fp_sreg(s, rd, tcg_single);
6663 break;
6664
6665 case 3: /* float16 */
6666 tcg_single = tcg_temp_new_i32();
6667 if (is_signed) {
6668 gen_helper_vfp_sqtoh(tcg_single, tcg_int,
6669 tcg_shift, tcg_fpstatus);
6670 } else {
6671 gen_helper_vfp_uqtoh(tcg_single, tcg_int,
6672 tcg_shift, tcg_fpstatus);
6673 }
6674 write_fp_sreg(s, rd, tcg_single);
6675 break;
6676
6677 default:
6678 g_assert_not_reached();
6679 }
6680 } else {
6681 TCGv_i64 tcg_int = cpu_reg(s, rd);
6682 TCGv_i32 tcg_rmode;
6683
6684 if (extract32(opcode, 2, 1)) {
6685 /* There are too many rounding modes to all fit into rmode,
6686 * so FCVTA[US] is a special case.
6687 */
6688 rmode = FPROUNDING_TIEAWAY;
6689 }
6690
6691 tcg_rmode = gen_set_rmode(rmode, tcg_fpstatus);
6692
6693 switch (type) {
6694 case 1: /* float64 */
6695 tcg_double = read_fp_dreg(s, rn);
6696 if (is_signed) {
6697 if (!sf) {
6698 gen_helper_vfp_tosld(tcg_int, tcg_double,
6699 tcg_shift, tcg_fpstatus);
6700 } else {
6701 gen_helper_vfp_tosqd(tcg_int, tcg_double,
6702 tcg_shift, tcg_fpstatus);
6703 }
6704 } else {
6705 if (!sf) {
6706 gen_helper_vfp_tould(tcg_int, tcg_double,
6707 tcg_shift, tcg_fpstatus);
6708 } else {
6709 gen_helper_vfp_touqd(tcg_int, tcg_double,
6710 tcg_shift, tcg_fpstatus);
6711 }
6712 }
6713 if (!sf) {
6714 tcg_gen_ext32u_i64(tcg_int, tcg_int);
6715 }
6716 break;
6717
6718 case 0: /* float32 */
6719 tcg_single = read_fp_sreg(s, rn);
6720 if (sf) {
6721 if (is_signed) {
6722 gen_helper_vfp_tosqs(tcg_int, tcg_single,
6723 tcg_shift, tcg_fpstatus);
6724 } else {
6725 gen_helper_vfp_touqs(tcg_int, tcg_single,
6726 tcg_shift, tcg_fpstatus);
6727 }
6728 } else {
6729 TCGv_i32 tcg_dest = tcg_temp_new_i32();
6730 if (is_signed) {
6731 gen_helper_vfp_tosls(tcg_dest, tcg_single,
6732 tcg_shift, tcg_fpstatus);
6733 } else {
6734 gen_helper_vfp_touls(tcg_dest, tcg_single,
6735 tcg_shift, tcg_fpstatus);
6736 }
6737 tcg_gen_extu_i32_i64(tcg_int, tcg_dest);
6738 }
6739 break;
6740
6741 case 3: /* float16 */
6742 tcg_single = read_fp_sreg(s, rn);
6743 if (sf) {
6744 if (is_signed) {
6745 gen_helper_vfp_tosqh(tcg_int, tcg_single,
6746 tcg_shift, tcg_fpstatus);
6747 } else {
6748 gen_helper_vfp_touqh(tcg_int, tcg_single,
6749 tcg_shift, tcg_fpstatus);
6750 }
6751 } else {
6752 TCGv_i32 tcg_dest = tcg_temp_new_i32();
6753 if (is_signed) {
6754 gen_helper_vfp_toslh(tcg_dest, tcg_single,
6755 tcg_shift, tcg_fpstatus);
6756 } else {
6757 gen_helper_vfp_toulh(tcg_dest, tcg_single,
6758 tcg_shift, tcg_fpstatus);
6759 }
6760 tcg_gen_extu_i32_i64(tcg_int, tcg_dest);
6761 }
6762 break;
6763
6764 default:
6765 g_assert_not_reached();
6766 }
6767
6768 gen_restore_rmode(tcg_rmode, tcg_fpstatus);
6769 }
6770 }
6771
6772 /* Floating point <-> fixed point conversions
6773 * 31 30 29 28 24 23 22 21 20 19 18 16 15 10 9 5 4 0
6774 * +----+---+---+-----------+------+---+-------+--------+-------+------+------+
6775 * | sf | 0 | S | 1 1 1 1 0 | type | 0 | rmode | opcode | scale | Rn | Rd |
6776 * +----+---+---+-----------+------+---+-------+--------+-------+------+------+
6777 */
disas_fp_fixed_conv(DisasContext * s,uint32_t insn)6778 static void disas_fp_fixed_conv(DisasContext *s, uint32_t insn)
6779 {
6780 int rd = extract32(insn, 0, 5);
6781 int rn = extract32(insn, 5, 5);
6782 int scale = extract32(insn, 10, 6);
6783 int opcode = extract32(insn, 16, 3);
6784 int rmode = extract32(insn, 19, 2);
6785 int type = extract32(insn, 22, 2);
6786 bool sbit = extract32(insn, 29, 1);
6787 bool sf = extract32(insn, 31, 1);
6788 bool itof;
6789
6790 if (sbit || (!sf && scale < 32)) {
6791 unallocated_encoding(s);
6792 return;
6793 }
6794
6795 switch (type) {
6796 case 0: /* float32 */
6797 case 1: /* float64 */
6798 break;
6799 case 3: /* float16 */
6800 if (dc_isar_feature(aa64_fp16, s)) {
6801 break;
6802 }
6803 /* fallthru */
6804 default:
6805 unallocated_encoding(s);
6806 return;
6807 }
6808
6809 switch ((rmode << 3) | opcode) {
6810 case 0x2: /* SCVTF */
6811 case 0x3: /* UCVTF */
6812 itof = true;
6813 break;
6814 case 0x18: /* FCVTZS */
6815 case 0x19: /* FCVTZU */
6816 itof = false;
6817 break;
6818 default:
6819 unallocated_encoding(s);
6820 return;
6821 }
6822
6823 if (!fp_access_check(s)) {
6824 return;
6825 }
6826
6827 handle_fpfpcvt(s, rd, rn, opcode, itof, FPROUNDING_ZERO, scale, sf, type);
6828 }
6829
handle_fmov(DisasContext * s,int rd,int rn,int type,bool itof)6830 static void handle_fmov(DisasContext *s, int rd, int rn, int type, bool itof)
6831 {
6832 /* FMOV: gpr to or from float, double, or top half of quad fp reg,
6833 * without conversion.
6834 */
6835
6836 if (itof) {
6837 TCGv_i64 tcg_rn = cpu_reg(s, rn);
6838 TCGv_i64 tmp;
6839
6840 switch (type) {
6841 case 0:
6842 /* 32 bit */
6843 tmp = tcg_temp_new_i64();
6844 tcg_gen_ext32u_i64(tmp, tcg_rn);
6845 write_fp_dreg(s, rd, tmp);
6846 break;
6847 case 1:
6848 /* 64 bit */
6849 write_fp_dreg(s, rd, tcg_rn);
6850 break;
6851 case 2:
6852 /* 64 bit to top half. */
6853 tcg_gen_st_i64(tcg_rn, tcg_env, fp_reg_hi_offset(s, rd));
6854 clear_vec_high(s, true, rd);
6855 break;
6856 case 3:
6857 /* 16 bit */
6858 tmp = tcg_temp_new_i64();
6859 tcg_gen_ext16u_i64(tmp, tcg_rn);
6860 write_fp_dreg(s, rd, tmp);
6861 break;
6862 default:
6863 g_assert_not_reached();
6864 }
6865 } else {
6866 TCGv_i64 tcg_rd = cpu_reg(s, rd);
6867
6868 switch (type) {
6869 case 0:
6870 /* 32 bit */
6871 tcg_gen_ld32u_i64(tcg_rd, tcg_env, fp_reg_offset(s, rn, MO_32));
6872 break;
6873 case 1:
6874 /* 64 bit */
6875 tcg_gen_ld_i64(tcg_rd, tcg_env, fp_reg_offset(s, rn, MO_64));
6876 break;
6877 case 2:
6878 /* 64 bits from top half */
6879 tcg_gen_ld_i64(tcg_rd, tcg_env, fp_reg_hi_offset(s, rn));
6880 break;
6881 case 3:
6882 /* 16 bit */
6883 tcg_gen_ld16u_i64(tcg_rd, tcg_env, fp_reg_offset(s, rn, MO_16));
6884 break;
6885 default:
6886 g_assert_not_reached();
6887 }
6888 }
6889 }
6890
handle_fjcvtzs(DisasContext * s,int rd,int rn)6891 static void handle_fjcvtzs(DisasContext *s, int rd, int rn)
6892 {
6893 TCGv_i64 t = read_fp_dreg(s, rn);
6894 TCGv_ptr fpstatus = fpstatus_ptr(FPST_FPCR);
6895
6896 gen_helper_fjcvtzs(t, t, fpstatus);
6897
6898 tcg_gen_ext32u_i64(cpu_reg(s, rd), t);
6899 tcg_gen_extrh_i64_i32(cpu_ZF, t);
6900 tcg_gen_movi_i32(cpu_CF, 0);
6901 tcg_gen_movi_i32(cpu_NF, 0);
6902 tcg_gen_movi_i32(cpu_VF, 0);
6903 }
6904
6905 /* Floating point <-> integer conversions
6906 * 31 30 29 28 24 23 22 21 20 19 18 16 15 10 9 5 4 0
6907 * +----+---+---+-----------+------+---+-------+-----+-------------+----+----+
6908 * | sf | 0 | S | 1 1 1 1 0 | type | 1 | rmode | opc | 0 0 0 0 0 0 | Rn | Rd |
6909 * +----+---+---+-----------+------+---+-------+-----+-------------+----+----+
6910 */
disas_fp_int_conv(DisasContext * s,uint32_t insn)6911 static void disas_fp_int_conv(DisasContext *s, uint32_t insn)
6912 {
6913 int rd = extract32(insn, 0, 5);
6914 int rn = extract32(insn, 5, 5);
6915 int opcode = extract32(insn, 16, 3);
6916 int rmode = extract32(insn, 19, 2);
6917 int type = extract32(insn, 22, 2);
6918 bool sbit = extract32(insn, 29, 1);
6919 bool sf = extract32(insn, 31, 1);
6920 bool itof = false;
6921
6922 if (sbit) {
6923 goto do_unallocated;
6924 }
6925
6926 switch (opcode) {
6927 case 2: /* SCVTF */
6928 case 3: /* UCVTF */
6929 itof = true;
6930 /* fallthru */
6931 case 4: /* FCVTAS */
6932 case 5: /* FCVTAU */
6933 if (rmode != 0) {
6934 goto do_unallocated;
6935 }
6936 /* fallthru */
6937 case 0: /* FCVT[NPMZ]S */
6938 case 1: /* FCVT[NPMZ]U */
6939 switch (type) {
6940 case 0: /* float32 */
6941 case 1: /* float64 */
6942 break;
6943 case 3: /* float16 */
6944 if (!dc_isar_feature(aa64_fp16, s)) {
6945 goto do_unallocated;
6946 }
6947 break;
6948 default:
6949 goto do_unallocated;
6950 }
6951 if (!fp_access_check(s)) {
6952 return;
6953 }
6954 handle_fpfpcvt(s, rd, rn, opcode, itof, rmode, 64, sf, type);
6955 break;
6956
6957 default:
6958 switch (sf << 7 | type << 5 | rmode << 3 | opcode) {
6959 case 0b01100110: /* FMOV half <-> 32-bit int */
6960 case 0b01100111:
6961 case 0b11100110: /* FMOV half <-> 64-bit int */
6962 case 0b11100111:
6963 if (!dc_isar_feature(aa64_fp16, s)) {
6964 goto do_unallocated;
6965 }
6966 /* fallthru */
6967 case 0b00000110: /* FMOV 32-bit */
6968 case 0b00000111:
6969 case 0b10100110: /* FMOV 64-bit */
6970 case 0b10100111:
6971 case 0b11001110: /* FMOV top half of 128-bit */
6972 case 0b11001111:
6973 if (!fp_access_check(s)) {
6974 return;
6975 }
6976 itof = opcode & 1;
6977 handle_fmov(s, rd, rn, type, itof);
6978 break;
6979
6980 case 0b00111110: /* FJCVTZS */
6981 if (!dc_isar_feature(aa64_jscvt, s)) {
6982 goto do_unallocated;
6983 } else if (fp_access_check(s)) {
6984 handle_fjcvtzs(s, rd, rn);
6985 }
6986 break;
6987
6988 default:
6989 do_unallocated:
6990 unallocated_encoding(s);
6991 return;
6992 }
6993 break;
6994 }
6995 }
6996
6997 /* FP-specific subcases of table C3-6 (SIMD and FP data processing)
6998 * 31 30 29 28 25 24 0
6999 * +---+---+---+---------+-----------------------------+
7000 * | | 0 | | 1 1 1 1 | |
7001 * +---+---+---+---------+-----------------------------+
7002 */
disas_data_proc_fp(DisasContext * s,uint32_t insn)7003 static void disas_data_proc_fp(DisasContext *s, uint32_t insn)
7004 {
7005 if (extract32(insn, 24, 1)) {
7006 /* Floating point data-processing (3 source) */
7007 disas_fp_3src(s, insn);
7008 } else if (extract32(insn, 21, 1) == 0) {
7009 /* Floating point to fixed point conversions */
7010 disas_fp_fixed_conv(s, insn);
7011 } else {
7012 switch (extract32(insn, 10, 2)) {
7013 case 1:
7014 /* Floating point conditional compare */
7015 disas_fp_ccomp(s, insn);
7016 break;
7017 case 2:
7018 /* Floating point data-processing (2 source) */
7019 disas_fp_2src(s, insn);
7020 break;
7021 case 3:
7022 /* Floating point conditional select */
7023 disas_fp_csel(s, insn);
7024 break;
7025 case 0:
7026 switch (ctz32(extract32(insn, 12, 4))) {
7027 case 0: /* [15:12] == xxx1 */
7028 /* Floating point immediate */
7029 disas_fp_imm(s, insn);
7030 break;
7031 case 1: /* [15:12] == xx10 */
7032 /* Floating point compare */
7033 disas_fp_compare(s, insn);
7034 break;
7035 case 2: /* [15:12] == x100 */
7036 /* Floating point data-processing (1 source) */
7037 disas_fp_1src(s, insn);
7038 break;
7039 case 3: /* [15:12] == 1000 */
7040 unallocated_encoding(s);
7041 break;
7042 default: /* [15:12] == 0000 */
7043 /* Floating point <-> integer conversions */
7044 disas_fp_int_conv(s, insn);
7045 break;
7046 }
7047 break;
7048 }
7049 }
7050 }
7051
do_ext64(DisasContext * s,TCGv_i64 tcg_left,TCGv_i64 tcg_right,int pos)7052 static void do_ext64(DisasContext *s, TCGv_i64 tcg_left, TCGv_i64 tcg_right,
7053 int pos)
7054 {
7055 /* Extract 64 bits from the middle of two concatenated 64 bit
7056 * vector register slices left:right. The extracted bits start
7057 * at 'pos' bits into the right (least significant) side.
7058 * We return the result in tcg_right, and guarantee not to
7059 * trash tcg_left.
7060 */
7061 TCGv_i64 tcg_tmp = tcg_temp_new_i64();
7062 assert(pos > 0 && pos < 64);
7063
7064 tcg_gen_shri_i64(tcg_right, tcg_right, pos);
7065 tcg_gen_shli_i64(tcg_tmp, tcg_left, 64 - pos);
7066 tcg_gen_or_i64(tcg_right, tcg_right, tcg_tmp);
7067 }
7068
7069 /* EXT
7070 * 31 30 29 24 23 22 21 20 16 15 14 11 10 9 5 4 0
7071 * +---+---+-------------+-----+---+------+---+------+---+------+------+
7072 * | 0 | Q | 1 0 1 1 1 0 | op2 | 0 | Rm | 0 | imm4 | 0 | Rn | Rd |
7073 * +---+---+-------------+-----+---+------+---+------+---+------+------+
7074 */
disas_simd_ext(DisasContext * s,uint32_t insn)7075 static void disas_simd_ext(DisasContext *s, uint32_t insn)
7076 {
7077 int is_q = extract32(insn, 30, 1);
7078 int op2 = extract32(insn, 22, 2);
7079 int imm4 = extract32(insn, 11, 4);
7080 int rm = extract32(insn, 16, 5);
7081 int rn = extract32(insn, 5, 5);
7082 int rd = extract32(insn, 0, 5);
7083 int pos = imm4 << 3;
7084 TCGv_i64 tcg_resl, tcg_resh;
7085
7086 if (op2 != 0 || (!is_q && extract32(imm4, 3, 1))) {
7087 unallocated_encoding(s);
7088 return;
7089 }
7090
7091 if (!fp_access_check(s)) {
7092 return;
7093 }
7094
7095 tcg_resh = tcg_temp_new_i64();
7096 tcg_resl = tcg_temp_new_i64();
7097
7098 /* Vd gets bits starting at pos bits into Vm:Vn. This is
7099 * either extracting 128 bits from a 128:128 concatenation, or
7100 * extracting 64 bits from a 64:64 concatenation.
7101 */
7102 if (!is_q) {
7103 read_vec_element(s, tcg_resl, rn, 0, MO_64);
7104 if (pos != 0) {
7105 read_vec_element(s, tcg_resh, rm, 0, MO_64);
7106 do_ext64(s, tcg_resh, tcg_resl, pos);
7107 }
7108 } else {
7109 TCGv_i64 tcg_hh;
7110 typedef struct {
7111 int reg;
7112 int elt;
7113 } EltPosns;
7114 EltPosns eltposns[] = { {rn, 0}, {rn, 1}, {rm, 0}, {rm, 1} };
7115 EltPosns *elt = eltposns;
7116
7117 if (pos >= 64) {
7118 elt++;
7119 pos -= 64;
7120 }
7121
7122 read_vec_element(s, tcg_resl, elt->reg, elt->elt, MO_64);
7123 elt++;
7124 read_vec_element(s, tcg_resh, elt->reg, elt->elt, MO_64);
7125 elt++;
7126 if (pos != 0) {
7127 do_ext64(s, tcg_resh, tcg_resl, pos);
7128 tcg_hh = tcg_temp_new_i64();
7129 read_vec_element(s, tcg_hh, elt->reg, elt->elt, MO_64);
7130 do_ext64(s, tcg_hh, tcg_resh, pos);
7131 }
7132 }
7133
7134 write_vec_element(s, tcg_resl, rd, 0, MO_64);
7135 if (is_q) {
7136 write_vec_element(s, tcg_resh, rd, 1, MO_64);
7137 }
7138 clear_vec_high(s, is_q, rd);
7139 }
7140
7141 /* TBL/TBX
7142 * 31 30 29 24 23 22 21 20 16 15 14 13 12 11 10 9 5 4 0
7143 * +---+---+-------------+-----+---+------+---+-----+----+-----+------+------+
7144 * | 0 | Q | 0 0 1 1 1 0 | op2 | 0 | Rm | 0 | len | op | 0 0 | Rn | Rd |
7145 * +---+---+-------------+-----+---+------+---+-----+----+-----+------+------+
7146 */
disas_simd_tb(DisasContext * s,uint32_t insn)7147 static void disas_simd_tb(DisasContext *s, uint32_t insn)
7148 {
7149 int op2 = extract32(insn, 22, 2);
7150 int is_q = extract32(insn, 30, 1);
7151 int rm = extract32(insn, 16, 5);
7152 int rn = extract32(insn, 5, 5);
7153 int rd = extract32(insn, 0, 5);
7154 int is_tbx = extract32(insn, 12, 1);
7155 int len = (extract32(insn, 13, 2) + 1) * 16;
7156
7157 if (op2 != 0) {
7158 unallocated_encoding(s);
7159 return;
7160 }
7161
7162 if (!fp_access_check(s)) {
7163 return;
7164 }
7165
7166 tcg_gen_gvec_2_ptr(vec_full_reg_offset(s, rd),
7167 vec_full_reg_offset(s, rm), tcg_env,
7168 is_q ? 16 : 8, vec_full_reg_size(s),
7169 (len << 6) | (is_tbx << 5) | rn,
7170 gen_helper_simd_tblx);
7171 }
7172
7173 /* ZIP/UZP/TRN
7174 * 31 30 29 24 23 22 21 20 16 15 14 12 11 10 9 5 4 0
7175 * +---+---+-------------+------+---+------+---+------------------+------+
7176 * | 0 | Q | 0 0 1 1 1 0 | size | 0 | Rm | 0 | opc | 1 0 | Rn | Rd |
7177 * +---+---+-------------+------+---+------+---+------------------+------+
7178 */
disas_simd_zip_trn(DisasContext * s,uint32_t insn)7179 static void disas_simd_zip_trn(DisasContext *s, uint32_t insn)
7180 {
7181 int rd = extract32(insn, 0, 5);
7182 int rn = extract32(insn, 5, 5);
7183 int rm = extract32(insn, 16, 5);
7184 int size = extract32(insn, 22, 2);
7185 /* opc field bits [1:0] indicate ZIP/UZP/TRN;
7186 * bit 2 indicates 1 vs 2 variant of the insn.
7187 */
7188 int opcode = extract32(insn, 12, 2);
7189 bool part = extract32(insn, 14, 1);
7190 bool is_q = extract32(insn, 30, 1);
7191 int esize = 8 << size;
7192 int i;
7193 int datasize = is_q ? 128 : 64;
7194 int elements = datasize / esize;
7195 TCGv_i64 tcg_res[2], tcg_ele;
7196
7197 if (opcode == 0 || (size == 3 && !is_q)) {
7198 unallocated_encoding(s);
7199 return;
7200 }
7201
7202 if (!fp_access_check(s)) {
7203 return;
7204 }
7205
7206 tcg_res[0] = tcg_temp_new_i64();
7207 tcg_res[1] = is_q ? tcg_temp_new_i64() : NULL;
7208 tcg_ele = tcg_temp_new_i64();
7209
7210 for (i = 0; i < elements; i++) {
7211 int o, w;
7212
7213 switch (opcode) {
7214 case 1: /* UZP1/2 */
7215 {
7216 int midpoint = elements / 2;
7217 if (i < midpoint) {
7218 read_vec_element(s, tcg_ele, rn, 2 * i + part, size);
7219 } else {
7220 read_vec_element(s, tcg_ele, rm,
7221 2 * (i - midpoint) + part, size);
7222 }
7223 break;
7224 }
7225 case 2: /* TRN1/2 */
7226 if (i & 1) {
7227 read_vec_element(s, tcg_ele, rm, (i & ~1) + part, size);
7228 } else {
7229 read_vec_element(s, tcg_ele, rn, (i & ~1) + part, size);
7230 }
7231 break;
7232 case 3: /* ZIP1/2 */
7233 {
7234 int base = part * elements / 2;
7235 if (i & 1) {
7236 read_vec_element(s, tcg_ele, rm, base + (i >> 1), size);
7237 } else {
7238 read_vec_element(s, tcg_ele, rn, base + (i >> 1), size);
7239 }
7240 break;
7241 }
7242 default:
7243 g_assert_not_reached();
7244 }
7245
7246 w = (i * esize) / 64;
7247 o = (i * esize) % 64;
7248 if (o == 0) {
7249 tcg_gen_mov_i64(tcg_res[w], tcg_ele);
7250 } else {
7251 tcg_gen_shli_i64(tcg_ele, tcg_ele, o);
7252 tcg_gen_or_i64(tcg_res[w], tcg_res[w], tcg_ele);
7253 }
7254 }
7255
7256 for (i = 0; i <= is_q; ++i) {
7257 write_vec_element(s, tcg_res[i], rd, i, MO_64);
7258 }
7259 clear_vec_high(s, is_q, rd);
7260 }
7261
7262 /*
7263 * do_reduction_op helper
7264 *
7265 * This mirrors the Reduce() pseudocode in the ARM ARM. It is
7266 * important for correct NaN propagation that we do these
7267 * operations in exactly the order specified by the pseudocode.
7268 *
7269 * This is a recursive function, TCG temps should be freed by the
7270 * calling function once it is done with the values.
7271 */
do_reduction_op(DisasContext * s,int fpopcode,int rn,int esize,int size,int vmap,TCGv_ptr fpst)7272 static TCGv_i32 do_reduction_op(DisasContext *s, int fpopcode, int rn,
7273 int esize, int size, int vmap, TCGv_ptr fpst)
7274 {
7275 if (esize == size) {
7276 int element;
7277 MemOp msize = esize == 16 ? MO_16 : MO_32;
7278 TCGv_i32 tcg_elem;
7279
7280 /* We should have one register left here */
7281 assert(ctpop8(vmap) == 1);
7282 element = ctz32(vmap);
7283 assert(element < 8);
7284
7285 tcg_elem = tcg_temp_new_i32();
7286 read_vec_element_i32(s, tcg_elem, rn, element, msize);
7287 return tcg_elem;
7288 } else {
7289 int bits = size / 2;
7290 int shift = ctpop8(vmap) / 2;
7291 int vmap_lo = (vmap >> shift) & vmap;
7292 int vmap_hi = (vmap & ~vmap_lo);
7293 TCGv_i32 tcg_hi, tcg_lo, tcg_res;
7294
7295 tcg_hi = do_reduction_op(s, fpopcode, rn, esize, bits, vmap_hi, fpst);
7296 tcg_lo = do_reduction_op(s, fpopcode, rn, esize, bits, vmap_lo, fpst);
7297 tcg_res = tcg_temp_new_i32();
7298
7299 switch (fpopcode) {
7300 case 0x0c: /* fmaxnmv half-precision */
7301 gen_helper_advsimd_maxnumh(tcg_res, tcg_lo, tcg_hi, fpst);
7302 break;
7303 case 0x0f: /* fmaxv half-precision */
7304 gen_helper_advsimd_maxh(tcg_res, tcg_lo, tcg_hi, fpst);
7305 break;
7306 case 0x1c: /* fminnmv half-precision */
7307 gen_helper_advsimd_minnumh(tcg_res, tcg_lo, tcg_hi, fpst);
7308 break;
7309 case 0x1f: /* fminv half-precision */
7310 gen_helper_advsimd_minh(tcg_res, tcg_lo, tcg_hi, fpst);
7311 break;
7312 case 0x2c: /* fmaxnmv */
7313 gen_helper_vfp_maxnums(tcg_res, tcg_lo, tcg_hi, fpst);
7314 break;
7315 case 0x2f: /* fmaxv */
7316 gen_helper_vfp_maxs(tcg_res, tcg_lo, tcg_hi, fpst);
7317 break;
7318 case 0x3c: /* fminnmv */
7319 gen_helper_vfp_minnums(tcg_res, tcg_lo, tcg_hi, fpst);
7320 break;
7321 case 0x3f: /* fminv */
7322 gen_helper_vfp_mins(tcg_res, tcg_lo, tcg_hi, fpst);
7323 break;
7324 default:
7325 g_assert_not_reached();
7326 }
7327 return tcg_res;
7328 }
7329 }
7330
7331 /* AdvSIMD across lanes
7332 * 31 30 29 28 24 23 22 21 17 16 12 11 10 9 5 4 0
7333 * +---+---+---+-----------+------+-----------+--------+-----+------+------+
7334 * | 0 | Q | U | 0 1 1 1 0 | size | 1 1 0 0 0 | opcode | 1 0 | Rn | Rd |
7335 * +---+---+---+-----------+------+-----------+--------+-----+------+------+
7336 */
disas_simd_across_lanes(DisasContext * s,uint32_t insn)7337 static void disas_simd_across_lanes(DisasContext *s, uint32_t insn)
7338 {
7339 int rd = extract32(insn, 0, 5);
7340 int rn = extract32(insn, 5, 5);
7341 int size = extract32(insn, 22, 2);
7342 int opcode = extract32(insn, 12, 5);
7343 bool is_q = extract32(insn, 30, 1);
7344 bool is_u = extract32(insn, 29, 1);
7345 bool is_fp = false;
7346 bool is_min = false;
7347 int esize;
7348 int elements;
7349 int i;
7350 TCGv_i64 tcg_res, tcg_elt;
7351
7352 switch (opcode) {
7353 case 0x1b: /* ADDV */
7354 if (is_u) {
7355 unallocated_encoding(s);
7356 return;
7357 }
7358 /* fall through */
7359 case 0x3: /* SADDLV, UADDLV */
7360 case 0xa: /* SMAXV, UMAXV */
7361 case 0x1a: /* SMINV, UMINV */
7362 if (size == 3 || (size == 2 && !is_q)) {
7363 unallocated_encoding(s);
7364 return;
7365 }
7366 break;
7367 case 0xc: /* FMAXNMV, FMINNMV */
7368 case 0xf: /* FMAXV, FMINV */
7369 /* Bit 1 of size field encodes min vs max and the actual size
7370 * depends on the encoding of the U bit. If not set (and FP16
7371 * enabled) then we do half-precision float instead of single
7372 * precision.
7373 */
7374 is_min = extract32(size, 1, 1);
7375 is_fp = true;
7376 if (!is_u && dc_isar_feature(aa64_fp16, s)) {
7377 size = 1;
7378 } else if (!is_u || !is_q || extract32(size, 0, 1)) {
7379 unallocated_encoding(s);
7380 return;
7381 } else {
7382 size = 2;
7383 }
7384 break;
7385 default:
7386 unallocated_encoding(s);
7387 return;
7388 }
7389
7390 if (!fp_access_check(s)) {
7391 return;
7392 }
7393
7394 esize = 8 << size;
7395 elements = (is_q ? 128 : 64) / esize;
7396
7397 tcg_res = tcg_temp_new_i64();
7398 tcg_elt = tcg_temp_new_i64();
7399
7400 /* These instructions operate across all lanes of a vector
7401 * to produce a single result. We can guarantee that a 64
7402 * bit intermediate is sufficient:
7403 * + for [US]ADDLV the maximum element size is 32 bits, and
7404 * the result type is 64 bits
7405 * + for FMAX*V, FMIN*V, ADDV the intermediate type is the
7406 * same as the element size, which is 32 bits at most
7407 * For the integer operations we can choose to work at 64
7408 * or 32 bits and truncate at the end; for simplicity
7409 * we use 64 bits always. The floating point
7410 * ops do require 32 bit intermediates, though.
7411 */
7412 if (!is_fp) {
7413 read_vec_element(s, tcg_res, rn, 0, size | (is_u ? 0 : MO_SIGN));
7414
7415 for (i = 1; i < elements; i++) {
7416 read_vec_element(s, tcg_elt, rn, i, size | (is_u ? 0 : MO_SIGN));
7417
7418 switch (opcode) {
7419 case 0x03: /* SADDLV / UADDLV */
7420 case 0x1b: /* ADDV */
7421 tcg_gen_add_i64(tcg_res, tcg_res, tcg_elt);
7422 break;
7423 case 0x0a: /* SMAXV / UMAXV */
7424 if (is_u) {
7425 tcg_gen_umax_i64(tcg_res, tcg_res, tcg_elt);
7426 } else {
7427 tcg_gen_smax_i64(tcg_res, tcg_res, tcg_elt);
7428 }
7429 break;
7430 case 0x1a: /* SMINV / UMINV */
7431 if (is_u) {
7432 tcg_gen_umin_i64(tcg_res, tcg_res, tcg_elt);
7433 } else {
7434 tcg_gen_smin_i64(tcg_res, tcg_res, tcg_elt);
7435 }
7436 break;
7437 default:
7438 g_assert_not_reached();
7439 }
7440
7441 }
7442 } else {
7443 /* Floating point vector reduction ops which work across 32
7444 * bit (single) or 16 bit (half-precision) intermediates.
7445 * Note that correct NaN propagation requires that we do these
7446 * operations in exactly the order specified by the pseudocode.
7447 */
7448 TCGv_ptr fpst = fpstatus_ptr(size == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
7449 int fpopcode = opcode | is_min << 4 | is_u << 5;
7450 int vmap = (1 << elements) - 1;
7451 TCGv_i32 tcg_res32 = do_reduction_op(s, fpopcode, rn, esize,
7452 (is_q ? 128 : 64), vmap, fpst);
7453 tcg_gen_extu_i32_i64(tcg_res, tcg_res32);
7454 }
7455
7456 /* Now truncate the result to the width required for the final output */
7457 if (opcode == 0x03) {
7458 /* SADDLV, UADDLV: result is 2*esize */
7459 size++;
7460 }
7461
7462 switch (size) {
7463 case 0:
7464 tcg_gen_ext8u_i64(tcg_res, tcg_res);
7465 break;
7466 case 1:
7467 tcg_gen_ext16u_i64(tcg_res, tcg_res);
7468 break;
7469 case 2:
7470 tcg_gen_ext32u_i64(tcg_res, tcg_res);
7471 break;
7472 case 3:
7473 break;
7474 default:
7475 g_assert_not_reached();
7476 }
7477
7478 write_fp_dreg(s, rd, tcg_res);
7479 }
7480
7481 /* DUP (Element, Vector)
7482 *
7483 * 31 30 29 21 20 16 15 10 9 5 4 0
7484 * +---+---+-------------------+--------+-------------+------+------+
7485 * | 0 | Q | 0 0 1 1 1 0 0 0 0 | imm5 | 0 0 0 0 0 1 | Rn | Rd |
7486 * +---+---+-------------------+--------+-------------+------+------+
7487 *
7488 * size: encoded in imm5 (see ARM ARM LowestSetBit())
7489 */
handle_simd_dupe(DisasContext * s,int is_q,int rd,int rn,int imm5)7490 static void handle_simd_dupe(DisasContext *s, int is_q, int rd, int rn,
7491 int imm5)
7492 {
7493 int size = ctz32(imm5);
7494 int index;
7495
7496 if (size > 3 || (size == 3 && !is_q)) {
7497 unallocated_encoding(s);
7498 return;
7499 }
7500
7501 if (!fp_access_check(s)) {
7502 return;
7503 }
7504
7505 index = imm5 >> (size + 1);
7506 tcg_gen_gvec_dup_mem(size, vec_full_reg_offset(s, rd),
7507 vec_reg_offset(s, rn, index, size),
7508 is_q ? 16 : 8, vec_full_reg_size(s));
7509 }
7510
7511 /* DUP (element, scalar)
7512 * 31 21 20 16 15 10 9 5 4 0
7513 * +-----------------------+--------+-------------+------+------+
7514 * | 0 1 0 1 1 1 1 0 0 0 0 | imm5 | 0 0 0 0 0 1 | Rn | Rd |
7515 * +-----------------------+--------+-------------+------+------+
7516 */
handle_simd_dupes(DisasContext * s,int rd,int rn,int imm5)7517 static void handle_simd_dupes(DisasContext *s, int rd, int rn,
7518 int imm5)
7519 {
7520 int size = ctz32(imm5);
7521 int index;
7522 TCGv_i64 tmp;
7523
7524 if (size > 3) {
7525 unallocated_encoding(s);
7526 return;
7527 }
7528
7529 if (!fp_access_check(s)) {
7530 return;
7531 }
7532
7533 index = imm5 >> (size + 1);
7534
7535 /* This instruction just extracts the specified element and
7536 * zero-extends it into the bottom of the destination register.
7537 */
7538 tmp = tcg_temp_new_i64();
7539 read_vec_element(s, tmp, rn, index, size);
7540 write_fp_dreg(s, rd, tmp);
7541 }
7542
7543 /* DUP (General)
7544 *
7545 * 31 30 29 21 20 16 15 10 9 5 4 0
7546 * +---+---+-------------------+--------+-------------+------+------+
7547 * | 0 | Q | 0 0 1 1 1 0 0 0 0 | imm5 | 0 0 0 0 1 1 | Rn | Rd |
7548 * +---+---+-------------------+--------+-------------+------+------+
7549 *
7550 * size: encoded in imm5 (see ARM ARM LowestSetBit())
7551 */
handle_simd_dupg(DisasContext * s,int is_q,int rd,int rn,int imm5)7552 static void handle_simd_dupg(DisasContext *s, int is_q, int rd, int rn,
7553 int imm5)
7554 {
7555 int size = ctz32(imm5);
7556 uint32_t dofs, oprsz, maxsz;
7557
7558 if (size > 3 || ((size == 3) && !is_q)) {
7559 unallocated_encoding(s);
7560 return;
7561 }
7562
7563 if (!fp_access_check(s)) {
7564 return;
7565 }
7566
7567 dofs = vec_full_reg_offset(s, rd);
7568 oprsz = is_q ? 16 : 8;
7569 maxsz = vec_full_reg_size(s);
7570
7571 tcg_gen_gvec_dup_i64(size, dofs, oprsz, maxsz, cpu_reg(s, rn));
7572 }
7573
7574 /* INS (Element)
7575 *
7576 * 31 21 20 16 15 14 11 10 9 5 4 0
7577 * +-----------------------+--------+------------+---+------+------+
7578 * | 0 1 1 0 1 1 1 0 0 0 0 | imm5 | 0 | imm4 | 1 | Rn | Rd |
7579 * +-----------------------+--------+------------+---+------+------+
7580 *
7581 * size: encoded in imm5 (see ARM ARM LowestSetBit())
7582 * index: encoded in imm5<4:size+1>
7583 */
handle_simd_inse(DisasContext * s,int rd,int rn,int imm4,int imm5)7584 static void handle_simd_inse(DisasContext *s, int rd, int rn,
7585 int imm4, int imm5)
7586 {
7587 int size = ctz32(imm5);
7588 int src_index, dst_index;
7589 TCGv_i64 tmp;
7590
7591 if (size > 3) {
7592 unallocated_encoding(s);
7593 return;
7594 }
7595
7596 if (!fp_access_check(s)) {
7597 return;
7598 }
7599
7600 dst_index = extract32(imm5, 1+size, 5);
7601 src_index = extract32(imm4, size, 4);
7602
7603 tmp = tcg_temp_new_i64();
7604
7605 read_vec_element(s, tmp, rn, src_index, size);
7606 write_vec_element(s, tmp, rd, dst_index, size);
7607
7608 /* INS is considered a 128-bit write for SVE. */
7609 clear_vec_high(s, true, rd);
7610 }
7611
7612
7613 /* INS (General)
7614 *
7615 * 31 21 20 16 15 10 9 5 4 0
7616 * +-----------------------+--------+-------------+------+------+
7617 * | 0 1 0 0 1 1 1 0 0 0 0 | imm5 | 0 0 0 1 1 1 | Rn | Rd |
7618 * +-----------------------+--------+-------------+------+------+
7619 *
7620 * size: encoded in imm5 (see ARM ARM LowestSetBit())
7621 * index: encoded in imm5<4:size+1>
7622 */
handle_simd_insg(DisasContext * s,int rd,int rn,int imm5)7623 static void handle_simd_insg(DisasContext *s, int rd, int rn, int imm5)
7624 {
7625 int size = ctz32(imm5);
7626 int idx;
7627
7628 if (size > 3) {
7629 unallocated_encoding(s);
7630 return;
7631 }
7632
7633 if (!fp_access_check(s)) {
7634 return;
7635 }
7636
7637 idx = extract32(imm5, 1 + size, 4 - size);
7638 write_vec_element(s, cpu_reg(s, rn), rd, idx, size);
7639
7640 /* INS is considered a 128-bit write for SVE. */
7641 clear_vec_high(s, true, rd);
7642 }
7643
7644 /*
7645 * UMOV (General)
7646 * SMOV (General)
7647 *
7648 * 31 30 29 21 20 16 15 12 10 9 5 4 0
7649 * +---+---+-------------------+--------+-------------+------+------+
7650 * | 0 | Q | 0 0 1 1 1 0 0 0 0 | imm5 | 0 0 1 U 1 1 | Rn | Rd |
7651 * +---+---+-------------------+--------+-------------+------+------+
7652 *
7653 * U: unsigned when set
7654 * size: encoded in imm5 (see ARM ARM LowestSetBit())
7655 */
handle_simd_umov_smov(DisasContext * s,int is_q,int is_signed,int rn,int rd,int imm5)7656 static void handle_simd_umov_smov(DisasContext *s, int is_q, int is_signed,
7657 int rn, int rd, int imm5)
7658 {
7659 int size = ctz32(imm5);
7660 int element;
7661 TCGv_i64 tcg_rd;
7662
7663 /* Check for UnallocatedEncodings */
7664 if (is_signed) {
7665 if (size > 2 || (size == 2 && !is_q)) {
7666 unallocated_encoding(s);
7667 return;
7668 }
7669 } else {
7670 if (size > 3
7671 || (size < 3 && is_q)
7672 || (size == 3 && !is_q)) {
7673 unallocated_encoding(s);
7674 return;
7675 }
7676 }
7677
7678 if (!fp_access_check(s)) {
7679 return;
7680 }
7681
7682 element = extract32(imm5, 1+size, 4);
7683
7684 tcg_rd = cpu_reg(s, rd);
7685 read_vec_element(s, tcg_rd, rn, element, size | (is_signed ? MO_SIGN : 0));
7686 if (is_signed && !is_q) {
7687 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
7688 }
7689 }
7690
7691 /* AdvSIMD copy
7692 * 31 30 29 28 21 20 16 15 14 11 10 9 5 4 0
7693 * +---+---+----+-----------------+------+---+------+---+------+------+
7694 * | 0 | Q | op | 0 1 1 1 0 0 0 0 | imm5 | 0 | imm4 | 1 | Rn | Rd |
7695 * +---+---+----+-----------------+------+---+------+---+------+------+
7696 */
disas_simd_copy(DisasContext * s,uint32_t insn)7697 static void disas_simd_copy(DisasContext *s, uint32_t insn)
7698 {
7699 int rd = extract32(insn, 0, 5);
7700 int rn = extract32(insn, 5, 5);
7701 int imm4 = extract32(insn, 11, 4);
7702 int op = extract32(insn, 29, 1);
7703 int is_q = extract32(insn, 30, 1);
7704 int imm5 = extract32(insn, 16, 5);
7705
7706 if (op) {
7707 if (is_q) {
7708 /* INS (element) */
7709 handle_simd_inse(s, rd, rn, imm4, imm5);
7710 } else {
7711 unallocated_encoding(s);
7712 }
7713 } else {
7714 switch (imm4) {
7715 case 0:
7716 /* DUP (element - vector) */
7717 handle_simd_dupe(s, is_q, rd, rn, imm5);
7718 break;
7719 case 1:
7720 /* DUP (general) */
7721 handle_simd_dupg(s, is_q, rd, rn, imm5);
7722 break;
7723 case 3:
7724 if (is_q) {
7725 /* INS (general) */
7726 handle_simd_insg(s, rd, rn, imm5);
7727 } else {
7728 unallocated_encoding(s);
7729 }
7730 break;
7731 case 5:
7732 case 7:
7733 /* UMOV/SMOV (is_q indicates 32/64; imm4 indicates signedness) */
7734 handle_simd_umov_smov(s, is_q, (imm4 == 5), rn, rd, imm5);
7735 break;
7736 default:
7737 unallocated_encoding(s);
7738 break;
7739 }
7740 }
7741 }
7742
7743 /* AdvSIMD modified immediate
7744 * 31 30 29 28 19 18 16 15 12 11 10 9 5 4 0
7745 * +---+---+----+---------------------+-----+-------+----+---+-------+------+
7746 * | 0 | Q | op | 0 1 1 1 1 0 0 0 0 0 | abc | cmode | o2 | 1 | defgh | Rd |
7747 * +---+---+----+---------------------+-----+-------+----+---+-------+------+
7748 *
7749 * There are a number of operations that can be carried out here:
7750 * MOVI - move (shifted) imm into register
7751 * MVNI - move inverted (shifted) imm into register
7752 * ORR - bitwise OR of (shifted) imm with register
7753 * BIC - bitwise clear of (shifted) imm with register
7754 * With ARMv8.2 we also have:
7755 * FMOV half-precision
7756 */
disas_simd_mod_imm(DisasContext * s,uint32_t insn)7757 static void disas_simd_mod_imm(DisasContext *s, uint32_t insn)
7758 {
7759 int rd = extract32(insn, 0, 5);
7760 int cmode = extract32(insn, 12, 4);
7761 int o2 = extract32(insn, 11, 1);
7762 uint64_t abcdefgh = extract32(insn, 5, 5) | (extract32(insn, 16, 3) << 5);
7763 bool is_neg = extract32(insn, 29, 1);
7764 bool is_q = extract32(insn, 30, 1);
7765 uint64_t imm = 0;
7766
7767 if (o2 != 0 || ((cmode == 0xf) && is_neg && !is_q)) {
7768 /* Check for FMOV (vector, immediate) - half-precision */
7769 if (!(dc_isar_feature(aa64_fp16, s) && o2 && cmode == 0xf)) {
7770 unallocated_encoding(s);
7771 return;
7772 }
7773 }
7774
7775 if (!fp_access_check(s)) {
7776 return;
7777 }
7778
7779 if (cmode == 15 && o2 && !is_neg) {
7780 /* FMOV (vector, immediate) - half-precision */
7781 imm = vfp_expand_imm(MO_16, abcdefgh);
7782 /* now duplicate across the lanes */
7783 imm = dup_const(MO_16, imm);
7784 } else {
7785 imm = asimd_imm_const(abcdefgh, cmode, is_neg);
7786 }
7787
7788 if (!((cmode & 0x9) == 0x1 || (cmode & 0xd) == 0x9)) {
7789 /* MOVI or MVNI, with MVNI negation handled above. */
7790 tcg_gen_gvec_dup_imm(MO_64, vec_full_reg_offset(s, rd), is_q ? 16 : 8,
7791 vec_full_reg_size(s), imm);
7792 } else {
7793 /* ORR or BIC, with BIC negation to AND handled above. */
7794 if (is_neg) {
7795 gen_gvec_fn2i(s, is_q, rd, rd, imm, tcg_gen_gvec_andi, MO_64);
7796 } else {
7797 gen_gvec_fn2i(s, is_q, rd, rd, imm, tcg_gen_gvec_ori, MO_64);
7798 }
7799 }
7800 }
7801
7802 /* AdvSIMD scalar copy
7803 * 31 30 29 28 21 20 16 15 14 11 10 9 5 4 0
7804 * +-----+----+-----------------+------+---+------+---+------+------+
7805 * | 0 1 | op | 1 1 1 1 0 0 0 0 | imm5 | 0 | imm4 | 1 | Rn | Rd |
7806 * +-----+----+-----------------+------+---+------+---+------+------+
7807 */
disas_simd_scalar_copy(DisasContext * s,uint32_t insn)7808 static void disas_simd_scalar_copy(DisasContext *s, uint32_t insn)
7809 {
7810 int rd = extract32(insn, 0, 5);
7811 int rn = extract32(insn, 5, 5);
7812 int imm4 = extract32(insn, 11, 4);
7813 int imm5 = extract32(insn, 16, 5);
7814 int op = extract32(insn, 29, 1);
7815
7816 if (op != 0 || imm4 != 0) {
7817 unallocated_encoding(s);
7818 return;
7819 }
7820
7821 /* DUP (element, scalar) */
7822 handle_simd_dupes(s, rd, rn, imm5);
7823 }
7824
7825 /* AdvSIMD scalar pairwise
7826 * 31 30 29 28 24 23 22 21 17 16 12 11 10 9 5 4 0
7827 * +-----+---+-----------+------+-----------+--------+-----+------+------+
7828 * | 0 1 | U | 1 1 1 1 0 | size | 1 1 0 0 0 | opcode | 1 0 | Rn | Rd |
7829 * +-----+---+-----------+------+-----------+--------+-----+------+------+
7830 */
disas_simd_scalar_pairwise(DisasContext * s,uint32_t insn)7831 static void disas_simd_scalar_pairwise(DisasContext *s, uint32_t insn)
7832 {
7833 int u = extract32(insn, 29, 1);
7834 int size = extract32(insn, 22, 2);
7835 int opcode = extract32(insn, 12, 5);
7836 int rn = extract32(insn, 5, 5);
7837 int rd = extract32(insn, 0, 5);
7838 TCGv_ptr fpst;
7839
7840 /* For some ops (the FP ones), size[1] is part of the encoding.
7841 * For ADDP strictly it is not but size[1] is always 1 for valid
7842 * encodings.
7843 */
7844 opcode |= (extract32(size, 1, 1) << 5);
7845
7846 switch (opcode) {
7847 case 0x3b: /* ADDP */
7848 if (u || size != 3) {
7849 unallocated_encoding(s);
7850 return;
7851 }
7852 if (!fp_access_check(s)) {
7853 return;
7854 }
7855
7856 fpst = NULL;
7857 break;
7858 case 0xc: /* FMAXNMP */
7859 case 0xd: /* FADDP */
7860 case 0xf: /* FMAXP */
7861 case 0x2c: /* FMINNMP */
7862 case 0x2f: /* FMINP */
7863 /* FP op, size[0] is 32 or 64 bit*/
7864 if (!u) {
7865 if (!dc_isar_feature(aa64_fp16, s)) {
7866 unallocated_encoding(s);
7867 return;
7868 } else {
7869 size = MO_16;
7870 }
7871 } else {
7872 size = extract32(size, 0, 1) ? MO_64 : MO_32;
7873 }
7874
7875 if (!fp_access_check(s)) {
7876 return;
7877 }
7878
7879 fpst = fpstatus_ptr(size == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
7880 break;
7881 default:
7882 unallocated_encoding(s);
7883 return;
7884 }
7885
7886 if (size == MO_64) {
7887 TCGv_i64 tcg_op1 = tcg_temp_new_i64();
7888 TCGv_i64 tcg_op2 = tcg_temp_new_i64();
7889 TCGv_i64 tcg_res = tcg_temp_new_i64();
7890
7891 read_vec_element(s, tcg_op1, rn, 0, MO_64);
7892 read_vec_element(s, tcg_op2, rn, 1, MO_64);
7893
7894 switch (opcode) {
7895 case 0x3b: /* ADDP */
7896 tcg_gen_add_i64(tcg_res, tcg_op1, tcg_op2);
7897 break;
7898 case 0xc: /* FMAXNMP */
7899 gen_helper_vfp_maxnumd(tcg_res, tcg_op1, tcg_op2, fpst);
7900 break;
7901 case 0xd: /* FADDP */
7902 gen_helper_vfp_addd(tcg_res, tcg_op1, tcg_op2, fpst);
7903 break;
7904 case 0xf: /* FMAXP */
7905 gen_helper_vfp_maxd(tcg_res, tcg_op1, tcg_op2, fpst);
7906 break;
7907 case 0x2c: /* FMINNMP */
7908 gen_helper_vfp_minnumd(tcg_res, tcg_op1, tcg_op2, fpst);
7909 break;
7910 case 0x2f: /* FMINP */
7911 gen_helper_vfp_mind(tcg_res, tcg_op1, tcg_op2, fpst);
7912 break;
7913 default:
7914 g_assert_not_reached();
7915 }
7916
7917 write_fp_dreg(s, rd, tcg_res);
7918 } else {
7919 TCGv_i32 tcg_op1 = tcg_temp_new_i32();
7920 TCGv_i32 tcg_op2 = tcg_temp_new_i32();
7921 TCGv_i32 tcg_res = tcg_temp_new_i32();
7922
7923 read_vec_element_i32(s, tcg_op1, rn, 0, size);
7924 read_vec_element_i32(s, tcg_op2, rn, 1, size);
7925
7926 if (size == MO_16) {
7927 switch (opcode) {
7928 case 0xc: /* FMAXNMP */
7929 gen_helper_advsimd_maxnumh(tcg_res, tcg_op1, tcg_op2, fpst);
7930 break;
7931 case 0xd: /* FADDP */
7932 gen_helper_advsimd_addh(tcg_res, tcg_op1, tcg_op2, fpst);
7933 break;
7934 case 0xf: /* FMAXP */
7935 gen_helper_advsimd_maxh(tcg_res, tcg_op1, tcg_op2, fpst);
7936 break;
7937 case 0x2c: /* FMINNMP */
7938 gen_helper_advsimd_minnumh(tcg_res, tcg_op1, tcg_op2, fpst);
7939 break;
7940 case 0x2f: /* FMINP */
7941 gen_helper_advsimd_minh(tcg_res, tcg_op1, tcg_op2, fpst);
7942 break;
7943 default:
7944 g_assert_not_reached();
7945 }
7946 } else {
7947 switch (opcode) {
7948 case 0xc: /* FMAXNMP */
7949 gen_helper_vfp_maxnums(tcg_res, tcg_op1, tcg_op2, fpst);
7950 break;
7951 case 0xd: /* FADDP */
7952 gen_helper_vfp_adds(tcg_res, tcg_op1, tcg_op2, fpst);
7953 break;
7954 case 0xf: /* FMAXP */
7955 gen_helper_vfp_maxs(tcg_res, tcg_op1, tcg_op2, fpst);
7956 break;
7957 case 0x2c: /* FMINNMP */
7958 gen_helper_vfp_minnums(tcg_res, tcg_op1, tcg_op2, fpst);
7959 break;
7960 case 0x2f: /* FMINP */
7961 gen_helper_vfp_mins(tcg_res, tcg_op1, tcg_op2, fpst);
7962 break;
7963 default:
7964 g_assert_not_reached();
7965 }
7966 }
7967
7968 write_fp_sreg(s, rd, tcg_res);
7969 }
7970 }
7971
7972 /*
7973 * Common SSHR[RA]/USHR[RA] - Shift right (optional rounding/accumulate)
7974 *
7975 * This code is handles the common shifting code and is used by both
7976 * the vector and scalar code.
7977 */
handle_shri_with_rndacc(TCGv_i64 tcg_res,TCGv_i64 tcg_src,TCGv_i64 tcg_rnd,bool accumulate,bool is_u,int size,int shift)7978 static void handle_shri_with_rndacc(TCGv_i64 tcg_res, TCGv_i64 tcg_src,
7979 TCGv_i64 tcg_rnd, bool accumulate,
7980 bool is_u, int size, int shift)
7981 {
7982 bool extended_result = false;
7983 bool round = tcg_rnd != NULL;
7984 int ext_lshift = 0;
7985 TCGv_i64 tcg_src_hi;
7986
7987 if (round && size == 3) {
7988 extended_result = true;
7989 ext_lshift = 64 - shift;
7990 tcg_src_hi = tcg_temp_new_i64();
7991 } else if (shift == 64) {
7992 if (!accumulate && is_u) {
7993 /* result is zero */
7994 tcg_gen_movi_i64(tcg_res, 0);
7995 return;
7996 }
7997 }
7998
7999 /* Deal with the rounding step */
8000 if (round) {
8001 if (extended_result) {
8002 TCGv_i64 tcg_zero = tcg_constant_i64(0);
8003 if (!is_u) {
8004 /* take care of sign extending tcg_res */
8005 tcg_gen_sari_i64(tcg_src_hi, tcg_src, 63);
8006 tcg_gen_add2_i64(tcg_src, tcg_src_hi,
8007 tcg_src, tcg_src_hi,
8008 tcg_rnd, tcg_zero);
8009 } else {
8010 tcg_gen_add2_i64(tcg_src, tcg_src_hi,
8011 tcg_src, tcg_zero,
8012 tcg_rnd, tcg_zero);
8013 }
8014 } else {
8015 tcg_gen_add_i64(tcg_src, tcg_src, tcg_rnd);
8016 }
8017 }
8018
8019 /* Now do the shift right */
8020 if (round && extended_result) {
8021 /* extended case, >64 bit precision required */
8022 if (ext_lshift == 0) {
8023 /* special case, only high bits matter */
8024 tcg_gen_mov_i64(tcg_src, tcg_src_hi);
8025 } else {
8026 tcg_gen_shri_i64(tcg_src, tcg_src, shift);
8027 tcg_gen_shli_i64(tcg_src_hi, tcg_src_hi, ext_lshift);
8028 tcg_gen_or_i64(tcg_src, tcg_src, tcg_src_hi);
8029 }
8030 } else {
8031 if (is_u) {
8032 if (shift == 64) {
8033 /* essentially shifting in 64 zeros */
8034 tcg_gen_movi_i64(tcg_src, 0);
8035 } else {
8036 tcg_gen_shri_i64(tcg_src, tcg_src, shift);
8037 }
8038 } else {
8039 if (shift == 64) {
8040 /* effectively extending the sign-bit */
8041 tcg_gen_sari_i64(tcg_src, tcg_src, 63);
8042 } else {
8043 tcg_gen_sari_i64(tcg_src, tcg_src, shift);
8044 }
8045 }
8046 }
8047
8048 if (accumulate) {
8049 tcg_gen_add_i64(tcg_res, tcg_res, tcg_src);
8050 } else {
8051 tcg_gen_mov_i64(tcg_res, tcg_src);
8052 }
8053 }
8054
8055 /* SSHR[RA]/USHR[RA] - Scalar shift right (optional rounding/accumulate) */
handle_scalar_simd_shri(DisasContext * s,bool is_u,int immh,int immb,int opcode,int rn,int rd)8056 static void handle_scalar_simd_shri(DisasContext *s,
8057 bool is_u, int immh, int immb,
8058 int opcode, int rn, int rd)
8059 {
8060 const int size = 3;
8061 int immhb = immh << 3 | immb;
8062 int shift = 2 * (8 << size) - immhb;
8063 bool accumulate = false;
8064 bool round = false;
8065 bool insert = false;
8066 TCGv_i64 tcg_rn;
8067 TCGv_i64 tcg_rd;
8068 TCGv_i64 tcg_round;
8069
8070 if (!extract32(immh, 3, 1)) {
8071 unallocated_encoding(s);
8072 return;
8073 }
8074
8075 if (!fp_access_check(s)) {
8076 return;
8077 }
8078
8079 switch (opcode) {
8080 case 0x02: /* SSRA / USRA (accumulate) */
8081 accumulate = true;
8082 break;
8083 case 0x04: /* SRSHR / URSHR (rounding) */
8084 round = true;
8085 break;
8086 case 0x06: /* SRSRA / URSRA (accum + rounding) */
8087 accumulate = round = true;
8088 break;
8089 case 0x08: /* SRI */
8090 insert = true;
8091 break;
8092 }
8093
8094 if (round) {
8095 tcg_round = tcg_constant_i64(1ULL << (shift - 1));
8096 } else {
8097 tcg_round = NULL;
8098 }
8099
8100 tcg_rn = read_fp_dreg(s, rn);
8101 tcg_rd = (accumulate || insert) ? read_fp_dreg(s, rd) : tcg_temp_new_i64();
8102
8103 if (insert) {
8104 /* shift count same as element size is valid but does nothing;
8105 * special case to avoid potential shift by 64.
8106 */
8107 int esize = 8 << size;
8108 if (shift != esize) {
8109 tcg_gen_shri_i64(tcg_rn, tcg_rn, shift);
8110 tcg_gen_deposit_i64(tcg_rd, tcg_rd, tcg_rn, 0, esize - shift);
8111 }
8112 } else {
8113 handle_shri_with_rndacc(tcg_rd, tcg_rn, tcg_round,
8114 accumulate, is_u, size, shift);
8115 }
8116
8117 write_fp_dreg(s, rd, tcg_rd);
8118 }
8119
8120 /* SHL/SLI - Scalar shift left */
handle_scalar_simd_shli(DisasContext * s,bool insert,int immh,int immb,int opcode,int rn,int rd)8121 static void handle_scalar_simd_shli(DisasContext *s, bool insert,
8122 int immh, int immb, int opcode,
8123 int rn, int rd)
8124 {
8125 int size = 32 - clz32(immh) - 1;
8126 int immhb = immh << 3 | immb;
8127 int shift = immhb - (8 << size);
8128 TCGv_i64 tcg_rn;
8129 TCGv_i64 tcg_rd;
8130
8131 if (!extract32(immh, 3, 1)) {
8132 unallocated_encoding(s);
8133 return;
8134 }
8135
8136 if (!fp_access_check(s)) {
8137 return;
8138 }
8139
8140 tcg_rn = read_fp_dreg(s, rn);
8141 tcg_rd = insert ? read_fp_dreg(s, rd) : tcg_temp_new_i64();
8142
8143 if (insert) {
8144 tcg_gen_deposit_i64(tcg_rd, tcg_rd, tcg_rn, shift, 64 - shift);
8145 } else {
8146 tcg_gen_shli_i64(tcg_rd, tcg_rn, shift);
8147 }
8148
8149 write_fp_dreg(s, rd, tcg_rd);
8150 }
8151
8152 /* SQSHRN/SQSHRUN - Saturating (signed/unsigned) shift right with
8153 * (signed/unsigned) narrowing */
handle_vec_simd_sqshrn(DisasContext * s,bool is_scalar,bool is_q,bool is_u_shift,bool is_u_narrow,int immh,int immb,int opcode,int rn,int rd)8154 static void handle_vec_simd_sqshrn(DisasContext *s, bool is_scalar, bool is_q,
8155 bool is_u_shift, bool is_u_narrow,
8156 int immh, int immb, int opcode,
8157 int rn, int rd)
8158 {
8159 int immhb = immh << 3 | immb;
8160 int size = 32 - clz32(immh) - 1;
8161 int esize = 8 << size;
8162 int shift = (2 * esize) - immhb;
8163 int elements = is_scalar ? 1 : (64 / esize);
8164 bool round = extract32(opcode, 0, 1);
8165 MemOp ldop = (size + 1) | (is_u_shift ? 0 : MO_SIGN);
8166 TCGv_i64 tcg_rn, tcg_rd, tcg_round;
8167 TCGv_i32 tcg_rd_narrowed;
8168 TCGv_i64 tcg_final;
8169
8170 static NeonGenNarrowEnvFn * const signed_narrow_fns[4][2] = {
8171 { gen_helper_neon_narrow_sat_s8,
8172 gen_helper_neon_unarrow_sat8 },
8173 { gen_helper_neon_narrow_sat_s16,
8174 gen_helper_neon_unarrow_sat16 },
8175 { gen_helper_neon_narrow_sat_s32,
8176 gen_helper_neon_unarrow_sat32 },
8177 { NULL, NULL },
8178 };
8179 static NeonGenNarrowEnvFn * const unsigned_narrow_fns[4] = {
8180 gen_helper_neon_narrow_sat_u8,
8181 gen_helper_neon_narrow_sat_u16,
8182 gen_helper_neon_narrow_sat_u32,
8183 NULL
8184 };
8185 NeonGenNarrowEnvFn *narrowfn;
8186
8187 int i;
8188
8189 assert(size < 4);
8190
8191 if (extract32(immh, 3, 1)) {
8192 unallocated_encoding(s);
8193 return;
8194 }
8195
8196 if (!fp_access_check(s)) {
8197 return;
8198 }
8199
8200 if (is_u_shift) {
8201 narrowfn = unsigned_narrow_fns[size];
8202 } else {
8203 narrowfn = signed_narrow_fns[size][is_u_narrow ? 1 : 0];
8204 }
8205
8206 tcg_rn = tcg_temp_new_i64();
8207 tcg_rd = tcg_temp_new_i64();
8208 tcg_rd_narrowed = tcg_temp_new_i32();
8209 tcg_final = tcg_temp_new_i64();
8210
8211 if (round) {
8212 tcg_round = tcg_constant_i64(1ULL << (shift - 1));
8213 } else {
8214 tcg_round = NULL;
8215 }
8216
8217 for (i = 0; i < elements; i++) {
8218 read_vec_element(s, tcg_rn, rn, i, ldop);
8219 handle_shri_with_rndacc(tcg_rd, tcg_rn, tcg_round,
8220 false, is_u_shift, size+1, shift);
8221 narrowfn(tcg_rd_narrowed, tcg_env, tcg_rd);
8222 tcg_gen_extu_i32_i64(tcg_rd, tcg_rd_narrowed);
8223 if (i == 0) {
8224 tcg_gen_extract_i64(tcg_final, tcg_rd, 0, esize);
8225 } else {
8226 tcg_gen_deposit_i64(tcg_final, tcg_final, tcg_rd, esize * i, esize);
8227 }
8228 }
8229
8230 if (!is_q) {
8231 write_vec_element(s, tcg_final, rd, 0, MO_64);
8232 } else {
8233 write_vec_element(s, tcg_final, rd, 1, MO_64);
8234 }
8235 clear_vec_high(s, is_q, rd);
8236 }
8237
8238 /* SQSHLU, UQSHL, SQSHL: saturating left shifts */
handle_simd_qshl(DisasContext * s,bool scalar,bool is_q,bool src_unsigned,bool dst_unsigned,int immh,int immb,int rn,int rd)8239 static void handle_simd_qshl(DisasContext *s, bool scalar, bool is_q,
8240 bool src_unsigned, bool dst_unsigned,
8241 int immh, int immb, int rn, int rd)
8242 {
8243 int immhb = immh << 3 | immb;
8244 int size = 32 - clz32(immh) - 1;
8245 int shift = immhb - (8 << size);
8246 int pass;
8247
8248 assert(immh != 0);
8249 assert(!(scalar && is_q));
8250
8251 if (!scalar) {
8252 if (!is_q && extract32(immh, 3, 1)) {
8253 unallocated_encoding(s);
8254 return;
8255 }
8256
8257 /* Since we use the variable-shift helpers we must
8258 * replicate the shift count into each element of
8259 * the tcg_shift value.
8260 */
8261 switch (size) {
8262 case 0:
8263 shift |= shift << 8;
8264 /* fall through */
8265 case 1:
8266 shift |= shift << 16;
8267 break;
8268 case 2:
8269 case 3:
8270 break;
8271 default:
8272 g_assert_not_reached();
8273 }
8274 }
8275
8276 if (!fp_access_check(s)) {
8277 return;
8278 }
8279
8280 if (size == 3) {
8281 TCGv_i64 tcg_shift = tcg_constant_i64(shift);
8282 static NeonGenTwo64OpEnvFn * const fns[2][2] = {
8283 { gen_helper_neon_qshl_s64, gen_helper_neon_qshlu_s64 },
8284 { NULL, gen_helper_neon_qshl_u64 },
8285 };
8286 NeonGenTwo64OpEnvFn *genfn = fns[src_unsigned][dst_unsigned];
8287 int maxpass = is_q ? 2 : 1;
8288
8289 for (pass = 0; pass < maxpass; pass++) {
8290 TCGv_i64 tcg_op = tcg_temp_new_i64();
8291
8292 read_vec_element(s, tcg_op, rn, pass, MO_64);
8293 genfn(tcg_op, tcg_env, tcg_op, tcg_shift);
8294 write_vec_element(s, tcg_op, rd, pass, MO_64);
8295 }
8296 clear_vec_high(s, is_q, rd);
8297 } else {
8298 TCGv_i32 tcg_shift = tcg_constant_i32(shift);
8299 static NeonGenTwoOpEnvFn * const fns[2][2][3] = {
8300 {
8301 { gen_helper_neon_qshl_s8,
8302 gen_helper_neon_qshl_s16,
8303 gen_helper_neon_qshl_s32 },
8304 { gen_helper_neon_qshlu_s8,
8305 gen_helper_neon_qshlu_s16,
8306 gen_helper_neon_qshlu_s32 }
8307 }, {
8308 { NULL, NULL, NULL },
8309 { gen_helper_neon_qshl_u8,
8310 gen_helper_neon_qshl_u16,
8311 gen_helper_neon_qshl_u32 }
8312 }
8313 };
8314 NeonGenTwoOpEnvFn *genfn = fns[src_unsigned][dst_unsigned][size];
8315 MemOp memop = scalar ? size : MO_32;
8316 int maxpass = scalar ? 1 : is_q ? 4 : 2;
8317
8318 for (pass = 0; pass < maxpass; pass++) {
8319 TCGv_i32 tcg_op = tcg_temp_new_i32();
8320
8321 read_vec_element_i32(s, tcg_op, rn, pass, memop);
8322 genfn(tcg_op, tcg_env, tcg_op, tcg_shift);
8323 if (scalar) {
8324 switch (size) {
8325 case 0:
8326 tcg_gen_ext8u_i32(tcg_op, tcg_op);
8327 break;
8328 case 1:
8329 tcg_gen_ext16u_i32(tcg_op, tcg_op);
8330 break;
8331 case 2:
8332 break;
8333 default:
8334 g_assert_not_reached();
8335 }
8336 write_fp_sreg(s, rd, tcg_op);
8337 } else {
8338 write_vec_element_i32(s, tcg_op, rd, pass, MO_32);
8339 }
8340 }
8341
8342 if (!scalar) {
8343 clear_vec_high(s, is_q, rd);
8344 }
8345 }
8346 }
8347
8348 /* Common vector code for handling integer to FP conversion */
handle_simd_intfp_conv(DisasContext * s,int rd,int rn,int elements,int is_signed,int fracbits,int size)8349 static void handle_simd_intfp_conv(DisasContext *s, int rd, int rn,
8350 int elements, int is_signed,
8351 int fracbits, int size)
8352 {
8353 TCGv_ptr tcg_fpst = fpstatus_ptr(size == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
8354 TCGv_i32 tcg_shift = NULL;
8355
8356 MemOp mop = size | (is_signed ? MO_SIGN : 0);
8357 int pass;
8358
8359 if (fracbits || size == MO_64) {
8360 tcg_shift = tcg_constant_i32(fracbits);
8361 }
8362
8363 if (size == MO_64) {
8364 TCGv_i64 tcg_int64 = tcg_temp_new_i64();
8365 TCGv_i64 tcg_double = tcg_temp_new_i64();
8366
8367 for (pass = 0; pass < elements; pass++) {
8368 read_vec_element(s, tcg_int64, rn, pass, mop);
8369
8370 if (is_signed) {
8371 gen_helper_vfp_sqtod(tcg_double, tcg_int64,
8372 tcg_shift, tcg_fpst);
8373 } else {
8374 gen_helper_vfp_uqtod(tcg_double, tcg_int64,
8375 tcg_shift, tcg_fpst);
8376 }
8377 if (elements == 1) {
8378 write_fp_dreg(s, rd, tcg_double);
8379 } else {
8380 write_vec_element(s, tcg_double, rd, pass, MO_64);
8381 }
8382 }
8383 } else {
8384 TCGv_i32 tcg_int32 = tcg_temp_new_i32();
8385 TCGv_i32 tcg_float = tcg_temp_new_i32();
8386
8387 for (pass = 0; pass < elements; pass++) {
8388 read_vec_element_i32(s, tcg_int32, rn, pass, mop);
8389
8390 switch (size) {
8391 case MO_32:
8392 if (fracbits) {
8393 if (is_signed) {
8394 gen_helper_vfp_sltos(tcg_float, tcg_int32,
8395 tcg_shift, tcg_fpst);
8396 } else {
8397 gen_helper_vfp_ultos(tcg_float, tcg_int32,
8398 tcg_shift, tcg_fpst);
8399 }
8400 } else {
8401 if (is_signed) {
8402 gen_helper_vfp_sitos(tcg_float, tcg_int32, tcg_fpst);
8403 } else {
8404 gen_helper_vfp_uitos(tcg_float, tcg_int32, tcg_fpst);
8405 }
8406 }
8407 break;
8408 case MO_16:
8409 if (fracbits) {
8410 if (is_signed) {
8411 gen_helper_vfp_sltoh(tcg_float, tcg_int32,
8412 tcg_shift, tcg_fpst);
8413 } else {
8414 gen_helper_vfp_ultoh(tcg_float, tcg_int32,
8415 tcg_shift, tcg_fpst);
8416 }
8417 } else {
8418 if (is_signed) {
8419 gen_helper_vfp_sitoh(tcg_float, tcg_int32, tcg_fpst);
8420 } else {
8421 gen_helper_vfp_uitoh(tcg_float, tcg_int32, tcg_fpst);
8422 }
8423 }
8424 break;
8425 default:
8426 g_assert_not_reached();
8427 }
8428
8429 if (elements == 1) {
8430 write_fp_sreg(s, rd, tcg_float);
8431 } else {
8432 write_vec_element_i32(s, tcg_float, rd, pass, size);
8433 }
8434 }
8435 }
8436
8437 clear_vec_high(s, elements << size == 16, rd);
8438 }
8439
8440 /* UCVTF/SCVTF - Integer to FP conversion */
handle_simd_shift_intfp_conv(DisasContext * s,bool is_scalar,bool is_q,bool is_u,int immh,int immb,int opcode,int rn,int rd)8441 static void handle_simd_shift_intfp_conv(DisasContext *s, bool is_scalar,
8442 bool is_q, bool is_u,
8443 int immh, int immb, int opcode,
8444 int rn, int rd)
8445 {
8446 int size, elements, fracbits;
8447 int immhb = immh << 3 | immb;
8448
8449 if (immh & 8) {
8450 size = MO_64;
8451 if (!is_scalar && !is_q) {
8452 unallocated_encoding(s);
8453 return;
8454 }
8455 } else if (immh & 4) {
8456 size = MO_32;
8457 } else if (immh & 2) {
8458 size = MO_16;
8459 if (!dc_isar_feature(aa64_fp16, s)) {
8460 unallocated_encoding(s);
8461 return;
8462 }
8463 } else {
8464 /* immh == 0 would be a failure of the decode logic */
8465 g_assert(immh == 1);
8466 unallocated_encoding(s);
8467 return;
8468 }
8469
8470 if (is_scalar) {
8471 elements = 1;
8472 } else {
8473 elements = (8 << is_q) >> size;
8474 }
8475 fracbits = (16 << size) - immhb;
8476
8477 if (!fp_access_check(s)) {
8478 return;
8479 }
8480
8481 handle_simd_intfp_conv(s, rd, rn, elements, !is_u, fracbits, size);
8482 }
8483
8484 /* FCVTZS, FVCVTZU - FP to fixedpoint conversion */
handle_simd_shift_fpint_conv(DisasContext * s,bool is_scalar,bool is_q,bool is_u,int immh,int immb,int rn,int rd)8485 static void handle_simd_shift_fpint_conv(DisasContext *s, bool is_scalar,
8486 bool is_q, bool is_u,
8487 int immh, int immb, int rn, int rd)
8488 {
8489 int immhb = immh << 3 | immb;
8490 int pass, size, fracbits;
8491 TCGv_ptr tcg_fpstatus;
8492 TCGv_i32 tcg_rmode, tcg_shift;
8493
8494 if (immh & 0x8) {
8495 size = MO_64;
8496 if (!is_scalar && !is_q) {
8497 unallocated_encoding(s);
8498 return;
8499 }
8500 } else if (immh & 0x4) {
8501 size = MO_32;
8502 } else if (immh & 0x2) {
8503 size = MO_16;
8504 if (!dc_isar_feature(aa64_fp16, s)) {
8505 unallocated_encoding(s);
8506 return;
8507 }
8508 } else {
8509 /* Should have split out AdvSIMD modified immediate earlier. */
8510 assert(immh == 1);
8511 unallocated_encoding(s);
8512 return;
8513 }
8514
8515 if (!fp_access_check(s)) {
8516 return;
8517 }
8518
8519 assert(!(is_scalar && is_q));
8520
8521 tcg_fpstatus = fpstatus_ptr(size == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
8522 tcg_rmode = gen_set_rmode(FPROUNDING_ZERO, tcg_fpstatus);
8523 fracbits = (16 << size) - immhb;
8524 tcg_shift = tcg_constant_i32(fracbits);
8525
8526 if (size == MO_64) {
8527 int maxpass = is_scalar ? 1 : 2;
8528
8529 for (pass = 0; pass < maxpass; pass++) {
8530 TCGv_i64 tcg_op = tcg_temp_new_i64();
8531
8532 read_vec_element(s, tcg_op, rn, pass, MO_64);
8533 if (is_u) {
8534 gen_helper_vfp_touqd(tcg_op, tcg_op, tcg_shift, tcg_fpstatus);
8535 } else {
8536 gen_helper_vfp_tosqd(tcg_op, tcg_op, tcg_shift, tcg_fpstatus);
8537 }
8538 write_vec_element(s, tcg_op, rd, pass, MO_64);
8539 }
8540 clear_vec_high(s, is_q, rd);
8541 } else {
8542 void (*fn)(TCGv_i32, TCGv_i32, TCGv_i32, TCGv_ptr);
8543 int maxpass = is_scalar ? 1 : ((8 << is_q) >> size);
8544
8545 switch (size) {
8546 case MO_16:
8547 if (is_u) {
8548 fn = gen_helper_vfp_touhh;
8549 } else {
8550 fn = gen_helper_vfp_toshh;
8551 }
8552 break;
8553 case MO_32:
8554 if (is_u) {
8555 fn = gen_helper_vfp_touls;
8556 } else {
8557 fn = gen_helper_vfp_tosls;
8558 }
8559 break;
8560 default:
8561 g_assert_not_reached();
8562 }
8563
8564 for (pass = 0; pass < maxpass; pass++) {
8565 TCGv_i32 tcg_op = tcg_temp_new_i32();
8566
8567 read_vec_element_i32(s, tcg_op, rn, pass, size);
8568 fn(tcg_op, tcg_op, tcg_shift, tcg_fpstatus);
8569 if (is_scalar) {
8570 write_fp_sreg(s, rd, tcg_op);
8571 } else {
8572 write_vec_element_i32(s, tcg_op, rd, pass, size);
8573 }
8574 }
8575 if (!is_scalar) {
8576 clear_vec_high(s, is_q, rd);
8577 }
8578 }
8579
8580 gen_restore_rmode(tcg_rmode, tcg_fpstatus);
8581 }
8582
8583 /* AdvSIMD scalar shift by immediate
8584 * 31 30 29 28 23 22 19 18 16 15 11 10 9 5 4 0
8585 * +-----+---+-------------+------+------+--------+---+------+------+
8586 * | 0 1 | U | 1 1 1 1 1 0 | immh | immb | opcode | 1 | Rn | Rd |
8587 * +-----+---+-------------+------+------+--------+---+------+------+
8588 *
8589 * This is the scalar version so it works on a fixed sized registers
8590 */
disas_simd_scalar_shift_imm(DisasContext * s,uint32_t insn)8591 static void disas_simd_scalar_shift_imm(DisasContext *s, uint32_t insn)
8592 {
8593 int rd = extract32(insn, 0, 5);
8594 int rn = extract32(insn, 5, 5);
8595 int opcode = extract32(insn, 11, 5);
8596 int immb = extract32(insn, 16, 3);
8597 int immh = extract32(insn, 19, 4);
8598 bool is_u = extract32(insn, 29, 1);
8599
8600 if (immh == 0) {
8601 unallocated_encoding(s);
8602 return;
8603 }
8604
8605 switch (opcode) {
8606 case 0x08: /* SRI */
8607 if (!is_u) {
8608 unallocated_encoding(s);
8609 return;
8610 }
8611 /* fall through */
8612 case 0x00: /* SSHR / USHR */
8613 case 0x02: /* SSRA / USRA */
8614 case 0x04: /* SRSHR / URSHR */
8615 case 0x06: /* SRSRA / URSRA */
8616 handle_scalar_simd_shri(s, is_u, immh, immb, opcode, rn, rd);
8617 break;
8618 case 0x0a: /* SHL / SLI */
8619 handle_scalar_simd_shli(s, is_u, immh, immb, opcode, rn, rd);
8620 break;
8621 case 0x1c: /* SCVTF, UCVTF */
8622 handle_simd_shift_intfp_conv(s, true, false, is_u, immh, immb,
8623 opcode, rn, rd);
8624 break;
8625 case 0x10: /* SQSHRUN, SQSHRUN2 */
8626 case 0x11: /* SQRSHRUN, SQRSHRUN2 */
8627 if (!is_u) {
8628 unallocated_encoding(s);
8629 return;
8630 }
8631 handle_vec_simd_sqshrn(s, true, false, false, true,
8632 immh, immb, opcode, rn, rd);
8633 break;
8634 case 0x12: /* SQSHRN, SQSHRN2, UQSHRN */
8635 case 0x13: /* SQRSHRN, SQRSHRN2, UQRSHRN, UQRSHRN2 */
8636 handle_vec_simd_sqshrn(s, true, false, is_u, is_u,
8637 immh, immb, opcode, rn, rd);
8638 break;
8639 case 0xc: /* SQSHLU */
8640 if (!is_u) {
8641 unallocated_encoding(s);
8642 return;
8643 }
8644 handle_simd_qshl(s, true, false, false, true, immh, immb, rn, rd);
8645 break;
8646 case 0xe: /* SQSHL, UQSHL */
8647 handle_simd_qshl(s, true, false, is_u, is_u, immh, immb, rn, rd);
8648 break;
8649 case 0x1f: /* FCVTZS, FCVTZU */
8650 handle_simd_shift_fpint_conv(s, true, false, is_u, immh, immb, rn, rd);
8651 break;
8652 default:
8653 unallocated_encoding(s);
8654 break;
8655 }
8656 }
8657
8658 /* AdvSIMD scalar three different
8659 * 31 30 29 28 24 23 22 21 20 16 15 12 11 10 9 5 4 0
8660 * +-----+---+-----------+------+---+------+--------+-----+------+------+
8661 * | 0 1 | U | 1 1 1 1 0 | size | 1 | Rm | opcode | 0 0 | Rn | Rd |
8662 * +-----+---+-----------+------+---+------+--------+-----+------+------+
8663 */
disas_simd_scalar_three_reg_diff(DisasContext * s,uint32_t insn)8664 static void disas_simd_scalar_three_reg_diff(DisasContext *s, uint32_t insn)
8665 {
8666 bool is_u = extract32(insn, 29, 1);
8667 int size = extract32(insn, 22, 2);
8668 int opcode = extract32(insn, 12, 4);
8669 int rm = extract32(insn, 16, 5);
8670 int rn = extract32(insn, 5, 5);
8671 int rd = extract32(insn, 0, 5);
8672
8673 if (is_u) {
8674 unallocated_encoding(s);
8675 return;
8676 }
8677
8678 switch (opcode) {
8679 case 0x9: /* SQDMLAL, SQDMLAL2 */
8680 case 0xb: /* SQDMLSL, SQDMLSL2 */
8681 case 0xd: /* SQDMULL, SQDMULL2 */
8682 if (size == 0 || size == 3) {
8683 unallocated_encoding(s);
8684 return;
8685 }
8686 break;
8687 default:
8688 unallocated_encoding(s);
8689 return;
8690 }
8691
8692 if (!fp_access_check(s)) {
8693 return;
8694 }
8695
8696 if (size == 2) {
8697 TCGv_i64 tcg_op1 = tcg_temp_new_i64();
8698 TCGv_i64 tcg_op2 = tcg_temp_new_i64();
8699 TCGv_i64 tcg_res = tcg_temp_new_i64();
8700
8701 read_vec_element(s, tcg_op1, rn, 0, MO_32 | MO_SIGN);
8702 read_vec_element(s, tcg_op2, rm, 0, MO_32 | MO_SIGN);
8703
8704 tcg_gen_mul_i64(tcg_res, tcg_op1, tcg_op2);
8705 gen_helper_neon_addl_saturate_s64(tcg_res, tcg_env, tcg_res, tcg_res);
8706
8707 switch (opcode) {
8708 case 0xd: /* SQDMULL, SQDMULL2 */
8709 break;
8710 case 0xb: /* SQDMLSL, SQDMLSL2 */
8711 tcg_gen_neg_i64(tcg_res, tcg_res);
8712 /* fall through */
8713 case 0x9: /* SQDMLAL, SQDMLAL2 */
8714 read_vec_element(s, tcg_op1, rd, 0, MO_64);
8715 gen_helper_neon_addl_saturate_s64(tcg_res, tcg_env,
8716 tcg_res, tcg_op1);
8717 break;
8718 default:
8719 g_assert_not_reached();
8720 }
8721
8722 write_fp_dreg(s, rd, tcg_res);
8723 } else {
8724 TCGv_i32 tcg_op1 = read_fp_hreg(s, rn);
8725 TCGv_i32 tcg_op2 = read_fp_hreg(s, rm);
8726 TCGv_i64 tcg_res = tcg_temp_new_i64();
8727
8728 gen_helper_neon_mull_s16(tcg_res, tcg_op1, tcg_op2);
8729 gen_helper_neon_addl_saturate_s32(tcg_res, tcg_env, tcg_res, tcg_res);
8730
8731 switch (opcode) {
8732 case 0xd: /* SQDMULL, SQDMULL2 */
8733 break;
8734 case 0xb: /* SQDMLSL, SQDMLSL2 */
8735 gen_helper_neon_negl_u32(tcg_res, tcg_res);
8736 /* fall through */
8737 case 0x9: /* SQDMLAL, SQDMLAL2 */
8738 {
8739 TCGv_i64 tcg_op3 = tcg_temp_new_i64();
8740 read_vec_element(s, tcg_op3, rd, 0, MO_32);
8741 gen_helper_neon_addl_saturate_s32(tcg_res, tcg_env,
8742 tcg_res, tcg_op3);
8743 break;
8744 }
8745 default:
8746 g_assert_not_reached();
8747 }
8748
8749 tcg_gen_ext32u_i64(tcg_res, tcg_res);
8750 write_fp_dreg(s, rd, tcg_res);
8751 }
8752 }
8753
handle_3same_64(DisasContext * s,int opcode,bool u,TCGv_i64 tcg_rd,TCGv_i64 tcg_rn,TCGv_i64 tcg_rm)8754 static void handle_3same_64(DisasContext *s, int opcode, bool u,
8755 TCGv_i64 tcg_rd, TCGv_i64 tcg_rn, TCGv_i64 tcg_rm)
8756 {
8757 /* Handle 64x64->64 opcodes which are shared between the scalar
8758 * and vector 3-same groups. We cover every opcode where size == 3
8759 * is valid in either the three-reg-same (integer, not pairwise)
8760 * or scalar-three-reg-same groups.
8761 */
8762 TCGCond cond;
8763
8764 switch (opcode) {
8765 case 0x1: /* SQADD */
8766 if (u) {
8767 gen_helper_neon_qadd_u64(tcg_rd, tcg_env, tcg_rn, tcg_rm);
8768 } else {
8769 gen_helper_neon_qadd_s64(tcg_rd, tcg_env, tcg_rn, tcg_rm);
8770 }
8771 break;
8772 case 0x5: /* SQSUB */
8773 if (u) {
8774 gen_helper_neon_qsub_u64(tcg_rd, tcg_env, tcg_rn, tcg_rm);
8775 } else {
8776 gen_helper_neon_qsub_s64(tcg_rd, tcg_env, tcg_rn, tcg_rm);
8777 }
8778 break;
8779 case 0x6: /* CMGT, CMHI */
8780 cond = u ? TCG_COND_GTU : TCG_COND_GT;
8781 do_cmop:
8782 /* 64 bit integer comparison, result = test ? -1 : 0. */
8783 tcg_gen_negsetcond_i64(cond, tcg_rd, tcg_rn, tcg_rm);
8784 break;
8785 case 0x7: /* CMGE, CMHS */
8786 cond = u ? TCG_COND_GEU : TCG_COND_GE;
8787 goto do_cmop;
8788 case 0x11: /* CMTST, CMEQ */
8789 if (u) {
8790 cond = TCG_COND_EQ;
8791 goto do_cmop;
8792 }
8793 gen_cmtst_i64(tcg_rd, tcg_rn, tcg_rm);
8794 break;
8795 case 0x8: /* SSHL, USHL */
8796 if (u) {
8797 gen_ushl_i64(tcg_rd, tcg_rn, tcg_rm);
8798 } else {
8799 gen_sshl_i64(tcg_rd, tcg_rn, tcg_rm);
8800 }
8801 break;
8802 case 0x9: /* SQSHL, UQSHL */
8803 if (u) {
8804 gen_helper_neon_qshl_u64(tcg_rd, tcg_env, tcg_rn, tcg_rm);
8805 } else {
8806 gen_helper_neon_qshl_s64(tcg_rd, tcg_env, tcg_rn, tcg_rm);
8807 }
8808 break;
8809 case 0xa: /* SRSHL, URSHL */
8810 if (u) {
8811 gen_helper_neon_rshl_u64(tcg_rd, tcg_rn, tcg_rm);
8812 } else {
8813 gen_helper_neon_rshl_s64(tcg_rd, tcg_rn, tcg_rm);
8814 }
8815 break;
8816 case 0xb: /* SQRSHL, UQRSHL */
8817 if (u) {
8818 gen_helper_neon_qrshl_u64(tcg_rd, tcg_env, tcg_rn, tcg_rm);
8819 } else {
8820 gen_helper_neon_qrshl_s64(tcg_rd, tcg_env, tcg_rn, tcg_rm);
8821 }
8822 break;
8823 case 0x10: /* ADD, SUB */
8824 if (u) {
8825 tcg_gen_sub_i64(tcg_rd, tcg_rn, tcg_rm);
8826 } else {
8827 tcg_gen_add_i64(tcg_rd, tcg_rn, tcg_rm);
8828 }
8829 break;
8830 default:
8831 g_assert_not_reached();
8832 }
8833 }
8834
8835 /* Handle the 3-same-operands float operations; shared by the scalar
8836 * and vector encodings. The caller must filter out any encodings
8837 * not allocated for the encoding it is dealing with.
8838 */
handle_3same_float(DisasContext * s,int size,int elements,int fpopcode,int rd,int rn,int rm)8839 static void handle_3same_float(DisasContext *s, int size, int elements,
8840 int fpopcode, int rd, int rn, int rm)
8841 {
8842 int pass;
8843 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR);
8844
8845 for (pass = 0; pass < elements; pass++) {
8846 if (size) {
8847 /* Double */
8848 TCGv_i64 tcg_op1 = tcg_temp_new_i64();
8849 TCGv_i64 tcg_op2 = tcg_temp_new_i64();
8850 TCGv_i64 tcg_res = tcg_temp_new_i64();
8851
8852 read_vec_element(s, tcg_op1, rn, pass, MO_64);
8853 read_vec_element(s, tcg_op2, rm, pass, MO_64);
8854
8855 switch (fpopcode) {
8856 case 0x39: /* FMLS */
8857 /* As usual for ARM, separate negation for fused multiply-add */
8858 gen_helper_vfp_negd(tcg_op1, tcg_op1);
8859 /* fall through */
8860 case 0x19: /* FMLA */
8861 read_vec_element(s, tcg_res, rd, pass, MO_64);
8862 gen_helper_vfp_muladdd(tcg_res, tcg_op1, tcg_op2,
8863 tcg_res, fpst);
8864 break;
8865 case 0x18: /* FMAXNM */
8866 gen_helper_vfp_maxnumd(tcg_res, tcg_op1, tcg_op2, fpst);
8867 break;
8868 case 0x1a: /* FADD */
8869 gen_helper_vfp_addd(tcg_res, tcg_op1, tcg_op2, fpst);
8870 break;
8871 case 0x1b: /* FMULX */
8872 gen_helper_vfp_mulxd(tcg_res, tcg_op1, tcg_op2, fpst);
8873 break;
8874 case 0x1c: /* FCMEQ */
8875 gen_helper_neon_ceq_f64(tcg_res, tcg_op1, tcg_op2, fpst);
8876 break;
8877 case 0x1e: /* FMAX */
8878 gen_helper_vfp_maxd(tcg_res, tcg_op1, tcg_op2, fpst);
8879 break;
8880 case 0x1f: /* FRECPS */
8881 gen_helper_recpsf_f64(tcg_res, tcg_op1, tcg_op2, fpst);
8882 break;
8883 case 0x38: /* FMINNM */
8884 gen_helper_vfp_minnumd(tcg_res, tcg_op1, tcg_op2, fpst);
8885 break;
8886 case 0x3a: /* FSUB */
8887 gen_helper_vfp_subd(tcg_res, tcg_op1, tcg_op2, fpst);
8888 break;
8889 case 0x3e: /* FMIN */
8890 gen_helper_vfp_mind(tcg_res, tcg_op1, tcg_op2, fpst);
8891 break;
8892 case 0x3f: /* FRSQRTS */
8893 gen_helper_rsqrtsf_f64(tcg_res, tcg_op1, tcg_op2, fpst);
8894 break;
8895 case 0x5b: /* FMUL */
8896 gen_helper_vfp_muld(tcg_res, tcg_op1, tcg_op2, fpst);
8897 break;
8898 case 0x5c: /* FCMGE */
8899 gen_helper_neon_cge_f64(tcg_res, tcg_op1, tcg_op2, fpst);
8900 break;
8901 case 0x5d: /* FACGE */
8902 gen_helper_neon_acge_f64(tcg_res, tcg_op1, tcg_op2, fpst);
8903 break;
8904 case 0x5f: /* FDIV */
8905 gen_helper_vfp_divd(tcg_res, tcg_op1, tcg_op2, fpst);
8906 break;
8907 case 0x7a: /* FABD */
8908 gen_helper_vfp_subd(tcg_res, tcg_op1, tcg_op2, fpst);
8909 gen_helper_vfp_absd(tcg_res, tcg_res);
8910 break;
8911 case 0x7c: /* FCMGT */
8912 gen_helper_neon_cgt_f64(tcg_res, tcg_op1, tcg_op2, fpst);
8913 break;
8914 case 0x7d: /* FACGT */
8915 gen_helper_neon_acgt_f64(tcg_res, tcg_op1, tcg_op2, fpst);
8916 break;
8917 default:
8918 g_assert_not_reached();
8919 }
8920
8921 write_vec_element(s, tcg_res, rd, pass, MO_64);
8922 } else {
8923 /* Single */
8924 TCGv_i32 tcg_op1 = tcg_temp_new_i32();
8925 TCGv_i32 tcg_op2 = tcg_temp_new_i32();
8926 TCGv_i32 tcg_res = tcg_temp_new_i32();
8927
8928 read_vec_element_i32(s, tcg_op1, rn, pass, MO_32);
8929 read_vec_element_i32(s, tcg_op2, rm, pass, MO_32);
8930
8931 switch (fpopcode) {
8932 case 0x39: /* FMLS */
8933 /* As usual for ARM, separate negation for fused multiply-add */
8934 gen_helper_vfp_negs(tcg_op1, tcg_op1);
8935 /* fall through */
8936 case 0x19: /* FMLA */
8937 read_vec_element_i32(s, tcg_res, rd, pass, MO_32);
8938 gen_helper_vfp_muladds(tcg_res, tcg_op1, tcg_op2,
8939 tcg_res, fpst);
8940 break;
8941 case 0x1a: /* FADD */
8942 gen_helper_vfp_adds(tcg_res, tcg_op1, tcg_op2, fpst);
8943 break;
8944 case 0x1b: /* FMULX */
8945 gen_helper_vfp_mulxs(tcg_res, tcg_op1, tcg_op2, fpst);
8946 break;
8947 case 0x1c: /* FCMEQ */
8948 gen_helper_neon_ceq_f32(tcg_res, tcg_op1, tcg_op2, fpst);
8949 break;
8950 case 0x1e: /* FMAX */
8951 gen_helper_vfp_maxs(tcg_res, tcg_op1, tcg_op2, fpst);
8952 break;
8953 case 0x1f: /* FRECPS */
8954 gen_helper_recpsf_f32(tcg_res, tcg_op1, tcg_op2, fpst);
8955 break;
8956 case 0x18: /* FMAXNM */
8957 gen_helper_vfp_maxnums(tcg_res, tcg_op1, tcg_op2, fpst);
8958 break;
8959 case 0x38: /* FMINNM */
8960 gen_helper_vfp_minnums(tcg_res, tcg_op1, tcg_op2, fpst);
8961 break;
8962 case 0x3a: /* FSUB */
8963 gen_helper_vfp_subs(tcg_res, tcg_op1, tcg_op2, fpst);
8964 break;
8965 case 0x3e: /* FMIN */
8966 gen_helper_vfp_mins(tcg_res, tcg_op1, tcg_op2, fpst);
8967 break;
8968 case 0x3f: /* FRSQRTS */
8969 gen_helper_rsqrtsf_f32(tcg_res, tcg_op1, tcg_op2, fpst);
8970 break;
8971 case 0x5b: /* FMUL */
8972 gen_helper_vfp_muls(tcg_res, tcg_op1, tcg_op2, fpst);
8973 break;
8974 case 0x5c: /* FCMGE */
8975 gen_helper_neon_cge_f32(tcg_res, tcg_op1, tcg_op2, fpst);
8976 break;
8977 case 0x5d: /* FACGE */
8978 gen_helper_neon_acge_f32(tcg_res, tcg_op1, tcg_op2, fpst);
8979 break;
8980 case 0x5f: /* FDIV */
8981 gen_helper_vfp_divs(tcg_res, tcg_op1, tcg_op2, fpst);
8982 break;
8983 case 0x7a: /* FABD */
8984 gen_helper_vfp_subs(tcg_res, tcg_op1, tcg_op2, fpst);
8985 gen_helper_vfp_abss(tcg_res, tcg_res);
8986 break;
8987 case 0x7c: /* FCMGT */
8988 gen_helper_neon_cgt_f32(tcg_res, tcg_op1, tcg_op2, fpst);
8989 break;
8990 case 0x7d: /* FACGT */
8991 gen_helper_neon_acgt_f32(tcg_res, tcg_op1, tcg_op2, fpst);
8992 break;
8993 default:
8994 g_assert_not_reached();
8995 }
8996
8997 if (elements == 1) {
8998 /* scalar single so clear high part */
8999 TCGv_i64 tcg_tmp = tcg_temp_new_i64();
9000
9001 tcg_gen_extu_i32_i64(tcg_tmp, tcg_res);
9002 write_vec_element(s, tcg_tmp, rd, pass, MO_64);
9003 } else {
9004 write_vec_element_i32(s, tcg_res, rd, pass, MO_32);
9005 }
9006 }
9007 }
9008
9009 clear_vec_high(s, elements * (size ? 8 : 4) > 8, rd);
9010 }
9011
9012 /* AdvSIMD scalar three same
9013 * 31 30 29 28 24 23 22 21 20 16 15 11 10 9 5 4 0
9014 * +-----+---+-----------+------+---+------+--------+---+------+------+
9015 * | 0 1 | U | 1 1 1 1 0 | size | 1 | Rm | opcode | 1 | Rn | Rd |
9016 * +-----+---+-----------+------+---+------+--------+---+------+------+
9017 */
disas_simd_scalar_three_reg_same(DisasContext * s,uint32_t insn)9018 static void disas_simd_scalar_three_reg_same(DisasContext *s, uint32_t insn)
9019 {
9020 int rd = extract32(insn, 0, 5);
9021 int rn = extract32(insn, 5, 5);
9022 int opcode = extract32(insn, 11, 5);
9023 int rm = extract32(insn, 16, 5);
9024 int size = extract32(insn, 22, 2);
9025 bool u = extract32(insn, 29, 1);
9026 TCGv_i64 tcg_rd;
9027
9028 if (opcode >= 0x18) {
9029 /* Floating point: U, size[1] and opcode indicate operation */
9030 int fpopcode = opcode | (extract32(size, 1, 1) << 5) | (u << 6);
9031 switch (fpopcode) {
9032 case 0x1b: /* FMULX */
9033 case 0x1f: /* FRECPS */
9034 case 0x3f: /* FRSQRTS */
9035 case 0x5d: /* FACGE */
9036 case 0x7d: /* FACGT */
9037 case 0x1c: /* FCMEQ */
9038 case 0x5c: /* FCMGE */
9039 case 0x7c: /* FCMGT */
9040 case 0x7a: /* FABD */
9041 break;
9042 default:
9043 unallocated_encoding(s);
9044 return;
9045 }
9046
9047 if (!fp_access_check(s)) {
9048 return;
9049 }
9050
9051 handle_3same_float(s, extract32(size, 0, 1), 1, fpopcode, rd, rn, rm);
9052 return;
9053 }
9054
9055 switch (opcode) {
9056 case 0x1: /* SQADD, UQADD */
9057 case 0x5: /* SQSUB, UQSUB */
9058 case 0x9: /* SQSHL, UQSHL */
9059 case 0xb: /* SQRSHL, UQRSHL */
9060 break;
9061 case 0x8: /* SSHL, USHL */
9062 case 0xa: /* SRSHL, URSHL */
9063 case 0x6: /* CMGT, CMHI */
9064 case 0x7: /* CMGE, CMHS */
9065 case 0x11: /* CMTST, CMEQ */
9066 case 0x10: /* ADD, SUB (vector) */
9067 if (size != 3) {
9068 unallocated_encoding(s);
9069 return;
9070 }
9071 break;
9072 case 0x16: /* SQDMULH, SQRDMULH (vector) */
9073 if (size != 1 && size != 2) {
9074 unallocated_encoding(s);
9075 return;
9076 }
9077 break;
9078 default:
9079 unallocated_encoding(s);
9080 return;
9081 }
9082
9083 if (!fp_access_check(s)) {
9084 return;
9085 }
9086
9087 tcg_rd = tcg_temp_new_i64();
9088
9089 if (size == 3) {
9090 TCGv_i64 tcg_rn = read_fp_dreg(s, rn);
9091 TCGv_i64 tcg_rm = read_fp_dreg(s, rm);
9092
9093 handle_3same_64(s, opcode, u, tcg_rd, tcg_rn, tcg_rm);
9094 } else {
9095 /* Do a single operation on the lowest element in the vector.
9096 * We use the standard Neon helpers and rely on 0 OP 0 == 0 with
9097 * no side effects for all these operations.
9098 * OPTME: special-purpose helpers would avoid doing some
9099 * unnecessary work in the helper for the 8 and 16 bit cases.
9100 */
9101 NeonGenTwoOpEnvFn *genenvfn;
9102 TCGv_i32 tcg_rn = tcg_temp_new_i32();
9103 TCGv_i32 tcg_rm = tcg_temp_new_i32();
9104 TCGv_i32 tcg_rd32 = tcg_temp_new_i32();
9105
9106 read_vec_element_i32(s, tcg_rn, rn, 0, size);
9107 read_vec_element_i32(s, tcg_rm, rm, 0, size);
9108
9109 switch (opcode) {
9110 case 0x1: /* SQADD, UQADD */
9111 {
9112 static NeonGenTwoOpEnvFn * const fns[3][2] = {
9113 { gen_helper_neon_qadd_s8, gen_helper_neon_qadd_u8 },
9114 { gen_helper_neon_qadd_s16, gen_helper_neon_qadd_u16 },
9115 { gen_helper_neon_qadd_s32, gen_helper_neon_qadd_u32 },
9116 };
9117 genenvfn = fns[size][u];
9118 break;
9119 }
9120 case 0x5: /* SQSUB, UQSUB */
9121 {
9122 static NeonGenTwoOpEnvFn * const fns[3][2] = {
9123 { gen_helper_neon_qsub_s8, gen_helper_neon_qsub_u8 },
9124 { gen_helper_neon_qsub_s16, gen_helper_neon_qsub_u16 },
9125 { gen_helper_neon_qsub_s32, gen_helper_neon_qsub_u32 },
9126 };
9127 genenvfn = fns[size][u];
9128 break;
9129 }
9130 case 0x9: /* SQSHL, UQSHL */
9131 {
9132 static NeonGenTwoOpEnvFn * const fns[3][2] = {
9133 { gen_helper_neon_qshl_s8, gen_helper_neon_qshl_u8 },
9134 { gen_helper_neon_qshl_s16, gen_helper_neon_qshl_u16 },
9135 { gen_helper_neon_qshl_s32, gen_helper_neon_qshl_u32 },
9136 };
9137 genenvfn = fns[size][u];
9138 break;
9139 }
9140 case 0xb: /* SQRSHL, UQRSHL */
9141 {
9142 static NeonGenTwoOpEnvFn * const fns[3][2] = {
9143 { gen_helper_neon_qrshl_s8, gen_helper_neon_qrshl_u8 },
9144 { gen_helper_neon_qrshl_s16, gen_helper_neon_qrshl_u16 },
9145 { gen_helper_neon_qrshl_s32, gen_helper_neon_qrshl_u32 },
9146 };
9147 genenvfn = fns[size][u];
9148 break;
9149 }
9150 case 0x16: /* SQDMULH, SQRDMULH */
9151 {
9152 static NeonGenTwoOpEnvFn * const fns[2][2] = {
9153 { gen_helper_neon_qdmulh_s16, gen_helper_neon_qrdmulh_s16 },
9154 { gen_helper_neon_qdmulh_s32, gen_helper_neon_qrdmulh_s32 },
9155 };
9156 assert(size == 1 || size == 2);
9157 genenvfn = fns[size - 1][u];
9158 break;
9159 }
9160 default:
9161 g_assert_not_reached();
9162 }
9163
9164 genenvfn(tcg_rd32, tcg_env, tcg_rn, tcg_rm);
9165 tcg_gen_extu_i32_i64(tcg_rd, tcg_rd32);
9166 }
9167
9168 write_fp_dreg(s, rd, tcg_rd);
9169 }
9170
9171 /* AdvSIMD scalar three same FP16
9172 * 31 30 29 28 24 23 22 21 20 16 15 14 13 11 10 9 5 4 0
9173 * +-----+---+-----------+---+-----+------+-----+--------+---+----+----+
9174 * | 0 1 | U | 1 1 1 1 0 | a | 1 0 | Rm | 0 0 | opcode | 1 | Rn | Rd |
9175 * +-----+---+-----------+---+-----+------+-----+--------+---+----+----+
9176 * v: 0101 1110 0100 0000 0000 0100 0000 0000 => 5e400400
9177 * m: 1101 1111 0110 0000 1100 0100 0000 0000 => df60c400
9178 */
disas_simd_scalar_three_reg_same_fp16(DisasContext * s,uint32_t insn)9179 static void disas_simd_scalar_three_reg_same_fp16(DisasContext *s,
9180 uint32_t insn)
9181 {
9182 int rd = extract32(insn, 0, 5);
9183 int rn = extract32(insn, 5, 5);
9184 int opcode = extract32(insn, 11, 3);
9185 int rm = extract32(insn, 16, 5);
9186 bool u = extract32(insn, 29, 1);
9187 bool a = extract32(insn, 23, 1);
9188 int fpopcode = opcode | (a << 3) | (u << 4);
9189 TCGv_ptr fpst;
9190 TCGv_i32 tcg_op1;
9191 TCGv_i32 tcg_op2;
9192 TCGv_i32 tcg_res;
9193
9194 switch (fpopcode) {
9195 case 0x03: /* FMULX */
9196 case 0x04: /* FCMEQ (reg) */
9197 case 0x07: /* FRECPS */
9198 case 0x0f: /* FRSQRTS */
9199 case 0x14: /* FCMGE (reg) */
9200 case 0x15: /* FACGE */
9201 case 0x1a: /* FABD */
9202 case 0x1c: /* FCMGT (reg) */
9203 case 0x1d: /* FACGT */
9204 break;
9205 default:
9206 unallocated_encoding(s);
9207 return;
9208 }
9209
9210 if (!dc_isar_feature(aa64_fp16, s)) {
9211 unallocated_encoding(s);
9212 }
9213
9214 if (!fp_access_check(s)) {
9215 return;
9216 }
9217
9218 fpst = fpstatus_ptr(FPST_FPCR_F16);
9219
9220 tcg_op1 = read_fp_hreg(s, rn);
9221 tcg_op2 = read_fp_hreg(s, rm);
9222 tcg_res = tcg_temp_new_i32();
9223
9224 switch (fpopcode) {
9225 case 0x03: /* FMULX */
9226 gen_helper_advsimd_mulxh(tcg_res, tcg_op1, tcg_op2, fpst);
9227 break;
9228 case 0x04: /* FCMEQ (reg) */
9229 gen_helper_advsimd_ceq_f16(tcg_res, tcg_op1, tcg_op2, fpst);
9230 break;
9231 case 0x07: /* FRECPS */
9232 gen_helper_recpsf_f16(tcg_res, tcg_op1, tcg_op2, fpst);
9233 break;
9234 case 0x0f: /* FRSQRTS */
9235 gen_helper_rsqrtsf_f16(tcg_res, tcg_op1, tcg_op2, fpst);
9236 break;
9237 case 0x14: /* FCMGE (reg) */
9238 gen_helper_advsimd_cge_f16(tcg_res, tcg_op1, tcg_op2, fpst);
9239 break;
9240 case 0x15: /* FACGE */
9241 gen_helper_advsimd_acge_f16(tcg_res, tcg_op1, tcg_op2, fpst);
9242 break;
9243 case 0x1a: /* FABD */
9244 gen_helper_advsimd_subh(tcg_res, tcg_op1, tcg_op2, fpst);
9245 tcg_gen_andi_i32(tcg_res, tcg_res, 0x7fff);
9246 break;
9247 case 0x1c: /* FCMGT (reg) */
9248 gen_helper_advsimd_cgt_f16(tcg_res, tcg_op1, tcg_op2, fpst);
9249 break;
9250 case 0x1d: /* FACGT */
9251 gen_helper_advsimd_acgt_f16(tcg_res, tcg_op1, tcg_op2, fpst);
9252 break;
9253 default:
9254 g_assert_not_reached();
9255 }
9256
9257 write_fp_sreg(s, rd, tcg_res);
9258 }
9259
9260 /* AdvSIMD scalar three same extra
9261 * 31 30 29 28 24 23 22 21 20 16 15 14 11 10 9 5 4 0
9262 * +-----+---+-----------+------+---+------+---+--------+---+----+----+
9263 * | 0 1 | U | 1 1 1 1 0 | size | 0 | Rm | 1 | opcode | 1 | Rn | Rd |
9264 * +-----+---+-----------+------+---+------+---+--------+---+----+----+
9265 */
disas_simd_scalar_three_reg_same_extra(DisasContext * s,uint32_t insn)9266 static void disas_simd_scalar_three_reg_same_extra(DisasContext *s,
9267 uint32_t insn)
9268 {
9269 int rd = extract32(insn, 0, 5);
9270 int rn = extract32(insn, 5, 5);
9271 int opcode = extract32(insn, 11, 4);
9272 int rm = extract32(insn, 16, 5);
9273 int size = extract32(insn, 22, 2);
9274 bool u = extract32(insn, 29, 1);
9275 TCGv_i32 ele1, ele2, ele3;
9276 TCGv_i64 res;
9277 bool feature;
9278
9279 switch (u * 16 + opcode) {
9280 case 0x10: /* SQRDMLAH (vector) */
9281 case 0x11: /* SQRDMLSH (vector) */
9282 if (size != 1 && size != 2) {
9283 unallocated_encoding(s);
9284 return;
9285 }
9286 feature = dc_isar_feature(aa64_rdm, s);
9287 break;
9288 default:
9289 unallocated_encoding(s);
9290 return;
9291 }
9292 if (!feature) {
9293 unallocated_encoding(s);
9294 return;
9295 }
9296 if (!fp_access_check(s)) {
9297 return;
9298 }
9299
9300 /* Do a single operation on the lowest element in the vector.
9301 * We use the standard Neon helpers and rely on 0 OP 0 == 0
9302 * with no side effects for all these operations.
9303 * OPTME: special-purpose helpers would avoid doing some
9304 * unnecessary work in the helper for the 16 bit cases.
9305 */
9306 ele1 = tcg_temp_new_i32();
9307 ele2 = tcg_temp_new_i32();
9308 ele3 = tcg_temp_new_i32();
9309
9310 read_vec_element_i32(s, ele1, rn, 0, size);
9311 read_vec_element_i32(s, ele2, rm, 0, size);
9312 read_vec_element_i32(s, ele3, rd, 0, size);
9313
9314 switch (opcode) {
9315 case 0x0: /* SQRDMLAH */
9316 if (size == 1) {
9317 gen_helper_neon_qrdmlah_s16(ele3, tcg_env, ele1, ele2, ele3);
9318 } else {
9319 gen_helper_neon_qrdmlah_s32(ele3, tcg_env, ele1, ele2, ele3);
9320 }
9321 break;
9322 case 0x1: /* SQRDMLSH */
9323 if (size == 1) {
9324 gen_helper_neon_qrdmlsh_s16(ele3, tcg_env, ele1, ele2, ele3);
9325 } else {
9326 gen_helper_neon_qrdmlsh_s32(ele3, tcg_env, ele1, ele2, ele3);
9327 }
9328 break;
9329 default:
9330 g_assert_not_reached();
9331 }
9332
9333 res = tcg_temp_new_i64();
9334 tcg_gen_extu_i32_i64(res, ele3);
9335 write_fp_dreg(s, rd, res);
9336 }
9337
handle_2misc_64(DisasContext * s,int opcode,bool u,TCGv_i64 tcg_rd,TCGv_i64 tcg_rn,TCGv_i32 tcg_rmode,TCGv_ptr tcg_fpstatus)9338 static void handle_2misc_64(DisasContext *s, int opcode, bool u,
9339 TCGv_i64 tcg_rd, TCGv_i64 tcg_rn,
9340 TCGv_i32 tcg_rmode, TCGv_ptr tcg_fpstatus)
9341 {
9342 /* Handle 64->64 opcodes which are shared between the scalar and
9343 * vector 2-reg-misc groups. We cover every integer opcode where size == 3
9344 * is valid in either group and also the double-precision fp ops.
9345 * The caller only need provide tcg_rmode and tcg_fpstatus if the op
9346 * requires them.
9347 */
9348 TCGCond cond;
9349
9350 switch (opcode) {
9351 case 0x4: /* CLS, CLZ */
9352 if (u) {
9353 tcg_gen_clzi_i64(tcg_rd, tcg_rn, 64);
9354 } else {
9355 tcg_gen_clrsb_i64(tcg_rd, tcg_rn);
9356 }
9357 break;
9358 case 0x5: /* NOT */
9359 /* This opcode is shared with CNT and RBIT but we have earlier
9360 * enforced that size == 3 if and only if this is the NOT insn.
9361 */
9362 tcg_gen_not_i64(tcg_rd, tcg_rn);
9363 break;
9364 case 0x7: /* SQABS, SQNEG */
9365 if (u) {
9366 gen_helper_neon_qneg_s64(tcg_rd, tcg_env, tcg_rn);
9367 } else {
9368 gen_helper_neon_qabs_s64(tcg_rd, tcg_env, tcg_rn);
9369 }
9370 break;
9371 case 0xa: /* CMLT */
9372 cond = TCG_COND_LT;
9373 do_cmop:
9374 /* 64 bit integer comparison against zero, result is test ? -1 : 0. */
9375 tcg_gen_negsetcond_i64(cond, tcg_rd, tcg_rn, tcg_constant_i64(0));
9376 break;
9377 case 0x8: /* CMGT, CMGE */
9378 cond = u ? TCG_COND_GE : TCG_COND_GT;
9379 goto do_cmop;
9380 case 0x9: /* CMEQ, CMLE */
9381 cond = u ? TCG_COND_LE : TCG_COND_EQ;
9382 goto do_cmop;
9383 case 0xb: /* ABS, NEG */
9384 if (u) {
9385 tcg_gen_neg_i64(tcg_rd, tcg_rn);
9386 } else {
9387 tcg_gen_abs_i64(tcg_rd, tcg_rn);
9388 }
9389 break;
9390 case 0x2f: /* FABS */
9391 gen_helper_vfp_absd(tcg_rd, tcg_rn);
9392 break;
9393 case 0x6f: /* FNEG */
9394 gen_helper_vfp_negd(tcg_rd, tcg_rn);
9395 break;
9396 case 0x7f: /* FSQRT */
9397 gen_helper_vfp_sqrtd(tcg_rd, tcg_rn, tcg_env);
9398 break;
9399 case 0x1a: /* FCVTNS */
9400 case 0x1b: /* FCVTMS */
9401 case 0x1c: /* FCVTAS */
9402 case 0x3a: /* FCVTPS */
9403 case 0x3b: /* FCVTZS */
9404 gen_helper_vfp_tosqd(tcg_rd, tcg_rn, tcg_constant_i32(0), tcg_fpstatus);
9405 break;
9406 case 0x5a: /* FCVTNU */
9407 case 0x5b: /* FCVTMU */
9408 case 0x5c: /* FCVTAU */
9409 case 0x7a: /* FCVTPU */
9410 case 0x7b: /* FCVTZU */
9411 gen_helper_vfp_touqd(tcg_rd, tcg_rn, tcg_constant_i32(0), tcg_fpstatus);
9412 break;
9413 case 0x18: /* FRINTN */
9414 case 0x19: /* FRINTM */
9415 case 0x38: /* FRINTP */
9416 case 0x39: /* FRINTZ */
9417 case 0x58: /* FRINTA */
9418 case 0x79: /* FRINTI */
9419 gen_helper_rintd(tcg_rd, tcg_rn, tcg_fpstatus);
9420 break;
9421 case 0x59: /* FRINTX */
9422 gen_helper_rintd_exact(tcg_rd, tcg_rn, tcg_fpstatus);
9423 break;
9424 case 0x1e: /* FRINT32Z */
9425 case 0x5e: /* FRINT32X */
9426 gen_helper_frint32_d(tcg_rd, tcg_rn, tcg_fpstatus);
9427 break;
9428 case 0x1f: /* FRINT64Z */
9429 case 0x5f: /* FRINT64X */
9430 gen_helper_frint64_d(tcg_rd, tcg_rn, tcg_fpstatus);
9431 break;
9432 default:
9433 g_assert_not_reached();
9434 }
9435 }
9436
handle_2misc_fcmp_zero(DisasContext * s,int opcode,bool is_scalar,bool is_u,bool is_q,int size,int rn,int rd)9437 static void handle_2misc_fcmp_zero(DisasContext *s, int opcode,
9438 bool is_scalar, bool is_u, bool is_q,
9439 int size, int rn, int rd)
9440 {
9441 bool is_double = (size == MO_64);
9442 TCGv_ptr fpst;
9443
9444 if (!fp_access_check(s)) {
9445 return;
9446 }
9447
9448 fpst = fpstatus_ptr(size == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
9449
9450 if (is_double) {
9451 TCGv_i64 tcg_op = tcg_temp_new_i64();
9452 TCGv_i64 tcg_zero = tcg_constant_i64(0);
9453 TCGv_i64 tcg_res = tcg_temp_new_i64();
9454 NeonGenTwoDoubleOpFn *genfn;
9455 bool swap = false;
9456 int pass;
9457
9458 switch (opcode) {
9459 case 0x2e: /* FCMLT (zero) */
9460 swap = true;
9461 /* fallthrough */
9462 case 0x2c: /* FCMGT (zero) */
9463 genfn = gen_helper_neon_cgt_f64;
9464 break;
9465 case 0x2d: /* FCMEQ (zero) */
9466 genfn = gen_helper_neon_ceq_f64;
9467 break;
9468 case 0x6d: /* FCMLE (zero) */
9469 swap = true;
9470 /* fall through */
9471 case 0x6c: /* FCMGE (zero) */
9472 genfn = gen_helper_neon_cge_f64;
9473 break;
9474 default:
9475 g_assert_not_reached();
9476 }
9477
9478 for (pass = 0; pass < (is_scalar ? 1 : 2); pass++) {
9479 read_vec_element(s, tcg_op, rn, pass, MO_64);
9480 if (swap) {
9481 genfn(tcg_res, tcg_zero, tcg_op, fpst);
9482 } else {
9483 genfn(tcg_res, tcg_op, tcg_zero, fpst);
9484 }
9485 write_vec_element(s, tcg_res, rd, pass, MO_64);
9486 }
9487
9488 clear_vec_high(s, !is_scalar, rd);
9489 } else {
9490 TCGv_i32 tcg_op = tcg_temp_new_i32();
9491 TCGv_i32 tcg_zero = tcg_constant_i32(0);
9492 TCGv_i32 tcg_res = tcg_temp_new_i32();
9493 NeonGenTwoSingleOpFn *genfn;
9494 bool swap = false;
9495 int pass, maxpasses;
9496
9497 if (size == MO_16) {
9498 switch (opcode) {
9499 case 0x2e: /* FCMLT (zero) */
9500 swap = true;
9501 /* fall through */
9502 case 0x2c: /* FCMGT (zero) */
9503 genfn = gen_helper_advsimd_cgt_f16;
9504 break;
9505 case 0x2d: /* FCMEQ (zero) */
9506 genfn = gen_helper_advsimd_ceq_f16;
9507 break;
9508 case 0x6d: /* FCMLE (zero) */
9509 swap = true;
9510 /* fall through */
9511 case 0x6c: /* FCMGE (zero) */
9512 genfn = gen_helper_advsimd_cge_f16;
9513 break;
9514 default:
9515 g_assert_not_reached();
9516 }
9517 } else {
9518 switch (opcode) {
9519 case 0x2e: /* FCMLT (zero) */
9520 swap = true;
9521 /* fall through */
9522 case 0x2c: /* FCMGT (zero) */
9523 genfn = gen_helper_neon_cgt_f32;
9524 break;
9525 case 0x2d: /* FCMEQ (zero) */
9526 genfn = gen_helper_neon_ceq_f32;
9527 break;
9528 case 0x6d: /* FCMLE (zero) */
9529 swap = true;
9530 /* fall through */
9531 case 0x6c: /* FCMGE (zero) */
9532 genfn = gen_helper_neon_cge_f32;
9533 break;
9534 default:
9535 g_assert_not_reached();
9536 }
9537 }
9538
9539 if (is_scalar) {
9540 maxpasses = 1;
9541 } else {
9542 int vector_size = 8 << is_q;
9543 maxpasses = vector_size >> size;
9544 }
9545
9546 for (pass = 0; pass < maxpasses; pass++) {
9547 read_vec_element_i32(s, tcg_op, rn, pass, size);
9548 if (swap) {
9549 genfn(tcg_res, tcg_zero, tcg_op, fpst);
9550 } else {
9551 genfn(tcg_res, tcg_op, tcg_zero, fpst);
9552 }
9553 if (is_scalar) {
9554 write_fp_sreg(s, rd, tcg_res);
9555 } else {
9556 write_vec_element_i32(s, tcg_res, rd, pass, size);
9557 }
9558 }
9559
9560 if (!is_scalar) {
9561 clear_vec_high(s, is_q, rd);
9562 }
9563 }
9564 }
9565
handle_2misc_reciprocal(DisasContext * s,int opcode,bool is_scalar,bool is_u,bool is_q,int size,int rn,int rd)9566 static void handle_2misc_reciprocal(DisasContext *s, int opcode,
9567 bool is_scalar, bool is_u, bool is_q,
9568 int size, int rn, int rd)
9569 {
9570 bool is_double = (size == 3);
9571 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR);
9572
9573 if (is_double) {
9574 TCGv_i64 tcg_op = tcg_temp_new_i64();
9575 TCGv_i64 tcg_res = tcg_temp_new_i64();
9576 int pass;
9577
9578 for (pass = 0; pass < (is_scalar ? 1 : 2); pass++) {
9579 read_vec_element(s, tcg_op, rn, pass, MO_64);
9580 switch (opcode) {
9581 case 0x3d: /* FRECPE */
9582 gen_helper_recpe_f64(tcg_res, tcg_op, fpst);
9583 break;
9584 case 0x3f: /* FRECPX */
9585 gen_helper_frecpx_f64(tcg_res, tcg_op, fpst);
9586 break;
9587 case 0x7d: /* FRSQRTE */
9588 gen_helper_rsqrte_f64(tcg_res, tcg_op, fpst);
9589 break;
9590 default:
9591 g_assert_not_reached();
9592 }
9593 write_vec_element(s, tcg_res, rd, pass, MO_64);
9594 }
9595 clear_vec_high(s, !is_scalar, rd);
9596 } else {
9597 TCGv_i32 tcg_op = tcg_temp_new_i32();
9598 TCGv_i32 tcg_res = tcg_temp_new_i32();
9599 int pass, maxpasses;
9600
9601 if (is_scalar) {
9602 maxpasses = 1;
9603 } else {
9604 maxpasses = is_q ? 4 : 2;
9605 }
9606
9607 for (pass = 0; pass < maxpasses; pass++) {
9608 read_vec_element_i32(s, tcg_op, rn, pass, MO_32);
9609
9610 switch (opcode) {
9611 case 0x3c: /* URECPE */
9612 gen_helper_recpe_u32(tcg_res, tcg_op);
9613 break;
9614 case 0x3d: /* FRECPE */
9615 gen_helper_recpe_f32(tcg_res, tcg_op, fpst);
9616 break;
9617 case 0x3f: /* FRECPX */
9618 gen_helper_frecpx_f32(tcg_res, tcg_op, fpst);
9619 break;
9620 case 0x7d: /* FRSQRTE */
9621 gen_helper_rsqrte_f32(tcg_res, tcg_op, fpst);
9622 break;
9623 default:
9624 g_assert_not_reached();
9625 }
9626
9627 if (is_scalar) {
9628 write_fp_sreg(s, rd, tcg_res);
9629 } else {
9630 write_vec_element_i32(s, tcg_res, rd, pass, MO_32);
9631 }
9632 }
9633 if (!is_scalar) {
9634 clear_vec_high(s, is_q, rd);
9635 }
9636 }
9637 }
9638
handle_2misc_narrow(DisasContext * s,bool scalar,int opcode,bool u,bool is_q,int size,int rn,int rd)9639 static void handle_2misc_narrow(DisasContext *s, bool scalar,
9640 int opcode, bool u, bool is_q,
9641 int size, int rn, int rd)
9642 {
9643 /* Handle 2-reg-misc ops which are narrowing (so each 2*size element
9644 * in the source becomes a size element in the destination).
9645 */
9646 int pass;
9647 TCGv_i32 tcg_res[2];
9648 int destelt = is_q ? 2 : 0;
9649 int passes = scalar ? 1 : 2;
9650
9651 if (scalar) {
9652 tcg_res[1] = tcg_constant_i32(0);
9653 }
9654
9655 for (pass = 0; pass < passes; pass++) {
9656 TCGv_i64 tcg_op = tcg_temp_new_i64();
9657 NeonGenNarrowFn *genfn = NULL;
9658 NeonGenNarrowEnvFn *genenvfn = NULL;
9659
9660 if (scalar) {
9661 read_vec_element(s, tcg_op, rn, pass, size + 1);
9662 } else {
9663 read_vec_element(s, tcg_op, rn, pass, MO_64);
9664 }
9665 tcg_res[pass] = tcg_temp_new_i32();
9666
9667 switch (opcode) {
9668 case 0x12: /* XTN, SQXTUN */
9669 {
9670 static NeonGenNarrowFn * const xtnfns[3] = {
9671 gen_helper_neon_narrow_u8,
9672 gen_helper_neon_narrow_u16,
9673 tcg_gen_extrl_i64_i32,
9674 };
9675 static NeonGenNarrowEnvFn * const sqxtunfns[3] = {
9676 gen_helper_neon_unarrow_sat8,
9677 gen_helper_neon_unarrow_sat16,
9678 gen_helper_neon_unarrow_sat32,
9679 };
9680 if (u) {
9681 genenvfn = sqxtunfns[size];
9682 } else {
9683 genfn = xtnfns[size];
9684 }
9685 break;
9686 }
9687 case 0x14: /* SQXTN, UQXTN */
9688 {
9689 static NeonGenNarrowEnvFn * const fns[3][2] = {
9690 { gen_helper_neon_narrow_sat_s8,
9691 gen_helper_neon_narrow_sat_u8 },
9692 { gen_helper_neon_narrow_sat_s16,
9693 gen_helper_neon_narrow_sat_u16 },
9694 { gen_helper_neon_narrow_sat_s32,
9695 gen_helper_neon_narrow_sat_u32 },
9696 };
9697 genenvfn = fns[size][u];
9698 break;
9699 }
9700 case 0x16: /* FCVTN, FCVTN2 */
9701 /* 32 bit to 16 bit or 64 bit to 32 bit float conversion */
9702 if (size == 2) {
9703 gen_helper_vfp_fcvtsd(tcg_res[pass], tcg_op, tcg_env);
9704 } else {
9705 TCGv_i32 tcg_lo = tcg_temp_new_i32();
9706 TCGv_i32 tcg_hi = tcg_temp_new_i32();
9707 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR);
9708 TCGv_i32 ahp = get_ahp_flag();
9709
9710 tcg_gen_extr_i64_i32(tcg_lo, tcg_hi, tcg_op);
9711 gen_helper_vfp_fcvt_f32_to_f16(tcg_lo, tcg_lo, fpst, ahp);
9712 gen_helper_vfp_fcvt_f32_to_f16(tcg_hi, tcg_hi, fpst, ahp);
9713 tcg_gen_deposit_i32(tcg_res[pass], tcg_lo, tcg_hi, 16, 16);
9714 }
9715 break;
9716 case 0x36: /* BFCVTN, BFCVTN2 */
9717 {
9718 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR);
9719 gen_helper_bfcvt_pair(tcg_res[pass], tcg_op, fpst);
9720 }
9721 break;
9722 case 0x56: /* FCVTXN, FCVTXN2 */
9723 /* 64 bit to 32 bit float conversion
9724 * with von Neumann rounding (round to odd)
9725 */
9726 assert(size == 2);
9727 gen_helper_fcvtx_f64_to_f32(tcg_res[pass], tcg_op, tcg_env);
9728 break;
9729 default:
9730 g_assert_not_reached();
9731 }
9732
9733 if (genfn) {
9734 genfn(tcg_res[pass], tcg_op);
9735 } else if (genenvfn) {
9736 genenvfn(tcg_res[pass], tcg_env, tcg_op);
9737 }
9738 }
9739
9740 for (pass = 0; pass < 2; pass++) {
9741 write_vec_element_i32(s, tcg_res[pass], rd, destelt + pass, MO_32);
9742 }
9743 clear_vec_high(s, is_q, rd);
9744 }
9745
9746 /* Remaining saturating accumulating ops */
handle_2misc_satacc(DisasContext * s,bool is_scalar,bool is_u,bool is_q,int size,int rn,int rd)9747 static void handle_2misc_satacc(DisasContext *s, bool is_scalar, bool is_u,
9748 bool is_q, int size, int rn, int rd)
9749 {
9750 bool is_double = (size == 3);
9751
9752 if (is_double) {
9753 TCGv_i64 tcg_rn = tcg_temp_new_i64();
9754 TCGv_i64 tcg_rd = tcg_temp_new_i64();
9755 int pass;
9756
9757 for (pass = 0; pass < (is_scalar ? 1 : 2); pass++) {
9758 read_vec_element(s, tcg_rn, rn, pass, MO_64);
9759 read_vec_element(s, tcg_rd, rd, pass, MO_64);
9760
9761 if (is_u) { /* USQADD */
9762 gen_helper_neon_uqadd_s64(tcg_rd, tcg_env, tcg_rn, tcg_rd);
9763 } else { /* SUQADD */
9764 gen_helper_neon_sqadd_u64(tcg_rd, tcg_env, tcg_rn, tcg_rd);
9765 }
9766 write_vec_element(s, tcg_rd, rd, pass, MO_64);
9767 }
9768 clear_vec_high(s, !is_scalar, rd);
9769 } else {
9770 TCGv_i32 tcg_rn = tcg_temp_new_i32();
9771 TCGv_i32 tcg_rd = tcg_temp_new_i32();
9772 int pass, maxpasses;
9773
9774 if (is_scalar) {
9775 maxpasses = 1;
9776 } else {
9777 maxpasses = is_q ? 4 : 2;
9778 }
9779
9780 for (pass = 0; pass < maxpasses; pass++) {
9781 if (is_scalar) {
9782 read_vec_element_i32(s, tcg_rn, rn, pass, size);
9783 read_vec_element_i32(s, tcg_rd, rd, pass, size);
9784 } else {
9785 read_vec_element_i32(s, tcg_rn, rn, pass, MO_32);
9786 read_vec_element_i32(s, tcg_rd, rd, pass, MO_32);
9787 }
9788
9789 if (is_u) { /* USQADD */
9790 switch (size) {
9791 case 0:
9792 gen_helper_neon_uqadd_s8(tcg_rd, tcg_env, tcg_rn, tcg_rd);
9793 break;
9794 case 1:
9795 gen_helper_neon_uqadd_s16(tcg_rd, tcg_env, tcg_rn, tcg_rd);
9796 break;
9797 case 2:
9798 gen_helper_neon_uqadd_s32(tcg_rd, tcg_env, tcg_rn, tcg_rd);
9799 break;
9800 default:
9801 g_assert_not_reached();
9802 }
9803 } else { /* SUQADD */
9804 switch (size) {
9805 case 0:
9806 gen_helper_neon_sqadd_u8(tcg_rd, tcg_env, tcg_rn, tcg_rd);
9807 break;
9808 case 1:
9809 gen_helper_neon_sqadd_u16(tcg_rd, tcg_env, tcg_rn, tcg_rd);
9810 break;
9811 case 2:
9812 gen_helper_neon_sqadd_u32(tcg_rd, tcg_env, tcg_rn, tcg_rd);
9813 break;
9814 default:
9815 g_assert_not_reached();
9816 }
9817 }
9818
9819 if (is_scalar) {
9820 write_vec_element(s, tcg_constant_i64(0), rd, 0, MO_64);
9821 }
9822 write_vec_element_i32(s, tcg_rd, rd, pass, MO_32);
9823 }
9824 clear_vec_high(s, is_q, rd);
9825 }
9826 }
9827
9828 /* AdvSIMD scalar two reg misc
9829 * 31 30 29 28 24 23 22 21 17 16 12 11 10 9 5 4 0
9830 * +-----+---+-----------+------+-----------+--------+-----+------+------+
9831 * | 0 1 | U | 1 1 1 1 0 | size | 1 0 0 0 0 | opcode | 1 0 | Rn | Rd |
9832 * +-----+---+-----------+------+-----------+--------+-----+------+------+
9833 */
disas_simd_scalar_two_reg_misc(DisasContext * s,uint32_t insn)9834 static void disas_simd_scalar_two_reg_misc(DisasContext *s, uint32_t insn)
9835 {
9836 int rd = extract32(insn, 0, 5);
9837 int rn = extract32(insn, 5, 5);
9838 int opcode = extract32(insn, 12, 5);
9839 int size = extract32(insn, 22, 2);
9840 bool u = extract32(insn, 29, 1);
9841 bool is_fcvt = false;
9842 int rmode;
9843 TCGv_i32 tcg_rmode;
9844 TCGv_ptr tcg_fpstatus;
9845
9846 switch (opcode) {
9847 case 0x3: /* USQADD / SUQADD*/
9848 if (!fp_access_check(s)) {
9849 return;
9850 }
9851 handle_2misc_satacc(s, true, u, false, size, rn, rd);
9852 return;
9853 case 0x7: /* SQABS / SQNEG */
9854 break;
9855 case 0xa: /* CMLT */
9856 if (u) {
9857 unallocated_encoding(s);
9858 return;
9859 }
9860 /* fall through */
9861 case 0x8: /* CMGT, CMGE */
9862 case 0x9: /* CMEQ, CMLE */
9863 case 0xb: /* ABS, NEG */
9864 if (size != 3) {
9865 unallocated_encoding(s);
9866 return;
9867 }
9868 break;
9869 case 0x12: /* SQXTUN */
9870 if (!u) {
9871 unallocated_encoding(s);
9872 return;
9873 }
9874 /* fall through */
9875 case 0x14: /* SQXTN, UQXTN */
9876 if (size == 3) {
9877 unallocated_encoding(s);
9878 return;
9879 }
9880 if (!fp_access_check(s)) {
9881 return;
9882 }
9883 handle_2misc_narrow(s, true, opcode, u, false, size, rn, rd);
9884 return;
9885 case 0xc ... 0xf:
9886 case 0x16 ... 0x1d:
9887 case 0x1f:
9888 /* Floating point: U, size[1] and opcode indicate operation;
9889 * size[0] indicates single or double precision.
9890 */
9891 opcode |= (extract32(size, 1, 1) << 5) | (u << 6);
9892 size = extract32(size, 0, 1) ? 3 : 2;
9893 switch (opcode) {
9894 case 0x2c: /* FCMGT (zero) */
9895 case 0x2d: /* FCMEQ (zero) */
9896 case 0x2e: /* FCMLT (zero) */
9897 case 0x6c: /* FCMGE (zero) */
9898 case 0x6d: /* FCMLE (zero) */
9899 handle_2misc_fcmp_zero(s, opcode, true, u, true, size, rn, rd);
9900 return;
9901 case 0x1d: /* SCVTF */
9902 case 0x5d: /* UCVTF */
9903 {
9904 bool is_signed = (opcode == 0x1d);
9905 if (!fp_access_check(s)) {
9906 return;
9907 }
9908 handle_simd_intfp_conv(s, rd, rn, 1, is_signed, 0, size);
9909 return;
9910 }
9911 case 0x3d: /* FRECPE */
9912 case 0x3f: /* FRECPX */
9913 case 0x7d: /* FRSQRTE */
9914 if (!fp_access_check(s)) {
9915 return;
9916 }
9917 handle_2misc_reciprocal(s, opcode, true, u, true, size, rn, rd);
9918 return;
9919 case 0x1a: /* FCVTNS */
9920 case 0x1b: /* FCVTMS */
9921 case 0x3a: /* FCVTPS */
9922 case 0x3b: /* FCVTZS */
9923 case 0x5a: /* FCVTNU */
9924 case 0x5b: /* FCVTMU */
9925 case 0x7a: /* FCVTPU */
9926 case 0x7b: /* FCVTZU */
9927 is_fcvt = true;
9928 rmode = extract32(opcode, 5, 1) | (extract32(opcode, 0, 1) << 1);
9929 break;
9930 case 0x1c: /* FCVTAS */
9931 case 0x5c: /* FCVTAU */
9932 /* TIEAWAY doesn't fit in the usual rounding mode encoding */
9933 is_fcvt = true;
9934 rmode = FPROUNDING_TIEAWAY;
9935 break;
9936 case 0x56: /* FCVTXN, FCVTXN2 */
9937 if (size == 2) {
9938 unallocated_encoding(s);
9939 return;
9940 }
9941 if (!fp_access_check(s)) {
9942 return;
9943 }
9944 handle_2misc_narrow(s, true, opcode, u, false, size - 1, rn, rd);
9945 return;
9946 default:
9947 unallocated_encoding(s);
9948 return;
9949 }
9950 break;
9951 default:
9952 unallocated_encoding(s);
9953 return;
9954 }
9955
9956 if (!fp_access_check(s)) {
9957 return;
9958 }
9959
9960 if (is_fcvt) {
9961 tcg_fpstatus = fpstatus_ptr(FPST_FPCR);
9962 tcg_rmode = gen_set_rmode(rmode, tcg_fpstatus);
9963 } else {
9964 tcg_fpstatus = NULL;
9965 tcg_rmode = NULL;
9966 }
9967
9968 if (size == 3) {
9969 TCGv_i64 tcg_rn = read_fp_dreg(s, rn);
9970 TCGv_i64 tcg_rd = tcg_temp_new_i64();
9971
9972 handle_2misc_64(s, opcode, u, tcg_rd, tcg_rn, tcg_rmode, tcg_fpstatus);
9973 write_fp_dreg(s, rd, tcg_rd);
9974 } else {
9975 TCGv_i32 tcg_rn = tcg_temp_new_i32();
9976 TCGv_i32 tcg_rd = tcg_temp_new_i32();
9977
9978 read_vec_element_i32(s, tcg_rn, rn, 0, size);
9979
9980 switch (opcode) {
9981 case 0x7: /* SQABS, SQNEG */
9982 {
9983 NeonGenOneOpEnvFn *genfn;
9984 static NeonGenOneOpEnvFn * const fns[3][2] = {
9985 { gen_helper_neon_qabs_s8, gen_helper_neon_qneg_s8 },
9986 { gen_helper_neon_qabs_s16, gen_helper_neon_qneg_s16 },
9987 { gen_helper_neon_qabs_s32, gen_helper_neon_qneg_s32 },
9988 };
9989 genfn = fns[size][u];
9990 genfn(tcg_rd, tcg_env, tcg_rn);
9991 break;
9992 }
9993 case 0x1a: /* FCVTNS */
9994 case 0x1b: /* FCVTMS */
9995 case 0x1c: /* FCVTAS */
9996 case 0x3a: /* FCVTPS */
9997 case 0x3b: /* FCVTZS */
9998 gen_helper_vfp_tosls(tcg_rd, tcg_rn, tcg_constant_i32(0),
9999 tcg_fpstatus);
10000 break;
10001 case 0x5a: /* FCVTNU */
10002 case 0x5b: /* FCVTMU */
10003 case 0x5c: /* FCVTAU */
10004 case 0x7a: /* FCVTPU */
10005 case 0x7b: /* FCVTZU */
10006 gen_helper_vfp_touls(tcg_rd, tcg_rn, tcg_constant_i32(0),
10007 tcg_fpstatus);
10008 break;
10009 default:
10010 g_assert_not_reached();
10011 }
10012
10013 write_fp_sreg(s, rd, tcg_rd);
10014 }
10015
10016 if (is_fcvt) {
10017 gen_restore_rmode(tcg_rmode, tcg_fpstatus);
10018 }
10019 }
10020
10021 /* SSHR[RA]/USHR[RA] - Vector shift right (optional rounding/accumulate) */
handle_vec_simd_shri(DisasContext * s,bool is_q,bool is_u,int immh,int immb,int opcode,int rn,int rd)10022 static void handle_vec_simd_shri(DisasContext *s, bool is_q, bool is_u,
10023 int immh, int immb, int opcode, int rn, int rd)
10024 {
10025 int size = 32 - clz32(immh) - 1;
10026 int immhb = immh << 3 | immb;
10027 int shift = 2 * (8 << size) - immhb;
10028 GVecGen2iFn *gvec_fn;
10029
10030 if (extract32(immh, 3, 1) && !is_q) {
10031 unallocated_encoding(s);
10032 return;
10033 }
10034 tcg_debug_assert(size <= 3);
10035
10036 if (!fp_access_check(s)) {
10037 return;
10038 }
10039
10040 switch (opcode) {
10041 case 0x02: /* SSRA / USRA (accumulate) */
10042 gvec_fn = is_u ? gen_gvec_usra : gen_gvec_ssra;
10043 break;
10044
10045 case 0x08: /* SRI */
10046 gvec_fn = gen_gvec_sri;
10047 break;
10048
10049 case 0x00: /* SSHR / USHR */
10050 if (is_u) {
10051 if (shift == 8 << size) {
10052 /* Shift count the same size as element size produces zero. */
10053 tcg_gen_gvec_dup_imm(size, vec_full_reg_offset(s, rd),
10054 is_q ? 16 : 8, vec_full_reg_size(s), 0);
10055 return;
10056 }
10057 gvec_fn = tcg_gen_gvec_shri;
10058 } else {
10059 /* Shift count the same size as element size produces all sign. */
10060 if (shift == 8 << size) {
10061 shift -= 1;
10062 }
10063 gvec_fn = tcg_gen_gvec_sari;
10064 }
10065 break;
10066
10067 case 0x04: /* SRSHR / URSHR (rounding) */
10068 gvec_fn = is_u ? gen_gvec_urshr : gen_gvec_srshr;
10069 break;
10070
10071 case 0x06: /* SRSRA / URSRA (accum + rounding) */
10072 gvec_fn = is_u ? gen_gvec_ursra : gen_gvec_srsra;
10073 break;
10074
10075 default:
10076 g_assert_not_reached();
10077 }
10078
10079 gen_gvec_fn2i(s, is_q, rd, rn, shift, gvec_fn, size);
10080 }
10081
10082 /* SHL/SLI - Vector shift left */
handle_vec_simd_shli(DisasContext * s,bool is_q,bool insert,int immh,int immb,int opcode,int rn,int rd)10083 static void handle_vec_simd_shli(DisasContext *s, bool is_q, bool insert,
10084 int immh, int immb, int opcode, int rn, int rd)
10085 {
10086 int size = 32 - clz32(immh) - 1;
10087 int immhb = immh << 3 | immb;
10088 int shift = immhb - (8 << size);
10089
10090 /* Range of size is limited by decode: immh is a non-zero 4 bit field */
10091 assert(size >= 0 && size <= 3);
10092
10093 if (extract32(immh, 3, 1) && !is_q) {
10094 unallocated_encoding(s);
10095 return;
10096 }
10097
10098 if (!fp_access_check(s)) {
10099 return;
10100 }
10101
10102 if (insert) {
10103 gen_gvec_fn2i(s, is_q, rd, rn, shift, gen_gvec_sli, size);
10104 } else {
10105 gen_gvec_fn2i(s, is_q, rd, rn, shift, tcg_gen_gvec_shli, size);
10106 }
10107 }
10108
10109 /* USHLL/SHLL - Vector shift left with widening */
handle_vec_simd_wshli(DisasContext * s,bool is_q,bool is_u,int immh,int immb,int opcode,int rn,int rd)10110 static void handle_vec_simd_wshli(DisasContext *s, bool is_q, bool is_u,
10111 int immh, int immb, int opcode, int rn, int rd)
10112 {
10113 int size = 32 - clz32(immh) - 1;
10114 int immhb = immh << 3 | immb;
10115 int shift = immhb - (8 << size);
10116 int dsize = 64;
10117 int esize = 8 << size;
10118 int elements = dsize/esize;
10119 TCGv_i64 tcg_rn = tcg_temp_new_i64();
10120 TCGv_i64 tcg_rd = tcg_temp_new_i64();
10121 int i;
10122
10123 if (size >= 3) {
10124 unallocated_encoding(s);
10125 return;
10126 }
10127
10128 if (!fp_access_check(s)) {
10129 return;
10130 }
10131
10132 /* For the LL variants the store is larger than the load,
10133 * so if rd == rn we would overwrite parts of our input.
10134 * So load everything right now and use shifts in the main loop.
10135 */
10136 read_vec_element(s, tcg_rn, rn, is_q ? 1 : 0, MO_64);
10137
10138 for (i = 0; i < elements; i++) {
10139 tcg_gen_shri_i64(tcg_rd, tcg_rn, i * esize);
10140 ext_and_shift_reg(tcg_rd, tcg_rd, size | (!is_u << 2), 0);
10141 tcg_gen_shli_i64(tcg_rd, tcg_rd, shift);
10142 write_vec_element(s, tcg_rd, rd, i, size + 1);
10143 }
10144 }
10145
10146 /* SHRN/RSHRN - Shift right with narrowing (and potential rounding) */
handle_vec_simd_shrn(DisasContext * s,bool is_q,int immh,int immb,int opcode,int rn,int rd)10147 static void handle_vec_simd_shrn(DisasContext *s, bool is_q,
10148 int immh, int immb, int opcode, int rn, int rd)
10149 {
10150 int immhb = immh << 3 | immb;
10151 int size = 32 - clz32(immh) - 1;
10152 int dsize = 64;
10153 int esize = 8 << size;
10154 int elements = dsize/esize;
10155 int shift = (2 * esize) - immhb;
10156 bool round = extract32(opcode, 0, 1);
10157 TCGv_i64 tcg_rn, tcg_rd, tcg_final;
10158 TCGv_i64 tcg_round;
10159 int i;
10160
10161 if (extract32(immh, 3, 1)) {
10162 unallocated_encoding(s);
10163 return;
10164 }
10165
10166 if (!fp_access_check(s)) {
10167 return;
10168 }
10169
10170 tcg_rn = tcg_temp_new_i64();
10171 tcg_rd = tcg_temp_new_i64();
10172 tcg_final = tcg_temp_new_i64();
10173 read_vec_element(s, tcg_final, rd, is_q ? 1 : 0, MO_64);
10174
10175 if (round) {
10176 tcg_round = tcg_constant_i64(1ULL << (shift - 1));
10177 } else {
10178 tcg_round = NULL;
10179 }
10180
10181 for (i = 0; i < elements; i++) {
10182 read_vec_element(s, tcg_rn, rn, i, size+1);
10183 handle_shri_with_rndacc(tcg_rd, tcg_rn, tcg_round,
10184 false, true, size+1, shift);
10185
10186 tcg_gen_deposit_i64(tcg_final, tcg_final, tcg_rd, esize * i, esize);
10187 }
10188
10189 if (!is_q) {
10190 write_vec_element(s, tcg_final, rd, 0, MO_64);
10191 } else {
10192 write_vec_element(s, tcg_final, rd, 1, MO_64);
10193 }
10194
10195 clear_vec_high(s, is_q, rd);
10196 }
10197
10198
10199 /* AdvSIMD shift by immediate
10200 * 31 30 29 28 23 22 19 18 16 15 11 10 9 5 4 0
10201 * +---+---+---+-------------+------+------+--------+---+------+------+
10202 * | 0 | Q | U | 0 1 1 1 1 0 | immh | immb | opcode | 1 | Rn | Rd |
10203 * +---+---+---+-------------+------+------+--------+---+------+------+
10204 */
disas_simd_shift_imm(DisasContext * s,uint32_t insn)10205 static void disas_simd_shift_imm(DisasContext *s, uint32_t insn)
10206 {
10207 int rd = extract32(insn, 0, 5);
10208 int rn = extract32(insn, 5, 5);
10209 int opcode = extract32(insn, 11, 5);
10210 int immb = extract32(insn, 16, 3);
10211 int immh = extract32(insn, 19, 4);
10212 bool is_u = extract32(insn, 29, 1);
10213 bool is_q = extract32(insn, 30, 1);
10214
10215 /* data_proc_simd[] has sent immh == 0 to disas_simd_mod_imm. */
10216 assert(immh != 0);
10217
10218 switch (opcode) {
10219 case 0x08: /* SRI */
10220 if (!is_u) {
10221 unallocated_encoding(s);
10222 return;
10223 }
10224 /* fall through */
10225 case 0x00: /* SSHR / USHR */
10226 case 0x02: /* SSRA / USRA (accumulate) */
10227 case 0x04: /* SRSHR / URSHR (rounding) */
10228 case 0x06: /* SRSRA / URSRA (accum + rounding) */
10229 handle_vec_simd_shri(s, is_q, is_u, immh, immb, opcode, rn, rd);
10230 break;
10231 case 0x0a: /* SHL / SLI */
10232 handle_vec_simd_shli(s, is_q, is_u, immh, immb, opcode, rn, rd);
10233 break;
10234 case 0x10: /* SHRN */
10235 case 0x11: /* RSHRN / SQRSHRUN */
10236 if (is_u) {
10237 handle_vec_simd_sqshrn(s, false, is_q, false, true, immh, immb,
10238 opcode, rn, rd);
10239 } else {
10240 handle_vec_simd_shrn(s, is_q, immh, immb, opcode, rn, rd);
10241 }
10242 break;
10243 case 0x12: /* SQSHRN / UQSHRN */
10244 case 0x13: /* SQRSHRN / UQRSHRN */
10245 handle_vec_simd_sqshrn(s, false, is_q, is_u, is_u, immh, immb,
10246 opcode, rn, rd);
10247 break;
10248 case 0x14: /* SSHLL / USHLL */
10249 handle_vec_simd_wshli(s, is_q, is_u, immh, immb, opcode, rn, rd);
10250 break;
10251 case 0x1c: /* SCVTF / UCVTF */
10252 handle_simd_shift_intfp_conv(s, false, is_q, is_u, immh, immb,
10253 opcode, rn, rd);
10254 break;
10255 case 0xc: /* SQSHLU */
10256 if (!is_u) {
10257 unallocated_encoding(s);
10258 return;
10259 }
10260 handle_simd_qshl(s, false, is_q, false, true, immh, immb, rn, rd);
10261 break;
10262 case 0xe: /* SQSHL, UQSHL */
10263 handle_simd_qshl(s, false, is_q, is_u, is_u, immh, immb, rn, rd);
10264 break;
10265 case 0x1f: /* FCVTZS/ FCVTZU */
10266 handle_simd_shift_fpint_conv(s, false, is_q, is_u, immh, immb, rn, rd);
10267 return;
10268 default:
10269 unallocated_encoding(s);
10270 return;
10271 }
10272 }
10273
10274 /* Generate code to do a "long" addition or subtraction, ie one done in
10275 * TCGv_i64 on vector lanes twice the width specified by size.
10276 */
gen_neon_addl(int size,bool is_sub,TCGv_i64 tcg_res,TCGv_i64 tcg_op1,TCGv_i64 tcg_op2)10277 static void gen_neon_addl(int size, bool is_sub, TCGv_i64 tcg_res,
10278 TCGv_i64 tcg_op1, TCGv_i64 tcg_op2)
10279 {
10280 static NeonGenTwo64OpFn * const fns[3][2] = {
10281 { gen_helper_neon_addl_u16, gen_helper_neon_subl_u16 },
10282 { gen_helper_neon_addl_u32, gen_helper_neon_subl_u32 },
10283 { tcg_gen_add_i64, tcg_gen_sub_i64 },
10284 };
10285 NeonGenTwo64OpFn *genfn;
10286 assert(size < 3);
10287
10288 genfn = fns[size][is_sub];
10289 genfn(tcg_res, tcg_op1, tcg_op2);
10290 }
10291
handle_3rd_widening(DisasContext * s,int is_q,int is_u,int size,int opcode,int rd,int rn,int rm)10292 static void handle_3rd_widening(DisasContext *s, int is_q, int is_u, int size,
10293 int opcode, int rd, int rn, int rm)
10294 {
10295 /* 3-reg-different widening insns: 64 x 64 -> 128 */
10296 TCGv_i64 tcg_res[2];
10297 int pass, accop;
10298
10299 tcg_res[0] = tcg_temp_new_i64();
10300 tcg_res[1] = tcg_temp_new_i64();
10301
10302 /* Does this op do an adding accumulate, a subtracting accumulate,
10303 * or no accumulate at all?
10304 */
10305 switch (opcode) {
10306 case 5:
10307 case 8:
10308 case 9:
10309 accop = 1;
10310 break;
10311 case 10:
10312 case 11:
10313 accop = -1;
10314 break;
10315 default:
10316 accop = 0;
10317 break;
10318 }
10319
10320 if (accop != 0) {
10321 read_vec_element(s, tcg_res[0], rd, 0, MO_64);
10322 read_vec_element(s, tcg_res[1], rd, 1, MO_64);
10323 }
10324
10325 /* size == 2 means two 32x32->64 operations; this is worth special
10326 * casing because we can generally handle it inline.
10327 */
10328 if (size == 2) {
10329 for (pass = 0; pass < 2; pass++) {
10330 TCGv_i64 tcg_op1 = tcg_temp_new_i64();
10331 TCGv_i64 tcg_op2 = tcg_temp_new_i64();
10332 TCGv_i64 tcg_passres;
10333 MemOp memop = MO_32 | (is_u ? 0 : MO_SIGN);
10334
10335 int elt = pass + is_q * 2;
10336
10337 read_vec_element(s, tcg_op1, rn, elt, memop);
10338 read_vec_element(s, tcg_op2, rm, elt, memop);
10339
10340 if (accop == 0) {
10341 tcg_passres = tcg_res[pass];
10342 } else {
10343 tcg_passres = tcg_temp_new_i64();
10344 }
10345
10346 switch (opcode) {
10347 case 0: /* SADDL, SADDL2, UADDL, UADDL2 */
10348 tcg_gen_add_i64(tcg_passres, tcg_op1, tcg_op2);
10349 break;
10350 case 2: /* SSUBL, SSUBL2, USUBL, USUBL2 */
10351 tcg_gen_sub_i64(tcg_passres, tcg_op1, tcg_op2);
10352 break;
10353 case 5: /* SABAL, SABAL2, UABAL, UABAL2 */
10354 case 7: /* SABDL, SABDL2, UABDL, UABDL2 */
10355 {
10356 TCGv_i64 tcg_tmp1 = tcg_temp_new_i64();
10357 TCGv_i64 tcg_tmp2 = tcg_temp_new_i64();
10358
10359 tcg_gen_sub_i64(tcg_tmp1, tcg_op1, tcg_op2);
10360 tcg_gen_sub_i64(tcg_tmp2, tcg_op2, tcg_op1);
10361 tcg_gen_movcond_i64(is_u ? TCG_COND_GEU : TCG_COND_GE,
10362 tcg_passres,
10363 tcg_op1, tcg_op2, tcg_tmp1, tcg_tmp2);
10364 break;
10365 }
10366 case 8: /* SMLAL, SMLAL2, UMLAL, UMLAL2 */
10367 case 10: /* SMLSL, SMLSL2, UMLSL, UMLSL2 */
10368 case 12: /* UMULL, UMULL2, SMULL, SMULL2 */
10369 tcg_gen_mul_i64(tcg_passres, tcg_op1, tcg_op2);
10370 break;
10371 case 9: /* SQDMLAL, SQDMLAL2 */
10372 case 11: /* SQDMLSL, SQDMLSL2 */
10373 case 13: /* SQDMULL, SQDMULL2 */
10374 tcg_gen_mul_i64(tcg_passres, tcg_op1, tcg_op2);
10375 gen_helper_neon_addl_saturate_s64(tcg_passres, tcg_env,
10376 tcg_passres, tcg_passres);
10377 break;
10378 default:
10379 g_assert_not_reached();
10380 }
10381
10382 if (opcode == 9 || opcode == 11) {
10383 /* saturating accumulate ops */
10384 if (accop < 0) {
10385 tcg_gen_neg_i64(tcg_passres, tcg_passres);
10386 }
10387 gen_helper_neon_addl_saturate_s64(tcg_res[pass], tcg_env,
10388 tcg_res[pass], tcg_passres);
10389 } else if (accop > 0) {
10390 tcg_gen_add_i64(tcg_res[pass], tcg_res[pass], tcg_passres);
10391 } else if (accop < 0) {
10392 tcg_gen_sub_i64(tcg_res[pass], tcg_res[pass], tcg_passres);
10393 }
10394 }
10395 } else {
10396 /* size 0 or 1, generally helper functions */
10397 for (pass = 0; pass < 2; pass++) {
10398 TCGv_i32 tcg_op1 = tcg_temp_new_i32();
10399 TCGv_i32 tcg_op2 = tcg_temp_new_i32();
10400 TCGv_i64 tcg_passres;
10401 int elt = pass + is_q * 2;
10402
10403 read_vec_element_i32(s, tcg_op1, rn, elt, MO_32);
10404 read_vec_element_i32(s, tcg_op2, rm, elt, MO_32);
10405
10406 if (accop == 0) {
10407 tcg_passres = tcg_res[pass];
10408 } else {
10409 tcg_passres = tcg_temp_new_i64();
10410 }
10411
10412 switch (opcode) {
10413 case 0: /* SADDL, SADDL2, UADDL, UADDL2 */
10414 case 2: /* SSUBL, SSUBL2, USUBL, USUBL2 */
10415 {
10416 TCGv_i64 tcg_op2_64 = tcg_temp_new_i64();
10417 static NeonGenWidenFn * const widenfns[2][2] = {
10418 { gen_helper_neon_widen_s8, gen_helper_neon_widen_u8 },
10419 { gen_helper_neon_widen_s16, gen_helper_neon_widen_u16 },
10420 };
10421 NeonGenWidenFn *widenfn = widenfns[size][is_u];
10422
10423 widenfn(tcg_op2_64, tcg_op2);
10424 widenfn(tcg_passres, tcg_op1);
10425 gen_neon_addl(size, (opcode == 2), tcg_passres,
10426 tcg_passres, tcg_op2_64);
10427 break;
10428 }
10429 case 5: /* SABAL, SABAL2, UABAL, UABAL2 */
10430 case 7: /* SABDL, SABDL2, UABDL, UABDL2 */
10431 if (size == 0) {
10432 if (is_u) {
10433 gen_helper_neon_abdl_u16(tcg_passres, tcg_op1, tcg_op2);
10434 } else {
10435 gen_helper_neon_abdl_s16(tcg_passres, tcg_op1, tcg_op2);
10436 }
10437 } else {
10438 if (is_u) {
10439 gen_helper_neon_abdl_u32(tcg_passres, tcg_op1, tcg_op2);
10440 } else {
10441 gen_helper_neon_abdl_s32(tcg_passres, tcg_op1, tcg_op2);
10442 }
10443 }
10444 break;
10445 case 8: /* SMLAL, SMLAL2, UMLAL, UMLAL2 */
10446 case 10: /* SMLSL, SMLSL2, UMLSL, UMLSL2 */
10447 case 12: /* UMULL, UMULL2, SMULL, SMULL2 */
10448 if (size == 0) {
10449 if (is_u) {
10450 gen_helper_neon_mull_u8(tcg_passres, tcg_op1, tcg_op2);
10451 } else {
10452 gen_helper_neon_mull_s8(tcg_passres, tcg_op1, tcg_op2);
10453 }
10454 } else {
10455 if (is_u) {
10456 gen_helper_neon_mull_u16(tcg_passres, tcg_op1, tcg_op2);
10457 } else {
10458 gen_helper_neon_mull_s16(tcg_passres, tcg_op1, tcg_op2);
10459 }
10460 }
10461 break;
10462 case 9: /* SQDMLAL, SQDMLAL2 */
10463 case 11: /* SQDMLSL, SQDMLSL2 */
10464 case 13: /* SQDMULL, SQDMULL2 */
10465 assert(size == 1);
10466 gen_helper_neon_mull_s16(tcg_passres, tcg_op1, tcg_op2);
10467 gen_helper_neon_addl_saturate_s32(tcg_passres, tcg_env,
10468 tcg_passres, tcg_passres);
10469 break;
10470 default:
10471 g_assert_not_reached();
10472 }
10473
10474 if (accop != 0) {
10475 if (opcode == 9 || opcode == 11) {
10476 /* saturating accumulate ops */
10477 if (accop < 0) {
10478 gen_helper_neon_negl_u32(tcg_passres, tcg_passres);
10479 }
10480 gen_helper_neon_addl_saturate_s32(tcg_res[pass], tcg_env,
10481 tcg_res[pass],
10482 tcg_passres);
10483 } else {
10484 gen_neon_addl(size, (accop < 0), tcg_res[pass],
10485 tcg_res[pass], tcg_passres);
10486 }
10487 }
10488 }
10489 }
10490
10491 write_vec_element(s, tcg_res[0], rd, 0, MO_64);
10492 write_vec_element(s, tcg_res[1], rd, 1, MO_64);
10493 }
10494
handle_3rd_wide(DisasContext * s,int is_q,int is_u,int size,int opcode,int rd,int rn,int rm)10495 static void handle_3rd_wide(DisasContext *s, int is_q, int is_u, int size,
10496 int opcode, int rd, int rn, int rm)
10497 {
10498 TCGv_i64 tcg_res[2];
10499 int part = is_q ? 2 : 0;
10500 int pass;
10501
10502 for (pass = 0; pass < 2; pass++) {
10503 TCGv_i64 tcg_op1 = tcg_temp_new_i64();
10504 TCGv_i32 tcg_op2 = tcg_temp_new_i32();
10505 TCGv_i64 tcg_op2_wide = tcg_temp_new_i64();
10506 static NeonGenWidenFn * const widenfns[3][2] = {
10507 { gen_helper_neon_widen_s8, gen_helper_neon_widen_u8 },
10508 { gen_helper_neon_widen_s16, gen_helper_neon_widen_u16 },
10509 { tcg_gen_ext_i32_i64, tcg_gen_extu_i32_i64 },
10510 };
10511 NeonGenWidenFn *widenfn = widenfns[size][is_u];
10512
10513 read_vec_element(s, tcg_op1, rn, pass, MO_64);
10514 read_vec_element_i32(s, tcg_op2, rm, part + pass, MO_32);
10515 widenfn(tcg_op2_wide, tcg_op2);
10516 tcg_res[pass] = tcg_temp_new_i64();
10517 gen_neon_addl(size, (opcode == 3),
10518 tcg_res[pass], tcg_op1, tcg_op2_wide);
10519 }
10520
10521 for (pass = 0; pass < 2; pass++) {
10522 write_vec_element(s, tcg_res[pass], rd, pass, MO_64);
10523 }
10524 }
10525
do_narrow_round_high_u32(TCGv_i32 res,TCGv_i64 in)10526 static void do_narrow_round_high_u32(TCGv_i32 res, TCGv_i64 in)
10527 {
10528 tcg_gen_addi_i64(in, in, 1U << 31);
10529 tcg_gen_extrh_i64_i32(res, in);
10530 }
10531
handle_3rd_narrowing(DisasContext * s,int is_q,int is_u,int size,int opcode,int rd,int rn,int rm)10532 static void handle_3rd_narrowing(DisasContext *s, int is_q, int is_u, int size,
10533 int opcode, int rd, int rn, int rm)
10534 {
10535 TCGv_i32 tcg_res[2];
10536 int part = is_q ? 2 : 0;
10537 int pass;
10538
10539 for (pass = 0; pass < 2; pass++) {
10540 TCGv_i64 tcg_op1 = tcg_temp_new_i64();
10541 TCGv_i64 tcg_op2 = tcg_temp_new_i64();
10542 TCGv_i64 tcg_wideres = tcg_temp_new_i64();
10543 static NeonGenNarrowFn * const narrowfns[3][2] = {
10544 { gen_helper_neon_narrow_high_u8,
10545 gen_helper_neon_narrow_round_high_u8 },
10546 { gen_helper_neon_narrow_high_u16,
10547 gen_helper_neon_narrow_round_high_u16 },
10548 { tcg_gen_extrh_i64_i32, do_narrow_round_high_u32 },
10549 };
10550 NeonGenNarrowFn *gennarrow = narrowfns[size][is_u];
10551
10552 read_vec_element(s, tcg_op1, rn, pass, MO_64);
10553 read_vec_element(s, tcg_op2, rm, pass, MO_64);
10554
10555 gen_neon_addl(size, (opcode == 6), tcg_wideres, tcg_op1, tcg_op2);
10556
10557 tcg_res[pass] = tcg_temp_new_i32();
10558 gennarrow(tcg_res[pass], tcg_wideres);
10559 }
10560
10561 for (pass = 0; pass < 2; pass++) {
10562 write_vec_element_i32(s, tcg_res[pass], rd, pass + part, MO_32);
10563 }
10564 clear_vec_high(s, is_q, rd);
10565 }
10566
10567 /* AdvSIMD three different
10568 * 31 30 29 28 24 23 22 21 20 16 15 12 11 10 9 5 4 0
10569 * +---+---+---+-----------+------+---+------+--------+-----+------+------+
10570 * | 0 | Q | U | 0 1 1 1 0 | size | 1 | Rm | opcode | 0 0 | Rn | Rd |
10571 * +---+---+---+-----------+------+---+------+--------+-----+------+------+
10572 */
disas_simd_three_reg_diff(DisasContext * s,uint32_t insn)10573 static void disas_simd_three_reg_diff(DisasContext *s, uint32_t insn)
10574 {
10575 /* Instructions in this group fall into three basic classes
10576 * (in each case with the operation working on each element in
10577 * the input vectors):
10578 * (1) widening 64 x 64 -> 128 (with possibly Vd as an extra
10579 * 128 bit input)
10580 * (2) wide 64 x 128 -> 128
10581 * (3) narrowing 128 x 128 -> 64
10582 * Here we do initial decode, catch unallocated cases and
10583 * dispatch to separate functions for each class.
10584 */
10585 int is_q = extract32(insn, 30, 1);
10586 int is_u = extract32(insn, 29, 1);
10587 int size = extract32(insn, 22, 2);
10588 int opcode = extract32(insn, 12, 4);
10589 int rm = extract32(insn, 16, 5);
10590 int rn = extract32(insn, 5, 5);
10591 int rd = extract32(insn, 0, 5);
10592
10593 switch (opcode) {
10594 case 1: /* SADDW, SADDW2, UADDW, UADDW2 */
10595 case 3: /* SSUBW, SSUBW2, USUBW, USUBW2 */
10596 /* 64 x 128 -> 128 */
10597 if (size == 3) {
10598 unallocated_encoding(s);
10599 return;
10600 }
10601 if (!fp_access_check(s)) {
10602 return;
10603 }
10604 handle_3rd_wide(s, is_q, is_u, size, opcode, rd, rn, rm);
10605 break;
10606 case 4: /* ADDHN, ADDHN2, RADDHN, RADDHN2 */
10607 case 6: /* SUBHN, SUBHN2, RSUBHN, RSUBHN2 */
10608 /* 128 x 128 -> 64 */
10609 if (size == 3) {
10610 unallocated_encoding(s);
10611 return;
10612 }
10613 if (!fp_access_check(s)) {
10614 return;
10615 }
10616 handle_3rd_narrowing(s, is_q, is_u, size, opcode, rd, rn, rm);
10617 break;
10618 case 14: /* PMULL, PMULL2 */
10619 if (is_u) {
10620 unallocated_encoding(s);
10621 return;
10622 }
10623 switch (size) {
10624 case 0: /* PMULL.P8 */
10625 if (!fp_access_check(s)) {
10626 return;
10627 }
10628 /* The Q field specifies lo/hi half input for this insn. */
10629 gen_gvec_op3_ool(s, true, rd, rn, rm, is_q,
10630 gen_helper_neon_pmull_h);
10631 break;
10632
10633 case 3: /* PMULL.P64 */
10634 if (!dc_isar_feature(aa64_pmull, s)) {
10635 unallocated_encoding(s);
10636 return;
10637 }
10638 if (!fp_access_check(s)) {
10639 return;
10640 }
10641 /* The Q field specifies lo/hi half input for this insn. */
10642 gen_gvec_op3_ool(s, true, rd, rn, rm, is_q,
10643 gen_helper_gvec_pmull_q);
10644 break;
10645
10646 default:
10647 unallocated_encoding(s);
10648 break;
10649 }
10650 return;
10651 case 9: /* SQDMLAL, SQDMLAL2 */
10652 case 11: /* SQDMLSL, SQDMLSL2 */
10653 case 13: /* SQDMULL, SQDMULL2 */
10654 if (is_u || size == 0) {
10655 unallocated_encoding(s);
10656 return;
10657 }
10658 /* fall through */
10659 case 0: /* SADDL, SADDL2, UADDL, UADDL2 */
10660 case 2: /* SSUBL, SSUBL2, USUBL, USUBL2 */
10661 case 5: /* SABAL, SABAL2, UABAL, UABAL2 */
10662 case 7: /* SABDL, SABDL2, UABDL, UABDL2 */
10663 case 8: /* SMLAL, SMLAL2, UMLAL, UMLAL2 */
10664 case 10: /* SMLSL, SMLSL2, UMLSL, UMLSL2 */
10665 case 12: /* SMULL, SMULL2, UMULL, UMULL2 */
10666 /* 64 x 64 -> 128 */
10667 if (size == 3) {
10668 unallocated_encoding(s);
10669 return;
10670 }
10671 if (!fp_access_check(s)) {
10672 return;
10673 }
10674
10675 handle_3rd_widening(s, is_q, is_u, size, opcode, rd, rn, rm);
10676 break;
10677 default:
10678 /* opcode 15 not allocated */
10679 unallocated_encoding(s);
10680 break;
10681 }
10682 }
10683
10684 /* Logic op (opcode == 3) subgroup of C3.6.16. */
disas_simd_3same_logic(DisasContext * s,uint32_t insn)10685 static void disas_simd_3same_logic(DisasContext *s, uint32_t insn)
10686 {
10687 int rd = extract32(insn, 0, 5);
10688 int rn = extract32(insn, 5, 5);
10689 int rm = extract32(insn, 16, 5);
10690 int size = extract32(insn, 22, 2);
10691 bool is_u = extract32(insn, 29, 1);
10692 bool is_q = extract32(insn, 30, 1);
10693
10694 if (!fp_access_check(s)) {
10695 return;
10696 }
10697
10698 switch (size + 4 * is_u) {
10699 case 0: /* AND */
10700 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_and, 0);
10701 return;
10702 case 1: /* BIC */
10703 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_andc, 0);
10704 return;
10705 case 2: /* ORR */
10706 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_or, 0);
10707 return;
10708 case 3: /* ORN */
10709 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_orc, 0);
10710 return;
10711 case 4: /* EOR */
10712 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_xor, 0);
10713 return;
10714
10715 case 5: /* BSL bitwise select */
10716 gen_gvec_fn4(s, is_q, rd, rd, rn, rm, tcg_gen_gvec_bitsel, 0);
10717 return;
10718 case 6: /* BIT, bitwise insert if true */
10719 gen_gvec_fn4(s, is_q, rd, rm, rn, rd, tcg_gen_gvec_bitsel, 0);
10720 return;
10721 case 7: /* BIF, bitwise insert if false */
10722 gen_gvec_fn4(s, is_q, rd, rm, rd, rn, tcg_gen_gvec_bitsel, 0);
10723 return;
10724
10725 default:
10726 g_assert_not_reached();
10727 }
10728 }
10729
10730 /* Pairwise op subgroup of C3.6.16.
10731 *
10732 * This is called directly or via the handle_3same_float for float pairwise
10733 * operations where the opcode and size are calculated differently.
10734 */
handle_simd_3same_pair(DisasContext * s,int is_q,int u,int opcode,int size,int rn,int rm,int rd)10735 static void handle_simd_3same_pair(DisasContext *s, int is_q, int u, int opcode,
10736 int size, int rn, int rm, int rd)
10737 {
10738 TCGv_ptr fpst;
10739 int pass;
10740
10741 /* Floating point operations need fpst */
10742 if (opcode >= 0x58) {
10743 fpst = fpstatus_ptr(FPST_FPCR);
10744 } else {
10745 fpst = NULL;
10746 }
10747
10748 if (!fp_access_check(s)) {
10749 return;
10750 }
10751
10752 /* These operations work on the concatenated rm:rn, with each pair of
10753 * adjacent elements being operated on to produce an element in the result.
10754 */
10755 if (size == 3) {
10756 TCGv_i64 tcg_res[2];
10757
10758 for (pass = 0; pass < 2; pass++) {
10759 TCGv_i64 tcg_op1 = tcg_temp_new_i64();
10760 TCGv_i64 tcg_op2 = tcg_temp_new_i64();
10761 int passreg = (pass == 0) ? rn : rm;
10762
10763 read_vec_element(s, tcg_op1, passreg, 0, MO_64);
10764 read_vec_element(s, tcg_op2, passreg, 1, MO_64);
10765 tcg_res[pass] = tcg_temp_new_i64();
10766
10767 switch (opcode) {
10768 case 0x17: /* ADDP */
10769 tcg_gen_add_i64(tcg_res[pass], tcg_op1, tcg_op2);
10770 break;
10771 case 0x58: /* FMAXNMP */
10772 gen_helper_vfp_maxnumd(tcg_res[pass], tcg_op1, tcg_op2, fpst);
10773 break;
10774 case 0x5a: /* FADDP */
10775 gen_helper_vfp_addd(tcg_res[pass], tcg_op1, tcg_op2, fpst);
10776 break;
10777 case 0x5e: /* FMAXP */
10778 gen_helper_vfp_maxd(tcg_res[pass], tcg_op1, tcg_op2, fpst);
10779 break;
10780 case 0x78: /* FMINNMP */
10781 gen_helper_vfp_minnumd(tcg_res[pass], tcg_op1, tcg_op2, fpst);
10782 break;
10783 case 0x7e: /* FMINP */
10784 gen_helper_vfp_mind(tcg_res[pass], tcg_op1, tcg_op2, fpst);
10785 break;
10786 default:
10787 g_assert_not_reached();
10788 }
10789 }
10790
10791 for (pass = 0; pass < 2; pass++) {
10792 write_vec_element(s, tcg_res[pass], rd, pass, MO_64);
10793 }
10794 } else {
10795 int maxpass = is_q ? 4 : 2;
10796 TCGv_i32 tcg_res[4];
10797
10798 for (pass = 0; pass < maxpass; pass++) {
10799 TCGv_i32 tcg_op1 = tcg_temp_new_i32();
10800 TCGv_i32 tcg_op2 = tcg_temp_new_i32();
10801 NeonGenTwoOpFn *genfn = NULL;
10802 int passreg = pass < (maxpass / 2) ? rn : rm;
10803 int passelt = (is_q && (pass & 1)) ? 2 : 0;
10804
10805 read_vec_element_i32(s, tcg_op1, passreg, passelt, MO_32);
10806 read_vec_element_i32(s, tcg_op2, passreg, passelt + 1, MO_32);
10807 tcg_res[pass] = tcg_temp_new_i32();
10808
10809 switch (opcode) {
10810 case 0x17: /* ADDP */
10811 {
10812 static NeonGenTwoOpFn * const fns[3] = {
10813 gen_helper_neon_padd_u8,
10814 gen_helper_neon_padd_u16,
10815 tcg_gen_add_i32,
10816 };
10817 genfn = fns[size];
10818 break;
10819 }
10820 case 0x14: /* SMAXP, UMAXP */
10821 {
10822 static NeonGenTwoOpFn * const fns[3][2] = {
10823 { gen_helper_neon_pmax_s8, gen_helper_neon_pmax_u8 },
10824 { gen_helper_neon_pmax_s16, gen_helper_neon_pmax_u16 },
10825 { tcg_gen_smax_i32, tcg_gen_umax_i32 },
10826 };
10827 genfn = fns[size][u];
10828 break;
10829 }
10830 case 0x15: /* SMINP, UMINP */
10831 {
10832 static NeonGenTwoOpFn * const fns[3][2] = {
10833 { gen_helper_neon_pmin_s8, gen_helper_neon_pmin_u8 },
10834 { gen_helper_neon_pmin_s16, gen_helper_neon_pmin_u16 },
10835 { tcg_gen_smin_i32, tcg_gen_umin_i32 },
10836 };
10837 genfn = fns[size][u];
10838 break;
10839 }
10840 /* The FP operations are all on single floats (32 bit) */
10841 case 0x58: /* FMAXNMP */
10842 gen_helper_vfp_maxnums(tcg_res[pass], tcg_op1, tcg_op2, fpst);
10843 break;
10844 case 0x5a: /* FADDP */
10845 gen_helper_vfp_adds(tcg_res[pass], tcg_op1, tcg_op2, fpst);
10846 break;
10847 case 0x5e: /* FMAXP */
10848 gen_helper_vfp_maxs(tcg_res[pass], tcg_op1, tcg_op2, fpst);
10849 break;
10850 case 0x78: /* FMINNMP */
10851 gen_helper_vfp_minnums(tcg_res[pass], tcg_op1, tcg_op2, fpst);
10852 break;
10853 case 0x7e: /* FMINP */
10854 gen_helper_vfp_mins(tcg_res[pass], tcg_op1, tcg_op2, fpst);
10855 break;
10856 default:
10857 g_assert_not_reached();
10858 }
10859
10860 /* FP ops called directly, otherwise call now */
10861 if (genfn) {
10862 genfn(tcg_res[pass], tcg_op1, tcg_op2);
10863 }
10864 }
10865
10866 for (pass = 0; pass < maxpass; pass++) {
10867 write_vec_element_i32(s, tcg_res[pass], rd, pass, MO_32);
10868 }
10869 clear_vec_high(s, is_q, rd);
10870 }
10871 }
10872
10873 /* Floating point op subgroup of C3.6.16. */
disas_simd_3same_float(DisasContext * s,uint32_t insn)10874 static void disas_simd_3same_float(DisasContext *s, uint32_t insn)
10875 {
10876 /* For floating point ops, the U, size[1] and opcode bits
10877 * together indicate the operation. size[0] indicates single
10878 * or double.
10879 */
10880 int fpopcode = extract32(insn, 11, 5)
10881 | (extract32(insn, 23, 1) << 5)
10882 | (extract32(insn, 29, 1) << 6);
10883 int is_q = extract32(insn, 30, 1);
10884 int size = extract32(insn, 22, 1);
10885 int rm = extract32(insn, 16, 5);
10886 int rn = extract32(insn, 5, 5);
10887 int rd = extract32(insn, 0, 5);
10888
10889 int datasize = is_q ? 128 : 64;
10890 int esize = 32 << size;
10891 int elements = datasize / esize;
10892
10893 if (size == 1 && !is_q) {
10894 unallocated_encoding(s);
10895 return;
10896 }
10897
10898 switch (fpopcode) {
10899 case 0x58: /* FMAXNMP */
10900 case 0x5a: /* FADDP */
10901 case 0x5e: /* FMAXP */
10902 case 0x78: /* FMINNMP */
10903 case 0x7e: /* FMINP */
10904 if (size && !is_q) {
10905 unallocated_encoding(s);
10906 return;
10907 }
10908 handle_simd_3same_pair(s, is_q, 0, fpopcode, size ? MO_64 : MO_32,
10909 rn, rm, rd);
10910 return;
10911 case 0x1b: /* FMULX */
10912 case 0x1f: /* FRECPS */
10913 case 0x3f: /* FRSQRTS */
10914 case 0x5d: /* FACGE */
10915 case 0x7d: /* FACGT */
10916 case 0x19: /* FMLA */
10917 case 0x39: /* FMLS */
10918 case 0x18: /* FMAXNM */
10919 case 0x1a: /* FADD */
10920 case 0x1c: /* FCMEQ */
10921 case 0x1e: /* FMAX */
10922 case 0x38: /* FMINNM */
10923 case 0x3a: /* FSUB */
10924 case 0x3e: /* FMIN */
10925 case 0x5b: /* FMUL */
10926 case 0x5c: /* FCMGE */
10927 case 0x5f: /* FDIV */
10928 case 0x7a: /* FABD */
10929 case 0x7c: /* FCMGT */
10930 if (!fp_access_check(s)) {
10931 return;
10932 }
10933 handle_3same_float(s, size, elements, fpopcode, rd, rn, rm);
10934 return;
10935
10936 case 0x1d: /* FMLAL */
10937 case 0x3d: /* FMLSL */
10938 case 0x59: /* FMLAL2 */
10939 case 0x79: /* FMLSL2 */
10940 if (size & 1 || !dc_isar_feature(aa64_fhm, s)) {
10941 unallocated_encoding(s);
10942 return;
10943 }
10944 if (fp_access_check(s)) {
10945 int is_s = extract32(insn, 23, 1);
10946 int is_2 = extract32(insn, 29, 1);
10947 int data = (is_2 << 1) | is_s;
10948 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd),
10949 vec_full_reg_offset(s, rn),
10950 vec_full_reg_offset(s, rm), tcg_env,
10951 is_q ? 16 : 8, vec_full_reg_size(s),
10952 data, gen_helper_gvec_fmlal_a64);
10953 }
10954 return;
10955
10956 default:
10957 unallocated_encoding(s);
10958 return;
10959 }
10960 }
10961
10962 /* Integer op subgroup of C3.6.16. */
disas_simd_3same_int(DisasContext * s,uint32_t insn)10963 static void disas_simd_3same_int(DisasContext *s, uint32_t insn)
10964 {
10965 int is_q = extract32(insn, 30, 1);
10966 int u = extract32(insn, 29, 1);
10967 int size = extract32(insn, 22, 2);
10968 int opcode = extract32(insn, 11, 5);
10969 int rm = extract32(insn, 16, 5);
10970 int rn = extract32(insn, 5, 5);
10971 int rd = extract32(insn, 0, 5);
10972 int pass;
10973 TCGCond cond;
10974
10975 switch (opcode) {
10976 case 0x13: /* MUL, PMUL */
10977 if (u && size != 0) {
10978 unallocated_encoding(s);
10979 return;
10980 }
10981 /* fall through */
10982 case 0x0: /* SHADD, UHADD */
10983 case 0x2: /* SRHADD, URHADD */
10984 case 0x4: /* SHSUB, UHSUB */
10985 case 0xc: /* SMAX, UMAX */
10986 case 0xd: /* SMIN, UMIN */
10987 case 0xe: /* SABD, UABD */
10988 case 0xf: /* SABA, UABA */
10989 case 0x12: /* MLA, MLS */
10990 if (size == 3) {
10991 unallocated_encoding(s);
10992 return;
10993 }
10994 break;
10995 case 0x16: /* SQDMULH, SQRDMULH */
10996 if (size == 0 || size == 3) {
10997 unallocated_encoding(s);
10998 return;
10999 }
11000 break;
11001 default:
11002 if (size == 3 && !is_q) {
11003 unallocated_encoding(s);
11004 return;
11005 }
11006 break;
11007 }
11008
11009 if (!fp_access_check(s)) {
11010 return;
11011 }
11012
11013 switch (opcode) {
11014 case 0x01: /* SQADD, UQADD */
11015 if (u) {
11016 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_uqadd_qc, size);
11017 } else {
11018 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_sqadd_qc, size);
11019 }
11020 return;
11021 case 0x05: /* SQSUB, UQSUB */
11022 if (u) {
11023 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_uqsub_qc, size);
11024 } else {
11025 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_sqsub_qc, size);
11026 }
11027 return;
11028 case 0x08: /* SSHL, USHL */
11029 if (u) {
11030 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_ushl, size);
11031 } else {
11032 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_sshl, size);
11033 }
11034 return;
11035 case 0x0c: /* SMAX, UMAX */
11036 if (u) {
11037 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_umax, size);
11038 } else {
11039 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_smax, size);
11040 }
11041 return;
11042 case 0x0d: /* SMIN, UMIN */
11043 if (u) {
11044 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_umin, size);
11045 } else {
11046 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_smin, size);
11047 }
11048 return;
11049 case 0xe: /* SABD, UABD */
11050 if (u) {
11051 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_uabd, size);
11052 } else {
11053 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_sabd, size);
11054 }
11055 return;
11056 case 0xf: /* SABA, UABA */
11057 if (u) {
11058 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_uaba, size);
11059 } else {
11060 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_saba, size);
11061 }
11062 return;
11063 case 0x10: /* ADD, SUB */
11064 if (u) {
11065 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_sub, size);
11066 } else {
11067 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_add, size);
11068 }
11069 return;
11070 case 0x13: /* MUL, PMUL */
11071 if (!u) { /* MUL */
11072 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_mul, size);
11073 } else { /* PMUL */
11074 gen_gvec_op3_ool(s, is_q, rd, rn, rm, 0, gen_helper_gvec_pmul_b);
11075 }
11076 return;
11077 case 0x12: /* MLA, MLS */
11078 if (u) {
11079 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_mls, size);
11080 } else {
11081 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_mla, size);
11082 }
11083 return;
11084 case 0x16: /* SQDMULH, SQRDMULH */
11085 {
11086 static gen_helper_gvec_3_ptr * const fns[2][2] = {
11087 { gen_helper_neon_sqdmulh_h, gen_helper_neon_sqrdmulh_h },
11088 { gen_helper_neon_sqdmulh_s, gen_helper_neon_sqrdmulh_s },
11089 };
11090 gen_gvec_op3_qc(s, is_q, rd, rn, rm, fns[size - 1][u]);
11091 }
11092 return;
11093 case 0x11:
11094 if (!u) { /* CMTST */
11095 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_cmtst, size);
11096 return;
11097 }
11098 /* else CMEQ */
11099 cond = TCG_COND_EQ;
11100 goto do_gvec_cmp;
11101 case 0x06: /* CMGT, CMHI */
11102 cond = u ? TCG_COND_GTU : TCG_COND_GT;
11103 goto do_gvec_cmp;
11104 case 0x07: /* CMGE, CMHS */
11105 cond = u ? TCG_COND_GEU : TCG_COND_GE;
11106 do_gvec_cmp:
11107 tcg_gen_gvec_cmp(cond, size, vec_full_reg_offset(s, rd),
11108 vec_full_reg_offset(s, rn),
11109 vec_full_reg_offset(s, rm),
11110 is_q ? 16 : 8, vec_full_reg_size(s));
11111 return;
11112 }
11113
11114 if (size == 3) {
11115 assert(is_q);
11116 for (pass = 0; pass < 2; pass++) {
11117 TCGv_i64 tcg_op1 = tcg_temp_new_i64();
11118 TCGv_i64 tcg_op2 = tcg_temp_new_i64();
11119 TCGv_i64 tcg_res = tcg_temp_new_i64();
11120
11121 read_vec_element(s, tcg_op1, rn, pass, MO_64);
11122 read_vec_element(s, tcg_op2, rm, pass, MO_64);
11123
11124 handle_3same_64(s, opcode, u, tcg_res, tcg_op1, tcg_op2);
11125
11126 write_vec_element(s, tcg_res, rd, pass, MO_64);
11127 }
11128 } else {
11129 for (pass = 0; pass < (is_q ? 4 : 2); pass++) {
11130 TCGv_i32 tcg_op1 = tcg_temp_new_i32();
11131 TCGv_i32 tcg_op2 = tcg_temp_new_i32();
11132 TCGv_i32 tcg_res = tcg_temp_new_i32();
11133 NeonGenTwoOpFn *genfn = NULL;
11134 NeonGenTwoOpEnvFn *genenvfn = NULL;
11135
11136 read_vec_element_i32(s, tcg_op1, rn, pass, MO_32);
11137 read_vec_element_i32(s, tcg_op2, rm, pass, MO_32);
11138
11139 switch (opcode) {
11140 case 0x0: /* SHADD, UHADD */
11141 {
11142 static NeonGenTwoOpFn * const fns[3][2] = {
11143 { gen_helper_neon_hadd_s8, gen_helper_neon_hadd_u8 },
11144 { gen_helper_neon_hadd_s16, gen_helper_neon_hadd_u16 },
11145 { gen_helper_neon_hadd_s32, gen_helper_neon_hadd_u32 },
11146 };
11147 genfn = fns[size][u];
11148 break;
11149 }
11150 case 0x2: /* SRHADD, URHADD */
11151 {
11152 static NeonGenTwoOpFn * const fns[3][2] = {
11153 { gen_helper_neon_rhadd_s8, gen_helper_neon_rhadd_u8 },
11154 { gen_helper_neon_rhadd_s16, gen_helper_neon_rhadd_u16 },
11155 { gen_helper_neon_rhadd_s32, gen_helper_neon_rhadd_u32 },
11156 };
11157 genfn = fns[size][u];
11158 break;
11159 }
11160 case 0x4: /* SHSUB, UHSUB */
11161 {
11162 static NeonGenTwoOpFn * const fns[3][2] = {
11163 { gen_helper_neon_hsub_s8, gen_helper_neon_hsub_u8 },
11164 { gen_helper_neon_hsub_s16, gen_helper_neon_hsub_u16 },
11165 { gen_helper_neon_hsub_s32, gen_helper_neon_hsub_u32 },
11166 };
11167 genfn = fns[size][u];
11168 break;
11169 }
11170 case 0x9: /* SQSHL, UQSHL */
11171 {
11172 static NeonGenTwoOpEnvFn * const fns[3][2] = {
11173 { gen_helper_neon_qshl_s8, gen_helper_neon_qshl_u8 },
11174 { gen_helper_neon_qshl_s16, gen_helper_neon_qshl_u16 },
11175 { gen_helper_neon_qshl_s32, gen_helper_neon_qshl_u32 },
11176 };
11177 genenvfn = fns[size][u];
11178 break;
11179 }
11180 case 0xa: /* SRSHL, URSHL */
11181 {
11182 static NeonGenTwoOpFn * const fns[3][2] = {
11183 { gen_helper_neon_rshl_s8, gen_helper_neon_rshl_u8 },
11184 { gen_helper_neon_rshl_s16, gen_helper_neon_rshl_u16 },
11185 { gen_helper_neon_rshl_s32, gen_helper_neon_rshl_u32 },
11186 };
11187 genfn = fns[size][u];
11188 break;
11189 }
11190 case 0xb: /* SQRSHL, UQRSHL */
11191 {
11192 static NeonGenTwoOpEnvFn * const fns[3][2] = {
11193 { gen_helper_neon_qrshl_s8, gen_helper_neon_qrshl_u8 },
11194 { gen_helper_neon_qrshl_s16, gen_helper_neon_qrshl_u16 },
11195 { gen_helper_neon_qrshl_s32, gen_helper_neon_qrshl_u32 },
11196 };
11197 genenvfn = fns[size][u];
11198 break;
11199 }
11200 default:
11201 g_assert_not_reached();
11202 }
11203
11204 if (genenvfn) {
11205 genenvfn(tcg_res, tcg_env, tcg_op1, tcg_op2);
11206 } else {
11207 genfn(tcg_res, tcg_op1, tcg_op2);
11208 }
11209
11210 write_vec_element_i32(s, tcg_res, rd, pass, MO_32);
11211 }
11212 }
11213 clear_vec_high(s, is_q, rd);
11214 }
11215
11216 /* AdvSIMD three same
11217 * 31 30 29 28 24 23 22 21 20 16 15 11 10 9 5 4 0
11218 * +---+---+---+-----------+------+---+------+--------+---+------+------+
11219 * | 0 | Q | U | 0 1 1 1 0 | size | 1 | Rm | opcode | 1 | Rn | Rd |
11220 * +---+---+---+-----------+------+---+------+--------+---+------+------+
11221 */
disas_simd_three_reg_same(DisasContext * s,uint32_t insn)11222 static void disas_simd_three_reg_same(DisasContext *s, uint32_t insn)
11223 {
11224 int opcode = extract32(insn, 11, 5);
11225
11226 switch (opcode) {
11227 case 0x3: /* logic ops */
11228 disas_simd_3same_logic(s, insn);
11229 break;
11230 case 0x17: /* ADDP */
11231 case 0x14: /* SMAXP, UMAXP */
11232 case 0x15: /* SMINP, UMINP */
11233 {
11234 /* Pairwise operations */
11235 int is_q = extract32(insn, 30, 1);
11236 int u = extract32(insn, 29, 1);
11237 int size = extract32(insn, 22, 2);
11238 int rm = extract32(insn, 16, 5);
11239 int rn = extract32(insn, 5, 5);
11240 int rd = extract32(insn, 0, 5);
11241 if (opcode == 0x17) {
11242 if (u || (size == 3 && !is_q)) {
11243 unallocated_encoding(s);
11244 return;
11245 }
11246 } else {
11247 if (size == 3) {
11248 unallocated_encoding(s);
11249 return;
11250 }
11251 }
11252 handle_simd_3same_pair(s, is_q, u, opcode, size, rn, rm, rd);
11253 break;
11254 }
11255 case 0x18 ... 0x31:
11256 /* floating point ops, sz[1] and U are part of opcode */
11257 disas_simd_3same_float(s, insn);
11258 break;
11259 default:
11260 disas_simd_3same_int(s, insn);
11261 break;
11262 }
11263 }
11264
11265 /*
11266 * Advanced SIMD three same (ARMv8.2 FP16 variants)
11267 *
11268 * 31 30 29 28 24 23 22 21 20 16 15 14 13 11 10 9 5 4 0
11269 * +---+---+---+-----------+---------+------+-----+--------+---+------+------+
11270 * | 0 | Q | U | 0 1 1 1 0 | a | 1 0 | Rm | 0 0 | opcode | 1 | Rn | Rd |
11271 * +---+---+---+-----------+---------+------+-----+--------+---+------+------+
11272 *
11273 * This includes FMULX, FCMEQ (register), FRECPS, FRSQRTS, FCMGE
11274 * (register), FACGE, FABD, FCMGT (register) and FACGT.
11275 *
11276 */
disas_simd_three_reg_same_fp16(DisasContext * s,uint32_t insn)11277 static void disas_simd_three_reg_same_fp16(DisasContext *s, uint32_t insn)
11278 {
11279 int opcode = extract32(insn, 11, 3);
11280 int u = extract32(insn, 29, 1);
11281 int a = extract32(insn, 23, 1);
11282 int is_q = extract32(insn, 30, 1);
11283 int rm = extract32(insn, 16, 5);
11284 int rn = extract32(insn, 5, 5);
11285 int rd = extract32(insn, 0, 5);
11286 /*
11287 * For these floating point ops, the U, a and opcode bits
11288 * together indicate the operation.
11289 */
11290 int fpopcode = opcode | (a << 3) | (u << 4);
11291 int datasize = is_q ? 128 : 64;
11292 int elements = datasize / 16;
11293 bool pairwise;
11294 TCGv_ptr fpst;
11295 int pass;
11296
11297 switch (fpopcode) {
11298 case 0x0: /* FMAXNM */
11299 case 0x1: /* FMLA */
11300 case 0x2: /* FADD */
11301 case 0x3: /* FMULX */
11302 case 0x4: /* FCMEQ */
11303 case 0x6: /* FMAX */
11304 case 0x7: /* FRECPS */
11305 case 0x8: /* FMINNM */
11306 case 0x9: /* FMLS */
11307 case 0xa: /* FSUB */
11308 case 0xe: /* FMIN */
11309 case 0xf: /* FRSQRTS */
11310 case 0x13: /* FMUL */
11311 case 0x14: /* FCMGE */
11312 case 0x15: /* FACGE */
11313 case 0x17: /* FDIV */
11314 case 0x1a: /* FABD */
11315 case 0x1c: /* FCMGT */
11316 case 0x1d: /* FACGT */
11317 pairwise = false;
11318 break;
11319 case 0x10: /* FMAXNMP */
11320 case 0x12: /* FADDP */
11321 case 0x16: /* FMAXP */
11322 case 0x18: /* FMINNMP */
11323 case 0x1e: /* FMINP */
11324 pairwise = true;
11325 break;
11326 default:
11327 unallocated_encoding(s);
11328 return;
11329 }
11330
11331 if (!dc_isar_feature(aa64_fp16, s)) {
11332 unallocated_encoding(s);
11333 return;
11334 }
11335
11336 if (!fp_access_check(s)) {
11337 return;
11338 }
11339
11340 fpst = fpstatus_ptr(FPST_FPCR_F16);
11341
11342 if (pairwise) {
11343 int maxpass = is_q ? 8 : 4;
11344 TCGv_i32 tcg_op1 = tcg_temp_new_i32();
11345 TCGv_i32 tcg_op2 = tcg_temp_new_i32();
11346 TCGv_i32 tcg_res[8];
11347
11348 for (pass = 0; pass < maxpass; pass++) {
11349 int passreg = pass < (maxpass / 2) ? rn : rm;
11350 int passelt = (pass << 1) & (maxpass - 1);
11351
11352 read_vec_element_i32(s, tcg_op1, passreg, passelt, MO_16);
11353 read_vec_element_i32(s, tcg_op2, passreg, passelt + 1, MO_16);
11354 tcg_res[pass] = tcg_temp_new_i32();
11355
11356 switch (fpopcode) {
11357 case 0x10: /* FMAXNMP */
11358 gen_helper_advsimd_maxnumh(tcg_res[pass], tcg_op1, tcg_op2,
11359 fpst);
11360 break;
11361 case 0x12: /* FADDP */
11362 gen_helper_advsimd_addh(tcg_res[pass], tcg_op1, tcg_op2, fpst);
11363 break;
11364 case 0x16: /* FMAXP */
11365 gen_helper_advsimd_maxh(tcg_res[pass], tcg_op1, tcg_op2, fpst);
11366 break;
11367 case 0x18: /* FMINNMP */
11368 gen_helper_advsimd_minnumh(tcg_res[pass], tcg_op1, tcg_op2,
11369 fpst);
11370 break;
11371 case 0x1e: /* FMINP */
11372 gen_helper_advsimd_minh(tcg_res[pass], tcg_op1, tcg_op2, fpst);
11373 break;
11374 default:
11375 g_assert_not_reached();
11376 }
11377 }
11378
11379 for (pass = 0; pass < maxpass; pass++) {
11380 write_vec_element_i32(s, tcg_res[pass], rd, pass, MO_16);
11381 }
11382 } else {
11383 for (pass = 0; pass < elements; pass++) {
11384 TCGv_i32 tcg_op1 = tcg_temp_new_i32();
11385 TCGv_i32 tcg_op2 = tcg_temp_new_i32();
11386 TCGv_i32 tcg_res = tcg_temp_new_i32();
11387
11388 read_vec_element_i32(s, tcg_op1, rn, pass, MO_16);
11389 read_vec_element_i32(s, tcg_op2, rm, pass, MO_16);
11390
11391 switch (fpopcode) {
11392 case 0x0: /* FMAXNM */
11393 gen_helper_advsimd_maxnumh(tcg_res, tcg_op1, tcg_op2, fpst);
11394 break;
11395 case 0x1: /* FMLA */
11396 read_vec_element_i32(s, tcg_res, rd, pass, MO_16);
11397 gen_helper_advsimd_muladdh(tcg_res, tcg_op1, tcg_op2, tcg_res,
11398 fpst);
11399 break;
11400 case 0x2: /* FADD */
11401 gen_helper_advsimd_addh(tcg_res, tcg_op1, tcg_op2, fpst);
11402 break;
11403 case 0x3: /* FMULX */
11404 gen_helper_advsimd_mulxh(tcg_res, tcg_op1, tcg_op2, fpst);
11405 break;
11406 case 0x4: /* FCMEQ */
11407 gen_helper_advsimd_ceq_f16(tcg_res, tcg_op1, tcg_op2, fpst);
11408 break;
11409 case 0x6: /* FMAX */
11410 gen_helper_advsimd_maxh(tcg_res, tcg_op1, tcg_op2, fpst);
11411 break;
11412 case 0x7: /* FRECPS */
11413 gen_helper_recpsf_f16(tcg_res, tcg_op1, tcg_op2, fpst);
11414 break;
11415 case 0x8: /* FMINNM */
11416 gen_helper_advsimd_minnumh(tcg_res, tcg_op1, tcg_op2, fpst);
11417 break;
11418 case 0x9: /* FMLS */
11419 /* As usual for ARM, separate negation for fused multiply-add */
11420 tcg_gen_xori_i32(tcg_op1, tcg_op1, 0x8000);
11421 read_vec_element_i32(s, tcg_res, rd, pass, MO_16);
11422 gen_helper_advsimd_muladdh(tcg_res, tcg_op1, tcg_op2, tcg_res,
11423 fpst);
11424 break;
11425 case 0xa: /* FSUB */
11426 gen_helper_advsimd_subh(tcg_res, tcg_op1, tcg_op2, fpst);
11427 break;
11428 case 0xe: /* FMIN */
11429 gen_helper_advsimd_minh(tcg_res, tcg_op1, tcg_op2, fpst);
11430 break;
11431 case 0xf: /* FRSQRTS */
11432 gen_helper_rsqrtsf_f16(tcg_res, tcg_op1, tcg_op2, fpst);
11433 break;
11434 case 0x13: /* FMUL */
11435 gen_helper_advsimd_mulh(tcg_res, tcg_op1, tcg_op2, fpst);
11436 break;
11437 case 0x14: /* FCMGE */
11438 gen_helper_advsimd_cge_f16(tcg_res, tcg_op1, tcg_op2, fpst);
11439 break;
11440 case 0x15: /* FACGE */
11441 gen_helper_advsimd_acge_f16(tcg_res, tcg_op1, tcg_op2, fpst);
11442 break;
11443 case 0x17: /* FDIV */
11444 gen_helper_advsimd_divh(tcg_res, tcg_op1, tcg_op2, fpst);
11445 break;
11446 case 0x1a: /* FABD */
11447 gen_helper_advsimd_subh(tcg_res, tcg_op1, tcg_op2, fpst);
11448 tcg_gen_andi_i32(tcg_res, tcg_res, 0x7fff);
11449 break;
11450 case 0x1c: /* FCMGT */
11451 gen_helper_advsimd_cgt_f16(tcg_res, tcg_op1, tcg_op2, fpst);
11452 break;
11453 case 0x1d: /* FACGT */
11454 gen_helper_advsimd_acgt_f16(tcg_res, tcg_op1, tcg_op2, fpst);
11455 break;
11456 default:
11457 g_assert_not_reached();
11458 }
11459
11460 write_vec_element_i32(s, tcg_res, rd, pass, MO_16);
11461 }
11462 }
11463
11464 clear_vec_high(s, is_q, rd);
11465 }
11466
11467 /* AdvSIMD three same extra
11468 * 31 30 29 28 24 23 22 21 20 16 15 14 11 10 9 5 4 0
11469 * +---+---+---+-----------+------+---+------+---+--------+---+----+----+
11470 * | 0 | Q | U | 0 1 1 1 0 | size | 0 | Rm | 1 | opcode | 1 | Rn | Rd |
11471 * +---+---+---+-----------+------+---+------+---+--------+---+----+----+
11472 */
disas_simd_three_reg_same_extra(DisasContext * s,uint32_t insn)11473 static void disas_simd_three_reg_same_extra(DisasContext *s, uint32_t insn)
11474 {
11475 int rd = extract32(insn, 0, 5);
11476 int rn = extract32(insn, 5, 5);
11477 int opcode = extract32(insn, 11, 4);
11478 int rm = extract32(insn, 16, 5);
11479 int size = extract32(insn, 22, 2);
11480 bool u = extract32(insn, 29, 1);
11481 bool is_q = extract32(insn, 30, 1);
11482 bool feature;
11483 int rot;
11484
11485 switch (u * 16 + opcode) {
11486 case 0x10: /* SQRDMLAH (vector) */
11487 case 0x11: /* SQRDMLSH (vector) */
11488 if (size != 1 && size != 2) {
11489 unallocated_encoding(s);
11490 return;
11491 }
11492 feature = dc_isar_feature(aa64_rdm, s);
11493 break;
11494 case 0x02: /* SDOT (vector) */
11495 case 0x12: /* UDOT (vector) */
11496 if (size != MO_32) {
11497 unallocated_encoding(s);
11498 return;
11499 }
11500 feature = dc_isar_feature(aa64_dp, s);
11501 break;
11502 case 0x03: /* USDOT */
11503 if (size != MO_32) {
11504 unallocated_encoding(s);
11505 return;
11506 }
11507 feature = dc_isar_feature(aa64_i8mm, s);
11508 break;
11509 case 0x04: /* SMMLA */
11510 case 0x14: /* UMMLA */
11511 case 0x05: /* USMMLA */
11512 if (!is_q || size != MO_32) {
11513 unallocated_encoding(s);
11514 return;
11515 }
11516 feature = dc_isar_feature(aa64_i8mm, s);
11517 break;
11518 case 0x18: /* FCMLA, #0 */
11519 case 0x19: /* FCMLA, #90 */
11520 case 0x1a: /* FCMLA, #180 */
11521 case 0x1b: /* FCMLA, #270 */
11522 case 0x1c: /* FCADD, #90 */
11523 case 0x1e: /* FCADD, #270 */
11524 if (size == 0
11525 || (size == 1 && !dc_isar_feature(aa64_fp16, s))
11526 || (size == 3 && !is_q)) {
11527 unallocated_encoding(s);
11528 return;
11529 }
11530 feature = dc_isar_feature(aa64_fcma, s);
11531 break;
11532 case 0x1d: /* BFMMLA */
11533 if (size != MO_16 || !is_q) {
11534 unallocated_encoding(s);
11535 return;
11536 }
11537 feature = dc_isar_feature(aa64_bf16, s);
11538 break;
11539 case 0x1f:
11540 switch (size) {
11541 case 1: /* BFDOT */
11542 case 3: /* BFMLAL{B,T} */
11543 feature = dc_isar_feature(aa64_bf16, s);
11544 break;
11545 default:
11546 unallocated_encoding(s);
11547 return;
11548 }
11549 break;
11550 default:
11551 unallocated_encoding(s);
11552 return;
11553 }
11554 if (!feature) {
11555 unallocated_encoding(s);
11556 return;
11557 }
11558 if (!fp_access_check(s)) {
11559 return;
11560 }
11561
11562 switch (opcode) {
11563 case 0x0: /* SQRDMLAH (vector) */
11564 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_sqrdmlah_qc, size);
11565 return;
11566
11567 case 0x1: /* SQRDMLSH (vector) */
11568 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_sqrdmlsh_qc, size);
11569 return;
11570
11571 case 0x2: /* SDOT / UDOT */
11572 gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, 0,
11573 u ? gen_helper_gvec_udot_b : gen_helper_gvec_sdot_b);
11574 return;
11575
11576 case 0x3: /* USDOT */
11577 gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, 0, gen_helper_gvec_usdot_b);
11578 return;
11579
11580 case 0x04: /* SMMLA, UMMLA */
11581 gen_gvec_op4_ool(s, 1, rd, rn, rm, rd, 0,
11582 u ? gen_helper_gvec_ummla_b
11583 : gen_helper_gvec_smmla_b);
11584 return;
11585 case 0x05: /* USMMLA */
11586 gen_gvec_op4_ool(s, 1, rd, rn, rm, rd, 0, gen_helper_gvec_usmmla_b);
11587 return;
11588
11589 case 0x8: /* FCMLA, #0 */
11590 case 0x9: /* FCMLA, #90 */
11591 case 0xa: /* FCMLA, #180 */
11592 case 0xb: /* FCMLA, #270 */
11593 rot = extract32(opcode, 0, 2);
11594 switch (size) {
11595 case 1:
11596 gen_gvec_op4_fpst(s, is_q, rd, rn, rm, rd, true, rot,
11597 gen_helper_gvec_fcmlah);
11598 break;
11599 case 2:
11600 gen_gvec_op4_fpst(s, is_q, rd, rn, rm, rd, false, rot,
11601 gen_helper_gvec_fcmlas);
11602 break;
11603 case 3:
11604 gen_gvec_op4_fpst(s, is_q, rd, rn, rm, rd, false, rot,
11605 gen_helper_gvec_fcmlad);
11606 break;
11607 default:
11608 g_assert_not_reached();
11609 }
11610 return;
11611
11612 case 0xc: /* FCADD, #90 */
11613 case 0xe: /* FCADD, #270 */
11614 rot = extract32(opcode, 1, 1);
11615 switch (size) {
11616 case 1:
11617 gen_gvec_op3_fpst(s, is_q, rd, rn, rm, size == 1, rot,
11618 gen_helper_gvec_fcaddh);
11619 break;
11620 case 2:
11621 gen_gvec_op3_fpst(s, is_q, rd, rn, rm, size == 1, rot,
11622 gen_helper_gvec_fcadds);
11623 break;
11624 case 3:
11625 gen_gvec_op3_fpst(s, is_q, rd, rn, rm, size == 1, rot,
11626 gen_helper_gvec_fcaddd);
11627 break;
11628 default:
11629 g_assert_not_reached();
11630 }
11631 return;
11632
11633 case 0xd: /* BFMMLA */
11634 gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, 0, gen_helper_gvec_bfmmla);
11635 return;
11636 case 0xf:
11637 switch (size) {
11638 case 1: /* BFDOT */
11639 gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, 0, gen_helper_gvec_bfdot);
11640 break;
11641 case 3: /* BFMLAL{B,T} */
11642 gen_gvec_op4_fpst(s, 1, rd, rn, rm, rd, false, is_q,
11643 gen_helper_gvec_bfmlal);
11644 break;
11645 default:
11646 g_assert_not_reached();
11647 }
11648 return;
11649
11650 default:
11651 g_assert_not_reached();
11652 }
11653 }
11654
handle_2misc_widening(DisasContext * s,int opcode,bool is_q,int size,int rn,int rd)11655 static void handle_2misc_widening(DisasContext *s, int opcode, bool is_q,
11656 int size, int rn, int rd)
11657 {
11658 /* Handle 2-reg-misc ops which are widening (so each size element
11659 * in the source becomes a 2*size element in the destination.
11660 * The only instruction like this is FCVTL.
11661 */
11662 int pass;
11663
11664 if (size == 3) {
11665 /* 32 -> 64 bit fp conversion */
11666 TCGv_i64 tcg_res[2];
11667 int srcelt = is_q ? 2 : 0;
11668
11669 for (pass = 0; pass < 2; pass++) {
11670 TCGv_i32 tcg_op = tcg_temp_new_i32();
11671 tcg_res[pass] = tcg_temp_new_i64();
11672
11673 read_vec_element_i32(s, tcg_op, rn, srcelt + pass, MO_32);
11674 gen_helper_vfp_fcvtds(tcg_res[pass], tcg_op, tcg_env);
11675 }
11676 for (pass = 0; pass < 2; pass++) {
11677 write_vec_element(s, tcg_res[pass], rd, pass, MO_64);
11678 }
11679 } else {
11680 /* 16 -> 32 bit fp conversion */
11681 int srcelt = is_q ? 4 : 0;
11682 TCGv_i32 tcg_res[4];
11683 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR);
11684 TCGv_i32 ahp = get_ahp_flag();
11685
11686 for (pass = 0; pass < 4; pass++) {
11687 tcg_res[pass] = tcg_temp_new_i32();
11688
11689 read_vec_element_i32(s, tcg_res[pass], rn, srcelt + pass, MO_16);
11690 gen_helper_vfp_fcvt_f16_to_f32(tcg_res[pass], tcg_res[pass],
11691 fpst, ahp);
11692 }
11693 for (pass = 0; pass < 4; pass++) {
11694 write_vec_element_i32(s, tcg_res[pass], rd, pass, MO_32);
11695 }
11696 }
11697 }
11698
handle_rev(DisasContext * s,int opcode,bool u,bool is_q,int size,int rn,int rd)11699 static void handle_rev(DisasContext *s, int opcode, bool u,
11700 bool is_q, int size, int rn, int rd)
11701 {
11702 int op = (opcode << 1) | u;
11703 int opsz = op + size;
11704 int grp_size = 3 - opsz;
11705 int dsize = is_q ? 128 : 64;
11706 int i;
11707
11708 if (opsz >= 3) {
11709 unallocated_encoding(s);
11710 return;
11711 }
11712
11713 if (!fp_access_check(s)) {
11714 return;
11715 }
11716
11717 if (size == 0) {
11718 /* Special case bytes, use bswap op on each group of elements */
11719 int groups = dsize / (8 << grp_size);
11720
11721 for (i = 0; i < groups; i++) {
11722 TCGv_i64 tcg_tmp = tcg_temp_new_i64();
11723
11724 read_vec_element(s, tcg_tmp, rn, i, grp_size);
11725 switch (grp_size) {
11726 case MO_16:
11727 tcg_gen_bswap16_i64(tcg_tmp, tcg_tmp, TCG_BSWAP_IZ);
11728 break;
11729 case MO_32:
11730 tcg_gen_bswap32_i64(tcg_tmp, tcg_tmp, TCG_BSWAP_IZ);
11731 break;
11732 case MO_64:
11733 tcg_gen_bswap64_i64(tcg_tmp, tcg_tmp);
11734 break;
11735 default:
11736 g_assert_not_reached();
11737 }
11738 write_vec_element(s, tcg_tmp, rd, i, grp_size);
11739 }
11740 clear_vec_high(s, is_q, rd);
11741 } else {
11742 int revmask = (1 << grp_size) - 1;
11743 int esize = 8 << size;
11744 int elements = dsize / esize;
11745 TCGv_i64 tcg_rn = tcg_temp_new_i64();
11746 TCGv_i64 tcg_rd[2];
11747
11748 for (i = 0; i < 2; i++) {
11749 tcg_rd[i] = tcg_temp_new_i64();
11750 tcg_gen_movi_i64(tcg_rd[i], 0);
11751 }
11752
11753 for (i = 0; i < elements; i++) {
11754 int e_rev = (i & 0xf) ^ revmask;
11755 int w = (e_rev * esize) / 64;
11756 int o = (e_rev * esize) % 64;
11757
11758 read_vec_element(s, tcg_rn, rn, i, size);
11759 tcg_gen_deposit_i64(tcg_rd[w], tcg_rd[w], tcg_rn, o, esize);
11760 }
11761
11762 for (i = 0; i < 2; i++) {
11763 write_vec_element(s, tcg_rd[i], rd, i, MO_64);
11764 }
11765 clear_vec_high(s, true, rd);
11766 }
11767 }
11768
handle_2misc_pairwise(DisasContext * s,int opcode,bool u,bool is_q,int size,int rn,int rd)11769 static void handle_2misc_pairwise(DisasContext *s, int opcode, bool u,
11770 bool is_q, int size, int rn, int rd)
11771 {
11772 /* Implement the pairwise operations from 2-misc:
11773 * SADDLP, UADDLP, SADALP, UADALP.
11774 * These all add pairs of elements in the input to produce a
11775 * double-width result element in the output (possibly accumulating).
11776 */
11777 bool accum = (opcode == 0x6);
11778 int maxpass = is_q ? 2 : 1;
11779 int pass;
11780 TCGv_i64 tcg_res[2];
11781
11782 if (size == 2) {
11783 /* 32 + 32 -> 64 op */
11784 MemOp memop = size + (u ? 0 : MO_SIGN);
11785
11786 for (pass = 0; pass < maxpass; pass++) {
11787 TCGv_i64 tcg_op1 = tcg_temp_new_i64();
11788 TCGv_i64 tcg_op2 = tcg_temp_new_i64();
11789
11790 tcg_res[pass] = tcg_temp_new_i64();
11791
11792 read_vec_element(s, tcg_op1, rn, pass * 2, memop);
11793 read_vec_element(s, tcg_op2, rn, pass * 2 + 1, memop);
11794 tcg_gen_add_i64(tcg_res[pass], tcg_op1, tcg_op2);
11795 if (accum) {
11796 read_vec_element(s, tcg_op1, rd, pass, MO_64);
11797 tcg_gen_add_i64(tcg_res[pass], tcg_res[pass], tcg_op1);
11798 }
11799 }
11800 } else {
11801 for (pass = 0; pass < maxpass; pass++) {
11802 TCGv_i64 tcg_op = tcg_temp_new_i64();
11803 NeonGenOne64OpFn *genfn;
11804 static NeonGenOne64OpFn * const fns[2][2] = {
11805 { gen_helper_neon_addlp_s8, gen_helper_neon_addlp_u8 },
11806 { gen_helper_neon_addlp_s16, gen_helper_neon_addlp_u16 },
11807 };
11808
11809 genfn = fns[size][u];
11810
11811 tcg_res[pass] = tcg_temp_new_i64();
11812
11813 read_vec_element(s, tcg_op, rn, pass, MO_64);
11814 genfn(tcg_res[pass], tcg_op);
11815
11816 if (accum) {
11817 read_vec_element(s, tcg_op, rd, pass, MO_64);
11818 if (size == 0) {
11819 gen_helper_neon_addl_u16(tcg_res[pass],
11820 tcg_res[pass], tcg_op);
11821 } else {
11822 gen_helper_neon_addl_u32(tcg_res[pass],
11823 tcg_res[pass], tcg_op);
11824 }
11825 }
11826 }
11827 }
11828 if (!is_q) {
11829 tcg_res[1] = tcg_constant_i64(0);
11830 }
11831 for (pass = 0; pass < 2; pass++) {
11832 write_vec_element(s, tcg_res[pass], rd, pass, MO_64);
11833 }
11834 }
11835
handle_shll(DisasContext * s,bool is_q,int size,int rn,int rd)11836 static void handle_shll(DisasContext *s, bool is_q, int size, int rn, int rd)
11837 {
11838 /* Implement SHLL and SHLL2 */
11839 int pass;
11840 int part = is_q ? 2 : 0;
11841 TCGv_i64 tcg_res[2];
11842
11843 for (pass = 0; pass < 2; pass++) {
11844 static NeonGenWidenFn * const widenfns[3] = {
11845 gen_helper_neon_widen_u8,
11846 gen_helper_neon_widen_u16,
11847 tcg_gen_extu_i32_i64,
11848 };
11849 NeonGenWidenFn *widenfn = widenfns[size];
11850 TCGv_i32 tcg_op = tcg_temp_new_i32();
11851
11852 read_vec_element_i32(s, tcg_op, rn, part + pass, MO_32);
11853 tcg_res[pass] = tcg_temp_new_i64();
11854 widenfn(tcg_res[pass], tcg_op);
11855 tcg_gen_shli_i64(tcg_res[pass], tcg_res[pass], 8 << size);
11856 }
11857
11858 for (pass = 0; pass < 2; pass++) {
11859 write_vec_element(s, tcg_res[pass], rd, pass, MO_64);
11860 }
11861 }
11862
11863 /* AdvSIMD two reg misc
11864 * 31 30 29 28 24 23 22 21 17 16 12 11 10 9 5 4 0
11865 * +---+---+---+-----------+------+-----------+--------+-----+------+------+
11866 * | 0 | Q | U | 0 1 1 1 0 | size | 1 0 0 0 0 | opcode | 1 0 | Rn | Rd |
11867 * +---+---+---+-----------+------+-----------+--------+-----+------+------+
11868 */
disas_simd_two_reg_misc(DisasContext * s,uint32_t insn)11869 static void disas_simd_two_reg_misc(DisasContext *s, uint32_t insn)
11870 {
11871 int size = extract32(insn, 22, 2);
11872 int opcode = extract32(insn, 12, 5);
11873 bool u = extract32(insn, 29, 1);
11874 bool is_q = extract32(insn, 30, 1);
11875 int rn = extract32(insn, 5, 5);
11876 int rd = extract32(insn, 0, 5);
11877 bool need_fpstatus = false;
11878 int rmode = -1;
11879 TCGv_i32 tcg_rmode;
11880 TCGv_ptr tcg_fpstatus;
11881
11882 switch (opcode) {
11883 case 0x0: /* REV64, REV32 */
11884 case 0x1: /* REV16 */
11885 handle_rev(s, opcode, u, is_q, size, rn, rd);
11886 return;
11887 case 0x5: /* CNT, NOT, RBIT */
11888 if (u && size == 0) {
11889 /* NOT */
11890 break;
11891 } else if (u && size == 1) {
11892 /* RBIT */
11893 break;
11894 } else if (!u && size == 0) {
11895 /* CNT */
11896 break;
11897 }
11898 unallocated_encoding(s);
11899 return;
11900 case 0x12: /* XTN, XTN2, SQXTUN, SQXTUN2 */
11901 case 0x14: /* SQXTN, SQXTN2, UQXTN, UQXTN2 */
11902 if (size == 3) {
11903 unallocated_encoding(s);
11904 return;
11905 }
11906 if (!fp_access_check(s)) {
11907 return;
11908 }
11909
11910 handle_2misc_narrow(s, false, opcode, u, is_q, size, rn, rd);
11911 return;
11912 case 0x4: /* CLS, CLZ */
11913 if (size == 3) {
11914 unallocated_encoding(s);
11915 return;
11916 }
11917 break;
11918 case 0x2: /* SADDLP, UADDLP */
11919 case 0x6: /* SADALP, UADALP */
11920 if (size == 3) {
11921 unallocated_encoding(s);
11922 return;
11923 }
11924 if (!fp_access_check(s)) {
11925 return;
11926 }
11927 handle_2misc_pairwise(s, opcode, u, is_q, size, rn, rd);
11928 return;
11929 case 0x13: /* SHLL, SHLL2 */
11930 if (u == 0 || size == 3) {
11931 unallocated_encoding(s);
11932 return;
11933 }
11934 if (!fp_access_check(s)) {
11935 return;
11936 }
11937 handle_shll(s, is_q, size, rn, rd);
11938 return;
11939 case 0xa: /* CMLT */
11940 if (u == 1) {
11941 unallocated_encoding(s);
11942 return;
11943 }
11944 /* fall through */
11945 case 0x8: /* CMGT, CMGE */
11946 case 0x9: /* CMEQ, CMLE */
11947 case 0xb: /* ABS, NEG */
11948 if (size == 3 && !is_q) {
11949 unallocated_encoding(s);
11950 return;
11951 }
11952 break;
11953 case 0x3: /* SUQADD, USQADD */
11954 if (size == 3 && !is_q) {
11955 unallocated_encoding(s);
11956 return;
11957 }
11958 if (!fp_access_check(s)) {
11959 return;
11960 }
11961 handle_2misc_satacc(s, false, u, is_q, size, rn, rd);
11962 return;
11963 case 0x7: /* SQABS, SQNEG */
11964 if (size == 3 && !is_q) {
11965 unallocated_encoding(s);
11966 return;
11967 }
11968 break;
11969 case 0xc ... 0xf:
11970 case 0x16 ... 0x1f:
11971 {
11972 /* Floating point: U, size[1] and opcode indicate operation;
11973 * size[0] indicates single or double precision.
11974 */
11975 int is_double = extract32(size, 0, 1);
11976 opcode |= (extract32(size, 1, 1) << 5) | (u << 6);
11977 size = is_double ? 3 : 2;
11978 switch (opcode) {
11979 case 0x2f: /* FABS */
11980 case 0x6f: /* FNEG */
11981 if (size == 3 && !is_q) {
11982 unallocated_encoding(s);
11983 return;
11984 }
11985 break;
11986 case 0x1d: /* SCVTF */
11987 case 0x5d: /* UCVTF */
11988 {
11989 bool is_signed = (opcode == 0x1d) ? true : false;
11990 int elements = is_double ? 2 : is_q ? 4 : 2;
11991 if (is_double && !is_q) {
11992 unallocated_encoding(s);
11993 return;
11994 }
11995 if (!fp_access_check(s)) {
11996 return;
11997 }
11998 handle_simd_intfp_conv(s, rd, rn, elements, is_signed, 0, size);
11999 return;
12000 }
12001 case 0x2c: /* FCMGT (zero) */
12002 case 0x2d: /* FCMEQ (zero) */
12003 case 0x2e: /* FCMLT (zero) */
12004 case 0x6c: /* FCMGE (zero) */
12005 case 0x6d: /* FCMLE (zero) */
12006 if (size == 3 && !is_q) {
12007 unallocated_encoding(s);
12008 return;
12009 }
12010 handle_2misc_fcmp_zero(s, opcode, false, u, is_q, size, rn, rd);
12011 return;
12012 case 0x7f: /* FSQRT */
12013 if (size == 3 && !is_q) {
12014 unallocated_encoding(s);
12015 return;
12016 }
12017 break;
12018 case 0x1a: /* FCVTNS */
12019 case 0x1b: /* FCVTMS */
12020 case 0x3a: /* FCVTPS */
12021 case 0x3b: /* FCVTZS */
12022 case 0x5a: /* FCVTNU */
12023 case 0x5b: /* FCVTMU */
12024 case 0x7a: /* FCVTPU */
12025 case 0x7b: /* FCVTZU */
12026 need_fpstatus = true;
12027 rmode = extract32(opcode, 5, 1) | (extract32(opcode, 0, 1) << 1);
12028 if (size == 3 && !is_q) {
12029 unallocated_encoding(s);
12030 return;
12031 }
12032 break;
12033 case 0x5c: /* FCVTAU */
12034 case 0x1c: /* FCVTAS */
12035 need_fpstatus = true;
12036 rmode = FPROUNDING_TIEAWAY;
12037 if (size == 3 && !is_q) {
12038 unallocated_encoding(s);
12039 return;
12040 }
12041 break;
12042 case 0x3c: /* URECPE */
12043 if (size == 3) {
12044 unallocated_encoding(s);
12045 return;
12046 }
12047 /* fall through */
12048 case 0x3d: /* FRECPE */
12049 case 0x7d: /* FRSQRTE */
12050 if (size == 3 && !is_q) {
12051 unallocated_encoding(s);
12052 return;
12053 }
12054 if (!fp_access_check(s)) {
12055 return;
12056 }
12057 handle_2misc_reciprocal(s, opcode, false, u, is_q, size, rn, rd);
12058 return;
12059 case 0x56: /* FCVTXN, FCVTXN2 */
12060 if (size == 2) {
12061 unallocated_encoding(s);
12062 return;
12063 }
12064 /* fall through */
12065 case 0x16: /* FCVTN, FCVTN2 */
12066 /* handle_2misc_narrow does a 2*size -> size operation, but these
12067 * instructions encode the source size rather than dest size.
12068 */
12069 if (!fp_access_check(s)) {
12070 return;
12071 }
12072 handle_2misc_narrow(s, false, opcode, 0, is_q, size - 1, rn, rd);
12073 return;
12074 case 0x36: /* BFCVTN, BFCVTN2 */
12075 if (!dc_isar_feature(aa64_bf16, s) || size != 2) {
12076 unallocated_encoding(s);
12077 return;
12078 }
12079 if (!fp_access_check(s)) {
12080 return;
12081 }
12082 handle_2misc_narrow(s, false, opcode, 0, is_q, size - 1, rn, rd);
12083 return;
12084 case 0x17: /* FCVTL, FCVTL2 */
12085 if (!fp_access_check(s)) {
12086 return;
12087 }
12088 handle_2misc_widening(s, opcode, is_q, size, rn, rd);
12089 return;
12090 case 0x18: /* FRINTN */
12091 case 0x19: /* FRINTM */
12092 case 0x38: /* FRINTP */
12093 case 0x39: /* FRINTZ */
12094 rmode = extract32(opcode, 5, 1) | (extract32(opcode, 0, 1) << 1);
12095 /* fall through */
12096 case 0x59: /* FRINTX */
12097 case 0x79: /* FRINTI */
12098 need_fpstatus = true;
12099 if (size == 3 && !is_q) {
12100 unallocated_encoding(s);
12101 return;
12102 }
12103 break;
12104 case 0x58: /* FRINTA */
12105 rmode = FPROUNDING_TIEAWAY;
12106 need_fpstatus = true;
12107 if (size == 3 && !is_q) {
12108 unallocated_encoding(s);
12109 return;
12110 }
12111 break;
12112 case 0x7c: /* URSQRTE */
12113 if (size == 3) {
12114 unallocated_encoding(s);
12115 return;
12116 }
12117 break;
12118 case 0x1e: /* FRINT32Z */
12119 case 0x1f: /* FRINT64Z */
12120 rmode = FPROUNDING_ZERO;
12121 /* fall through */
12122 case 0x5e: /* FRINT32X */
12123 case 0x5f: /* FRINT64X */
12124 need_fpstatus = true;
12125 if ((size == 3 && !is_q) || !dc_isar_feature(aa64_frint, s)) {
12126 unallocated_encoding(s);
12127 return;
12128 }
12129 break;
12130 default:
12131 unallocated_encoding(s);
12132 return;
12133 }
12134 break;
12135 }
12136 default:
12137 unallocated_encoding(s);
12138 return;
12139 }
12140
12141 if (!fp_access_check(s)) {
12142 return;
12143 }
12144
12145 if (need_fpstatus || rmode >= 0) {
12146 tcg_fpstatus = fpstatus_ptr(FPST_FPCR);
12147 } else {
12148 tcg_fpstatus = NULL;
12149 }
12150 if (rmode >= 0) {
12151 tcg_rmode = gen_set_rmode(rmode, tcg_fpstatus);
12152 } else {
12153 tcg_rmode = NULL;
12154 }
12155
12156 switch (opcode) {
12157 case 0x5:
12158 if (u && size == 0) { /* NOT */
12159 gen_gvec_fn2(s, is_q, rd, rn, tcg_gen_gvec_not, 0);
12160 return;
12161 }
12162 break;
12163 case 0x8: /* CMGT, CMGE */
12164 if (u) {
12165 gen_gvec_fn2(s, is_q, rd, rn, gen_gvec_cge0, size);
12166 } else {
12167 gen_gvec_fn2(s, is_q, rd, rn, gen_gvec_cgt0, size);
12168 }
12169 return;
12170 case 0x9: /* CMEQ, CMLE */
12171 if (u) {
12172 gen_gvec_fn2(s, is_q, rd, rn, gen_gvec_cle0, size);
12173 } else {
12174 gen_gvec_fn2(s, is_q, rd, rn, gen_gvec_ceq0, size);
12175 }
12176 return;
12177 case 0xa: /* CMLT */
12178 gen_gvec_fn2(s, is_q, rd, rn, gen_gvec_clt0, size);
12179 return;
12180 case 0xb:
12181 if (u) { /* ABS, NEG */
12182 gen_gvec_fn2(s, is_q, rd, rn, tcg_gen_gvec_neg, size);
12183 } else {
12184 gen_gvec_fn2(s, is_q, rd, rn, tcg_gen_gvec_abs, size);
12185 }
12186 return;
12187 }
12188
12189 if (size == 3) {
12190 /* All 64-bit element operations can be shared with scalar 2misc */
12191 int pass;
12192
12193 /* Coverity claims (size == 3 && !is_q) has been eliminated
12194 * from all paths leading to here.
12195 */
12196 tcg_debug_assert(is_q);
12197 for (pass = 0; pass < 2; pass++) {
12198 TCGv_i64 tcg_op = tcg_temp_new_i64();
12199 TCGv_i64 tcg_res = tcg_temp_new_i64();
12200
12201 read_vec_element(s, tcg_op, rn, pass, MO_64);
12202
12203 handle_2misc_64(s, opcode, u, tcg_res, tcg_op,
12204 tcg_rmode, tcg_fpstatus);
12205
12206 write_vec_element(s, tcg_res, rd, pass, MO_64);
12207 }
12208 } else {
12209 int pass;
12210
12211 for (pass = 0; pass < (is_q ? 4 : 2); pass++) {
12212 TCGv_i32 tcg_op = tcg_temp_new_i32();
12213 TCGv_i32 tcg_res = tcg_temp_new_i32();
12214
12215 read_vec_element_i32(s, tcg_op, rn, pass, MO_32);
12216
12217 if (size == 2) {
12218 /* Special cases for 32 bit elements */
12219 switch (opcode) {
12220 case 0x4: /* CLS */
12221 if (u) {
12222 tcg_gen_clzi_i32(tcg_res, tcg_op, 32);
12223 } else {
12224 tcg_gen_clrsb_i32(tcg_res, tcg_op);
12225 }
12226 break;
12227 case 0x7: /* SQABS, SQNEG */
12228 if (u) {
12229 gen_helper_neon_qneg_s32(tcg_res, tcg_env, tcg_op);
12230 } else {
12231 gen_helper_neon_qabs_s32(tcg_res, tcg_env, tcg_op);
12232 }
12233 break;
12234 case 0x2f: /* FABS */
12235 gen_helper_vfp_abss(tcg_res, tcg_op);
12236 break;
12237 case 0x6f: /* FNEG */
12238 gen_helper_vfp_negs(tcg_res, tcg_op);
12239 break;
12240 case 0x7f: /* FSQRT */
12241 gen_helper_vfp_sqrts(tcg_res, tcg_op, tcg_env);
12242 break;
12243 case 0x1a: /* FCVTNS */
12244 case 0x1b: /* FCVTMS */
12245 case 0x1c: /* FCVTAS */
12246 case 0x3a: /* FCVTPS */
12247 case 0x3b: /* FCVTZS */
12248 gen_helper_vfp_tosls(tcg_res, tcg_op,
12249 tcg_constant_i32(0), tcg_fpstatus);
12250 break;
12251 case 0x5a: /* FCVTNU */
12252 case 0x5b: /* FCVTMU */
12253 case 0x5c: /* FCVTAU */
12254 case 0x7a: /* FCVTPU */
12255 case 0x7b: /* FCVTZU */
12256 gen_helper_vfp_touls(tcg_res, tcg_op,
12257 tcg_constant_i32(0), tcg_fpstatus);
12258 break;
12259 case 0x18: /* FRINTN */
12260 case 0x19: /* FRINTM */
12261 case 0x38: /* FRINTP */
12262 case 0x39: /* FRINTZ */
12263 case 0x58: /* FRINTA */
12264 case 0x79: /* FRINTI */
12265 gen_helper_rints(tcg_res, tcg_op, tcg_fpstatus);
12266 break;
12267 case 0x59: /* FRINTX */
12268 gen_helper_rints_exact(tcg_res, tcg_op, tcg_fpstatus);
12269 break;
12270 case 0x7c: /* URSQRTE */
12271 gen_helper_rsqrte_u32(tcg_res, tcg_op);
12272 break;
12273 case 0x1e: /* FRINT32Z */
12274 case 0x5e: /* FRINT32X */
12275 gen_helper_frint32_s(tcg_res, tcg_op, tcg_fpstatus);
12276 break;
12277 case 0x1f: /* FRINT64Z */
12278 case 0x5f: /* FRINT64X */
12279 gen_helper_frint64_s(tcg_res, tcg_op, tcg_fpstatus);
12280 break;
12281 default:
12282 g_assert_not_reached();
12283 }
12284 } else {
12285 /* Use helpers for 8 and 16 bit elements */
12286 switch (opcode) {
12287 case 0x5: /* CNT, RBIT */
12288 /* For these two insns size is part of the opcode specifier
12289 * (handled earlier); they always operate on byte elements.
12290 */
12291 if (u) {
12292 gen_helper_neon_rbit_u8(tcg_res, tcg_op);
12293 } else {
12294 gen_helper_neon_cnt_u8(tcg_res, tcg_op);
12295 }
12296 break;
12297 case 0x7: /* SQABS, SQNEG */
12298 {
12299 NeonGenOneOpEnvFn *genfn;
12300 static NeonGenOneOpEnvFn * const fns[2][2] = {
12301 { gen_helper_neon_qabs_s8, gen_helper_neon_qneg_s8 },
12302 { gen_helper_neon_qabs_s16, gen_helper_neon_qneg_s16 },
12303 };
12304 genfn = fns[size][u];
12305 genfn(tcg_res, tcg_env, tcg_op);
12306 break;
12307 }
12308 case 0x4: /* CLS, CLZ */
12309 if (u) {
12310 if (size == 0) {
12311 gen_helper_neon_clz_u8(tcg_res, tcg_op);
12312 } else {
12313 gen_helper_neon_clz_u16(tcg_res, tcg_op);
12314 }
12315 } else {
12316 if (size == 0) {
12317 gen_helper_neon_cls_s8(tcg_res, tcg_op);
12318 } else {
12319 gen_helper_neon_cls_s16(tcg_res, tcg_op);
12320 }
12321 }
12322 break;
12323 default:
12324 g_assert_not_reached();
12325 }
12326 }
12327
12328 write_vec_element_i32(s, tcg_res, rd, pass, MO_32);
12329 }
12330 }
12331 clear_vec_high(s, is_q, rd);
12332
12333 if (tcg_rmode) {
12334 gen_restore_rmode(tcg_rmode, tcg_fpstatus);
12335 }
12336 }
12337
12338 /* AdvSIMD [scalar] two register miscellaneous (FP16)
12339 *
12340 * 31 30 29 28 27 24 23 22 21 17 16 12 11 10 9 5 4 0
12341 * +---+---+---+---+---------+---+-------------+--------+-----+------+------+
12342 * | 0 | Q | U | S | 1 1 1 0 | a | 1 1 1 1 0 0 | opcode | 1 0 | Rn | Rd |
12343 * +---+---+---+---+---------+---+-------------+--------+-----+------+------+
12344 * mask: 1000 1111 0111 1110 0000 1100 0000 0000 0x8f7e 0c00
12345 * val: 0000 1110 0111 1000 0000 1000 0000 0000 0x0e78 0800
12346 *
12347 * This actually covers two groups where scalar access is governed by
12348 * bit 28. A bunch of the instructions (float to integral) only exist
12349 * in the vector form and are un-allocated for the scalar decode. Also
12350 * in the scalar decode Q is always 1.
12351 */
disas_simd_two_reg_misc_fp16(DisasContext * s,uint32_t insn)12352 static void disas_simd_two_reg_misc_fp16(DisasContext *s, uint32_t insn)
12353 {
12354 int fpop, opcode, a, u;
12355 int rn, rd;
12356 bool is_q;
12357 bool is_scalar;
12358 bool only_in_vector = false;
12359
12360 int pass;
12361 TCGv_i32 tcg_rmode = NULL;
12362 TCGv_ptr tcg_fpstatus = NULL;
12363 bool need_fpst = true;
12364 int rmode = -1;
12365
12366 if (!dc_isar_feature(aa64_fp16, s)) {
12367 unallocated_encoding(s);
12368 return;
12369 }
12370
12371 rd = extract32(insn, 0, 5);
12372 rn = extract32(insn, 5, 5);
12373
12374 a = extract32(insn, 23, 1);
12375 u = extract32(insn, 29, 1);
12376 is_scalar = extract32(insn, 28, 1);
12377 is_q = extract32(insn, 30, 1);
12378
12379 opcode = extract32(insn, 12, 5);
12380 fpop = deposit32(opcode, 5, 1, a);
12381 fpop = deposit32(fpop, 6, 1, u);
12382
12383 switch (fpop) {
12384 case 0x1d: /* SCVTF */
12385 case 0x5d: /* UCVTF */
12386 {
12387 int elements;
12388
12389 if (is_scalar) {
12390 elements = 1;
12391 } else {
12392 elements = (is_q ? 8 : 4);
12393 }
12394
12395 if (!fp_access_check(s)) {
12396 return;
12397 }
12398 handle_simd_intfp_conv(s, rd, rn, elements, !u, 0, MO_16);
12399 return;
12400 }
12401 break;
12402 case 0x2c: /* FCMGT (zero) */
12403 case 0x2d: /* FCMEQ (zero) */
12404 case 0x2e: /* FCMLT (zero) */
12405 case 0x6c: /* FCMGE (zero) */
12406 case 0x6d: /* FCMLE (zero) */
12407 handle_2misc_fcmp_zero(s, fpop, is_scalar, 0, is_q, MO_16, rn, rd);
12408 return;
12409 case 0x3d: /* FRECPE */
12410 case 0x3f: /* FRECPX */
12411 break;
12412 case 0x18: /* FRINTN */
12413 only_in_vector = true;
12414 rmode = FPROUNDING_TIEEVEN;
12415 break;
12416 case 0x19: /* FRINTM */
12417 only_in_vector = true;
12418 rmode = FPROUNDING_NEGINF;
12419 break;
12420 case 0x38: /* FRINTP */
12421 only_in_vector = true;
12422 rmode = FPROUNDING_POSINF;
12423 break;
12424 case 0x39: /* FRINTZ */
12425 only_in_vector = true;
12426 rmode = FPROUNDING_ZERO;
12427 break;
12428 case 0x58: /* FRINTA */
12429 only_in_vector = true;
12430 rmode = FPROUNDING_TIEAWAY;
12431 break;
12432 case 0x59: /* FRINTX */
12433 case 0x79: /* FRINTI */
12434 only_in_vector = true;
12435 /* current rounding mode */
12436 break;
12437 case 0x1a: /* FCVTNS */
12438 rmode = FPROUNDING_TIEEVEN;
12439 break;
12440 case 0x1b: /* FCVTMS */
12441 rmode = FPROUNDING_NEGINF;
12442 break;
12443 case 0x1c: /* FCVTAS */
12444 rmode = FPROUNDING_TIEAWAY;
12445 break;
12446 case 0x3a: /* FCVTPS */
12447 rmode = FPROUNDING_POSINF;
12448 break;
12449 case 0x3b: /* FCVTZS */
12450 rmode = FPROUNDING_ZERO;
12451 break;
12452 case 0x5a: /* FCVTNU */
12453 rmode = FPROUNDING_TIEEVEN;
12454 break;
12455 case 0x5b: /* FCVTMU */
12456 rmode = FPROUNDING_NEGINF;
12457 break;
12458 case 0x5c: /* FCVTAU */
12459 rmode = FPROUNDING_TIEAWAY;
12460 break;
12461 case 0x7a: /* FCVTPU */
12462 rmode = FPROUNDING_POSINF;
12463 break;
12464 case 0x7b: /* FCVTZU */
12465 rmode = FPROUNDING_ZERO;
12466 break;
12467 case 0x2f: /* FABS */
12468 case 0x6f: /* FNEG */
12469 need_fpst = false;
12470 break;
12471 case 0x7d: /* FRSQRTE */
12472 case 0x7f: /* FSQRT (vector) */
12473 break;
12474 default:
12475 unallocated_encoding(s);
12476 return;
12477 }
12478
12479
12480 /* Check additional constraints for the scalar encoding */
12481 if (is_scalar) {
12482 if (!is_q) {
12483 unallocated_encoding(s);
12484 return;
12485 }
12486 /* FRINTxx is only in the vector form */
12487 if (only_in_vector) {
12488 unallocated_encoding(s);
12489 return;
12490 }
12491 }
12492
12493 if (!fp_access_check(s)) {
12494 return;
12495 }
12496
12497 if (rmode >= 0 || need_fpst) {
12498 tcg_fpstatus = fpstatus_ptr(FPST_FPCR_F16);
12499 }
12500
12501 if (rmode >= 0) {
12502 tcg_rmode = gen_set_rmode(rmode, tcg_fpstatus);
12503 }
12504
12505 if (is_scalar) {
12506 TCGv_i32 tcg_op = read_fp_hreg(s, rn);
12507 TCGv_i32 tcg_res = tcg_temp_new_i32();
12508
12509 switch (fpop) {
12510 case 0x1a: /* FCVTNS */
12511 case 0x1b: /* FCVTMS */
12512 case 0x1c: /* FCVTAS */
12513 case 0x3a: /* FCVTPS */
12514 case 0x3b: /* FCVTZS */
12515 gen_helper_advsimd_f16tosinth(tcg_res, tcg_op, tcg_fpstatus);
12516 break;
12517 case 0x3d: /* FRECPE */
12518 gen_helper_recpe_f16(tcg_res, tcg_op, tcg_fpstatus);
12519 break;
12520 case 0x3f: /* FRECPX */
12521 gen_helper_frecpx_f16(tcg_res, tcg_op, tcg_fpstatus);
12522 break;
12523 case 0x5a: /* FCVTNU */
12524 case 0x5b: /* FCVTMU */
12525 case 0x5c: /* FCVTAU */
12526 case 0x7a: /* FCVTPU */
12527 case 0x7b: /* FCVTZU */
12528 gen_helper_advsimd_f16touinth(tcg_res, tcg_op, tcg_fpstatus);
12529 break;
12530 case 0x6f: /* FNEG */
12531 tcg_gen_xori_i32(tcg_res, tcg_op, 0x8000);
12532 break;
12533 case 0x7d: /* FRSQRTE */
12534 gen_helper_rsqrte_f16(tcg_res, tcg_op, tcg_fpstatus);
12535 break;
12536 default:
12537 g_assert_not_reached();
12538 }
12539
12540 /* limit any sign extension going on */
12541 tcg_gen_andi_i32(tcg_res, tcg_res, 0xffff);
12542 write_fp_sreg(s, rd, tcg_res);
12543 } else {
12544 for (pass = 0; pass < (is_q ? 8 : 4); pass++) {
12545 TCGv_i32 tcg_op = tcg_temp_new_i32();
12546 TCGv_i32 tcg_res = tcg_temp_new_i32();
12547
12548 read_vec_element_i32(s, tcg_op, rn, pass, MO_16);
12549
12550 switch (fpop) {
12551 case 0x1a: /* FCVTNS */
12552 case 0x1b: /* FCVTMS */
12553 case 0x1c: /* FCVTAS */
12554 case 0x3a: /* FCVTPS */
12555 case 0x3b: /* FCVTZS */
12556 gen_helper_advsimd_f16tosinth(tcg_res, tcg_op, tcg_fpstatus);
12557 break;
12558 case 0x3d: /* FRECPE */
12559 gen_helper_recpe_f16(tcg_res, tcg_op, tcg_fpstatus);
12560 break;
12561 case 0x5a: /* FCVTNU */
12562 case 0x5b: /* FCVTMU */
12563 case 0x5c: /* FCVTAU */
12564 case 0x7a: /* FCVTPU */
12565 case 0x7b: /* FCVTZU */
12566 gen_helper_advsimd_f16touinth(tcg_res, tcg_op, tcg_fpstatus);
12567 break;
12568 case 0x18: /* FRINTN */
12569 case 0x19: /* FRINTM */
12570 case 0x38: /* FRINTP */
12571 case 0x39: /* FRINTZ */
12572 case 0x58: /* FRINTA */
12573 case 0x79: /* FRINTI */
12574 gen_helper_advsimd_rinth(tcg_res, tcg_op, tcg_fpstatus);
12575 break;
12576 case 0x59: /* FRINTX */
12577 gen_helper_advsimd_rinth_exact(tcg_res, tcg_op, tcg_fpstatus);
12578 break;
12579 case 0x2f: /* FABS */
12580 tcg_gen_andi_i32(tcg_res, tcg_op, 0x7fff);
12581 break;
12582 case 0x6f: /* FNEG */
12583 tcg_gen_xori_i32(tcg_res, tcg_op, 0x8000);
12584 break;
12585 case 0x7d: /* FRSQRTE */
12586 gen_helper_rsqrte_f16(tcg_res, tcg_op, tcg_fpstatus);
12587 break;
12588 case 0x7f: /* FSQRT */
12589 gen_helper_sqrt_f16(tcg_res, tcg_op, tcg_fpstatus);
12590 break;
12591 default:
12592 g_assert_not_reached();
12593 }
12594
12595 write_vec_element_i32(s, tcg_res, rd, pass, MO_16);
12596 }
12597
12598 clear_vec_high(s, is_q, rd);
12599 }
12600
12601 if (tcg_rmode) {
12602 gen_restore_rmode(tcg_rmode, tcg_fpstatus);
12603 }
12604 }
12605
12606 /* AdvSIMD scalar x indexed element
12607 * 31 30 29 28 24 23 22 21 20 19 16 15 12 11 10 9 5 4 0
12608 * +-----+---+-----------+------+---+---+------+-----+---+---+------+------+
12609 * | 0 1 | U | 1 1 1 1 1 | size | L | M | Rm | opc | H | 0 | Rn | Rd |
12610 * +-----+---+-----------+------+---+---+------+-----+---+---+------+------+
12611 * AdvSIMD vector x indexed element
12612 * 31 30 29 28 24 23 22 21 20 19 16 15 12 11 10 9 5 4 0
12613 * +---+---+---+-----------+------+---+---+------+-----+---+---+------+------+
12614 * | 0 | Q | U | 0 1 1 1 1 | size | L | M | Rm | opc | H | 0 | Rn | Rd |
12615 * +---+---+---+-----------+------+---+---+------+-----+---+---+------+------+
12616 */
disas_simd_indexed(DisasContext * s,uint32_t insn)12617 static void disas_simd_indexed(DisasContext *s, uint32_t insn)
12618 {
12619 /* This encoding has two kinds of instruction:
12620 * normal, where we perform elt x idxelt => elt for each
12621 * element in the vector
12622 * long, where we perform elt x idxelt and generate a result of
12623 * double the width of the input element
12624 * The long ops have a 'part' specifier (ie come in INSN, INSN2 pairs).
12625 */
12626 bool is_scalar = extract32(insn, 28, 1);
12627 bool is_q = extract32(insn, 30, 1);
12628 bool u = extract32(insn, 29, 1);
12629 int size = extract32(insn, 22, 2);
12630 int l = extract32(insn, 21, 1);
12631 int m = extract32(insn, 20, 1);
12632 /* Note that the Rm field here is only 4 bits, not 5 as it usually is */
12633 int rm = extract32(insn, 16, 4);
12634 int opcode = extract32(insn, 12, 4);
12635 int h = extract32(insn, 11, 1);
12636 int rn = extract32(insn, 5, 5);
12637 int rd = extract32(insn, 0, 5);
12638 bool is_long = false;
12639 int is_fp = 0;
12640 bool is_fp16 = false;
12641 int index;
12642 TCGv_ptr fpst;
12643
12644 switch (16 * u + opcode) {
12645 case 0x08: /* MUL */
12646 case 0x10: /* MLA */
12647 case 0x14: /* MLS */
12648 if (is_scalar) {
12649 unallocated_encoding(s);
12650 return;
12651 }
12652 break;
12653 case 0x02: /* SMLAL, SMLAL2 */
12654 case 0x12: /* UMLAL, UMLAL2 */
12655 case 0x06: /* SMLSL, SMLSL2 */
12656 case 0x16: /* UMLSL, UMLSL2 */
12657 case 0x0a: /* SMULL, SMULL2 */
12658 case 0x1a: /* UMULL, UMULL2 */
12659 if (is_scalar) {
12660 unallocated_encoding(s);
12661 return;
12662 }
12663 is_long = true;
12664 break;
12665 case 0x03: /* SQDMLAL, SQDMLAL2 */
12666 case 0x07: /* SQDMLSL, SQDMLSL2 */
12667 case 0x0b: /* SQDMULL, SQDMULL2 */
12668 is_long = true;
12669 break;
12670 case 0x0c: /* SQDMULH */
12671 case 0x0d: /* SQRDMULH */
12672 break;
12673 case 0x01: /* FMLA */
12674 case 0x05: /* FMLS */
12675 case 0x09: /* FMUL */
12676 case 0x19: /* FMULX */
12677 is_fp = 1;
12678 break;
12679 case 0x1d: /* SQRDMLAH */
12680 case 0x1f: /* SQRDMLSH */
12681 if (!dc_isar_feature(aa64_rdm, s)) {
12682 unallocated_encoding(s);
12683 return;
12684 }
12685 break;
12686 case 0x0e: /* SDOT */
12687 case 0x1e: /* UDOT */
12688 if (is_scalar || size != MO_32 || !dc_isar_feature(aa64_dp, s)) {
12689 unallocated_encoding(s);
12690 return;
12691 }
12692 break;
12693 case 0x0f:
12694 switch (size) {
12695 case 0: /* SUDOT */
12696 case 2: /* USDOT */
12697 if (is_scalar || !dc_isar_feature(aa64_i8mm, s)) {
12698 unallocated_encoding(s);
12699 return;
12700 }
12701 size = MO_32;
12702 break;
12703 case 1: /* BFDOT */
12704 if (is_scalar || !dc_isar_feature(aa64_bf16, s)) {
12705 unallocated_encoding(s);
12706 return;
12707 }
12708 size = MO_32;
12709 break;
12710 case 3: /* BFMLAL{B,T} */
12711 if (is_scalar || !dc_isar_feature(aa64_bf16, s)) {
12712 unallocated_encoding(s);
12713 return;
12714 }
12715 /* can't set is_fp without other incorrect size checks */
12716 size = MO_16;
12717 break;
12718 default:
12719 unallocated_encoding(s);
12720 return;
12721 }
12722 break;
12723 case 0x11: /* FCMLA #0 */
12724 case 0x13: /* FCMLA #90 */
12725 case 0x15: /* FCMLA #180 */
12726 case 0x17: /* FCMLA #270 */
12727 if (is_scalar || !dc_isar_feature(aa64_fcma, s)) {
12728 unallocated_encoding(s);
12729 return;
12730 }
12731 is_fp = 2;
12732 break;
12733 case 0x00: /* FMLAL */
12734 case 0x04: /* FMLSL */
12735 case 0x18: /* FMLAL2 */
12736 case 0x1c: /* FMLSL2 */
12737 if (is_scalar || size != MO_32 || !dc_isar_feature(aa64_fhm, s)) {
12738 unallocated_encoding(s);
12739 return;
12740 }
12741 size = MO_16;
12742 /* is_fp, but we pass tcg_env not fp_status. */
12743 break;
12744 default:
12745 unallocated_encoding(s);
12746 return;
12747 }
12748
12749 switch (is_fp) {
12750 case 1: /* normal fp */
12751 /* convert insn encoded size to MemOp size */
12752 switch (size) {
12753 case 0: /* half-precision */
12754 size = MO_16;
12755 is_fp16 = true;
12756 break;
12757 case MO_32: /* single precision */
12758 case MO_64: /* double precision */
12759 break;
12760 default:
12761 unallocated_encoding(s);
12762 return;
12763 }
12764 break;
12765
12766 case 2: /* complex fp */
12767 /* Each indexable element is a complex pair. */
12768 size += 1;
12769 switch (size) {
12770 case MO_32:
12771 if (h && !is_q) {
12772 unallocated_encoding(s);
12773 return;
12774 }
12775 is_fp16 = true;
12776 break;
12777 case MO_64:
12778 break;
12779 default:
12780 unallocated_encoding(s);
12781 return;
12782 }
12783 break;
12784
12785 default: /* integer */
12786 switch (size) {
12787 case MO_8:
12788 case MO_64:
12789 unallocated_encoding(s);
12790 return;
12791 }
12792 break;
12793 }
12794 if (is_fp16 && !dc_isar_feature(aa64_fp16, s)) {
12795 unallocated_encoding(s);
12796 return;
12797 }
12798
12799 /* Given MemOp size, adjust register and indexing. */
12800 switch (size) {
12801 case MO_16:
12802 index = h << 2 | l << 1 | m;
12803 break;
12804 case MO_32:
12805 index = h << 1 | l;
12806 rm |= m << 4;
12807 break;
12808 case MO_64:
12809 if (l || !is_q) {
12810 unallocated_encoding(s);
12811 return;
12812 }
12813 index = h;
12814 rm |= m << 4;
12815 break;
12816 default:
12817 g_assert_not_reached();
12818 }
12819
12820 if (!fp_access_check(s)) {
12821 return;
12822 }
12823
12824 if (is_fp) {
12825 fpst = fpstatus_ptr(is_fp16 ? FPST_FPCR_F16 : FPST_FPCR);
12826 } else {
12827 fpst = NULL;
12828 }
12829
12830 switch (16 * u + opcode) {
12831 case 0x0e: /* SDOT */
12832 case 0x1e: /* UDOT */
12833 gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, index,
12834 u ? gen_helper_gvec_udot_idx_b
12835 : gen_helper_gvec_sdot_idx_b);
12836 return;
12837 case 0x0f:
12838 switch (extract32(insn, 22, 2)) {
12839 case 0: /* SUDOT */
12840 gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, index,
12841 gen_helper_gvec_sudot_idx_b);
12842 return;
12843 case 1: /* BFDOT */
12844 gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, index,
12845 gen_helper_gvec_bfdot_idx);
12846 return;
12847 case 2: /* USDOT */
12848 gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, index,
12849 gen_helper_gvec_usdot_idx_b);
12850 return;
12851 case 3: /* BFMLAL{B,T} */
12852 gen_gvec_op4_fpst(s, 1, rd, rn, rm, rd, 0, (index << 1) | is_q,
12853 gen_helper_gvec_bfmlal_idx);
12854 return;
12855 }
12856 g_assert_not_reached();
12857 case 0x11: /* FCMLA #0 */
12858 case 0x13: /* FCMLA #90 */
12859 case 0x15: /* FCMLA #180 */
12860 case 0x17: /* FCMLA #270 */
12861 {
12862 int rot = extract32(insn, 13, 2);
12863 int data = (index << 2) | rot;
12864 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, rd),
12865 vec_full_reg_offset(s, rn),
12866 vec_full_reg_offset(s, rm),
12867 vec_full_reg_offset(s, rd), fpst,
12868 is_q ? 16 : 8, vec_full_reg_size(s), data,
12869 size == MO_64
12870 ? gen_helper_gvec_fcmlas_idx
12871 : gen_helper_gvec_fcmlah_idx);
12872 }
12873 return;
12874
12875 case 0x00: /* FMLAL */
12876 case 0x04: /* FMLSL */
12877 case 0x18: /* FMLAL2 */
12878 case 0x1c: /* FMLSL2 */
12879 {
12880 int is_s = extract32(opcode, 2, 1);
12881 int is_2 = u;
12882 int data = (index << 2) | (is_2 << 1) | is_s;
12883 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd),
12884 vec_full_reg_offset(s, rn),
12885 vec_full_reg_offset(s, rm), tcg_env,
12886 is_q ? 16 : 8, vec_full_reg_size(s),
12887 data, gen_helper_gvec_fmlal_idx_a64);
12888 }
12889 return;
12890
12891 case 0x08: /* MUL */
12892 if (!is_long && !is_scalar) {
12893 static gen_helper_gvec_3 * const fns[3] = {
12894 gen_helper_gvec_mul_idx_h,
12895 gen_helper_gvec_mul_idx_s,
12896 gen_helper_gvec_mul_idx_d,
12897 };
12898 tcg_gen_gvec_3_ool(vec_full_reg_offset(s, rd),
12899 vec_full_reg_offset(s, rn),
12900 vec_full_reg_offset(s, rm),
12901 is_q ? 16 : 8, vec_full_reg_size(s),
12902 index, fns[size - 1]);
12903 return;
12904 }
12905 break;
12906
12907 case 0x10: /* MLA */
12908 if (!is_long && !is_scalar) {
12909 static gen_helper_gvec_4 * const fns[3] = {
12910 gen_helper_gvec_mla_idx_h,
12911 gen_helper_gvec_mla_idx_s,
12912 gen_helper_gvec_mla_idx_d,
12913 };
12914 tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd),
12915 vec_full_reg_offset(s, rn),
12916 vec_full_reg_offset(s, rm),
12917 vec_full_reg_offset(s, rd),
12918 is_q ? 16 : 8, vec_full_reg_size(s),
12919 index, fns[size - 1]);
12920 return;
12921 }
12922 break;
12923
12924 case 0x14: /* MLS */
12925 if (!is_long && !is_scalar) {
12926 static gen_helper_gvec_4 * const fns[3] = {
12927 gen_helper_gvec_mls_idx_h,
12928 gen_helper_gvec_mls_idx_s,
12929 gen_helper_gvec_mls_idx_d,
12930 };
12931 tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd),
12932 vec_full_reg_offset(s, rn),
12933 vec_full_reg_offset(s, rm),
12934 vec_full_reg_offset(s, rd),
12935 is_q ? 16 : 8, vec_full_reg_size(s),
12936 index, fns[size - 1]);
12937 return;
12938 }
12939 break;
12940 }
12941
12942 if (size == 3) {
12943 TCGv_i64 tcg_idx = tcg_temp_new_i64();
12944 int pass;
12945
12946 assert(is_fp && is_q && !is_long);
12947
12948 read_vec_element(s, tcg_idx, rm, index, MO_64);
12949
12950 for (pass = 0; pass < (is_scalar ? 1 : 2); pass++) {
12951 TCGv_i64 tcg_op = tcg_temp_new_i64();
12952 TCGv_i64 tcg_res = tcg_temp_new_i64();
12953
12954 read_vec_element(s, tcg_op, rn, pass, MO_64);
12955
12956 switch (16 * u + opcode) {
12957 case 0x05: /* FMLS */
12958 /* As usual for ARM, separate negation for fused multiply-add */
12959 gen_helper_vfp_negd(tcg_op, tcg_op);
12960 /* fall through */
12961 case 0x01: /* FMLA */
12962 read_vec_element(s, tcg_res, rd, pass, MO_64);
12963 gen_helper_vfp_muladdd(tcg_res, tcg_op, tcg_idx, tcg_res, fpst);
12964 break;
12965 case 0x09: /* FMUL */
12966 gen_helper_vfp_muld(tcg_res, tcg_op, tcg_idx, fpst);
12967 break;
12968 case 0x19: /* FMULX */
12969 gen_helper_vfp_mulxd(tcg_res, tcg_op, tcg_idx, fpst);
12970 break;
12971 default:
12972 g_assert_not_reached();
12973 }
12974
12975 write_vec_element(s, tcg_res, rd, pass, MO_64);
12976 }
12977
12978 clear_vec_high(s, !is_scalar, rd);
12979 } else if (!is_long) {
12980 /* 32 bit floating point, or 16 or 32 bit integer.
12981 * For the 16 bit scalar case we use the usual Neon helpers and
12982 * rely on the fact that 0 op 0 == 0 with no side effects.
12983 */
12984 TCGv_i32 tcg_idx = tcg_temp_new_i32();
12985 int pass, maxpasses;
12986
12987 if (is_scalar) {
12988 maxpasses = 1;
12989 } else {
12990 maxpasses = is_q ? 4 : 2;
12991 }
12992
12993 read_vec_element_i32(s, tcg_idx, rm, index, size);
12994
12995 if (size == 1 && !is_scalar) {
12996 /* The simplest way to handle the 16x16 indexed ops is to duplicate
12997 * the index into both halves of the 32 bit tcg_idx and then use
12998 * the usual Neon helpers.
12999 */
13000 tcg_gen_deposit_i32(tcg_idx, tcg_idx, tcg_idx, 16, 16);
13001 }
13002
13003 for (pass = 0; pass < maxpasses; pass++) {
13004 TCGv_i32 tcg_op = tcg_temp_new_i32();
13005 TCGv_i32 tcg_res = tcg_temp_new_i32();
13006
13007 read_vec_element_i32(s, tcg_op, rn, pass, is_scalar ? size : MO_32);
13008
13009 switch (16 * u + opcode) {
13010 case 0x08: /* MUL */
13011 case 0x10: /* MLA */
13012 case 0x14: /* MLS */
13013 {
13014 static NeonGenTwoOpFn * const fns[2][2] = {
13015 { gen_helper_neon_add_u16, gen_helper_neon_sub_u16 },
13016 { tcg_gen_add_i32, tcg_gen_sub_i32 },
13017 };
13018 NeonGenTwoOpFn *genfn;
13019 bool is_sub = opcode == 0x4;
13020
13021 if (size == 1) {
13022 gen_helper_neon_mul_u16(tcg_res, tcg_op, tcg_idx);
13023 } else {
13024 tcg_gen_mul_i32(tcg_res, tcg_op, tcg_idx);
13025 }
13026 if (opcode == 0x8) {
13027 break;
13028 }
13029 read_vec_element_i32(s, tcg_op, rd, pass, MO_32);
13030 genfn = fns[size - 1][is_sub];
13031 genfn(tcg_res, tcg_op, tcg_res);
13032 break;
13033 }
13034 case 0x05: /* FMLS */
13035 case 0x01: /* FMLA */
13036 read_vec_element_i32(s, tcg_res, rd, pass,
13037 is_scalar ? size : MO_32);
13038 switch (size) {
13039 case 1:
13040 if (opcode == 0x5) {
13041 /* As usual for ARM, separate negation for fused
13042 * multiply-add */
13043 tcg_gen_xori_i32(tcg_op, tcg_op, 0x80008000);
13044 }
13045 if (is_scalar) {
13046 gen_helper_advsimd_muladdh(tcg_res, tcg_op, tcg_idx,
13047 tcg_res, fpst);
13048 } else {
13049 gen_helper_advsimd_muladd2h(tcg_res, tcg_op, tcg_idx,
13050 tcg_res, fpst);
13051 }
13052 break;
13053 case 2:
13054 if (opcode == 0x5) {
13055 /* As usual for ARM, separate negation for
13056 * fused multiply-add */
13057 tcg_gen_xori_i32(tcg_op, tcg_op, 0x80000000);
13058 }
13059 gen_helper_vfp_muladds(tcg_res, tcg_op, tcg_idx,
13060 tcg_res, fpst);
13061 break;
13062 default:
13063 g_assert_not_reached();
13064 }
13065 break;
13066 case 0x09: /* FMUL */
13067 switch (size) {
13068 case 1:
13069 if (is_scalar) {
13070 gen_helper_advsimd_mulh(tcg_res, tcg_op,
13071 tcg_idx, fpst);
13072 } else {
13073 gen_helper_advsimd_mul2h(tcg_res, tcg_op,
13074 tcg_idx, fpst);
13075 }
13076 break;
13077 case 2:
13078 gen_helper_vfp_muls(tcg_res, tcg_op, tcg_idx, fpst);
13079 break;
13080 default:
13081 g_assert_not_reached();
13082 }
13083 break;
13084 case 0x19: /* FMULX */
13085 switch (size) {
13086 case 1:
13087 if (is_scalar) {
13088 gen_helper_advsimd_mulxh(tcg_res, tcg_op,
13089 tcg_idx, fpst);
13090 } else {
13091 gen_helper_advsimd_mulx2h(tcg_res, tcg_op,
13092 tcg_idx, fpst);
13093 }
13094 break;
13095 case 2:
13096 gen_helper_vfp_mulxs(tcg_res, tcg_op, tcg_idx, fpst);
13097 break;
13098 default:
13099 g_assert_not_reached();
13100 }
13101 break;
13102 case 0x0c: /* SQDMULH */
13103 if (size == 1) {
13104 gen_helper_neon_qdmulh_s16(tcg_res, tcg_env,
13105 tcg_op, tcg_idx);
13106 } else {
13107 gen_helper_neon_qdmulh_s32(tcg_res, tcg_env,
13108 tcg_op, tcg_idx);
13109 }
13110 break;
13111 case 0x0d: /* SQRDMULH */
13112 if (size == 1) {
13113 gen_helper_neon_qrdmulh_s16(tcg_res, tcg_env,
13114 tcg_op, tcg_idx);
13115 } else {
13116 gen_helper_neon_qrdmulh_s32(tcg_res, tcg_env,
13117 tcg_op, tcg_idx);
13118 }
13119 break;
13120 case 0x1d: /* SQRDMLAH */
13121 read_vec_element_i32(s, tcg_res, rd, pass,
13122 is_scalar ? size : MO_32);
13123 if (size == 1) {
13124 gen_helper_neon_qrdmlah_s16(tcg_res, tcg_env,
13125 tcg_op, tcg_idx, tcg_res);
13126 } else {
13127 gen_helper_neon_qrdmlah_s32(tcg_res, tcg_env,
13128 tcg_op, tcg_idx, tcg_res);
13129 }
13130 break;
13131 case 0x1f: /* SQRDMLSH */
13132 read_vec_element_i32(s, tcg_res, rd, pass,
13133 is_scalar ? size : MO_32);
13134 if (size == 1) {
13135 gen_helper_neon_qrdmlsh_s16(tcg_res, tcg_env,
13136 tcg_op, tcg_idx, tcg_res);
13137 } else {
13138 gen_helper_neon_qrdmlsh_s32(tcg_res, tcg_env,
13139 tcg_op, tcg_idx, tcg_res);
13140 }
13141 break;
13142 default:
13143 g_assert_not_reached();
13144 }
13145
13146 if (is_scalar) {
13147 write_fp_sreg(s, rd, tcg_res);
13148 } else {
13149 write_vec_element_i32(s, tcg_res, rd, pass, MO_32);
13150 }
13151 }
13152
13153 clear_vec_high(s, is_q, rd);
13154 } else {
13155 /* long ops: 16x16->32 or 32x32->64 */
13156 TCGv_i64 tcg_res[2];
13157 int pass;
13158 bool satop = extract32(opcode, 0, 1);
13159 MemOp memop = MO_32;
13160
13161 if (satop || !u) {
13162 memop |= MO_SIGN;
13163 }
13164
13165 if (size == 2) {
13166 TCGv_i64 tcg_idx = tcg_temp_new_i64();
13167
13168 read_vec_element(s, tcg_idx, rm, index, memop);
13169
13170 for (pass = 0; pass < (is_scalar ? 1 : 2); pass++) {
13171 TCGv_i64 tcg_op = tcg_temp_new_i64();
13172 TCGv_i64 tcg_passres;
13173 int passelt;
13174
13175 if (is_scalar) {
13176 passelt = 0;
13177 } else {
13178 passelt = pass + (is_q * 2);
13179 }
13180
13181 read_vec_element(s, tcg_op, rn, passelt, memop);
13182
13183 tcg_res[pass] = tcg_temp_new_i64();
13184
13185 if (opcode == 0xa || opcode == 0xb) {
13186 /* Non-accumulating ops */
13187 tcg_passres = tcg_res[pass];
13188 } else {
13189 tcg_passres = tcg_temp_new_i64();
13190 }
13191
13192 tcg_gen_mul_i64(tcg_passres, tcg_op, tcg_idx);
13193
13194 if (satop) {
13195 /* saturating, doubling */
13196 gen_helper_neon_addl_saturate_s64(tcg_passres, tcg_env,
13197 tcg_passres, tcg_passres);
13198 }
13199
13200 if (opcode == 0xa || opcode == 0xb) {
13201 continue;
13202 }
13203
13204 /* Accumulating op: handle accumulate step */
13205 read_vec_element(s, tcg_res[pass], rd, pass, MO_64);
13206
13207 switch (opcode) {
13208 case 0x2: /* SMLAL, SMLAL2, UMLAL, UMLAL2 */
13209 tcg_gen_add_i64(tcg_res[pass], tcg_res[pass], tcg_passres);
13210 break;
13211 case 0x6: /* SMLSL, SMLSL2, UMLSL, UMLSL2 */
13212 tcg_gen_sub_i64(tcg_res[pass], tcg_res[pass], tcg_passres);
13213 break;
13214 case 0x7: /* SQDMLSL, SQDMLSL2 */
13215 tcg_gen_neg_i64(tcg_passres, tcg_passres);
13216 /* fall through */
13217 case 0x3: /* SQDMLAL, SQDMLAL2 */
13218 gen_helper_neon_addl_saturate_s64(tcg_res[pass], tcg_env,
13219 tcg_res[pass],
13220 tcg_passres);
13221 break;
13222 default:
13223 g_assert_not_reached();
13224 }
13225 }
13226
13227 clear_vec_high(s, !is_scalar, rd);
13228 } else {
13229 TCGv_i32 tcg_idx = tcg_temp_new_i32();
13230
13231 assert(size == 1);
13232 read_vec_element_i32(s, tcg_idx, rm, index, size);
13233
13234 if (!is_scalar) {
13235 /* The simplest way to handle the 16x16 indexed ops is to
13236 * duplicate the index into both halves of the 32 bit tcg_idx
13237 * and then use the usual Neon helpers.
13238 */
13239 tcg_gen_deposit_i32(tcg_idx, tcg_idx, tcg_idx, 16, 16);
13240 }
13241
13242 for (pass = 0; pass < (is_scalar ? 1 : 2); pass++) {
13243 TCGv_i32 tcg_op = tcg_temp_new_i32();
13244 TCGv_i64 tcg_passres;
13245
13246 if (is_scalar) {
13247 read_vec_element_i32(s, tcg_op, rn, pass, size);
13248 } else {
13249 read_vec_element_i32(s, tcg_op, rn,
13250 pass + (is_q * 2), MO_32);
13251 }
13252
13253 tcg_res[pass] = tcg_temp_new_i64();
13254
13255 if (opcode == 0xa || opcode == 0xb) {
13256 /* Non-accumulating ops */
13257 tcg_passres = tcg_res[pass];
13258 } else {
13259 tcg_passres = tcg_temp_new_i64();
13260 }
13261
13262 if (memop & MO_SIGN) {
13263 gen_helper_neon_mull_s16(tcg_passres, tcg_op, tcg_idx);
13264 } else {
13265 gen_helper_neon_mull_u16(tcg_passres, tcg_op, tcg_idx);
13266 }
13267 if (satop) {
13268 gen_helper_neon_addl_saturate_s32(tcg_passres, tcg_env,
13269 tcg_passres, tcg_passres);
13270 }
13271
13272 if (opcode == 0xa || opcode == 0xb) {
13273 continue;
13274 }
13275
13276 /* Accumulating op: handle accumulate step */
13277 read_vec_element(s, tcg_res[pass], rd, pass, MO_64);
13278
13279 switch (opcode) {
13280 case 0x2: /* SMLAL, SMLAL2, UMLAL, UMLAL2 */
13281 gen_helper_neon_addl_u32(tcg_res[pass], tcg_res[pass],
13282 tcg_passres);
13283 break;
13284 case 0x6: /* SMLSL, SMLSL2, UMLSL, UMLSL2 */
13285 gen_helper_neon_subl_u32(tcg_res[pass], tcg_res[pass],
13286 tcg_passres);
13287 break;
13288 case 0x7: /* SQDMLSL, SQDMLSL2 */
13289 gen_helper_neon_negl_u32(tcg_passres, tcg_passres);
13290 /* fall through */
13291 case 0x3: /* SQDMLAL, SQDMLAL2 */
13292 gen_helper_neon_addl_saturate_s32(tcg_res[pass], tcg_env,
13293 tcg_res[pass],
13294 tcg_passres);
13295 break;
13296 default:
13297 g_assert_not_reached();
13298 }
13299 }
13300
13301 if (is_scalar) {
13302 tcg_gen_ext32u_i64(tcg_res[0], tcg_res[0]);
13303 }
13304 }
13305
13306 if (is_scalar) {
13307 tcg_res[1] = tcg_constant_i64(0);
13308 }
13309
13310 for (pass = 0; pass < 2; pass++) {
13311 write_vec_element(s, tcg_res[pass], rd, pass, MO_64);
13312 }
13313 }
13314 }
13315
13316 /* Crypto AES
13317 * 31 24 23 22 21 17 16 12 11 10 9 5 4 0
13318 * +-----------------+------+-----------+--------+-----+------+------+
13319 * | 0 1 0 0 1 1 1 0 | size | 1 0 1 0 0 | opcode | 1 0 | Rn | Rd |
13320 * +-----------------+------+-----------+--------+-----+------+------+
13321 */
disas_crypto_aes(DisasContext * s,uint32_t insn)13322 static void disas_crypto_aes(DisasContext *s, uint32_t insn)
13323 {
13324 int size = extract32(insn, 22, 2);
13325 int opcode = extract32(insn, 12, 5);
13326 int rn = extract32(insn, 5, 5);
13327 int rd = extract32(insn, 0, 5);
13328 gen_helper_gvec_2 *genfn2 = NULL;
13329 gen_helper_gvec_3 *genfn3 = NULL;
13330
13331 if (!dc_isar_feature(aa64_aes, s) || size != 0) {
13332 unallocated_encoding(s);
13333 return;
13334 }
13335
13336 switch (opcode) {
13337 case 0x4: /* AESE */
13338 genfn3 = gen_helper_crypto_aese;
13339 break;
13340 case 0x6: /* AESMC */
13341 genfn2 = gen_helper_crypto_aesmc;
13342 break;
13343 case 0x5: /* AESD */
13344 genfn3 = gen_helper_crypto_aesd;
13345 break;
13346 case 0x7: /* AESIMC */
13347 genfn2 = gen_helper_crypto_aesimc;
13348 break;
13349 default:
13350 unallocated_encoding(s);
13351 return;
13352 }
13353
13354 if (!fp_access_check(s)) {
13355 return;
13356 }
13357 if (genfn2) {
13358 gen_gvec_op2_ool(s, true, rd, rn, 0, genfn2);
13359 } else {
13360 gen_gvec_op3_ool(s, true, rd, rd, rn, 0, genfn3);
13361 }
13362 }
13363
13364 /* Crypto three-reg SHA
13365 * 31 24 23 22 21 20 16 15 14 12 11 10 9 5 4 0
13366 * +-----------------+------+---+------+---+--------+-----+------+------+
13367 * | 0 1 0 1 1 1 1 0 | size | 0 | Rm | 0 | opcode | 0 0 | Rn | Rd |
13368 * +-----------------+------+---+------+---+--------+-----+------+------+
13369 */
disas_crypto_three_reg_sha(DisasContext * s,uint32_t insn)13370 static void disas_crypto_three_reg_sha(DisasContext *s, uint32_t insn)
13371 {
13372 int size = extract32(insn, 22, 2);
13373 int opcode = extract32(insn, 12, 3);
13374 int rm = extract32(insn, 16, 5);
13375 int rn = extract32(insn, 5, 5);
13376 int rd = extract32(insn, 0, 5);
13377 gen_helper_gvec_3 *genfn;
13378 bool feature;
13379
13380 if (size != 0) {
13381 unallocated_encoding(s);
13382 return;
13383 }
13384
13385 switch (opcode) {
13386 case 0: /* SHA1C */
13387 genfn = gen_helper_crypto_sha1c;
13388 feature = dc_isar_feature(aa64_sha1, s);
13389 break;
13390 case 1: /* SHA1P */
13391 genfn = gen_helper_crypto_sha1p;
13392 feature = dc_isar_feature(aa64_sha1, s);
13393 break;
13394 case 2: /* SHA1M */
13395 genfn = gen_helper_crypto_sha1m;
13396 feature = dc_isar_feature(aa64_sha1, s);
13397 break;
13398 case 3: /* SHA1SU0 */
13399 genfn = gen_helper_crypto_sha1su0;
13400 feature = dc_isar_feature(aa64_sha1, s);
13401 break;
13402 case 4: /* SHA256H */
13403 genfn = gen_helper_crypto_sha256h;
13404 feature = dc_isar_feature(aa64_sha256, s);
13405 break;
13406 case 5: /* SHA256H2 */
13407 genfn = gen_helper_crypto_sha256h2;
13408 feature = dc_isar_feature(aa64_sha256, s);
13409 break;
13410 case 6: /* SHA256SU1 */
13411 genfn = gen_helper_crypto_sha256su1;
13412 feature = dc_isar_feature(aa64_sha256, s);
13413 break;
13414 default:
13415 unallocated_encoding(s);
13416 return;
13417 }
13418
13419 if (!feature) {
13420 unallocated_encoding(s);
13421 return;
13422 }
13423
13424 if (!fp_access_check(s)) {
13425 return;
13426 }
13427 gen_gvec_op3_ool(s, true, rd, rn, rm, 0, genfn);
13428 }
13429
13430 /* Crypto two-reg SHA
13431 * 31 24 23 22 21 17 16 12 11 10 9 5 4 0
13432 * +-----------------+------+-----------+--------+-----+------+------+
13433 * | 0 1 0 1 1 1 1 0 | size | 1 0 1 0 0 | opcode | 1 0 | Rn | Rd |
13434 * +-----------------+------+-----------+--------+-----+------+------+
13435 */
disas_crypto_two_reg_sha(DisasContext * s,uint32_t insn)13436 static void disas_crypto_two_reg_sha(DisasContext *s, uint32_t insn)
13437 {
13438 int size = extract32(insn, 22, 2);
13439 int opcode = extract32(insn, 12, 5);
13440 int rn = extract32(insn, 5, 5);
13441 int rd = extract32(insn, 0, 5);
13442 gen_helper_gvec_2 *genfn;
13443 bool feature;
13444
13445 if (size != 0) {
13446 unallocated_encoding(s);
13447 return;
13448 }
13449
13450 switch (opcode) {
13451 case 0: /* SHA1H */
13452 feature = dc_isar_feature(aa64_sha1, s);
13453 genfn = gen_helper_crypto_sha1h;
13454 break;
13455 case 1: /* SHA1SU1 */
13456 feature = dc_isar_feature(aa64_sha1, s);
13457 genfn = gen_helper_crypto_sha1su1;
13458 break;
13459 case 2: /* SHA256SU0 */
13460 feature = dc_isar_feature(aa64_sha256, s);
13461 genfn = gen_helper_crypto_sha256su0;
13462 break;
13463 default:
13464 unallocated_encoding(s);
13465 return;
13466 }
13467
13468 if (!feature) {
13469 unallocated_encoding(s);
13470 return;
13471 }
13472
13473 if (!fp_access_check(s)) {
13474 return;
13475 }
13476 gen_gvec_op2_ool(s, true, rd, rn, 0, genfn);
13477 }
13478
gen_rax1_i64(TCGv_i64 d,TCGv_i64 n,TCGv_i64 m)13479 static void gen_rax1_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m)
13480 {
13481 tcg_gen_rotli_i64(d, m, 1);
13482 tcg_gen_xor_i64(d, d, n);
13483 }
13484
gen_rax1_vec(unsigned vece,TCGv_vec d,TCGv_vec n,TCGv_vec m)13485 static void gen_rax1_vec(unsigned vece, TCGv_vec d, TCGv_vec n, TCGv_vec m)
13486 {
13487 tcg_gen_rotli_vec(vece, d, m, 1);
13488 tcg_gen_xor_vec(vece, d, d, n);
13489 }
13490
gen_gvec_rax1(unsigned vece,uint32_t rd_ofs,uint32_t rn_ofs,uint32_t rm_ofs,uint32_t opr_sz,uint32_t max_sz)13491 void gen_gvec_rax1(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
13492 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
13493 {
13494 static const TCGOpcode vecop_list[] = { INDEX_op_rotli_vec, 0 };
13495 static const GVecGen3 op = {
13496 .fni8 = gen_rax1_i64,
13497 .fniv = gen_rax1_vec,
13498 .opt_opc = vecop_list,
13499 .fno = gen_helper_crypto_rax1,
13500 .vece = MO_64,
13501 };
13502 tcg_gen_gvec_3(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, &op);
13503 }
13504
13505 /* Crypto three-reg SHA512
13506 * 31 21 20 16 15 14 13 12 11 10 9 5 4 0
13507 * +-----------------------+------+---+---+-----+--------+------+------+
13508 * | 1 1 0 0 1 1 1 0 0 1 1 | Rm | 1 | O | 0 0 | opcode | Rn | Rd |
13509 * +-----------------------+------+---+---+-----+--------+------+------+
13510 */
disas_crypto_three_reg_sha512(DisasContext * s,uint32_t insn)13511 static void disas_crypto_three_reg_sha512(DisasContext *s, uint32_t insn)
13512 {
13513 int opcode = extract32(insn, 10, 2);
13514 int o = extract32(insn, 14, 1);
13515 int rm = extract32(insn, 16, 5);
13516 int rn = extract32(insn, 5, 5);
13517 int rd = extract32(insn, 0, 5);
13518 bool feature;
13519 gen_helper_gvec_3 *oolfn = NULL;
13520 GVecGen3Fn *gvecfn = NULL;
13521
13522 if (o == 0) {
13523 switch (opcode) {
13524 case 0: /* SHA512H */
13525 feature = dc_isar_feature(aa64_sha512, s);
13526 oolfn = gen_helper_crypto_sha512h;
13527 break;
13528 case 1: /* SHA512H2 */
13529 feature = dc_isar_feature(aa64_sha512, s);
13530 oolfn = gen_helper_crypto_sha512h2;
13531 break;
13532 case 2: /* SHA512SU1 */
13533 feature = dc_isar_feature(aa64_sha512, s);
13534 oolfn = gen_helper_crypto_sha512su1;
13535 break;
13536 case 3: /* RAX1 */
13537 feature = dc_isar_feature(aa64_sha3, s);
13538 gvecfn = gen_gvec_rax1;
13539 break;
13540 default:
13541 g_assert_not_reached();
13542 }
13543 } else {
13544 switch (opcode) {
13545 case 0: /* SM3PARTW1 */
13546 feature = dc_isar_feature(aa64_sm3, s);
13547 oolfn = gen_helper_crypto_sm3partw1;
13548 break;
13549 case 1: /* SM3PARTW2 */
13550 feature = dc_isar_feature(aa64_sm3, s);
13551 oolfn = gen_helper_crypto_sm3partw2;
13552 break;
13553 case 2: /* SM4EKEY */
13554 feature = dc_isar_feature(aa64_sm4, s);
13555 oolfn = gen_helper_crypto_sm4ekey;
13556 break;
13557 default:
13558 unallocated_encoding(s);
13559 return;
13560 }
13561 }
13562
13563 if (!feature) {
13564 unallocated_encoding(s);
13565 return;
13566 }
13567
13568 if (!fp_access_check(s)) {
13569 return;
13570 }
13571
13572 if (oolfn) {
13573 gen_gvec_op3_ool(s, true, rd, rn, rm, 0, oolfn);
13574 } else {
13575 gen_gvec_fn3(s, true, rd, rn, rm, gvecfn, MO_64);
13576 }
13577 }
13578
13579 /* Crypto two-reg SHA512
13580 * 31 12 11 10 9 5 4 0
13581 * +-----------------------------------------+--------+------+------+
13582 * | 1 1 0 0 1 1 1 0 1 1 0 0 0 0 0 0 1 0 0 0 | opcode | Rn | Rd |
13583 * +-----------------------------------------+--------+------+------+
13584 */
disas_crypto_two_reg_sha512(DisasContext * s,uint32_t insn)13585 static void disas_crypto_two_reg_sha512(DisasContext *s, uint32_t insn)
13586 {
13587 int opcode = extract32(insn, 10, 2);
13588 int rn = extract32(insn, 5, 5);
13589 int rd = extract32(insn, 0, 5);
13590 bool feature;
13591
13592 switch (opcode) {
13593 case 0: /* SHA512SU0 */
13594 feature = dc_isar_feature(aa64_sha512, s);
13595 break;
13596 case 1: /* SM4E */
13597 feature = dc_isar_feature(aa64_sm4, s);
13598 break;
13599 default:
13600 unallocated_encoding(s);
13601 return;
13602 }
13603
13604 if (!feature) {
13605 unallocated_encoding(s);
13606 return;
13607 }
13608
13609 if (!fp_access_check(s)) {
13610 return;
13611 }
13612
13613 switch (opcode) {
13614 case 0: /* SHA512SU0 */
13615 gen_gvec_op2_ool(s, true, rd, rn, 0, gen_helper_crypto_sha512su0);
13616 break;
13617 case 1: /* SM4E */
13618 gen_gvec_op3_ool(s, true, rd, rd, rn, 0, gen_helper_crypto_sm4e);
13619 break;
13620 default:
13621 g_assert_not_reached();
13622 }
13623 }
13624
13625 /* Crypto four-register
13626 * 31 23 22 21 20 16 15 14 10 9 5 4 0
13627 * +-------------------+-----+------+---+------+------+------+
13628 * | 1 1 0 0 1 1 1 0 0 | Op0 | Rm | 0 | Ra | Rn | Rd |
13629 * +-------------------+-----+------+---+------+------+------+
13630 */
disas_crypto_four_reg(DisasContext * s,uint32_t insn)13631 static void disas_crypto_four_reg(DisasContext *s, uint32_t insn)
13632 {
13633 int op0 = extract32(insn, 21, 2);
13634 int rm = extract32(insn, 16, 5);
13635 int ra = extract32(insn, 10, 5);
13636 int rn = extract32(insn, 5, 5);
13637 int rd = extract32(insn, 0, 5);
13638 bool feature;
13639
13640 switch (op0) {
13641 case 0: /* EOR3 */
13642 case 1: /* BCAX */
13643 feature = dc_isar_feature(aa64_sha3, s);
13644 break;
13645 case 2: /* SM3SS1 */
13646 feature = dc_isar_feature(aa64_sm3, s);
13647 break;
13648 default:
13649 unallocated_encoding(s);
13650 return;
13651 }
13652
13653 if (!feature) {
13654 unallocated_encoding(s);
13655 return;
13656 }
13657
13658 if (!fp_access_check(s)) {
13659 return;
13660 }
13661
13662 if (op0 < 2) {
13663 TCGv_i64 tcg_op1, tcg_op2, tcg_op3, tcg_res[2];
13664 int pass;
13665
13666 tcg_op1 = tcg_temp_new_i64();
13667 tcg_op2 = tcg_temp_new_i64();
13668 tcg_op3 = tcg_temp_new_i64();
13669 tcg_res[0] = tcg_temp_new_i64();
13670 tcg_res[1] = tcg_temp_new_i64();
13671
13672 for (pass = 0; pass < 2; pass++) {
13673 read_vec_element(s, tcg_op1, rn, pass, MO_64);
13674 read_vec_element(s, tcg_op2, rm, pass, MO_64);
13675 read_vec_element(s, tcg_op3, ra, pass, MO_64);
13676
13677 if (op0 == 0) {
13678 /* EOR3 */
13679 tcg_gen_xor_i64(tcg_res[pass], tcg_op2, tcg_op3);
13680 } else {
13681 /* BCAX */
13682 tcg_gen_andc_i64(tcg_res[pass], tcg_op2, tcg_op3);
13683 }
13684 tcg_gen_xor_i64(tcg_res[pass], tcg_res[pass], tcg_op1);
13685 }
13686 write_vec_element(s, tcg_res[0], rd, 0, MO_64);
13687 write_vec_element(s, tcg_res[1], rd, 1, MO_64);
13688 } else {
13689 TCGv_i32 tcg_op1, tcg_op2, tcg_op3, tcg_res, tcg_zero;
13690
13691 tcg_op1 = tcg_temp_new_i32();
13692 tcg_op2 = tcg_temp_new_i32();
13693 tcg_op3 = tcg_temp_new_i32();
13694 tcg_res = tcg_temp_new_i32();
13695 tcg_zero = tcg_constant_i32(0);
13696
13697 read_vec_element_i32(s, tcg_op1, rn, 3, MO_32);
13698 read_vec_element_i32(s, tcg_op2, rm, 3, MO_32);
13699 read_vec_element_i32(s, tcg_op3, ra, 3, MO_32);
13700
13701 tcg_gen_rotri_i32(tcg_res, tcg_op1, 20);
13702 tcg_gen_add_i32(tcg_res, tcg_res, tcg_op2);
13703 tcg_gen_add_i32(tcg_res, tcg_res, tcg_op3);
13704 tcg_gen_rotri_i32(tcg_res, tcg_res, 25);
13705
13706 write_vec_element_i32(s, tcg_zero, rd, 0, MO_32);
13707 write_vec_element_i32(s, tcg_zero, rd, 1, MO_32);
13708 write_vec_element_i32(s, tcg_zero, rd, 2, MO_32);
13709 write_vec_element_i32(s, tcg_res, rd, 3, MO_32);
13710 }
13711 }
13712
13713 /* Crypto XAR
13714 * 31 21 20 16 15 10 9 5 4 0
13715 * +-----------------------+------+--------+------+------+
13716 * | 1 1 0 0 1 1 1 0 1 0 0 | Rm | imm6 | Rn | Rd |
13717 * +-----------------------+------+--------+------+------+
13718 */
disas_crypto_xar(DisasContext * s,uint32_t insn)13719 static void disas_crypto_xar(DisasContext *s, uint32_t insn)
13720 {
13721 int rm = extract32(insn, 16, 5);
13722 int imm6 = extract32(insn, 10, 6);
13723 int rn = extract32(insn, 5, 5);
13724 int rd = extract32(insn, 0, 5);
13725
13726 if (!dc_isar_feature(aa64_sha3, s)) {
13727 unallocated_encoding(s);
13728 return;
13729 }
13730
13731 if (!fp_access_check(s)) {
13732 return;
13733 }
13734
13735 gen_gvec_xar(MO_64, vec_full_reg_offset(s, rd),
13736 vec_full_reg_offset(s, rn),
13737 vec_full_reg_offset(s, rm), imm6, 16,
13738 vec_full_reg_size(s));
13739 }
13740
13741 /* Crypto three-reg imm2
13742 * 31 21 20 16 15 14 13 12 11 10 9 5 4 0
13743 * +-----------------------+------+-----+------+--------+------+------+
13744 * | 1 1 0 0 1 1 1 0 0 1 0 | Rm | 1 0 | imm2 | opcode | Rn | Rd |
13745 * +-----------------------+------+-----+------+--------+------+------+
13746 */
disas_crypto_three_reg_imm2(DisasContext * s,uint32_t insn)13747 static void disas_crypto_three_reg_imm2(DisasContext *s, uint32_t insn)
13748 {
13749 static gen_helper_gvec_3 * const fns[4] = {
13750 gen_helper_crypto_sm3tt1a, gen_helper_crypto_sm3tt1b,
13751 gen_helper_crypto_sm3tt2a, gen_helper_crypto_sm3tt2b,
13752 };
13753 int opcode = extract32(insn, 10, 2);
13754 int imm2 = extract32(insn, 12, 2);
13755 int rm = extract32(insn, 16, 5);
13756 int rn = extract32(insn, 5, 5);
13757 int rd = extract32(insn, 0, 5);
13758
13759 if (!dc_isar_feature(aa64_sm3, s)) {
13760 unallocated_encoding(s);
13761 return;
13762 }
13763
13764 if (!fp_access_check(s)) {
13765 return;
13766 }
13767
13768 gen_gvec_op3_ool(s, true, rd, rn, rm, imm2, fns[opcode]);
13769 }
13770
13771 /* C3.6 Data processing - SIMD, inc Crypto
13772 *
13773 * As the decode gets a little complex we are using a table based
13774 * approach for this part of the decode.
13775 */
13776 static const AArch64DecodeTable data_proc_simd[] = {
13777 /* pattern , mask , fn */
13778 { 0x0e200400, 0x9f200400, disas_simd_three_reg_same },
13779 { 0x0e008400, 0x9f208400, disas_simd_three_reg_same_extra },
13780 { 0x0e200000, 0x9f200c00, disas_simd_three_reg_diff },
13781 { 0x0e200800, 0x9f3e0c00, disas_simd_two_reg_misc },
13782 { 0x0e300800, 0x9f3e0c00, disas_simd_across_lanes },
13783 { 0x0e000400, 0x9fe08400, disas_simd_copy },
13784 { 0x0f000000, 0x9f000400, disas_simd_indexed }, /* vector indexed */
13785 /* simd_mod_imm decode is a subset of simd_shift_imm, so must precede it */
13786 { 0x0f000400, 0x9ff80400, disas_simd_mod_imm },
13787 { 0x0f000400, 0x9f800400, disas_simd_shift_imm },
13788 { 0x0e000000, 0xbf208c00, disas_simd_tb },
13789 { 0x0e000800, 0xbf208c00, disas_simd_zip_trn },
13790 { 0x2e000000, 0xbf208400, disas_simd_ext },
13791 { 0x5e200400, 0xdf200400, disas_simd_scalar_three_reg_same },
13792 { 0x5e008400, 0xdf208400, disas_simd_scalar_three_reg_same_extra },
13793 { 0x5e200000, 0xdf200c00, disas_simd_scalar_three_reg_diff },
13794 { 0x5e200800, 0xdf3e0c00, disas_simd_scalar_two_reg_misc },
13795 { 0x5e300800, 0xdf3e0c00, disas_simd_scalar_pairwise },
13796 { 0x5e000400, 0xdfe08400, disas_simd_scalar_copy },
13797 { 0x5f000000, 0xdf000400, disas_simd_indexed }, /* scalar indexed */
13798 { 0x5f000400, 0xdf800400, disas_simd_scalar_shift_imm },
13799 { 0x4e280800, 0xff3e0c00, disas_crypto_aes },
13800 { 0x5e000000, 0xff208c00, disas_crypto_three_reg_sha },
13801 { 0x5e280800, 0xff3e0c00, disas_crypto_two_reg_sha },
13802 { 0xce608000, 0xffe0b000, disas_crypto_three_reg_sha512 },
13803 { 0xcec08000, 0xfffff000, disas_crypto_two_reg_sha512 },
13804 { 0xce000000, 0xff808000, disas_crypto_four_reg },
13805 { 0xce800000, 0xffe00000, disas_crypto_xar },
13806 { 0xce408000, 0xffe0c000, disas_crypto_three_reg_imm2 },
13807 { 0x0e400400, 0x9f60c400, disas_simd_three_reg_same_fp16 },
13808 { 0x0e780800, 0x8f7e0c00, disas_simd_two_reg_misc_fp16 },
13809 { 0x5e400400, 0xdf60c400, disas_simd_scalar_three_reg_same_fp16 },
13810 { 0x00000000, 0x00000000, NULL }
13811 };
13812
disas_data_proc_simd(DisasContext * s,uint32_t insn)13813 static void disas_data_proc_simd(DisasContext *s, uint32_t insn)
13814 {
13815 /* Note that this is called with all non-FP cases from
13816 * table C3-6 so it must UNDEF for entries not specifically
13817 * allocated to instructions in that table.
13818 */
13819 AArch64DecodeFn *fn = lookup_disas_fn(&data_proc_simd[0], insn);
13820 if (fn) {
13821 fn(s, insn);
13822 } else {
13823 unallocated_encoding(s);
13824 }
13825 }
13826
13827 /* C3.6 Data processing - SIMD and floating point */
disas_data_proc_simd_fp(DisasContext * s,uint32_t insn)13828 static void disas_data_proc_simd_fp(DisasContext *s, uint32_t insn)
13829 {
13830 if (extract32(insn, 28, 1) == 1 && extract32(insn, 30, 1) == 0) {
13831 disas_data_proc_fp(s, insn);
13832 } else {
13833 /* SIMD, including crypto */
13834 disas_data_proc_simd(s, insn);
13835 }
13836 }
13837
trans_OK(DisasContext * s,arg_OK * a)13838 static bool trans_OK(DisasContext *s, arg_OK *a)
13839 {
13840 return true;
13841 }
13842
trans_FAIL(DisasContext * s,arg_OK * a)13843 static bool trans_FAIL(DisasContext *s, arg_OK *a)
13844 {
13845 s->is_nonstreaming = true;
13846 return true;
13847 }
13848
13849 /**
13850 * is_guarded_page:
13851 * @env: The cpu environment
13852 * @s: The DisasContext
13853 *
13854 * Return true if the page is guarded.
13855 */
is_guarded_page(CPUARMState * env,DisasContext * s)13856 static bool is_guarded_page(CPUARMState *env, DisasContext *s)
13857 {
13858 uint64_t addr = s->base.pc_first;
13859 #ifdef CONFIG_USER_ONLY
13860 return page_get_flags(addr) & PAGE_BTI;
13861 #else
13862 CPUTLBEntryFull *full;
13863 void *host;
13864 int mmu_idx = arm_to_core_mmu_idx(s->mmu_idx);
13865 int flags;
13866
13867 /*
13868 * We test this immediately after reading an insn, which means
13869 * that the TLB entry must be present and valid, and thus this
13870 * access will never raise an exception.
13871 */
13872 flags = probe_access_full(env, addr, 0, MMU_INST_FETCH, mmu_idx,
13873 false, &host, &full, 0);
13874 assert(!(flags & TLB_INVALID_MASK));
13875
13876 return full->extra.arm.guarded;
13877 #endif
13878 }
13879
13880 /**
13881 * btype_destination_ok:
13882 * @insn: The instruction at the branch destination
13883 * @bt: SCTLR_ELx.BT
13884 * @btype: PSTATE.BTYPE, and is non-zero
13885 *
13886 * On a guarded page, there are a limited number of insns
13887 * that may be present at the branch target:
13888 * - branch target identifiers,
13889 * - paciasp, pacibsp,
13890 * - BRK insn
13891 * - HLT insn
13892 * Anything else causes a Branch Target Exception.
13893 *
13894 * Return true if the branch is compatible, false to raise BTITRAP.
13895 */
btype_destination_ok(uint32_t insn,bool bt,int btype)13896 static bool btype_destination_ok(uint32_t insn, bool bt, int btype)
13897 {
13898 if ((insn & 0xfffff01fu) == 0xd503201fu) {
13899 /* HINT space */
13900 switch (extract32(insn, 5, 7)) {
13901 case 0b011001: /* PACIASP */
13902 case 0b011011: /* PACIBSP */
13903 /*
13904 * If SCTLR_ELx.BT, then PACI*SP are not compatible
13905 * with btype == 3. Otherwise all btype are ok.
13906 */
13907 return !bt || btype != 3;
13908 case 0b100000: /* BTI */
13909 /* Not compatible with any btype. */
13910 return false;
13911 case 0b100010: /* BTI c */
13912 /* Not compatible with btype == 3 */
13913 return btype != 3;
13914 case 0b100100: /* BTI j */
13915 /* Not compatible with btype == 2 */
13916 return btype != 2;
13917 case 0b100110: /* BTI jc */
13918 /* Compatible with any btype. */
13919 return true;
13920 }
13921 } else {
13922 switch (insn & 0xffe0001fu) {
13923 case 0xd4200000u: /* BRK */
13924 case 0xd4400000u: /* HLT */
13925 /* Give priority to the breakpoint exception. */
13926 return true;
13927 }
13928 }
13929 return false;
13930 }
13931
13932 /* C3.1 A64 instruction index by encoding */
disas_a64_legacy(DisasContext * s,uint32_t insn)13933 static void disas_a64_legacy(DisasContext *s, uint32_t insn)
13934 {
13935 switch (extract32(insn, 25, 4)) {
13936 case 0x5:
13937 case 0xd: /* Data processing - register */
13938 disas_data_proc_reg(s, insn);
13939 break;
13940 case 0x7:
13941 case 0xf: /* Data processing - SIMD and floating point */
13942 disas_data_proc_simd_fp(s, insn);
13943 break;
13944 default:
13945 unallocated_encoding(s);
13946 break;
13947 }
13948 }
13949
aarch64_tr_init_disas_context(DisasContextBase * dcbase,CPUState * cpu)13950 static void aarch64_tr_init_disas_context(DisasContextBase *dcbase,
13951 CPUState *cpu)
13952 {
13953 DisasContext *dc = container_of(dcbase, DisasContext, base);
13954 CPUARMState *env = cpu_env(cpu);
13955 ARMCPU *arm_cpu = env_archcpu(env);
13956 CPUARMTBFlags tb_flags = arm_tbflags_from_tb(dc->base.tb);
13957 int bound, core_mmu_idx;
13958
13959 dc->isar = &arm_cpu->isar;
13960 dc->condjmp = 0;
13961 dc->pc_save = dc->base.pc_first;
13962 dc->aarch64 = true;
13963 dc->thumb = false;
13964 dc->sctlr_b = 0;
13965 dc->be_data = EX_TBFLAG_ANY(tb_flags, BE_DATA) ? MO_BE : MO_LE;
13966 dc->condexec_mask = 0;
13967 dc->condexec_cond = 0;
13968 core_mmu_idx = EX_TBFLAG_ANY(tb_flags, MMUIDX);
13969 dc->mmu_idx = core_to_aa64_mmu_idx(core_mmu_idx);
13970 dc->tbii = EX_TBFLAG_A64(tb_flags, TBII);
13971 dc->tbid = EX_TBFLAG_A64(tb_flags, TBID);
13972 dc->tcma = EX_TBFLAG_A64(tb_flags, TCMA);
13973 dc->current_el = arm_mmu_idx_to_el(dc->mmu_idx);
13974 #if !defined(CONFIG_USER_ONLY)
13975 dc->user = (dc->current_el == 0);
13976 #endif
13977 dc->fp_excp_el = EX_TBFLAG_ANY(tb_flags, FPEXC_EL);
13978 dc->align_mem = EX_TBFLAG_ANY(tb_flags, ALIGN_MEM);
13979 dc->pstate_il = EX_TBFLAG_ANY(tb_flags, PSTATE__IL);
13980 dc->fgt_active = EX_TBFLAG_ANY(tb_flags, FGT_ACTIVE);
13981 dc->fgt_svc = EX_TBFLAG_ANY(tb_flags, FGT_SVC);
13982 dc->fgt_eret = EX_TBFLAG_A64(tb_flags, FGT_ERET);
13983 dc->sve_excp_el = EX_TBFLAG_A64(tb_flags, SVEEXC_EL);
13984 dc->sme_excp_el = EX_TBFLAG_A64(tb_flags, SMEEXC_EL);
13985 dc->vl = (EX_TBFLAG_A64(tb_flags, VL) + 1) * 16;
13986 dc->svl = (EX_TBFLAG_A64(tb_flags, SVL) + 1) * 16;
13987 dc->pauth_active = EX_TBFLAG_A64(tb_flags, PAUTH_ACTIVE);
13988 dc->bt = EX_TBFLAG_A64(tb_flags, BT);
13989 dc->btype = EX_TBFLAG_A64(tb_flags, BTYPE);
13990 dc->unpriv = EX_TBFLAG_A64(tb_flags, UNPRIV);
13991 dc->ata[0] = EX_TBFLAG_A64(tb_flags, ATA);
13992 dc->ata[1] = EX_TBFLAG_A64(tb_flags, ATA0);
13993 dc->mte_active[0] = EX_TBFLAG_A64(tb_flags, MTE_ACTIVE);
13994 dc->mte_active[1] = EX_TBFLAG_A64(tb_flags, MTE0_ACTIVE);
13995 dc->pstate_sm = EX_TBFLAG_A64(tb_flags, PSTATE_SM);
13996 dc->pstate_za = EX_TBFLAG_A64(tb_flags, PSTATE_ZA);
13997 dc->sme_trap_nonstreaming = EX_TBFLAG_A64(tb_flags, SME_TRAP_NONSTREAMING);
13998 dc->naa = EX_TBFLAG_A64(tb_flags, NAA);
13999 dc->vec_len = 0;
14000 dc->vec_stride = 0;
14001 dc->cp_regs = arm_cpu->cp_regs;
14002 dc->features = env->features;
14003 dc->dcz_blocksize = arm_cpu->dcz_blocksize;
14004 dc->gm_blocksize = arm_cpu->gm_blocksize;
14005
14006 #ifdef CONFIG_USER_ONLY
14007 /* In sve_probe_page, we assume TBI is enabled. */
14008 tcg_debug_assert(dc->tbid & 1);
14009 #endif
14010
14011 dc->lse2 = dc_isar_feature(aa64_lse2, dc);
14012
14013 /* Single step state. The code-generation logic here is:
14014 * SS_ACTIVE == 0:
14015 * generate code with no special handling for single-stepping (except
14016 * that anything that can make us go to SS_ACTIVE == 1 must end the TB;
14017 * this happens anyway because those changes are all system register or
14018 * PSTATE writes).
14019 * SS_ACTIVE == 1, PSTATE.SS == 1: (active-not-pending)
14020 * emit code for one insn
14021 * emit code to clear PSTATE.SS
14022 * emit code to generate software step exception for completed step
14023 * end TB (as usual for having generated an exception)
14024 * SS_ACTIVE == 1, PSTATE.SS == 0: (active-pending)
14025 * emit code to generate a software step exception
14026 * end the TB
14027 */
14028 dc->ss_active = EX_TBFLAG_ANY(tb_flags, SS_ACTIVE);
14029 dc->pstate_ss = EX_TBFLAG_ANY(tb_flags, PSTATE__SS);
14030 dc->is_ldex = false;
14031
14032 /* Bound the number of insns to execute to those left on the page. */
14033 bound = -(dc->base.pc_first | TARGET_PAGE_MASK) / 4;
14034
14035 /* If architectural single step active, limit to 1. */
14036 if (dc->ss_active) {
14037 bound = 1;
14038 }
14039 dc->base.max_insns = MIN(dc->base.max_insns, bound);
14040 }
14041
aarch64_tr_tb_start(DisasContextBase * db,CPUState * cpu)14042 static void aarch64_tr_tb_start(DisasContextBase *db, CPUState *cpu)
14043 {
14044 }
14045
aarch64_tr_insn_start(DisasContextBase * dcbase,CPUState * cpu)14046 static void aarch64_tr_insn_start(DisasContextBase *dcbase, CPUState *cpu)
14047 {
14048 DisasContext *dc = container_of(dcbase, DisasContext, base);
14049 target_ulong pc_arg = dc->base.pc_next;
14050
14051 if (tb_cflags(dcbase->tb) & CF_PCREL) {
14052 pc_arg &= ~TARGET_PAGE_MASK;
14053 }
14054 tcg_gen_insn_start(pc_arg, 0, 0);
14055 dc->insn_start = tcg_last_op();
14056 }
14057
aarch64_tr_translate_insn(DisasContextBase * dcbase,CPUState * cpu)14058 static void aarch64_tr_translate_insn(DisasContextBase *dcbase, CPUState *cpu)
14059 {
14060 DisasContext *s = container_of(dcbase, DisasContext, base);
14061 CPUARMState *env = cpu_env(cpu);
14062 uint64_t pc = s->base.pc_next;
14063 uint32_t insn;
14064
14065 /* Singlestep exceptions have the highest priority. */
14066 if (s->ss_active && !s->pstate_ss) {
14067 /* Singlestep state is Active-pending.
14068 * If we're in this state at the start of a TB then either
14069 * a) we just took an exception to an EL which is being debugged
14070 * and this is the first insn in the exception handler
14071 * b) debug exceptions were masked and we just unmasked them
14072 * without changing EL (eg by clearing PSTATE.D)
14073 * In either case we're going to take a swstep exception in the
14074 * "did not step an insn" case, and so the syndrome ISV and EX
14075 * bits should be zero.
14076 */
14077 assert(s->base.num_insns == 1);
14078 gen_swstep_exception(s, 0, 0);
14079 s->base.is_jmp = DISAS_NORETURN;
14080 s->base.pc_next = pc + 4;
14081 return;
14082 }
14083
14084 if (pc & 3) {
14085 /*
14086 * PC alignment fault. This has priority over the instruction abort
14087 * that we would receive from a translation fault via arm_ldl_code.
14088 * This should only be possible after an indirect branch, at the
14089 * start of the TB.
14090 */
14091 assert(s->base.num_insns == 1);
14092 gen_helper_exception_pc_alignment(tcg_env, tcg_constant_tl(pc));
14093 s->base.is_jmp = DISAS_NORETURN;
14094 s->base.pc_next = QEMU_ALIGN_UP(pc, 4);
14095 return;
14096 }
14097
14098 s->pc_curr = pc;
14099 insn = arm_ldl_code(env, &s->base, pc, s->sctlr_b);
14100 s->insn = insn;
14101 s->base.pc_next = pc + 4;
14102
14103 s->fp_access_checked = false;
14104 s->sve_access_checked = false;
14105
14106 if (s->pstate_il) {
14107 /*
14108 * Illegal execution state. This has priority over BTI
14109 * exceptions, but comes after instruction abort exceptions.
14110 */
14111 gen_exception_insn(s, 0, EXCP_UDEF, syn_illegalstate());
14112 return;
14113 }
14114
14115 if (dc_isar_feature(aa64_bti, s)) {
14116 if (s->base.num_insns == 1) {
14117 /*
14118 * At the first insn of the TB, compute s->guarded_page.
14119 * We delayed computing this until successfully reading
14120 * the first insn of the TB, above. This (mostly) ensures
14121 * that the softmmu tlb entry has been populated, and the
14122 * page table GP bit is available.
14123 *
14124 * Note that we need to compute this even if btype == 0,
14125 * because this value is used for BR instructions later
14126 * where ENV is not available.
14127 */
14128 s->guarded_page = is_guarded_page(env, s);
14129
14130 /* First insn can have btype set to non-zero. */
14131 tcg_debug_assert(s->btype >= 0);
14132
14133 /*
14134 * Note that the Branch Target Exception has fairly high
14135 * priority -- below debugging exceptions but above most
14136 * everything else. This allows us to handle this now
14137 * instead of waiting until the insn is otherwise decoded.
14138 */
14139 if (s->btype != 0
14140 && s->guarded_page
14141 && !btype_destination_ok(insn, s->bt, s->btype)) {
14142 gen_exception_insn(s, 0, EXCP_UDEF, syn_btitrap(s->btype));
14143 return;
14144 }
14145 } else {
14146 /* Not the first insn: btype must be 0. */
14147 tcg_debug_assert(s->btype == 0);
14148 }
14149 }
14150
14151 s->is_nonstreaming = false;
14152 if (s->sme_trap_nonstreaming) {
14153 disas_sme_fa64(s, insn);
14154 }
14155
14156 if (!disas_a64(s, insn) &&
14157 !disas_sme(s, insn) &&
14158 !disas_sve(s, insn)) {
14159 disas_a64_legacy(s, insn);
14160 }
14161
14162 /*
14163 * After execution of most insns, btype is reset to 0.
14164 * Note that we set btype == -1 when the insn sets btype.
14165 */
14166 if (s->btype > 0 && s->base.is_jmp != DISAS_NORETURN) {
14167 reset_btype(s);
14168 }
14169 }
14170
aarch64_tr_tb_stop(DisasContextBase * dcbase,CPUState * cpu)14171 static void aarch64_tr_tb_stop(DisasContextBase *dcbase, CPUState *cpu)
14172 {
14173 DisasContext *dc = container_of(dcbase, DisasContext, base);
14174
14175 if (unlikely(dc->ss_active)) {
14176 /* Note that this means single stepping WFI doesn't halt the CPU.
14177 * For conditional branch insns this is harmless unreachable code as
14178 * gen_goto_tb() has already handled emitting the debug exception
14179 * (and thus a tb-jump is not possible when singlestepping).
14180 */
14181 switch (dc->base.is_jmp) {
14182 default:
14183 gen_a64_update_pc(dc, 4);
14184 /* fall through */
14185 case DISAS_EXIT:
14186 case DISAS_JUMP:
14187 gen_step_complete_exception(dc);
14188 break;
14189 case DISAS_NORETURN:
14190 break;
14191 }
14192 } else {
14193 switch (dc->base.is_jmp) {
14194 case DISAS_NEXT:
14195 case DISAS_TOO_MANY:
14196 gen_goto_tb(dc, 1, 4);
14197 break;
14198 default:
14199 case DISAS_UPDATE_EXIT:
14200 gen_a64_update_pc(dc, 4);
14201 /* fall through */
14202 case DISAS_EXIT:
14203 tcg_gen_exit_tb(NULL, 0);
14204 break;
14205 case DISAS_UPDATE_NOCHAIN:
14206 gen_a64_update_pc(dc, 4);
14207 /* fall through */
14208 case DISAS_JUMP:
14209 tcg_gen_lookup_and_goto_ptr();
14210 break;
14211 case DISAS_NORETURN:
14212 case DISAS_SWI:
14213 break;
14214 case DISAS_WFE:
14215 gen_a64_update_pc(dc, 4);
14216 gen_helper_wfe(tcg_env);
14217 break;
14218 case DISAS_YIELD:
14219 gen_a64_update_pc(dc, 4);
14220 gen_helper_yield(tcg_env);
14221 break;
14222 case DISAS_WFI:
14223 /*
14224 * This is a special case because we don't want to just halt
14225 * the CPU if trying to debug across a WFI.
14226 */
14227 gen_a64_update_pc(dc, 4);
14228 gen_helper_wfi(tcg_env, tcg_constant_i32(4));
14229 /*
14230 * The helper doesn't necessarily throw an exception, but we
14231 * must go back to the main loop to check for interrupts anyway.
14232 */
14233 tcg_gen_exit_tb(NULL, 0);
14234 break;
14235 }
14236 }
14237 }
14238
aarch64_tr_disas_log(const DisasContextBase * dcbase,CPUState * cpu,FILE * logfile)14239 static void aarch64_tr_disas_log(const DisasContextBase *dcbase,
14240 CPUState *cpu, FILE *logfile)
14241 {
14242 DisasContext *dc = container_of(dcbase, DisasContext, base);
14243
14244 fprintf(logfile, "IN: %s\n", lookup_symbol(dc->base.pc_first));
14245 target_disas(logfile, cpu, dc->base.pc_first, dc->base.tb->size);
14246 }
14247
14248 const TranslatorOps aarch64_translator_ops = {
14249 .init_disas_context = aarch64_tr_init_disas_context,
14250 .tb_start = aarch64_tr_tb_start,
14251 .insn_start = aarch64_tr_insn_start,
14252 .translate_insn = aarch64_tr_translate_insn,
14253 .tb_stop = aarch64_tr_tb_stop,
14254 .disas_log = aarch64_tr_disas_log,
14255 };
14256