xref: /openbmc/qemu/target/arm/helper.c (revision b68a92f4cb16115025f41bc59e1b2f182a610370)
1 /*
2  * ARM generic helpers.
3  *
4  * This code is licensed under the GNU GPL v2 or later.
5  *
6  * SPDX-License-Identifier: GPL-2.0-or-later
7  */
8 
9 #include "qemu/osdep.h"
10 #include "qemu/units.h"
11 #include "target/arm/idau.h"
12 #include "trace.h"
13 #include "cpu.h"
14 #include "internals.h"
15 #include "exec/gdbstub.h"
16 #include "exec/helper-proto.h"
17 #include "qemu/host-utils.h"
18 #include "qemu/main-loop.h"
19 #include "qemu/bitops.h"
20 #include "qemu/crc32c.h"
21 #include "qemu/qemu-print.h"
22 #include "exec/exec-all.h"
23 #include <zlib.h> /* For crc32 */
24 #include "hw/irq.h"
25 #include "hw/semihosting/semihost.h"
26 #include "sysemu/cpus.h"
27 #include "sysemu/cpu-timers.h"
28 #include "sysemu/kvm.h"
29 #include "sysemu/tcg.h"
30 #include "qemu/range.h"
31 #include "qapi/qapi-commands-machine-target.h"
32 #include "qapi/error.h"
33 #include "qemu/guest-random.h"
34 #ifdef CONFIG_TCG
35 #include "arm_ldst.h"
36 #include "exec/cpu_ldst.h"
37 #endif
38 
39 #define ARM_CPU_FREQ 1000000000 /* FIXME: 1 GHz, should be configurable */
40 
41 #ifndef CONFIG_USER_ONLY
42 
43 static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address,
44                                MMUAccessType access_type, ARMMMUIdx mmu_idx,
45                                bool s1_is_el0,
46                                hwaddr *phys_ptr, MemTxAttrs *txattrs, int *prot,
47                                target_ulong *page_size_ptr,
48                                ARMMMUFaultInfo *fi, ARMCacheAttrs *cacheattrs)
49     __attribute__((nonnull));
50 #endif
51 
52 static void switch_mode(CPUARMState *env, int mode);
53 static int aa64_va_parameter_tbi(uint64_t tcr, ARMMMUIdx mmu_idx);
54 
55 static int vfp_gdb_get_reg(CPUARMState *env, GByteArray *buf, int reg)
56 {
57     ARMCPU *cpu = env_archcpu(env);
58     int nregs = cpu_isar_feature(aa32_simd_r32, cpu) ? 32 : 16;
59 
60     /* VFP data registers are always little-endian.  */
61     if (reg < nregs) {
62         return gdb_get_reg64(buf, *aa32_vfp_dreg(env, reg));
63     }
64     if (arm_feature(env, ARM_FEATURE_NEON)) {
65         /* Aliases for Q regs.  */
66         nregs += 16;
67         if (reg < nregs) {
68             uint64_t *q = aa32_vfp_qreg(env, reg - 32);
69             return gdb_get_reg128(buf, q[0], q[1]);
70         }
71     }
72     switch (reg - nregs) {
73     case 0: return gdb_get_reg32(buf, env->vfp.xregs[ARM_VFP_FPSID]); break;
74     case 1: return gdb_get_reg32(buf, vfp_get_fpscr(env)); break;
75     case 2: return gdb_get_reg32(buf, env->vfp.xregs[ARM_VFP_FPEXC]); break;
76     }
77     return 0;
78 }
79 
80 static int vfp_gdb_set_reg(CPUARMState *env, uint8_t *buf, int reg)
81 {
82     ARMCPU *cpu = env_archcpu(env);
83     int nregs = cpu_isar_feature(aa32_simd_r32, cpu) ? 32 : 16;
84 
85     if (reg < nregs) {
86         *aa32_vfp_dreg(env, reg) = ldq_le_p(buf);
87         return 8;
88     }
89     if (arm_feature(env, ARM_FEATURE_NEON)) {
90         nregs += 16;
91         if (reg < nregs) {
92             uint64_t *q = aa32_vfp_qreg(env, reg - 32);
93             q[0] = ldq_le_p(buf);
94             q[1] = ldq_le_p(buf + 8);
95             return 16;
96         }
97     }
98     switch (reg - nregs) {
99     case 0: env->vfp.xregs[ARM_VFP_FPSID] = ldl_p(buf); return 4;
100     case 1: vfp_set_fpscr(env, ldl_p(buf)); return 4;
101     case 2: env->vfp.xregs[ARM_VFP_FPEXC] = ldl_p(buf) & (1 << 30); return 4;
102     }
103     return 0;
104 }
105 
106 static int aarch64_fpu_gdb_get_reg(CPUARMState *env, GByteArray *buf, int reg)
107 {
108     switch (reg) {
109     case 0 ... 31:
110     {
111         /* 128 bit FP register - quads are in LE order */
112         uint64_t *q = aa64_vfp_qreg(env, reg);
113         return gdb_get_reg128(buf, q[1], q[0]);
114     }
115     case 32:
116         /* FPSR */
117         return gdb_get_reg32(buf, vfp_get_fpsr(env));
118     case 33:
119         /* FPCR */
120         return gdb_get_reg32(buf,vfp_get_fpcr(env));
121     default:
122         return 0;
123     }
124 }
125 
126 static int aarch64_fpu_gdb_set_reg(CPUARMState *env, uint8_t *buf, int reg)
127 {
128     switch (reg) {
129     case 0 ... 31:
130         /* 128 bit FP register */
131         {
132             uint64_t *q = aa64_vfp_qreg(env, reg);
133             q[0] = ldq_le_p(buf);
134             q[1] = ldq_le_p(buf + 8);
135             return 16;
136         }
137     case 32:
138         /* FPSR */
139         vfp_set_fpsr(env, ldl_p(buf));
140         return 4;
141     case 33:
142         /* FPCR */
143         vfp_set_fpcr(env, ldl_p(buf));
144         return 4;
145     default:
146         return 0;
147     }
148 }
149 
150 static uint64_t raw_read(CPUARMState *env, const ARMCPRegInfo *ri)
151 {
152     assert(ri->fieldoffset);
153     if (cpreg_field_is_64bit(ri)) {
154         return CPREG_FIELD64(env, ri);
155     } else {
156         return CPREG_FIELD32(env, ri);
157     }
158 }
159 
160 static void raw_write(CPUARMState *env, const ARMCPRegInfo *ri,
161                       uint64_t value)
162 {
163     assert(ri->fieldoffset);
164     if (cpreg_field_is_64bit(ri)) {
165         CPREG_FIELD64(env, ri) = value;
166     } else {
167         CPREG_FIELD32(env, ri) = value;
168     }
169 }
170 
171 static void *raw_ptr(CPUARMState *env, const ARMCPRegInfo *ri)
172 {
173     return (char *)env + ri->fieldoffset;
174 }
175 
176 uint64_t read_raw_cp_reg(CPUARMState *env, const ARMCPRegInfo *ri)
177 {
178     /* Raw read of a coprocessor register (as needed for migration, etc). */
179     if (ri->type & ARM_CP_CONST) {
180         return ri->resetvalue;
181     } else if (ri->raw_readfn) {
182         return ri->raw_readfn(env, ri);
183     } else if (ri->readfn) {
184         return ri->readfn(env, ri);
185     } else {
186         return raw_read(env, ri);
187     }
188 }
189 
190 static void write_raw_cp_reg(CPUARMState *env, const ARMCPRegInfo *ri,
191                              uint64_t v)
192 {
193     /* Raw write of a coprocessor register (as needed for migration, etc).
194      * Note that constant registers are treated as write-ignored; the
195      * caller should check for success by whether a readback gives the
196      * value written.
197      */
198     if (ri->type & ARM_CP_CONST) {
199         return;
200     } else if (ri->raw_writefn) {
201         ri->raw_writefn(env, ri, v);
202     } else if (ri->writefn) {
203         ri->writefn(env, ri, v);
204     } else {
205         raw_write(env, ri, v);
206     }
207 }
208 
209 /**
210  * arm_get/set_gdb_*: get/set a gdb register
211  * @env: the CPU state
212  * @buf: a buffer to copy to/from
213  * @reg: register number (offset from start of group)
214  *
215  * We return the number of bytes copied
216  */
217 
218 static int arm_gdb_get_sysreg(CPUARMState *env, GByteArray *buf, int reg)
219 {
220     ARMCPU *cpu = env_archcpu(env);
221     const ARMCPRegInfo *ri;
222     uint32_t key;
223 
224     key = cpu->dyn_sysreg_xml.data.cpregs.keys[reg];
225     ri = get_arm_cp_reginfo(cpu->cp_regs, key);
226     if (ri) {
227         if (cpreg_field_is_64bit(ri)) {
228             return gdb_get_reg64(buf, (uint64_t)read_raw_cp_reg(env, ri));
229         } else {
230             return gdb_get_reg32(buf, (uint32_t)read_raw_cp_reg(env, ri));
231         }
232     }
233     return 0;
234 }
235 
236 static int arm_gdb_set_sysreg(CPUARMState *env, uint8_t *buf, int reg)
237 {
238     return 0;
239 }
240 
241 #ifdef TARGET_AARCH64
242 static int arm_gdb_get_svereg(CPUARMState *env, GByteArray *buf, int reg)
243 {
244     ARMCPU *cpu = env_archcpu(env);
245 
246     switch (reg) {
247     /* The first 32 registers are the zregs */
248     case 0 ... 31:
249     {
250         int vq, len = 0;
251         for (vq = 0; vq < cpu->sve_max_vq; vq++) {
252             len += gdb_get_reg128(buf,
253                                   env->vfp.zregs[reg].d[vq * 2 + 1],
254                                   env->vfp.zregs[reg].d[vq * 2]);
255         }
256         return len;
257     }
258     case 32:
259         return gdb_get_reg32(buf, vfp_get_fpsr(env));
260     case 33:
261         return gdb_get_reg32(buf, vfp_get_fpcr(env));
262     /* then 16 predicates and the ffr */
263     case 34 ... 50:
264     {
265         int preg = reg - 34;
266         int vq, len = 0;
267         for (vq = 0; vq < cpu->sve_max_vq; vq = vq + 4) {
268             len += gdb_get_reg64(buf, env->vfp.pregs[preg].p[vq / 4]);
269         }
270         return len;
271     }
272     case 51:
273     {
274         /*
275          * We report in Vector Granules (VG) which is 64bit in a Z reg
276          * while the ZCR works in Vector Quads (VQ) which is 128bit chunks.
277          */
278         int vq = sve_zcr_len_for_el(env, arm_current_el(env)) + 1;
279         return gdb_get_reg32(buf, vq * 2);
280     }
281     default:
282         /* gdbstub asked for something out our range */
283         qemu_log_mask(LOG_UNIMP, "%s: out of range register %d", __func__, reg);
284         break;
285     }
286 
287     return 0;
288 }
289 
290 static int arm_gdb_set_svereg(CPUARMState *env, uint8_t *buf, int reg)
291 {
292     ARMCPU *cpu = env_archcpu(env);
293 
294     /* The first 32 registers are the zregs */
295     switch (reg) {
296     /* The first 32 registers are the zregs */
297     case 0 ... 31:
298     {
299         int vq, len = 0;
300         uint64_t *p = (uint64_t *) buf;
301         for (vq = 0; vq < cpu->sve_max_vq; vq++) {
302             env->vfp.zregs[reg].d[vq * 2 + 1] = *p++;
303             env->vfp.zregs[reg].d[vq * 2] = *p++;
304             len += 16;
305         }
306         return len;
307     }
308     case 32:
309         vfp_set_fpsr(env, *(uint32_t *)buf);
310         return 4;
311     case 33:
312         vfp_set_fpcr(env, *(uint32_t *)buf);
313         return 4;
314     case 34 ... 50:
315     {
316         int preg = reg - 34;
317         int vq, len = 0;
318         uint64_t *p = (uint64_t *) buf;
319         for (vq = 0; vq < cpu->sve_max_vq; vq = vq + 4) {
320             env->vfp.pregs[preg].p[vq / 4] = *p++;
321             len += 8;
322         }
323         return len;
324     }
325     case 51:
326         /* cannot set vg via gdbstub */
327         return 0;
328     default:
329         /* gdbstub asked for something out our range */
330         break;
331     }
332 
333     return 0;
334 }
335 #endif /* TARGET_AARCH64 */
336 
337 static bool raw_accessors_invalid(const ARMCPRegInfo *ri)
338 {
339    /* Return true if the regdef would cause an assertion if you called
340     * read_raw_cp_reg() or write_raw_cp_reg() on it (ie if it is a
341     * program bug for it not to have the NO_RAW flag).
342     * NB that returning false here doesn't necessarily mean that calling
343     * read/write_raw_cp_reg() is safe, because we can't distinguish "has
344     * read/write access functions which are safe for raw use" from "has
345     * read/write access functions which have side effects but has forgotten
346     * to provide raw access functions".
347     * The tests here line up with the conditions in read/write_raw_cp_reg()
348     * and assertions in raw_read()/raw_write().
349     */
350     if ((ri->type & ARM_CP_CONST) ||
351         ri->fieldoffset ||
352         ((ri->raw_writefn || ri->writefn) && (ri->raw_readfn || ri->readfn))) {
353         return false;
354     }
355     return true;
356 }
357 
358 bool write_cpustate_to_list(ARMCPU *cpu, bool kvm_sync)
359 {
360     /* Write the coprocessor state from cpu->env to the (index,value) list. */
361     int i;
362     bool ok = true;
363 
364     for (i = 0; i < cpu->cpreg_array_len; i++) {
365         uint32_t regidx = kvm_to_cpreg_id(cpu->cpreg_indexes[i]);
366         const ARMCPRegInfo *ri;
367         uint64_t newval;
368 
369         ri = get_arm_cp_reginfo(cpu->cp_regs, regidx);
370         if (!ri) {
371             ok = false;
372             continue;
373         }
374         if (ri->type & ARM_CP_NO_RAW) {
375             continue;
376         }
377 
378         newval = read_raw_cp_reg(&cpu->env, ri);
379         if (kvm_sync) {
380             /*
381              * Only sync if the previous list->cpustate sync succeeded.
382              * Rather than tracking the success/failure state for every
383              * item in the list, we just recheck "does the raw write we must
384              * have made in write_list_to_cpustate() read back OK" here.
385              */
386             uint64_t oldval = cpu->cpreg_values[i];
387 
388             if (oldval == newval) {
389                 continue;
390             }
391 
392             write_raw_cp_reg(&cpu->env, ri, oldval);
393             if (read_raw_cp_reg(&cpu->env, ri) != oldval) {
394                 continue;
395             }
396 
397             write_raw_cp_reg(&cpu->env, ri, newval);
398         }
399         cpu->cpreg_values[i] = newval;
400     }
401     return ok;
402 }
403 
404 bool write_list_to_cpustate(ARMCPU *cpu)
405 {
406     int i;
407     bool ok = true;
408 
409     for (i = 0; i < cpu->cpreg_array_len; i++) {
410         uint32_t regidx = kvm_to_cpreg_id(cpu->cpreg_indexes[i]);
411         uint64_t v = cpu->cpreg_values[i];
412         const ARMCPRegInfo *ri;
413 
414         ri = get_arm_cp_reginfo(cpu->cp_regs, regidx);
415         if (!ri) {
416             ok = false;
417             continue;
418         }
419         if (ri->type & ARM_CP_NO_RAW) {
420             continue;
421         }
422         /* Write value and confirm it reads back as written
423          * (to catch read-only registers and partially read-only
424          * registers where the incoming migration value doesn't match)
425          */
426         write_raw_cp_reg(&cpu->env, ri, v);
427         if (read_raw_cp_reg(&cpu->env, ri) != v) {
428             ok = false;
429         }
430     }
431     return ok;
432 }
433 
434 static void add_cpreg_to_list(gpointer key, gpointer opaque)
435 {
436     ARMCPU *cpu = opaque;
437     uint64_t regidx;
438     const ARMCPRegInfo *ri;
439 
440     regidx = *(uint32_t *)key;
441     ri = get_arm_cp_reginfo(cpu->cp_regs, regidx);
442 
443     if (!(ri->type & (ARM_CP_NO_RAW|ARM_CP_ALIAS))) {
444         cpu->cpreg_indexes[cpu->cpreg_array_len] = cpreg_to_kvm_id(regidx);
445         /* The value array need not be initialized at this point */
446         cpu->cpreg_array_len++;
447     }
448 }
449 
450 static void count_cpreg(gpointer key, gpointer opaque)
451 {
452     ARMCPU *cpu = opaque;
453     uint64_t regidx;
454     const ARMCPRegInfo *ri;
455 
456     regidx = *(uint32_t *)key;
457     ri = get_arm_cp_reginfo(cpu->cp_regs, regidx);
458 
459     if (!(ri->type & (ARM_CP_NO_RAW|ARM_CP_ALIAS))) {
460         cpu->cpreg_array_len++;
461     }
462 }
463 
464 static gint cpreg_key_compare(gconstpointer a, gconstpointer b)
465 {
466     uint64_t aidx = cpreg_to_kvm_id(*(uint32_t *)a);
467     uint64_t bidx = cpreg_to_kvm_id(*(uint32_t *)b);
468 
469     if (aidx > bidx) {
470         return 1;
471     }
472     if (aidx < bidx) {
473         return -1;
474     }
475     return 0;
476 }
477 
478 void init_cpreg_list(ARMCPU *cpu)
479 {
480     /* Initialise the cpreg_tuples[] array based on the cp_regs hash.
481      * Note that we require cpreg_tuples[] to be sorted by key ID.
482      */
483     GList *keys;
484     int arraylen;
485 
486     keys = g_hash_table_get_keys(cpu->cp_regs);
487     keys = g_list_sort(keys, cpreg_key_compare);
488 
489     cpu->cpreg_array_len = 0;
490 
491     g_list_foreach(keys, count_cpreg, cpu);
492 
493     arraylen = cpu->cpreg_array_len;
494     cpu->cpreg_indexes = g_new(uint64_t, arraylen);
495     cpu->cpreg_values = g_new(uint64_t, arraylen);
496     cpu->cpreg_vmstate_indexes = g_new(uint64_t, arraylen);
497     cpu->cpreg_vmstate_values = g_new(uint64_t, arraylen);
498     cpu->cpreg_vmstate_array_len = cpu->cpreg_array_len;
499     cpu->cpreg_array_len = 0;
500 
501     g_list_foreach(keys, add_cpreg_to_list, cpu);
502 
503     assert(cpu->cpreg_array_len == arraylen);
504 
505     g_list_free(keys);
506 }
507 
508 /*
509  * Some registers are not accessible from AArch32 EL3 if SCR.NS == 0.
510  */
511 static CPAccessResult access_el3_aa32ns(CPUARMState *env,
512                                         const ARMCPRegInfo *ri,
513                                         bool isread)
514 {
515     if (!is_a64(env) && arm_current_el(env) == 3 &&
516         arm_is_secure_below_el3(env)) {
517         return CP_ACCESS_TRAP_UNCATEGORIZED;
518     }
519     return CP_ACCESS_OK;
520 }
521 
522 /* Some secure-only AArch32 registers trap to EL3 if used from
523  * Secure EL1 (but are just ordinary UNDEF in other non-EL3 contexts).
524  * Note that an access from Secure EL1 can only happen if EL3 is AArch64.
525  * We assume that the .access field is set to PL1_RW.
526  */
527 static CPAccessResult access_trap_aa32s_el1(CPUARMState *env,
528                                             const ARMCPRegInfo *ri,
529                                             bool isread)
530 {
531     if (arm_current_el(env) == 3) {
532         return CP_ACCESS_OK;
533     }
534     if (arm_is_secure_below_el3(env)) {
535         return CP_ACCESS_TRAP_EL3;
536     }
537     /* This will be EL1 NS and EL2 NS, which just UNDEF */
538     return CP_ACCESS_TRAP_UNCATEGORIZED;
539 }
540 
541 /* Check for traps to "powerdown debug" registers, which are controlled
542  * by MDCR.TDOSA
543  */
544 static CPAccessResult access_tdosa(CPUARMState *env, const ARMCPRegInfo *ri,
545                                    bool isread)
546 {
547     int el = arm_current_el(env);
548     bool mdcr_el2_tdosa = (env->cp15.mdcr_el2 & MDCR_TDOSA) ||
549         (env->cp15.mdcr_el2 & MDCR_TDE) ||
550         (arm_hcr_el2_eff(env) & HCR_TGE);
551 
552     if (el < 2 && mdcr_el2_tdosa && !arm_is_secure_below_el3(env)) {
553         return CP_ACCESS_TRAP_EL2;
554     }
555     if (el < 3 && (env->cp15.mdcr_el3 & MDCR_TDOSA)) {
556         return CP_ACCESS_TRAP_EL3;
557     }
558     return CP_ACCESS_OK;
559 }
560 
561 /* Check for traps to "debug ROM" registers, which are controlled
562  * by MDCR_EL2.TDRA for EL2 but by the more general MDCR_EL3.TDA for EL3.
563  */
564 static CPAccessResult access_tdra(CPUARMState *env, const ARMCPRegInfo *ri,
565                                   bool isread)
566 {
567     int el = arm_current_el(env);
568     bool mdcr_el2_tdra = (env->cp15.mdcr_el2 & MDCR_TDRA) ||
569         (env->cp15.mdcr_el2 & MDCR_TDE) ||
570         (arm_hcr_el2_eff(env) & HCR_TGE);
571 
572     if (el < 2 && mdcr_el2_tdra && !arm_is_secure_below_el3(env)) {
573         return CP_ACCESS_TRAP_EL2;
574     }
575     if (el < 3 && (env->cp15.mdcr_el3 & MDCR_TDA)) {
576         return CP_ACCESS_TRAP_EL3;
577     }
578     return CP_ACCESS_OK;
579 }
580 
581 /* Check for traps to general debug registers, which are controlled
582  * by MDCR_EL2.TDA for EL2 and MDCR_EL3.TDA for EL3.
583  */
584 static CPAccessResult access_tda(CPUARMState *env, const ARMCPRegInfo *ri,
585                                   bool isread)
586 {
587     int el = arm_current_el(env);
588     bool mdcr_el2_tda = (env->cp15.mdcr_el2 & MDCR_TDA) ||
589         (env->cp15.mdcr_el2 & MDCR_TDE) ||
590         (arm_hcr_el2_eff(env) & HCR_TGE);
591 
592     if (el < 2 && mdcr_el2_tda && !arm_is_secure_below_el3(env)) {
593         return CP_ACCESS_TRAP_EL2;
594     }
595     if (el < 3 && (env->cp15.mdcr_el3 & MDCR_TDA)) {
596         return CP_ACCESS_TRAP_EL3;
597     }
598     return CP_ACCESS_OK;
599 }
600 
601 /* Check for traps to performance monitor registers, which are controlled
602  * by MDCR_EL2.TPM for EL2 and MDCR_EL3.TPM for EL3.
603  */
604 static CPAccessResult access_tpm(CPUARMState *env, const ARMCPRegInfo *ri,
605                                  bool isread)
606 {
607     int el = arm_current_el(env);
608 
609     if (el < 2 && (env->cp15.mdcr_el2 & MDCR_TPM)
610         && !arm_is_secure_below_el3(env)) {
611         return CP_ACCESS_TRAP_EL2;
612     }
613     if (el < 3 && (env->cp15.mdcr_el3 & MDCR_TPM)) {
614         return CP_ACCESS_TRAP_EL3;
615     }
616     return CP_ACCESS_OK;
617 }
618 
619 /* Check for traps from EL1 due to HCR_EL2.TVM and HCR_EL2.TRVM.  */
620 static CPAccessResult access_tvm_trvm(CPUARMState *env, const ARMCPRegInfo *ri,
621                                       bool isread)
622 {
623     if (arm_current_el(env) == 1) {
624         uint64_t trap = isread ? HCR_TRVM : HCR_TVM;
625         if (arm_hcr_el2_eff(env) & trap) {
626             return CP_ACCESS_TRAP_EL2;
627         }
628     }
629     return CP_ACCESS_OK;
630 }
631 
632 /* Check for traps from EL1 due to HCR_EL2.TSW.  */
633 static CPAccessResult access_tsw(CPUARMState *env, const ARMCPRegInfo *ri,
634                                  bool isread)
635 {
636     if (arm_current_el(env) == 1 && (arm_hcr_el2_eff(env) & HCR_TSW)) {
637         return CP_ACCESS_TRAP_EL2;
638     }
639     return CP_ACCESS_OK;
640 }
641 
642 /* Check for traps from EL1 due to HCR_EL2.TACR.  */
643 static CPAccessResult access_tacr(CPUARMState *env, const ARMCPRegInfo *ri,
644                                   bool isread)
645 {
646     if (arm_current_el(env) == 1 && (arm_hcr_el2_eff(env) & HCR_TACR)) {
647         return CP_ACCESS_TRAP_EL2;
648     }
649     return CP_ACCESS_OK;
650 }
651 
652 /* Check for traps from EL1 due to HCR_EL2.TTLB. */
653 static CPAccessResult access_ttlb(CPUARMState *env, const ARMCPRegInfo *ri,
654                                   bool isread)
655 {
656     if (arm_current_el(env) == 1 && (arm_hcr_el2_eff(env) & HCR_TTLB)) {
657         return CP_ACCESS_TRAP_EL2;
658     }
659     return CP_ACCESS_OK;
660 }
661 
662 static void dacr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
663 {
664     ARMCPU *cpu = env_archcpu(env);
665 
666     raw_write(env, ri, value);
667     tlb_flush(CPU(cpu)); /* Flush TLB as domain not tracked in TLB */
668 }
669 
670 static void fcse_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
671 {
672     ARMCPU *cpu = env_archcpu(env);
673 
674     if (raw_read(env, ri) != value) {
675         /* Unlike real hardware the qemu TLB uses virtual addresses,
676          * not modified virtual addresses, so this causes a TLB flush.
677          */
678         tlb_flush(CPU(cpu));
679         raw_write(env, ri, value);
680     }
681 }
682 
683 static void contextidr_write(CPUARMState *env, const ARMCPRegInfo *ri,
684                              uint64_t value)
685 {
686     ARMCPU *cpu = env_archcpu(env);
687 
688     if (raw_read(env, ri) != value && !arm_feature(env, ARM_FEATURE_PMSA)
689         && !extended_addresses_enabled(env)) {
690         /* For VMSA (when not using the LPAE long descriptor page table
691          * format) this register includes the ASID, so do a TLB flush.
692          * For PMSA it is purely a process ID and no action is needed.
693          */
694         tlb_flush(CPU(cpu));
695     }
696     raw_write(env, ri, value);
697 }
698 
699 /* IS variants of TLB operations must affect all cores */
700 static void tlbiall_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
701                              uint64_t value)
702 {
703     CPUState *cs = env_cpu(env);
704 
705     tlb_flush_all_cpus_synced(cs);
706 }
707 
708 static void tlbiasid_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
709                              uint64_t value)
710 {
711     CPUState *cs = env_cpu(env);
712 
713     tlb_flush_all_cpus_synced(cs);
714 }
715 
716 static void tlbimva_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
717                              uint64_t value)
718 {
719     CPUState *cs = env_cpu(env);
720 
721     tlb_flush_page_all_cpus_synced(cs, value & TARGET_PAGE_MASK);
722 }
723 
724 static void tlbimvaa_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
725                              uint64_t value)
726 {
727     CPUState *cs = env_cpu(env);
728 
729     tlb_flush_page_all_cpus_synced(cs, value & TARGET_PAGE_MASK);
730 }
731 
732 /*
733  * Non-IS variants of TLB operations are upgraded to
734  * IS versions if we are at NS EL1 and HCR_EL2.FB is set to
735  * force broadcast of these operations.
736  */
737 static bool tlb_force_broadcast(CPUARMState *env)
738 {
739     return (env->cp15.hcr_el2 & HCR_FB) &&
740         arm_current_el(env) == 1 && arm_is_secure_below_el3(env);
741 }
742 
743 static void tlbiall_write(CPUARMState *env, const ARMCPRegInfo *ri,
744                           uint64_t value)
745 {
746     /* Invalidate all (TLBIALL) */
747     CPUState *cs = env_cpu(env);
748 
749     if (tlb_force_broadcast(env)) {
750         tlb_flush_all_cpus_synced(cs);
751     } else {
752         tlb_flush(cs);
753     }
754 }
755 
756 static void tlbimva_write(CPUARMState *env, const ARMCPRegInfo *ri,
757                           uint64_t value)
758 {
759     /* Invalidate single TLB entry by MVA and ASID (TLBIMVA) */
760     CPUState *cs = env_cpu(env);
761 
762     value &= TARGET_PAGE_MASK;
763     if (tlb_force_broadcast(env)) {
764         tlb_flush_page_all_cpus_synced(cs, value);
765     } else {
766         tlb_flush_page(cs, value);
767     }
768 }
769 
770 static void tlbiasid_write(CPUARMState *env, const ARMCPRegInfo *ri,
771                            uint64_t value)
772 {
773     /* Invalidate by ASID (TLBIASID) */
774     CPUState *cs = env_cpu(env);
775 
776     if (tlb_force_broadcast(env)) {
777         tlb_flush_all_cpus_synced(cs);
778     } else {
779         tlb_flush(cs);
780     }
781 }
782 
783 static void tlbimvaa_write(CPUARMState *env, const ARMCPRegInfo *ri,
784                            uint64_t value)
785 {
786     /* Invalidate single entry by MVA, all ASIDs (TLBIMVAA) */
787     CPUState *cs = env_cpu(env);
788 
789     value &= TARGET_PAGE_MASK;
790     if (tlb_force_broadcast(env)) {
791         tlb_flush_page_all_cpus_synced(cs, value);
792     } else {
793         tlb_flush_page(cs, value);
794     }
795 }
796 
797 static void tlbiall_nsnh_write(CPUARMState *env, const ARMCPRegInfo *ri,
798                                uint64_t value)
799 {
800     CPUState *cs = env_cpu(env);
801 
802     tlb_flush_by_mmuidx(cs,
803                         ARMMMUIdxBit_E10_1 |
804                         ARMMMUIdxBit_E10_1_PAN |
805                         ARMMMUIdxBit_E10_0);
806 }
807 
808 static void tlbiall_nsnh_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
809                                   uint64_t value)
810 {
811     CPUState *cs = env_cpu(env);
812 
813     tlb_flush_by_mmuidx_all_cpus_synced(cs,
814                                         ARMMMUIdxBit_E10_1 |
815                                         ARMMMUIdxBit_E10_1_PAN |
816                                         ARMMMUIdxBit_E10_0);
817 }
818 
819 
820 static void tlbiall_hyp_write(CPUARMState *env, const ARMCPRegInfo *ri,
821                               uint64_t value)
822 {
823     CPUState *cs = env_cpu(env);
824 
825     tlb_flush_by_mmuidx(cs, ARMMMUIdxBit_E2);
826 }
827 
828 static void tlbiall_hyp_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
829                                  uint64_t value)
830 {
831     CPUState *cs = env_cpu(env);
832 
833     tlb_flush_by_mmuidx_all_cpus_synced(cs, ARMMMUIdxBit_E2);
834 }
835 
836 static void tlbimva_hyp_write(CPUARMState *env, const ARMCPRegInfo *ri,
837                               uint64_t value)
838 {
839     CPUState *cs = env_cpu(env);
840     uint64_t pageaddr = value & ~MAKE_64BIT_MASK(0, 12);
841 
842     tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdxBit_E2);
843 }
844 
845 static void tlbimva_hyp_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
846                                  uint64_t value)
847 {
848     CPUState *cs = env_cpu(env);
849     uint64_t pageaddr = value & ~MAKE_64BIT_MASK(0, 12);
850 
851     tlb_flush_page_by_mmuidx_all_cpus_synced(cs, pageaddr,
852                                              ARMMMUIdxBit_E2);
853 }
854 
855 static const ARMCPRegInfo cp_reginfo[] = {
856     /* Define the secure and non-secure FCSE identifier CP registers
857      * separately because there is no secure bank in V8 (no _EL3).  This allows
858      * the secure register to be properly reset and migrated. There is also no
859      * v8 EL1 version of the register so the non-secure instance stands alone.
860      */
861     { .name = "FCSEIDR",
862       .cp = 15, .opc1 = 0, .crn = 13, .crm = 0, .opc2 = 0,
863       .access = PL1_RW, .secure = ARM_CP_SECSTATE_NS,
864       .fieldoffset = offsetof(CPUARMState, cp15.fcseidr_ns),
865       .resetvalue = 0, .writefn = fcse_write, .raw_writefn = raw_write, },
866     { .name = "FCSEIDR_S",
867       .cp = 15, .opc1 = 0, .crn = 13, .crm = 0, .opc2 = 0,
868       .access = PL1_RW, .secure = ARM_CP_SECSTATE_S,
869       .fieldoffset = offsetof(CPUARMState, cp15.fcseidr_s),
870       .resetvalue = 0, .writefn = fcse_write, .raw_writefn = raw_write, },
871     /* Define the secure and non-secure context identifier CP registers
872      * separately because there is no secure bank in V8 (no _EL3).  This allows
873      * the secure register to be properly reset and migrated.  In the
874      * non-secure case, the 32-bit register will have reset and migration
875      * disabled during registration as it is handled by the 64-bit instance.
876      */
877     { .name = "CONTEXTIDR_EL1", .state = ARM_CP_STATE_BOTH,
878       .opc0 = 3, .opc1 = 0, .crn = 13, .crm = 0, .opc2 = 1,
879       .access = PL1_RW, .accessfn = access_tvm_trvm,
880       .secure = ARM_CP_SECSTATE_NS,
881       .fieldoffset = offsetof(CPUARMState, cp15.contextidr_el[1]),
882       .resetvalue = 0, .writefn = contextidr_write, .raw_writefn = raw_write, },
883     { .name = "CONTEXTIDR_S", .state = ARM_CP_STATE_AA32,
884       .cp = 15, .opc1 = 0, .crn = 13, .crm = 0, .opc2 = 1,
885       .access = PL1_RW, .accessfn = access_tvm_trvm,
886       .secure = ARM_CP_SECSTATE_S,
887       .fieldoffset = offsetof(CPUARMState, cp15.contextidr_s),
888       .resetvalue = 0, .writefn = contextidr_write, .raw_writefn = raw_write, },
889     REGINFO_SENTINEL
890 };
891 
892 static const ARMCPRegInfo not_v8_cp_reginfo[] = {
893     /* NB: Some of these registers exist in v8 but with more precise
894      * definitions that don't use CP_ANY wildcards (mostly in v8_cp_reginfo[]).
895      */
896     /* MMU Domain access control / MPU write buffer control */
897     { .name = "DACR",
898       .cp = 15, .opc1 = CP_ANY, .crn = 3, .crm = CP_ANY, .opc2 = CP_ANY,
899       .access = PL1_RW, .accessfn = access_tvm_trvm, .resetvalue = 0,
900       .writefn = dacr_write, .raw_writefn = raw_write,
901       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.dacr_s),
902                              offsetoflow32(CPUARMState, cp15.dacr_ns) } },
903     /* ARMv7 allocates a range of implementation defined TLB LOCKDOWN regs.
904      * For v6 and v5, these mappings are overly broad.
905      */
906     { .name = "TLB_LOCKDOWN", .cp = 15, .crn = 10, .crm = 0,
907       .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_NOP },
908     { .name = "TLB_LOCKDOWN", .cp = 15, .crn = 10, .crm = 1,
909       .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_NOP },
910     { .name = "TLB_LOCKDOWN", .cp = 15, .crn = 10, .crm = 4,
911       .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_NOP },
912     { .name = "TLB_LOCKDOWN", .cp = 15, .crn = 10, .crm = 8,
913       .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_NOP },
914     /* Cache maintenance ops; some of this space may be overridden later. */
915     { .name = "CACHEMAINT", .cp = 15, .crn = 7, .crm = CP_ANY,
916       .opc1 = 0, .opc2 = CP_ANY, .access = PL1_W,
917       .type = ARM_CP_NOP | ARM_CP_OVERRIDE },
918     REGINFO_SENTINEL
919 };
920 
921 static const ARMCPRegInfo not_v6_cp_reginfo[] = {
922     /* Not all pre-v6 cores implemented this WFI, so this is slightly
923      * over-broad.
924      */
925     { .name = "WFI_v5", .cp = 15, .crn = 7, .crm = 8, .opc1 = 0, .opc2 = 2,
926       .access = PL1_W, .type = ARM_CP_WFI },
927     REGINFO_SENTINEL
928 };
929 
930 static const ARMCPRegInfo not_v7_cp_reginfo[] = {
931     /* Standard v6 WFI (also used in some pre-v6 cores); not in v7 (which
932      * is UNPREDICTABLE; we choose to NOP as most implementations do).
933      */
934     { .name = "WFI_v6", .cp = 15, .crn = 7, .crm = 0, .opc1 = 0, .opc2 = 4,
935       .access = PL1_W, .type = ARM_CP_WFI },
936     /* L1 cache lockdown. Not architectural in v6 and earlier but in practice
937      * implemented in 926, 946, 1026, 1136, 1176 and 11MPCore. StrongARM and
938      * OMAPCP will override this space.
939      */
940     { .name = "DLOCKDOWN", .cp = 15, .crn = 9, .crm = 0, .opc1 = 0, .opc2 = 0,
941       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c9_data),
942       .resetvalue = 0 },
943     { .name = "ILOCKDOWN", .cp = 15, .crn = 9, .crm = 0, .opc1 = 0, .opc2 = 1,
944       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c9_insn),
945       .resetvalue = 0 },
946     /* v6 doesn't have the cache ID registers but Linux reads them anyway */
947     { .name = "DUMMY", .cp = 15, .crn = 0, .crm = 0, .opc1 = 1, .opc2 = CP_ANY,
948       .access = PL1_R, .type = ARM_CP_CONST | ARM_CP_NO_RAW,
949       .resetvalue = 0 },
950     /* We don't implement pre-v7 debug but most CPUs had at least a DBGDIDR;
951      * implementing it as RAZ means the "debug architecture version" bits
952      * will read as a reserved value, which should cause Linux to not try
953      * to use the debug hardware.
954      */
955     { .name = "DBGDIDR", .cp = 14, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 0,
956       .access = PL0_R, .type = ARM_CP_CONST, .resetvalue = 0 },
957     /* MMU TLB control. Note that the wildcarding means we cover not just
958      * the unified TLB ops but also the dside/iside/inner-shareable variants.
959      */
960     { .name = "TLBIALL", .cp = 15, .crn = 8, .crm = CP_ANY,
961       .opc1 = CP_ANY, .opc2 = 0, .access = PL1_W, .writefn = tlbiall_write,
962       .type = ARM_CP_NO_RAW },
963     { .name = "TLBIMVA", .cp = 15, .crn = 8, .crm = CP_ANY,
964       .opc1 = CP_ANY, .opc2 = 1, .access = PL1_W, .writefn = tlbimva_write,
965       .type = ARM_CP_NO_RAW },
966     { .name = "TLBIASID", .cp = 15, .crn = 8, .crm = CP_ANY,
967       .opc1 = CP_ANY, .opc2 = 2, .access = PL1_W, .writefn = tlbiasid_write,
968       .type = ARM_CP_NO_RAW },
969     { .name = "TLBIMVAA", .cp = 15, .crn = 8, .crm = CP_ANY,
970       .opc1 = CP_ANY, .opc2 = 3, .access = PL1_W, .writefn = tlbimvaa_write,
971       .type = ARM_CP_NO_RAW },
972     { .name = "PRRR", .cp = 15, .crn = 10, .crm = 2,
973       .opc1 = 0, .opc2 = 0, .access = PL1_RW, .type = ARM_CP_NOP },
974     { .name = "NMRR", .cp = 15, .crn = 10, .crm = 2,
975       .opc1 = 0, .opc2 = 1, .access = PL1_RW, .type = ARM_CP_NOP },
976     REGINFO_SENTINEL
977 };
978 
979 static void cpacr_write(CPUARMState *env, const ARMCPRegInfo *ri,
980                         uint64_t value)
981 {
982     uint32_t mask = 0;
983 
984     /* In ARMv8 most bits of CPACR_EL1 are RES0. */
985     if (!arm_feature(env, ARM_FEATURE_V8)) {
986         /* ARMv7 defines bits for unimplemented coprocessors as RAZ/WI.
987          * ASEDIS [31] and D32DIS [30] are both UNK/SBZP without VFP.
988          * TRCDIS [28] is RAZ/WI since we do not implement a trace macrocell.
989          */
990         if (cpu_isar_feature(aa32_vfp_simd, env_archcpu(env))) {
991             /* VFP coprocessor: cp10 & cp11 [23:20] */
992             mask |= (1 << 31) | (1 << 30) | (0xf << 20);
993 
994             if (!arm_feature(env, ARM_FEATURE_NEON)) {
995                 /* ASEDIS [31] bit is RAO/WI */
996                 value |= (1 << 31);
997             }
998 
999             /* VFPv3 and upwards with NEON implement 32 double precision
1000              * registers (D0-D31).
1001              */
1002             if (!cpu_isar_feature(aa32_simd_r32, env_archcpu(env))) {
1003                 /* D32DIS [30] is RAO/WI if D16-31 are not implemented. */
1004                 value |= (1 << 30);
1005             }
1006         }
1007         value &= mask;
1008     }
1009 
1010     /*
1011      * For A-profile AArch32 EL3 (but not M-profile secure mode), if NSACR.CP10
1012      * is 0 then CPACR.{CP11,CP10} ignore writes and read as 0b00.
1013      */
1014     if (arm_feature(env, ARM_FEATURE_EL3) && !arm_el_is_aa64(env, 3) &&
1015         !arm_is_secure(env) && !extract32(env->cp15.nsacr, 10, 1)) {
1016         value &= ~(0xf << 20);
1017         value |= env->cp15.cpacr_el1 & (0xf << 20);
1018     }
1019 
1020     env->cp15.cpacr_el1 = value;
1021 }
1022 
1023 static uint64_t cpacr_read(CPUARMState *env, const ARMCPRegInfo *ri)
1024 {
1025     /*
1026      * For A-profile AArch32 EL3 (but not M-profile secure mode), if NSACR.CP10
1027      * is 0 then CPACR.{CP11,CP10} ignore writes and read as 0b00.
1028      */
1029     uint64_t value = env->cp15.cpacr_el1;
1030 
1031     if (arm_feature(env, ARM_FEATURE_EL3) && !arm_el_is_aa64(env, 3) &&
1032         !arm_is_secure(env) && !extract32(env->cp15.nsacr, 10, 1)) {
1033         value &= ~(0xf << 20);
1034     }
1035     return value;
1036 }
1037 
1038 
1039 static void cpacr_reset(CPUARMState *env, const ARMCPRegInfo *ri)
1040 {
1041     /* Call cpacr_write() so that we reset with the correct RAO bits set
1042      * for our CPU features.
1043      */
1044     cpacr_write(env, ri, 0);
1045 }
1046 
1047 static CPAccessResult cpacr_access(CPUARMState *env, const ARMCPRegInfo *ri,
1048                                    bool isread)
1049 {
1050     if (arm_feature(env, ARM_FEATURE_V8)) {
1051         /* Check if CPACR accesses are to be trapped to EL2 */
1052         if (arm_current_el(env) == 1 &&
1053             (env->cp15.cptr_el[2] & CPTR_TCPAC) && !arm_is_secure(env)) {
1054             return CP_ACCESS_TRAP_EL2;
1055         /* Check if CPACR accesses are to be trapped to EL3 */
1056         } else if (arm_current_el(env) < 3 &&
1057                    (env->cp15.cptr_el[3] & CPTR_TCPAC)) {
1058             return CP_ACCESS_TRAP_EL3;
1059         }
1060     }
1061 
1062     return CP_ACCESS_OK;
1063 }
1064 
1065 static CPAccessResult cptr_access(CPUARMState *env, const ARMCPRegInfo *ri,
1066                                   bool isread)
1067 {
1068     /* Check if CPTR accesses are set to trap to EL3 */
1069     if (arm_current_el(env) == 2 && (env->cp15.cptr_el[3] & CPTR_TCPAC)) {
1070         return CP_ACCESS_TRAP_EL3;
1071     }
1072 
1073     return CP_ACCESS_OK;
1074 }
1075 
1076 static const ARMCPRegInfo v6_cp_reginfo[] = {
1077     /* prefetch by MVA in v6, NOP in v7 */
1078     { .name = "MVA_prefetch",
1079       .cp = 15, .crn = 7, .crm = 13, .opc1 = 0, .opc2 = 1,
1080       .access = PL1_W, .type = ARM_CP_NOP },
1081     /* We need to break the TB after ISB to execute self-modifying code
1082      * correctly and also to take any pending interrupts immediately.
1083      * So use arm_cp_write_ignore() function instead of ARM_CP_NOP flag.
1084      */
1085     { .name = "ISB", .cp = 15, .crn = 7, .crm = 5, .opc1 = 0, .opc2 = 4,
1086       .access = PL0_W, .type = ARM_CP_NO_RAW, .writefn = arm_cp_write_ignore },
1087     { .name = "DSB", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 4,
1088       .access = PL0_W, .type = ARM_CP_NOP },
1089     { .name = "DMB", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 5,
1090       .access = PL0_W, .type = ARM_CP_NOP },
1091     { .name = "IFAR", .cp = 15, .crn = 6, .crm = 0, .opc1 = 0, .opc2 = 2,
1092       .access = PL1_RW, .accessfn = access_tvm_trvm,
1093       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ifar_s),
1094                              offsetof(CPUARMState, cp15.ifar_ns) },
1095       .resetvalue = 0, },
1096     /* Watchpoint Fault Address Register : should actually only be present
1097      * for 1136, 1176, 11MPCore.
1098      */
1099     { .name = "WFAR", .cp = 15, .crn = 6, .crm = 0, .opc1 = 0, .opc2 = 1,
1100       .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0, },
1101     { .name = "CPACR", .state = ARM_CP_STATE_BOTH, .opc0 = 3,
1102       .crn = 1, .crm = 0, .opc1 = 0, .opc2 = 2, .accessfn = cpacr_access,
1103       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.cpacr_el1),
1104       .resetfn = cpacr_reset, .writefn = cpacr_write, .readfn = cpacr_read },
1105     REGINFO_SENTINEL
1106 };
1107 
1108 /* Definitions for the PMU registers */
1109 #define PMCRN_MASK  0xf800
1110 #define PMCRN_SHIFT 11
1111 #define PMCRLC  0x40
1112 #define PMCRDP  0x20
1113 #define PMCRX   0x10
1114 #define PMCRD   0x8
1115 #define PMCRC   0x4
1116 #define PMCRP   0x2
1117 #define PMCRE   0x1
1118 /*
1119  * Mask of PMCR bits writeable by guest (not including WO bits like C, P,
1120  * which can be written as 1 to trigger behaviour but which stay RAZ).
1121  */
1122 #define PMCR_WRITEABLE_MASK (PMCRLC | PMCRDP | PMCRX | PMCRD | PMCRE)
1123 
1124 #define PMXEVTYPER_P          0x80000000
1125 #define PMXEVTYPER_U          0x40000000
1126 #define PMXEVTYPER_NSK        0x20000000
1127 #define PMXEVTYPER_NSU        0x10000000
1128 #define PMXEVTYPER_NSH        0x08000000
1129 #define PMXEVTYPER_M          0x04000000
1130 #define PMXEVTYPER_MT         0x02000000
1131 #define PMXEVTYPER_EVTCOUNT   0x0000ffff
1132 #define PMXEVTYPER_MASK       (PMXEVTYPER_P | PMXEVTYPER_U | PMXEVTYPER_NSK | \
1133                                PMXEVTYPER_NSU | PMXEVTYPER_NSH | \
1134                                PMXEVTYPER_M | PMXEVTYPER_MT | \
1135                                PMXEVTYPER_EVTCOUNT)
1136 
1137 #define PMCCFILTR             0xf8000000
1138 #define PMCCFILTR_M           PMXEVTYPER_M
1139 #define PMCCFILTR_EL0         (PMCCFILTR | PMCCFILTR_M)
1140 
1141 static inline uint32_t pmu_num_counters(CPUARMState *env)
1142 {
1143   return (env->cp15.c9_pmcr & PMCRN_MASK) >> PMCRN_SHIFT;
1144 }
1145 
1146 /* Bits allowed to be set/cleared for PMCNTEN* and PMINTEN* */
1147 static inline uint64_t pmu_counter_mask(CPUARMState *env)
1148 {
1149   return (1 << 31) | ((1 << pmu_num_counters(env)) - 1);
1150 }
1151 
1152 typedef struct pm_event {
1153     uint16_t number; /* PMEVTYPER.evtCount is 16 bits wide */
1154     /* If the event is supported on this CPU (used to generate PMCEID[01]) */
1155     bool (*supported)(CPUARMState *);
1156     /*
1157      * Retrieve the current count of the underlying event. The programmed
1158      * counters hold a difference from the return value from this function
1159      */
1160     uint64_t (*get_count)(CPUARMState *);
1161     /*
1162      * Return how many nanoseconds it will take (at a minimum) for count events
1163      * to occur. A negative value indicates the counter will never overflow, or
1164      * that the counter has otherwise arranged for the overflow bit to be set
1165      * and the PMU interrupt to be raised on overflow.
1166      */
1167     int64_t (*ns_per_count)(uint64_t);
1168 } pm_event;
1169 
1170 static bool event_always_supported(CPUARMState *env)
1171 {
1172     return true;
1173 }
1174 
1175 static uint64_t swinc_get_count(CPUARMState *env)
1176 {
1177     /*
1178      * SW_INCR events are written directly to the pmevcntr's by writes to
1179      * PMSWINC, so there is no underlying count maintained by the PMU itself
1180      */
1181     return 0;
1182 }
1183 
1184 static int64_t swinc_ns_per(uint64_t ignored)
1185 {
1186     return -1;
1187 }
1188 
1189 /*
1190  * Return the underlying cycle count for the PMU cycle counters. If we're in
1191  * usermode, simply return 0.
1192  */
1193 static uint64_t cycles_get_count(CPUARMState *env)
1194 {
1195 #ifndef CONFIG_USER_ONLY
1196     return muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL),
1197                    ARM_CPU_FREQ, NANOSECONDS_PER_SECOND);
1198 #else
1199     return cpu_get_host_ticks();
1200 #endif
1201 }
1202 
1203 #ifndef CONFIG_USER_ONLY
1204 static int64_t cycles_ns_per(uint64_t cycles)
1205 {
1206     return (ARM_CPU_FREQ / NANOSECONDS_PER_SECOND) * cycles;
1207 }
1208 
1209 static bool instructions_supported(CPUARMState *env)
1210 {
1211     return icount_enabled() == 1; /* Precise instruction counting */
1212 }
1213 
1214 static uint64_t instructions_get_count(CPUARMState *env)
1215 {
1216     return (uint64_t)icount_get_raw();
1217 }
1218 
1219 static int64_t instructions_ns_per(uint64_t icount)
1220 {
1221     return icount_to_ns((int64_t)icount);
1222 }
1223 #endif
1224 
1225 static bool pmu_8_1_events_supported(CPUARMState *env)
1226 {
1227     /* For events which are supported in any v8.1 PMU */
1228     return cpu_isar_feature(any_pmu_8_1, env_archcpu(env));
1229 }
1230 
1231 static bool pmu_8_4_events_supported(CPUARMState *env)
1232 {
1233     /* For events which are supported in any v8.1 PMU */
1234     return cpu_isar_feature(any_pmu_8_4, env_archcpu(env));
1235 }
1236 
1237 static uint64_t zero_event_get_count(CPUARMState *env)
1238 {
1239     /* For events which on QEMU never fire, so their count is always zero */
1240     return 0;
1241 }
1242 
1243 static int64_t zero_event_ns_per(uint64_t cycles)
1244 {
1245     /* An event which never fires can never overflow */
1246     return -1;
1247 }
1248 
1249 static const pm_event pm_events[] = {
1250     { .number = 0x000, /* SW_INCR */
1251       .supported = event_always_supported,
1252       .get_count = swinc_get_count,
1253       .ns_per_count = swinc_ns_per,
1254     },
1255 #ifndef CONFIG_USER_ONLY
1256     { .number = 0x008, /* INST_RETIRED, Instruction architecturally executed */
1257       .supported = instructions_supported,
1258       .get_count = instructions_get_count,
1259       .ns_per_count = instructions_ns_per,
1260     },
1261     { .number = 0x011, /* CPU_CYCLES, Cycle */
1262       .supported = event_always_supported,
1263       .get_count = cycles_get_count,
1264       .ns_per_count = cycles_ns_per,
1265     },
1266 #endif
1267     { .number = 0x023, /* STALL_FRONTEND */
1268       .supported = pmu_8_1_events_supported,
1269       .get_count = zero_event_get_count,
1270       .ns_per_count = zero_event_ns_per,
1271     },
1272     { .number = 0x024, /* STALL_BACKEND */
1273       .supported = pmu_8_1_events_supported,
1274       .get_count = zero_event_get_count,
1275       .ns_per_count = zero_event_ns_per,
1276     },
1277     { .number = 0x03c, /* STALL */
1278       .supported = pmu_8_4_events_supported,
1279       .get_count = zero_event_get_count,
1280       .ns_per_count = zero_event_ns_per,
1281     },
1282 };
1283 
1284 /*
1285  * Note: Before increasing MAX_EVENT_ID beyond 0x3f into the 0x40xx range of
1286  * events (i.e. the statistical profiling extension), this implementation
1287  * should first be updated to something sparse instead of the current
1288  * supported_event_map[] array.
1289  */
1290 #define MAX_EVENT_ID 0x3c
1291 #define UNSUPPORTED_EVENT UINT16_MAX
1292 static uint16_t supported_event_map[MAX_EVENT_ID + 1];
1293 
1294 /*
1295  * Called upon CPU initialization to initialize PMCEID[01]_EL0 and build a map
1296  * of ARM event numbers to indices in our pm_events array.
1297  *
1298  * Note: Events in the 0x40XX range are not currently supported.
1299  */
1300 void pmu_init(ARMCPU *cpu)
1301 {
1302     unsigned int i;
1303 
1304     /*
1305      * Empty supported_event_map and cpu->pmceid[01] before adding supported
1306      * events to them
1307      */
1308     for (i = 0; i < ARRAY_SIZE(supported_event_map); i++) {
1309         supported_event_map[i] = UNSUPPORTED_EVENT;
1310     }
1311     cpu->pmceid0 = 0;
1312     cpu->pmceid1 = 0;
1313 
1314     for (i = 0; i < ARRAY_SIZE(pm_events); i++) {
1315         const pm_event *cnt = &pm_events[i];
1316         assert(cnt->number <= MAX_EVENT_ID);
1317         /* We do not currently support events in the 0x40xx range */
1318         assert(cnt->number <= 0x3f);
1319 
1320         if (cnt->supported(&cpu->env)) {
1321             supported_event_map[cnt->number] = i;
1322             uint64_t event_mask = 1ULL << (cnt->number & 0x1f);
1323             if (cnt->number & 0x20) {
1324                 cpu->pmceid1 |= event_mask;
1325             } else {
1326                 cpu->pmceid0 |= event_mask;
1327             }
1328         }
1329     }
1330 }
1331 
1332 /*
1333  * Check at runtime whether a PMU event is supported for the current machine
1334  */
1335 static bool event_supported(uint16_t number)
1336 {
1337     if (number > MAX_EVENT_ID) {
1338         return false;
1339     }
1340     return supported_event_map[number] != UNSUPPORTED_EVENT;
1341 }
1342 
1343 static CPAccessResult pmreg_access(CPUARMState *env, const ARMCPRegInfo *ri,
1344                                    bool isread)
1345 {
1346     /* Performance monitor registers user accessibility is controlled
1347      * by PMUSERENR. MDCR_EL2.TPM and MDCR_EL3.TPM allow configurable
1348      * trapping to EL2 or EL3 for other accesses.
1349      */
1350     int el = arm_current_el(env);
1351 
1352     if (el == 0 && !(env->cp15.c9_pmuserenr & 1)) {
1353         return CP_ACCESS_TRAP;
1354     }
1355     if (el < 2 && (env->cp15.mdcr_el2 & MDCR_TPM)
1356         && !arm_is_secure_below_el3(env)) {
1357         return CP_ACCESS_TRAP_EL2;
1358     }
1359     if (el < 3 && (env->cp15.mdcr_el3 & MDCR_TPM)) {
1360         return CP_ACCESS_TRAP_EL3;
1361     }
1362 
1363     return CP_ACCESS_OK;
1364 }
1365 
1366 static CPAccessResult pmreg_access_xevcntr(CPUARMState *env,
1367                                            const ARMCPRegInfo *ri,
1368                                            bool isread)
1369 {
1370     /* ER: event counter read trap control */
1371     if (arm_feature(env, ARM_FEATURE_V8)
1372         && arm_current_el(env) == 0
1373         && (env->cp15.c9_pmuserenr & (1 << 3)) != 0
1374         && isread) {
1375         return CP_ACCESS_OK;
1376     }
1377 
1378     return pmreg_access(env, ri, isread);
1379 }
1380 
1381 static CPAccessResult pmreg_access_swinc(CPUARMState *env,
1382                                          const ARMCPRegInfo *ri,
1383                                          bool isread)
1384 {
1385     /* SW: software increment write trap control */
1386     if (arm_feature(env, ARM_FEATURE_V8)
1387         && arm_current_el(env) == 0
1388         && (env->cp15.c9_pmuserenr & (1 << 1)) != 0
1389         && !isread) {
1390         return CP_ACCESS_OK;
1391     }
1392 
1393     return pmreg_access(env, ri, isread);
1394 }
1395 
1396 static CPAccessResult pmreg_access_selr(CPUARMState *env,
1397                                         const ARMCPRegInfo *ri,
1398                                         bool isread)
1399 {
1400     /* ER: event counter read trap control */
1401     if (arm_feature(env, ARM_FEATURE_V8)
1402         && arm_current_el(env) == 0
1403         && (env->cp15.c9_pmuserenr & (1 << 3)) != 0) {
1404         return CP_ACCESS_OK;
1405     }
1406 
1407     return pmreg_access(env, ri, isread);
1408 }
1409 
1410 static CPAccessResult pmreg_access_ccntr(CPUARMState *env,
1411                                          const ARMCPRegInfo *ri,
1412                                          bool isread)
1413 {
1414     /* CR: cycle counter read trap control */
1415     if (arm_feature(env, ARM_FEATURE_V8)
1416         && arm_current_el(env) == 0
1417         && (env->cp15.c9_pmuserenr & (1 << 2)) != 0
1418         && isread) {
1419         return CP_ACCESS_OK;
1420     }
1421 
1422     return pmreg_access(env, ri, isread);
1423 }
1424 
1425 /* Returns true if the counter (pass 31 for PMCCNTR) should count events using
1426  * the current EL, security state, and register configuration.
1427  */
1428 static bool pmu_counter_enabled(CPUARMState *env, uint8_t counter)
1429 {
1430     uint64_t filter;
1431     bool e, p, u, nsk, nsu, nsh, m;
1432     bool enabled, prohibited, filtered;
1433     bool secure = arm_is_secure(env);
1434     int el = arm_current_el(env);
1435     uint8_t hpmn = env->cp15.mdcr_el2 & MDCR_HPMN;
1436 
1437     if (!arm_feature(env, ARM_FEATURE_PMU)) {
1438         return false;
1439     }
1440 
1441     if (!arm_feature(env, ARM_FEATURE_EL2) ||
1442             (counter < hpmn || counter == 31)) {
1443         e = env->cp15.c9_pmcr & PMCRE;
1444     } else {
1445         e = env->cp15.mdcr_el2 & MDCR_HPME;
1446     }
1447     enabled = e && (env->cp15.c9_pmcnten & (1 << counter));
1448 
1449     if (!secure) {
1450         if (el == 2 && (counter < hpmn || counter == 31)) {
1451             prohibited = env->cp15.mdcr_el2 & MDCR_HPMD;
1452         } else {
1453             prohibited = false;
1454         }
1455     } else {
1456         prohibited = arm_feature(env, ARM_FEATURE_EL3) &&
1457            !(env->cp15.mdcr_el3 & MDCR_SPME);
1458     }
1459 
1460     if (prohibited && counter == 31) {
1461         prohibited = env->cp15.c9_pmcr & PMCRDP;
1462     }
1463 
1464     if (counter == 31) {
1465         filter = env->cp15.pmccfiltr_el0;
1466     } else {
1467         filter = env->cp15.c14_pmevtyper[counter];
1468     }
1469 
1470     p   = filter & PMXEVTYPER_P;
1471     u   = filter & PMXEVTYPER_U;
1472     nsk = arm_feature(env, ARM_FEATURE_EL3) && (filter & PMXEVTYPER_NSK);
1473     nsu = arm_feature(env, ARM_FEATURE_EL3) && (filter & PMXEVTYPER_NSU);
1474     nsh = arm_feature(env, ARM_FEATURE_EL2) && (filter & PMXEVTYPER_NSH);
1475     m   = arm_el_is_aa64(env, 1) &&
1476               arm_feature(env, ARM_FEATURE_EL3) && (filter & PMXEVTYPER_M);
1477 
1478     if (el == 0) {
1479         filtered = secure ? u : u != nsu;
1480     } else if (el == 1) {
1481         filtered = secure ? p : p != nsk;
1482     } else if (el == 2) {
1483         filtered = !nsh;
1484     } else { /* EL3 */
1485         filtered = m != p;
1486     }
1487 
1488     if (counter != 31) {
1489         /*
1490          * If not checking PMCCNTR, ensure the counter is setup to an event we
1491          * support
1492          */
1493         uint16_t event = filter & PMXEVTYPER_EVTCOUNT;
1494         if (!event_supported(event)) {
1495             return false;
1496         }
1497     }
1498 
1499     return enabled && !prohibited && !filtered;
1500 }
1501 
1502 static void pmu_update_irq(CPUARMState *env)
1503 {
1504     ARMCPU *cpu = env_archcpu(env);
1505     qemu_set_irq(cpu->pmu_interrupt, (env->cp15.c9_pmcr & PMCRE) &&
1506             (env->cp15.c9_pminten & env->cp15.c9_pmovsr));
1507 }
1508 
1509 /*
1510  * Ensure c15_ccnt is the guest-visible count so that operations such as
1511  * enabling/disabling the counter or filtering, modifying the count itself,
1512  * etc. can be done logically. This is essentially a no-op if the counter is
1513  * not enabled at the time of the call.
1514  */
1515 static void pmccntr_op_start(CPUARMState *env)
1516 {
1517     uint64_t cycles = cycles_get_count(env);
1518 
1519     if (pmu_counter_enabled(env, 31)) {
1520         uint64_t eff_cycles = cycles;
1521         if (env->cp15.c9_pmcr & PMCRD) {
1522             /* Increment once every 64 processor clock cycles */
1523             eff_cycles /= 64;
1524         }
1525 
1526         uint64_t new_pmccntr = eff_cycles - env->cp15.c15_ccnt_delta;
1527 
1528         uint64_t overflow_mask = env->cp15.c9_pmcr & PMCRLC ? \
1529                                  1ull << 63 : 1ull << 31;
1530         if (env->cp15.c15_ccnt & ~new_pmccntr & overflow_mask) {
1531             env->cp15.c9_pmovsr |= (1 << 31);
1532             pmu_update_irq(env);
1533         }
1534 
1535         env->cp15.c15_ccnt = new_pmccntr;
1536     }
1537     env->cp15.c15_ccnt_delta = cycles;
1538 }
1539 
1540 /*
1541  * If PMCCNTR is enabled, recalculate the delta between the clock and the
1542  * guest-visible count. A call to pmccntr_op_finish should follow every call to
1543  * pmccntr_op_start.
1544  */
1545 static void pmccntr_op_finish(CPUARMState *env)
1546 {
1547     if (pmu_counter_enabled(env, 31)) {
1548 #ifndef CONFIG_USER_ONLY
1549         /* Calculate when the counter will next overflow */
1550         uint64_t remaining_cycles = -env->cp15.c15_ccnt;
1551         if (!(env->cp15.c9_pmcr & PMCRLC)) {
1552             remaining_cycles = (uint32_t)remaining_cycles;
1553         }
1554         int64_t overflow_in = cycles_ns_per(remaining_cycles);
1555 
1556         if (overflow_in > 0) {
1557             int64_t overflow_at = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
1558                 overflow_in;
1559             ARMCPU *cpu = env_archcpu(env);
1560             timer_mod_anticipate_ns(cpu->pmu_timer, overflow_at);
1561         }
1562 #endif
1563 
1564         uint64_t prev_cycles = env->cp15.c15_ccnt_delta;
1565         if (env->cp15.c9_pmcr & PMCRD) {
1566             /* Increment once every 64 processor clock cycles */
1567             prev_cycles /= 64;
1568         }
1569         env->cp15.c15_ccnt_delta = prev_cycles - env->cp15.c15_ccnt;
1570     }
1571 }
1572 
1573 static void pmevcntr_op_start(CPUARMState *env, uint8_t counter)
1574 {
1575 
1576     uint16_t event = env->cp15.c14_pmevtyper[counter] & PMXEVTYPER_EVTCOUNT;
1577     uint64_t count = 0;
1578     if (event_supported(event)) {
1579         uint16_t event_idx = supported_event_map[event];
1580         count = pm_events[event_idx].get_count(env);
1581     }
1582 
1583     if (pmu_counter_enabled(env, counter)) {
1584         uint32_t new_pmevcntr = count - env->cp15.c14_pmevcntr_delta[counter];
1585 
1586         if (env->cp15.c14_pmevcntr[counter] & ~new_pmevcntr & INT32_MIN) {
1587             env->cp15.c9_pmovsr |= (1 << counter);
1588             pmu_update_irq(env);
1589         }
1590         env->cp15.c14_pmevcntr[counter] = new_pmevcntr;
1591     }
1592     env->cp15.c14_pmevcntr_delta[counter] = count;
1593 }
1594 
1595 static void pmevcntr_op_finish(CPUARMState *env, uint8_t counter)
1596 {
1597     if (pmu_counter_enabled(env, counter)) {
1598 #ifndef CONFIG_USER_ONLY
1599         uint16_t event = env->cp15.c14_pmevtyper[counter] & PMXEVTYPER_EVTCOUNT;
1600         uint16_t event_idx = supported_event_map[event];
1601         uint64_t delta = UINT32_MAX -
1602             (uint32_t)env->cp15.c14_pmevcntr[counter] + 1;
1603         int64_t overflow_in = pm_events[event_idx].ns_per_count(delta);
1604 
1605         if (overflow_in > 0) {
1606             int64_t overflow_at = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
1607                 overflow_in;
1608             ARMCPU *cpu = env_archcpu(env);
1609             timer_mod_anticipate_ns(cpu->pmu_timer, overflow_at);
1610         }
1611 #endif
1612 
1613         env->cp15.c14_pmevcntr_delta[counter] -=
1614             env->cp15.c14_pmevcntr[counter];
1615     }
1616 }
1617 
1618 void pmu_op_start(CPUARMState *env)
1619 {
1620     unsigned int i;
1621     pmccntr_op_start(env);
1622     for (i = 0; i < pmu_num_counters(env); i++) {
1623         pmevcntr_op_start(env, i);
1624     }
1625 }
1626 
1627 void pmu_op_finish(CPUARMState *env)
1628 {
1629     unsigned int i;
1630     pmccntr_op_finish(env);
1631     for (i = 0; i < pmu_num_counters(env); i++) {
1632         pmevcntr_op_finish(env, i);
1633     }
1634 }
1635 
1636 void pmu_pre_el_change(ARMCPU *cpu, void *ignored)
1637 {
1638     pmu_op_start(&cpu->env);
1639 }
1640 
1641 void pmu_post_el_change(ARMCPU *cpu, void *ignored)
1642 {
1643     pmu_op_finish(&cpu->env);
1644 }
1645 
1646 void arm_pmu_timer_cb(void *opaque)
1647 {
1648     ARMCPU *cpu = opaque;
1649 
1650     /*
1651      * Update all the counter values based on the current underlying counts,
1652      * triggering interrupts to be raised, if necessary. pmu_op_finish() also
1653      * has the effect of setting the cpu->pmu_timer to the next earliest time a
1654      * counter may expire.
1655      */
1656     pmu_op_start(&cpu->env);
1657     pmu_op_finish(&cpu->env);
1658 }
1659 
1660 static void pmcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1661                        uint64_t value)
1662 {
1663     pmu_op_start(env);
1664 
1665     if (value & PMCRC) {
1666         /* The counter has been reset */
1667         env->cp15.c15_ccnt = 0;
1668     }
1669 
1670     if (value & PMCRP) {
1671         unsigned int i;
1672         for (i = 0; i < pmu_num_counters(env); i++) {
1673             env->cp15.c14_pmevcntr[i] = 0;
1674         }
1675     }
1676 
1677     env->cp15.c9_pmcr &= ~PMCR_WRITEABLE_MASK;
1678     env->cp15.c9_pmcr |= (value & PMCR_WRITEABLE_MASK);
1679 
1680     pmu_op_finish(env);
1681 }
1682 
1683 static void pmswinc_write(CPUARMState *env, const ARMCPRegInfo *ri,
1684                           uint64_t value)
1685 {
1686     unsigned int i;
1687     for (i = 0; i < pmu_num_counters(env); i++) {
1688         /* Increment a counter's count iff: */
1689         if ((value & (1 << i)) && /* counter's bit is set */
1690                 /* counter is enabled and not filtered */
1691                 pmu_counter_enabled(env, i) &&
1692                 /* counter is SW_INCR */
1693                 (env->cp15.c14_pmevtyper[i] & PMXEVTYPER_EVTCOUNT) == 0x0) {
1694             pmevcntr_op_start(env, i);
1695 
1696             /*
1697              * Detect if this write causes an overflow since we can't predict
1698              * PMSWINC overflows like we can for other events
1699              */
1700             uint32_t new_pmswinc = env->cp15.c14_pmevcntr[i] + 1;
1701 
1702             if (env->cp15.c14_pmevcntr[i] & ~new_pmswinc & INT32_MIN) {
1703                 env->cp15.c9_pmovsr |= (1 << i);
1704                 pmu_update_irq(env);
1705             }
1706 
1707             env->cp15.c14_pmevcntr[i] = new_pmswinc;
1708 
1709             pmevcntr_op_finish(env, i);
1710         }
1711     }
1712 }
1713 
1714 static uint64_t pmccntr_read(CPUARMState *env, const ARMCPRegInfo *ri)
1715 {
1716     uint64_t ret;
1717     pmccntr_op_start(env);
1718     ret = env->cp15.c15_ccnt;
1719     pmccntr_op_finish(env);
1720     return ret;
1721 }
1722 
1723 static void pmselr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1724                          uint64_t value)
1725 {
1726     /* The value of PMSELR.SEL affects the behavior of PMXEVTYPER and
1727      * PMXEVCNTR. We allow [0..31] to be written to PMSELR here; in the
1728      * meanwhile, we check PMSELR.SEL when PMXEVTYPER and PMXEVCNTR are
1729      * accessed.
1730      */
1731     env->cp15.c9_pmselr = value & 0x1f;
1732 }
1733 
1734 static void pmccntr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1735                         uint64_t value)
1736 {
1737     pmccntr_op_start(env);
1738     env->cp15.c15_ccnt = value;
1739     pmccntr_op_finish(env);
1740 }
1741 
1742 static void pmccntr_write32(CPUARMState *env, const ARMCPRegInfo *ri,
1743                             uint64_t value)
1744 {
1745     uint64_t cur_val = pmccntr_read(env, NULL);
1746 
1747     pmccntr_write(env, ri, deposit64(cur_val, 0, 32, value));
1748 }
1749 
1750 static void pmccfiltr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1751                             uint64_t value)
1752 {
1753     pmccntr_op_start(env);
1754     env->cp15.pmccfiltr_el0 = value & PMCCFILTR_EL0;
1755     pmccntr_op_finish(env);
1756 }
1757 
1758 static void pmccfiltr_write_a32(CPUARMState *env, const ARMCPRegInfo *ri,
1759                             uint64_t value)
1760 {
1761     pmccntr_op_start(env);
1762     /* M is not accessible from AArch32 */
1763     env->cp15.pmccfiltr_el0 = (env->cp15.pmccfiltr_el0 & PMCCFILTR_M) |
1764         (value & PMCCFILTR);
1765     pmccntr_op_finish(env);
1766 }
1767 
1768 static uint64_t pmccfiltr_read_a32(CPUARMState *env, const ARMCPRegInfo *ri)
1769 {
1770     /* M is not visible in AArch32 */
1771     return env->cp15.pmccfiltr_el0 & PMCCFILTR;
1772 }
1773 
1774 static void pmcntenset_write(CPUARMState *env, const ARMCPRegInfo *ri,
1775                             uint64_t value)
1776 {
1777     value &= pmu_counter_mask(env);
1778     env->cp15.c9_pmcnten |= value;
1779 }
1780 
1781 static void pmcntenclr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1782                              uint64_t value)
1783 {
1784     value &= pmu_counter_mask(env);
1785     env->cp15.c9_pmcnten &= ~value;
1786 }
1787 
1788 static void pmovsr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1789                          uint64_t value)
1790 {
1791     value &= pmu_counter_mask(env);
1792     env->cp15.c9_pmovsr &= ~value;
1793     pmu_update_irq(env);
1794 }
1795 
1796 static void pmovsset_write(CPUARMState *env, const ARMCPRegInfo *ri,
1797                          uint64_t value)
1798 {
1799     value &= pmu_counter_mask(env);
1800     env->cp15.c9_pmovsr |= value;
1801     pmu_update_irq(env);
1802 }
1803 
1804 static void pmevtyper_write(CPUARMState *env, const ARMCPRegInfo *ri,
1805                              uint64_t value, const uint8_t counter)
1806 {
1807     if (counter == 31) {
1808         pmccfiltr_write(env, ri, value);
1809     } else if (counter < pmu_num_counters(env)) {
1810         pmevcntr_op_start(env, counter);
1811 
1812         /*
1813          * If this counter's event type is changing, store the current
1814          * underlying count for the new type in c14_pmevcntr_delta[counter] so
1815          * pmevcntr_op_finish has the correct baseline when it converts back to
1816          * a delta.
1817          */
1818         uint16_t old_event = env->cp15.c14_pmevtyper[counter] &
1819             PMXEVTYPER_EVTCOUNT;
1820         uint16_t new_event = value & PMXEVTYPER_EVTCOUNT;
1821         if (old_event != new_event) {
1822             uint64_t count = 0;
1823             if (event_supported(new_event)) {
1824                 uint16_t event_idx = supported_event_map[new_event];
1825                 count = pm_events[event_idx].get_count(env);
1826             }
1827             env->cp15.c14_pmevcntr_delta[counter] = count;
1828         }
1829 
1830         env->cp15.c14_pmevtyper[counter] = value & PMXEVTYPER_MASK;
1831         pmevcntr_op_finish(env, counter);
1832     }
1833     /* Attempts to access PMXEVTYPER are CONSTRAINED UNPREDICTABLE when
1834      * PMSELR value is equal to or greater than the number of implemented
1835      * counters, but not equal to 0x1f. We opt to behave as a RAZ/WI.
1836      */
1837 }
1838 
1839 static uint64_t pmevtyper_read(CPUARMState *env, const ARMCPRegInfo *ri,
1840                                const uint8_t counter)
1841 {
1842     if (counter == 31) {
1843         return env->cp15.pmccfiltr_el0;
1844     } else if (counter < pmu_num_counters(env)) {
1845         return env->cp15.c14_pmevtyper[counter];
1846     } else {
1847       /*
1848        * We opt to behave as a RAZ/WI when attempts to access PMXEVTYPER
1849        * are CONSTRAINED UNPREDICTABLE. See comments in pmevtyper_write().
1850        */
1851         return 0;
1852     }
1853 }
1854 
1855 static void pmevtyper_writefn(CPUARMState *env, const ARMCPRegInfo *ri,
1856                               uint64_t value)
1857 {
1858     uint8_t counter = ((ri->crm & 3) << 3) | (ri->opc2 & 7);
1859     pmevtyper_write(env, ri, value, counter);
1860 }
1861 
1862 static void pmevtyper_rawwrite(CPUARMState *env, const ARMCPRegInfo *ri,
1863                                uint64_t value)
1864 {
1865     uint8_t counter = ((ri->crm & 3) << 3) | (ri->opc2 & 7);
1866     env->cp15.c14_pmevtyper[counter] = value;
1867 
1868     /*
1869      * pmevtyper_rawwrite is called between a pair of pmu_op_start and
1870      * pmu_op_finish calls when loading saved state for a migration. Because
1871      * we're potentially updating the type of event here, the value written to
1872      * c14_pmevcntr_delta by the preceeding pmu_op_start call may be for a
1873      * different counter type. Therefore, we need to set this value to the
1874      * current count for the counter type we're writing so that pmu_op_finish
1875      * has the correct count for its calculation.
1876      */
1877     uint16_t event = value & PMXEVTYPER_EVTCOUNT;
1878     if (event_supported(event)) {
1879         uint16_t event_idx = supported_event_map[event];
1880         env->cp15.c14_pmevcntr_delta[counter] =
1881             pm_events[event_idx].get_count(env);
1882     }
1883 }
1884 
1885 static uint64_t pmevtyper_readfn(CPUARMState *env, const ARMCPRegInfo *ri)
1886 {
1887     uint8_t counter = ((ri->crm & 3) << 3) | (ri->opc2 & 7);
1888     return pmevtyper_read(env, ri, counter);
1889 }
1890 
1891 static void pmxevtyper_write(CPUARMState *env, const ARMCPRegInfo *ri,
1892                              uint64_t value)
1893 {
1894     pmevtyper_write(env, ri, value, env->cp15.c9_pmselr & 31);
1895 }
1896 
1897 static uint64_t pmxevtyper_read(CPUARMState *env, const ARMCPRegInfo *ri)
1898 {
1899     return pmevtyper_read(env, ri, env->cp15.c9_pmselr & 31);
1900 }
1901 
1902 static void pmevcntr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1903                              uint64_t value, uint8_t counter)
1904 {
1905     if (counter < pmu_num_counters(env)) {
1906         pmevcntr_op_start(env, counter);
1907         env->cp15.c14_pmevcntr[counter] = value;
1908         pmevcntr_op_finish(env, counter);
1909     }
1910     /*
1911      * We opt to behave as a RAZ/WI when attempts to access PM[X]EVCNTR
1912      * are CONSTRAINED UNPREDICTABLE.
1913      */
1914 }
1915 
1916 static uint64_t pmevcntr_read(CPUARMState *env, const ARMCPRegInfo *ri,
1917                               uint8_t counter)
1918 {
1919     if (counter < pmu_num_counters(env)) {
1920         uint64_t ret;
1921         pmevcntr_op_start(env, counter);
1922         ret = env->cp15.c14_pmevcntr[counter];
1923         pmevcntr_op_finish(env, counter);
1924         return ret;
1925     } else {
1926       /* We opt to behave as a RAZ/WI when attempts to access PM[X]EVCNTR
1927        * are CONSTRAINED UNPREDICTABLE. */
1928         return 0;
1929     }
1930 }
1931 
1932 static void pmevcntr_writefn(CPUARMState *env, const ARMCPRegInfo *ri,
1933                              uint64_t value)
1934 {
1935     uint8_t counter = ((ri->crm & 3) << 3) | (ri->opc2 & 7);
1936     pmevcntr_write(env, ri, value, counter);
1937 }
1938 
1939 static uint64_t pmevcntr_readfn(CPUARMState *env, const ARMCPRegInfo *ri)
1940 {
1941     uint8_t counter = ((ri->crm & 3) << 3) | (ri->opc2 & 7);
1942     return pmevcntr_read(env, ri, counter);
1943 }
1944 
1945 static void pmevcntr_rawwrite(CPUARMState *env, const ARMCPRegInfo *ri,
1946                              uint64_t value)
1947 {
1948     uint8_t counter = ((ri->crm & 3) << 3) | (ri->opc2 & 7);
1949     assert(counter < pmu_num_counters(env));
1950     env->cp15.c14_pmevcntr[counter] = value;
1951     pmevcntr_write(env, ri, value, counter);
1952 }
1953 
1954 static uint64_t pmevcntr_rawread(CPUARMState *env, const ARMCPRegInfo *ri)
1955 {
1956     uint8_t counter = ((ri->crm & 3) << 3) | (ri->opc2 & 7);
1957     assert(counter < pmu_num_counters(env));
1958     return env->cp15.c14_pmevcntr[counter];
1959 }
1960 
1961 static void pmxevcntr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1962                              uint64_t value)
1963 {
1964     pmevcntr_write(env, ri, value, env->cp15.c9_pmselr & 31);
1965 }
1966 
1967 static uint64_t pmxevcntr_read(CPUARMState *env, const ARMCPRegInfo *ri)
1968 {
1969     return pmevcntr_read(env, ri, env->cp15.c9_pmselr & 31);
1970 }
1971 
1972 static void pmuserenr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1973                             uint64_t value)
1974 {
1975     if (arm_feature(env, ARM_FEATURE_V8)) {
1976         env->cp15.c9_pmuserenr = value & 0xf;
1977     } else {
1978         env->cp15.c9_pmuserenr = value & 1;
1979     }
1980 }
1981 
1982 static void pmintenset_write(CPUARMState *env, const ARMCPRegInfo *ri,
1983                              uint64_t value)
1984 {
1985     /* We have no event counters so only the C bit can be changed */
1986     value &= pmu_counter_mask(env);
1987     env->cp15.c9_pminten |= value;
1988     pmu_update_irq(env);
1989 }
1990 
1991 static void pmintenclr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1992                              uint64_t value)
1993 {
1994     value &= pmu_counter_mask(env);
1995     env->cp15.c9_pminten &= ~value;
1996     pmu_update_irq(env);
1997 }
1998 
1999 static void vbar_write(CPUARMState *env, const ARMCPRegInfo *ri,
2000                        uint64_t value)
2001 {
2002     /* Note that even though the AArch64 view of this register has bits
2003      * [10:0] all RES0 we can only mask the bottom 5, to comply with the
2004      * architectural requirements for bits which are RES0 only in some
2005      * contexts. (ARMv8 would permit us to do no masking at all, but ARMv7
2006      * requires the bottom five bits to be RAZ/WI because they're UNK/SBZP.)
2007      */
2008     raw_write(env, ri, value & ~0x1FULL);
2009 }
2010 
2011 static void scr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
2012 {
2013     /* Begin with base v8.0 state.  */
2014     uint32_t valid_mask = 0x3fff;
2015     ARMCPU *cpu = env_archcpu(env);
2016 
2017     if (ri->state == ARM_CP_STATE_AA64) {
2018         value |= SCR_FW | SCR_AW;   /* these two bits are RES1.  */
2019         valid_mask &= ~SCR_NET;
2020 
2021         if (cpu_isar_feature(aa64_lor, cpu)) {
2022             valid_mask |= SCR_TLOR;
2023         }
2024         if (cpu_isar_feature(aa64_pauth, cpu)) {
2025             valid_mask |= SCR_API | SCR_APK;
2026         }
2027         if (cpu_isar_feature(aa64_mte, cpu)) {
2028             valid_mask |= SCR_ATA;
2029         }
2030     } else {
2031         valid_mask &= ~(SCR_RW | SCR_ST);
2032     }
2033 
2034     if (!arm_feature(env, ARM_FEATURE_EL2)) {
2035         valid_mask &= ~SCR_HCE;
2036 
2037         /* On ARMv7, SMD (or SCD as it is called in v7) is only
2038          * supported if EL2 exists. The bit is UNK/SBZP when
2039          * EL2 is unavailable. In QEMU ARMv7, we force it to always zero
2040          * when EL2 is unavailable.
2041          * On ARMv8, this bit is always available.
2042          */
2043         if (arm_feature(env, ARM_FEATURE_V7) &&
2044             !arm_feature(env, ARM_FEATURE_V8)) {
2045             valid_mask &= ~SCR_SMD;
2046         }
2047     }
2048 
2049     /* Clear all-context RES0 bits.  */
2050     value &= valid_mask;
2051     raw_write(env, ri, value);
2052 }
2053 
2054 static CPAccessResult access_aa64_tid2(CPUARMState *env,
2055                                        const ARMCPRegInfo *ri,
2056                                        bool isread)
2057 {
2058     if (arm_current_el(env) == 1 && (arm_hcr_el2_eff(env) & HCR_TID2)) {
2059         return CP_ACCESS_TRAP_EL2;
2060     }
2061 
2062     return CP_ACCESS_OK;
2063 }
2064 
2065 static uint64_t ccsidr_read(CPUARMState *env, const ARMCPRegInfo *ri)
2066 {
2067     ARMCPU *cpu = env_archcpu(env);
2068 
2069     /* Acquire the CSSELR index from the bank corresponding to the CCSIDR
2070      * bank
2071      */
2072     uint32_t index = A32_BANKED_REG_GET(env, csselr,
2073                                         ri->secure & ARM_CP_SECSTATE_S);
2074 
2075     return cpu->ccsidr[index];
2076 }
2077 
2078 static void csselr_write(CPUARMState *env, const ARMCPRegInfo *ri,
2079                          uint64_t value)
2080 {
2081     raw_write(env, ri, value & 0xf);
2082 }
2083 
2084 static uint64_t isr_read(CPUARMState *env, const ARMCPRegInfo *ri)
2085 {
2086     CPUState *cs = env_cpu(env);
2087     uint64_t hcr_el2 = arm_hcr_el2_eff(env);
2088     uint64_t ret = 0;
2089     bool allow_virt = (arm_current_el(env) == 1 &&
2090                        (!arm_is_secure_below_el3(env) ||
2091                         (env->cp15.scr_el3 & SCR_EEL2)));
2092 
2093     if (allow_virt && (hcr_el2 & HCR_IMO)) {
2094         if (cs->interrupt_request & CPU_INTERRUPT_VIRQ) {
2095             ret |= CPSR_I;
2096         }
2097     } else {
2098         if (cs->interrupt_request & CPU_INTERRUPT_HARD) {
2099             ret |= CPSR_I;
2100         }
2101     }
2102 
2103     if (allow_virt && (hcr_el2 & HCR_FMO)) {
2104         if (cs->interrupt_request & CPU_INTERRUPT_VFIQ) {
2105             ret |= CPSR_F;
2106         }
2107     } else {
2108         if (cs->interrupt_request & CPU_INTERRUPT_FIQ) {
2109             ret |= CPSR_F;
2110         }
2111     }
2112 
2113     /* External aborts are not possible in QEMU so A bit is always clear */
2114     return ret;
2115 }
2116 
2117 static CPAccessResult access_aa64_tid1(CPUARMState *env, const ARMCPRegInfo *ri,
2118                                        bool isread)
2119 {
2120     if (arm_current_el(env) == 1 && (arm_hcr_el2_eff(env) & HCR_TID1)) {
2121         return CP_ACCESS_TRAP_EL2;
2122     }
2123 
2124     return CP_ACCESS_OK;
2125 }
2126 
2127 static CPAccessResult access_aa32_tid1(CPUARMState *env, const ARMCPRegInfo *ri,
2128                                        bool isread)
2129 {
2130     if (arm_feature(env, ARM_FEATURE_V8)) {
2131         return access_aa64_tid1(env, ri, isread);
2132     }
2133 
2134     return CP_ACCESS_OK;
2135 }
2136 
2137 static const ARMCPRegInfo v7_cp_reginfo[] = {
2138     /* the old v6 WFI, UNPREDICTABLE in v7 but we choose to NOP */
2139     { .name = "NOP", .cp = 15, .crn = 7, .crm = 0, .opc1 = 0, .opc2 = 4,
2140       .access = PL1_W, .type = ARM_CP_NOP },
2141     /* Performance monitors are implementation defined in v7,
2142      * but with an ARM recommended set of registers, which we
2143      * follow.
2144      *
2145      * Performance registers fall into three categories:
2146      *  (a) always UNDEF in PL0, RW in PL1 (PMINTENSET, PMINTENCLR)
2147      *  (b) RO in PL0 (ie UNDEF on write), RW in PL1 (PMUSERENR)
2148      *  (c) UNDEF in PL0 if PMUSERENR.EN==0, otherwise accessible (all others)
2149      * For the cases controlled by PMUSERENR we must set .access to PL0_RW
2150      * or PL0_RO as appropriate and then check PMUSERENR in the helper fn.
2151      */
2152     { .name = "PMCNTENSET", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 1,
2153       .access = PL0_RW, .type = ARM_CP_ALIAS,
2154       .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmcnten),
2155       .writefn = pmcntenset_write,
2156       .accessfn = pmreg_access,
2157       .raw_writefn = raw_write },
2158     { .name = "PMCNTENSET_EL0", .state = ARM_CP_STATE_AA64,
2159       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 1,
2160       .access = PL0_RW, .accessfn = pmreg_access,
2161       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmcnten), .resetvalue = 0,
2162       .writefn = pmcntenset_write, .raw_writefn = raw_write },
2163     { .name = "PMCNTENCLR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 2,
2164       .access = PL0_RW,
2165       .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmcnten),
2166       .accessfn = pmreg_access,
2167       .writefn = pmcntenclr_write,
2168       .type = ARM_CP_ALIAS },
2169     { .name = "PMCNTENCLR_EL0", .state = ARM_CP_STATE_AA64,
2170       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 2,
2171       .access = PL0_RW, .accessfn = pmreg_access,
2172       .type = ARM_CP_ALIAS,
2173       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmcnten),
2174       .writefn = pmcntenclr_write },
2175     { .name = "PMOVSR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 3,
2176       .access = PL0_RW, .type = ARM_CP_IO,
2177       .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmovsr),
2178       .accessfn = pmreg_access,
2179       .writefn = pmovsr_write,
2180       .raw_writefn = raw_write },
2181     { .name = "PMOVSCLR_EL0", .state = ARM_CP_STATE_AA64,
2182       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 3,
2183       .access = PL0_RW, .accessfn = pmreg_access,
2184       .type = ARM_CP_ALIAS | ARM_CP_IO,
2185       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmovsr),
2186       .writefn = pmovsr_write,
2187       .raw_writefn = raw_write },
2188     { .name = "PMSWINC", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 4,
2189       .access = PL0_W, .accessfn = pmreg_access_swinc,
2190       .type = ARM_CP_NO_RAW | ARM_CP_IO,
2191       .writefn = pmswinc_write },
2192     { .name = "PMSWINC_EL0", .state = ARM_CP_STATE_AA64,
2193       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 4,
2194       .access = PL0_W, .accessfn = pmreg_access_swinc,
2195       .type = ARM_CP_NO_RAW | ARM_CP_IO,
2196       .writefn = pmswinc_write },
2197     { .name = "PMSELR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 5,
2198       .access = PL0_RW, .type = ARM_CP_ALIAS,
2199       .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmselr),
2200       .accessfn = pmreg_access_selr, .writefn = pmselr_write,
2201       .raw_writefn = raw_write},
2202     { .name = "PMSELR_EL0", .state = ARM_CP_STATE_AA64,
2203       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 5,
2204       .access = PL0_RW, .accessfn = pmreg_access_selr,
2205       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmselr),
2206       .writefn = pmselr_write, .raw_writefn = raw_write, },
2207     { .name = "PMCCNTR", .cp = 15, .crn = 9, .crm = 13, .opc1 = 0, .opc2 = 0,
2208       .access = PL0_RW, .resetvalue = 0, .type = ARM_CP_ALIAS | ARM_CP_IO,
2209       .readfn = pmccntr_read, .writefn = pmccntr_write32,
2210       .accessfn = pmreg_access_ccntr },
2211     { .name = "PMCCNTR_EL0", .state = ARM_CP_STATE_AA64,
2212       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 13, .opc2 = 0,
2213       .access = PL0_RW, .accessfn = pmreg_access_ccntr,
2214       .type = ARM_CP_IO,
2215       .fieldoffset = offsetof(CPUARMState, cp15.c15_ccnt),
2216       .readfn = pmccntr_read, .writefn = pmccntr_write,
2217       .raw_readfn = raw_read, .raw_writefn = raw_write, },
2218     { .name = "PMCCFILTR", .cp = 15, .opc1 = 0, .crn = 14, .crm = 15, .opc2 = 7,
2219       .writefn = pmccfiltr_write_a32, .readfn = pmccfiltr_read_a32,
2220       .access = PL0_RW, .accessfn = pmreg_access,
2221       .type = ARM_CP_ALIAS | ARM_CP_IO,
2222       .resetvalue = 0, },
2223     { .name = "PMCCFILTR_EL0", .state = ARM_CP_STATE_AA64,
2224       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 15, .opc2 = 7,
2225       .writefn = pmccfiltr_write, .raw_writefn = raw_write,
2226       .access = PL0_RW, .accessfn = pmreg_access,
2227       .type = ARM_CP_IO,
2228       .fieldoffset = offsetof(CPUARMState, cp15.pmccfiltr_el0),
2229       .resetvalue = 0, },
2230     { .name = "PMXEVTYPER", .cp = 15, .crn = 9, .crm = 13, .opc1 = 0, .opc2 = 1,
2231       .access = PL0_RW, .type = ARM_CP_NO_RAW | ARM_CP_IO,
2232       .accessfn = pmreg_access,
2233       .writefn = pmxevtyper_write, .readfn = pmxevtyper_read },
2234     { .name = "PMXEVTYPER_EL0", .state = ARM_CP_STATE_AA64,
2235       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 13, .opc2 = 1,
2236       .access = PL0_RW, .type = ARM_CP_NO_RAW | ARM_CP_IO,
2237       .accessfn = pmreg_access,
2238       .writefn = pmxevtyper_write, .readfn = pmxevtyper_read },
2239     { .name = "PMXEVCNTR", .cp = 15, .crn = 9, .crm = 13, .opc1 = 0, .opc2 = 2,
2240       .access = PL0_RW, .type = ARM_CP_NO_RAW | ARM_CP_IO,
2241       .accessfn = pmreg_access_xevcntr,
2242       .writefn = pmxevcntr_write, .readfn = pmxevcntr_read },
2243     { .name = "PMXEVCNTR_EL0", .state = ARM_CP_STATE_AA64,
2244       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 13, .opc2 = 2,
2245       .access = PL0_RW, .type = ARM_CP_NO_RAW | ARM_CP_IO,
2246       .accessfn = pmreg_access_xevcntr,
2247       .writefn = pmxevcntr_write, .readfn = pmxevcntr_read },
2248     { .name = "PMUSERENR", .cp = 15, .crn = 9, .crm = 14, .opc1 = 0, .opc2 = 0,
2249       .access = PL0_R | PL1_RW, .accessfn = access_tpm,
2250       .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmuserenr),
2251       .resetvalue = 0,
2252       .writefn = pmuserenr_write, .raw_writefn = raw_write },
2253     { .name = "PMUSERENR_EL0", .state = ARM_CP_STATE_AA64,
2254       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 14, .opc2 = 0,
2255       .access = PL0_R | PL1_RW, .accessfn = access_tpm, .type = ARM_CP_ALIAS,
2256       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmuserenr),
2257       .resetvalue = 0,
2258       .writefn = pmuserenr_write, .raw_writefn = raw_write },
2259     { .name = "PMINTENSET", .cp = 15, .crn = 9, .crm = 14, .opc1 = 0, .opc2 = 1,
2260       .access = PL1_RW, .accessfn = access_tpm,
2261       .type = ARM_CP_ALIAS | ARM_CP_IO,
2262       .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pminten),
2263       .resetvalue = 0,
2264       .writefn = pmintenset_write, .raw_writefn = raw_write },
2265     { .name = "PMINTENSET_EL1", .state = ARM_CP_STATE_AA64,
2266       .opc0 = 3, .opc1 = 0, .crn = 9, .crm = 14, .opc2 = 1,
2267       .access = PL1_RW, .accessfn = access_tpm,
2268       .type = ARM_CP_IO,
2269       .fieldoffset = offsetof(CPUARMState, cp15.c9_pminten),
2270       .writefn = pmintenset_write, .raw_writefn = raw_write,
2271       .resetvalue = 0x0 },
2272     { .name = "PMINTENCLR", .cp = 15, .crn = 9, .crm = 14, .opc1 = 0, .opc2 = 2,
2273       .access = PL1_RW, .accessfn = access_tpm,
2274       .type = ARM_CP_ALIAS | ARM_CP_IO | ARM_CP_NO_RAW,
2275       .fieldoffset = offsetof(CPUARMState, cp15.c9_pminten),
2276       .writefn = pmintenclr_write, },
2277     { .name = "PMINTENCLR_EL1", .state = ARM_CP_STATE_AA64,
2278       .opc0 = 3, .opc1 = 0, .crn = 9, .crm = 14, .opc2 = 2,
2279       .access = PL1_RW, .accessfn = access_tpm,
2280       .type = ARM_CP_ALIAS | ARM_CP_IO | ARM_CP_NO_RAW,
2281       .fieldoffset = offsetof(CPUARMState, cp15.c9_pminten),
2282       .writefn = pmintenclr_write },
2283     { .name = "CCSIDR", .state = ARM_CP_STATE_BOTH,
2284       .opc0 = 3, .crn = 0, .crm = 0, .opc1 = 1, .opc2 = 0,
2285       .access = PL1_R,
2286       .accessfn = access_aa64_tid2,
2287       .readfn = ccsidr_read, .type = ARM_CP_NO_RAW },
2288     { .name = "CSSELR", .state = ARM_CP_STATE_BOTH,
2289       .opc0 = 3, .crn = 0, .crm = 0, .opc1 = 2, .opc2 = 0,
2290       .access = PL1_RW,
2291       .accessfn = access_aa64_tid2,
2292       .writefn = csselr_write, .resetvalue = 0,
2293       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.csselr_s),
2294                              offsetof(CPUARMState, cp15.csselr_ns) } },
2295     /* Auxiliary ID register: this actually has an IMPDEF value but for now
2296      * just RAZ for all cores:
2297      */
2298     { .name = "AIDR", .state = ARM_CP_STATE_BOTH,
2299       .opc0 = 3, .opc1 = 1, .crn = 0, .crm = 0, .opc2 = 7,
2300       .access = PL1_R, .type = ARM_CP_CONST,
2301       .accessfn = access_aa64_tid1,
2302       .resetvalue = 0 },
2303     /* Auxiliary fault status registers: these also are IMPDEF, and we
2304      * choose to RAZ/WI for all cores.
2305      */
2306     { .name = "AFSR0_EL1", .state = ARM_CP_STATE_BOTH,
2307       .opc0 = 3, .opc1 = 0, .crn = 5, .crm = 1, .opc2 = 0,
2308       .access = PL1_RW, .accessfn = access_tvm_trvm,
2309       .type = ARM_CP_CONST, .resetvalue = 0 },
2310     { .name = "AFSR1_EL1", .state = ARM_CP_STATE_BOTH,
2311       .opc0 = 3, .opc1 = 0, .crn = 5, .crm = 1, .opc2 = 1,
2312       .access = PL1_RW, .accessfn = access_tvm_trvm,
2313       .type = ARM_CP_CONST, .resetvalue = 0 },
2314     /* MAIR can just read-as-written because we don't implement caches
2315      * and so don't need to care about memory attributes.
2316      */
2317     { .name = "MAIR_EL1", .state = ARM_CP_STATE_AA64,
2318       .opc0 = 3, .opc1 = 0, .crn = 10, .crm = 2, .opc2 = 0,
2319       .access = PL1_RW, .accessfn = access_tvm_trvm,
2320       .fieldoffset = offsetof(CPUARMState, cp15.mair_el[1]),
2321       .resetvalue = 0 },
2322     { .name = "MAIR_EL3", .state = ARM_CP_STATE_AA64,
2323       .opc0 = 3, .opc1 = 6, .crn = 10, .crm = 2, .opc2 = 0,
2324       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.mair_el[3]),
2325       .resetvalue = 0 },
2326     /* For non-long-descriptor page tables these are PRRR and NMRR;
2327      * regardless they still act as reads-as-written for QEMU.
2328      */
2329      /* MAIR0/1 are defined separately from their 64-bit counterpart which
2330       * allows them to assign the correct fieldoffset based on the endianness
2331       * handled in the field definitions.
2332       */
2333     { .name = "MAIR0", .state = ARM_CP_STATE_AA32,
2334       .cp = 15, .opc1 = 0, .crn = 10, .crm = 2, .opc2 = 0,
2335       .access = PL1_RW, .accessfn = access_tvm_trvm,
2336       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.mair0_s),
2337                              offsetof(CPUARMState, cp15.mair0_ns) },
2338       .resetfn = arm_cp_reset_ignore },
2339     { .name = "MAIR1", .state = ARM_CP_STATE_AA32,
2340       .cp = 15, .opc1 = 0, .crn = 10, .crm = 2, .opc2 = 1,
2341       .access = PL1_RW, .accessfn = access_tvm_trvm,
2342       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.mair1_s),
2343                              offsetof(CPUARMState, cp15.mair1_ns) },
2344       .resetfn = arm_cp_reset_ignore },
2345     { .name = "ISR_EL1", .state = ARM_CP_STATE_BOTH,
2346       .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 1, .opc2 = 0,
2347       .type = ARM_CP_NO_RAW, .access = PL1_R, .readfn = isr_read },
2348     /* 32 bit ITLB invalidates */
2349     { .name = "ITLBIALL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 0,
2350       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlb,
2351       .writefn = tlbiall_write },
2352     { .name = "ITLBIMVA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 1,
2353       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlb,
2354       .writefn = tlbimva_write },
2355     { .name = "ITLBIASID", .cp = 15, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 2,
2356       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlb,
2357       .writefn = tlbiasid_write },
2358     /* 32 bit DTLB invalidates */
2359     { .name = "DTLBIALL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 0,
2360       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlb,
2361       .writefn = tlbiall_write },
2362     { .name = "DTLBIMVA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 1,
2363       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlb,
2364       .writefn = tlbimva_write },
2365     { .name = "DTLBIASID", .cp = 15, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 2,
2366       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlb,
2367       .writefn = tlbiasid_write },
2368     /* 32 bit TLB invalidates */
2369     { .name = "TLBIALL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 0,
2370       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlb,
2371       .writefn = tlbiall_write },
2372     { .name = "TLBIMVA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 1,
2373       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlb,
2374       .writefn = tlbimva_write },
2375     { .name = "TLBIASID", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 2,
2376       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlb,
2377       .writefn = tlbiasid_write },
2378     { .name = "TLBIMVAA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 3,
2379       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlb,
2380       .writefn = tlbimvaa_write },
2381     REGINFO_SENTINEL
2382 };
2383 
2384 static const ARMCPRegInfo v7mp_cp_reginfo[] = {
2385     /* 32 bit TLB invalidates, Inner Shareable */
2386     { .name = "TLBIALLIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 0,
2387       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlb,
2388       .writefn = tlbiall_is_write },
2389     { .name = "TLBIMVAIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 1,
2390       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlb,
2391       .writefn = tlbimva_is_write },
2392     { .name = "TLBIASIDIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 2,
2393       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlb,
2394       .writefn = tlbiasid_is_write },
2395     { .name = "TLBIMVAAIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 3,
2396       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlb,
2397       .writefn = tlbimvaa_is_write },
2398     REGINFO_SENTINEL
2399 };
2400 
2401 static const ARMCPRegInfo pmovsset_cp_reginfo[] = {
2402     /* PMOVSSET is not implemented in v7 before v7ve */
2403     { .name = "PMOVSSET", .cp = 15, .opc1 = 0, .crn = 9, .crm = 14, .opc2 = 3,
2404       .access = PL0_RW, .accessfn = pmreg_access,
2405       .type = ARM_CP_ALIAS | ARM_CP_IO,
2406       .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmovsr),
2407       .writefn = pmovsset_write,
2408       .raw_writefn = raw_write },
2409     { .name = "PMOVSSET_EL0", .state = ARM_CP_STATE_AA64,
2410       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 14, .opc2 = 3,
2411       .access = PL0_RW, .accessfn = pmreg_access,
2412       .type = ARM_CP_ALIAS | ARM_CP_IO,
2413       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmovsr),
2414       .writefn = pmovsset_write,
2415       .raw_writefn = raw_write },
2416     REGINFO_SENTINEL
2417 };
2418 
2419 static void teecr_write(CPUARMState *env, const ARMCPRegInfo *ri,
2420                         uint64_t value)
2421 {
2422     value &= 1;
2423     env->teecr = value;
2424 }
2425 
2426 static CPAccessResult teehbr_access(CPUARMState *env, const ARMCPRegInfo *ri,
2427                                     bool isread)
2428 {
2429     if (arm_current_el(env) == 0 && (env->teecr & 1)) {
2430         return CP_ACCESS_TRAP;
2431     }
2432     return CP_ACCESS_OK;
2433 }
2434 
2435 static const ARMCPRegInfo t2ee_cp_reginfo[] = {
2436     { .name = "TEECR", .cp = 14, .crn = 0, .crm = 0, .opc1 = 6, .opc2 = 0,
2437       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, teecr),
2438       .resetvalue = 0,
2439       .writefn = teecr_write },
2440     { .name = "TEEHBR", .cp = 14, .crn = 1, .crm = 0, .opc1 = 6, .opc2 = 0,
2441       .access = PL0_RW, .fieldoffset = offsetof(CPUARMState, teehbr),
2442       .accessfn = teehbr_access, .resetvalue = 0 },
2443     REGINFO_SENTINEL
2444 };
2445 
2446 static const ARMCPRegInfo v6k_cp_reginfo[] = {
2447     { .name = "TPIDR_EL0", .state = ARM_CP_STATE_AA64,
2448       .opc0 = 3, .opc1 = 3, .opc2 = 2, .crn = 13, .crm = 0,
2449       .access = PL0_RW,
2450       .fieldoffset = offsetof(CPUARMState, cp15.tpidr_el[0]), .resetvalue = 0 },
2451     { .name = "TPIDRURW", .cp = 15, .crn = 13, .crm = 0, .opc1 = 0, .opc2 = 2,
2452       .access = PL0_RW,
2453       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.tpidrurw_s),
2454                              offsetoflow32(CPUARMState, cp15.tpidrurw_ns) },
2455       .resetfn = arm_cp_reset_ignore },
2456     { .name = "TPIDRRO_EL0", .state = ARM_CP_STATE_AA64,
2457       .opc0 = 3, .opc1 = 3, .opc2 = 3, .crn = 13, .crm = 0,
2458       .access = PL0_R|PL1_W,
2459       .fieldoffset = offsetof(CPUARMState, cp15.tpidrro_el[0]),
2460       .resetvalue = 0},
2461     { .name = "TPIDRURO", .cp = 15, .crn = 13, .crm = 0, .opc1 = 0, .opc2 = 3,
2462       .access = PL0_R|PL1_W,
2463       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.tpidruro_s),
2464                              offsetoflow32(CPUARMState, cp15.tpidruro_ns) },
2465       .resetfn = arm_cp_reset_ignore },
2466     { .name = "TPIDR_EL1", .state = ARM_CP_STATE_AA64,
2467       .opc0 = 3, .opc1 = 0, .opc2 = 4, .crn = 13, .crm = 0,
2468       .access = PL1_RW,
2469       .fieldoffset = offsetof(CPUARMState, cp15.tpidr_el[1]), .resetvalue = 0 },
2470     { .name = "TPIDRPRW", .opc1 = 0, .cp = 15, .crn = 13, .crm = 0, .opc2 = 4,
2471       .access = PL1_RW,
2472       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.tpidrprw_s),
2473                              offsetoflow32(CPUARMState, cp15.tpidrprw_ns) },
2474       .resetvalue = 0 },
2475     REGINFO_SENTINEL
2476 };
2477 
2478 #ifndef CONFIG_USER_ONLY
2479 
2480 static CPAccessResult gt_cntfrq_access(CPUARMState *env, const ARMCPRegInfo *ri,
2481                                        bool isread)
2482 {
2483     /* CNTFRQ: not visible from PL0 if both PL0PCTEN and PL0VCTEN are zero.
2484      * Writable only at the highest implemented exception level.
2485      */
2486     int el = arm_current_el(env);
2487     uint64_t hcr;
2488     uint32_t cntkctl;
2489 
2490     switch (el) {
2491     case 0:
2492         hcr = arm_hcr_el2_eff(env);
2493         if ((hcr & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)) {
2494             cntkctl = env->cp15.cnthctl_el2;
2495         } else {
2496             cntkctl = env->cp15.c14_cntkctl;
2497         }
2498         if (!extract32(cntkctl, 0, 2)) {
2499             return CP_ACCESS_TRAP;
2500         }
2501         break;
2502     case 1:
2503         if (!isread && ri->state == ARM_CP_STATE_AA32 &&
2504             arm_is_secure_below_el3(env)) {
2505             /* Accesses from 32-bit Secure EL1 UNDEF (*not* trap to EL3!) */
2506             return CP_ACCESS_TRAP_UNCATEGORIZED;
2507         }
2508         break;
2509     case 2:
2510     case 3:
2511         break;
2512     }
2513 
2514     if (!isread && el < arm_highest_el(env)) {
2515         return CP_ACCESS_TRAP_UNCATEGORIZED;
2516     }
2517 
2518     return CP_ACCESS_OK;
2519 }
2520 
2521 static CPAccessResult gt_counter_access(CPUARMState *env, int timeridx,
2522                                         bool isread)
2523 {
2524     unsigned int cur_el = arm_current_el(env);
2525     bool secure = arm_is_secure(env);
2526     uint64_t hcr = arm_hcr_el2_eff(env);
2527 
2528     switch (cur_el) {
2529     case 0:
2530         /* If HCR_EL2.<E2H,TGE> == '11': check CNTHCTL_EL2.EL0[PV]CTEN. */
2531         if ((hcr & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)) {
2532             return (extract32(env->cp15.cnthctl_el2, timeridx, 1)
2533                     ? CP_ACCESS_OK : CP_ACCESS_TRAP_EL2);
2534         }
2535 
2536         /* CNT[PV]CT: not visible from PL0 if EL0[PV]CTEN is zero */
2537         if (!extract32(env->cp15.c14_cntkctl, timeridx, 1)) {
2538             return CP_ACCESS_TRAP;
2539         }
2540 
2541         /* If HCR_EL2.<E2H,TGE> == '10': check CNTHCTL_EL2.EL1PCTEN. */
2542         if (hcr & HCR_E2H) {
2543             if (timeridx == GTIMER_PHYS &&
2544                 !extract32(env->cp15.cnthctl_el2, 10, 1)) {
2545                 return CP_ACCESS_TRAP_EL2;
2546             }
2547         } else {
2548             /* If HCR_EL2.<E2H> == 0: check CNTHCTL_EL2.EL1PCEN. */
2549             if (arm_feature(env, ARM_FEATURE_EL2) &&
2550                 timeridx == GTIMER_PHYS && !secure &&
2551                 !extract32(env->cp15.cnthctl_el2, 1, 1)) {
2552                 return CP_ACCESS_TRAP_EL2;
2553             }
2554         }
2555         break;
2556 
2557     case 1:
2558         /* Check CNTHCTL_EL2.EL1PCTEN, which changes location based on E2H. */
2559         if (arm_feature(env, ARM_FEATURE_EL2) &&
2560             timeridx == GTIMER_PHYS && !secure &&
2561             (hcr & HCR_E2H
2562              ? !extract32(env->cp15.cnthctl_el2, 10, 1)
2563              : !extract32(env->cp15.cnthctl_el2, 0, 1))) {
2564             return CP_ACCESS_TRAP_EL2;
2565         }
2566         break;
2567     }
2568     return CP_ACCESS_OK;
2569 }
2570 
2571 static CPAccessResult gt_timer_access(CPUARMState *env, int timeridx,
2572                                       bool isread)
2573 {
2574     unsigned int cur_el = arm_current_el(env);
2575     bool secure = arm_is_secure(env);
2576     uint64_t hcr = arm_hcr_el2_eff(env);
2577 
2578     switch (cur_el) {
2579     case 0:
2580         if ((hcr & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)) {
2581             /* If HCR_EL2.<E2H,TGE> == '11': check CNTHCTL_EL2.EL0[PV]TEN. */
2582             return (extract32(env->cp15.cnthctl_el2, 9 - timeridx, 1)
2583                     ? CP_ACCESS_OK : CP_ACCESS_TRAP_EL2);
2584         }
2585 
2586         /*
2587          * CNT[PV]_CVAL, CNT[PV]_CTL, CNT[PV]_TVAL: not visible from
2588          * EL0 if EL0[PV]TEN is zero.
2589          */
2590         if (!extract32(env->cp15.c14_cntkctl, 9 - timeridx, 1)) {
2591             return CP_ACCESS_TRAP;
2592         }
2593         /* fall through */
2594 
2595     case 1:
2596         if (arm_feature(env, ARM_FEATURE_EL2) &&
2597             timeridx == GTIMER_PHYS && !secure) {
2598             if (hcr & HCR_E2H) {
2599                 /* If HCR_EL2.<E2H,TGE> == '10': check CNTHCTL_EL2.EL1PTEN. */
2600                 if (!extract32(env->cp15.cnthctl_el2, 11, 1)) {
2601                     return CP_ACCESS_TRAP_EL2;
2602                 }
2603             } else {
2604                 /* If HCR_EL2.<E2H> == 0: check CNTHCTL_EL2.EL1PCEN. */
2605                 if (!extract32(env->cp15.cnthctl_el2, 1, 1)) {
2606                     return CP_ACCESS_TRAP_EL2;
2607                 }
2608             }
2609         }
2610         break;
2611     }
2612     return CP_ACCESS_OK;
2613 }
2614 
2615 static CPAccessResult gt_pct_access(CPUARMState *env,
2616                                     const ARMCPRegInfo *ri,
2617                                     bool isread)
2618 {
2619     return gt_counter_access(env, GTIMER_PHYS, isread);
2620 }
2621 
2622 static CPAccessResult gt_vct_access(CPUARMState *env,
2623                                     const ARMCPRegInfo *ri,
2624                                     bool isread)
2625 {
2626     return gt_counter_access(env, GTIMER_VIRT, isread);
2627 }
2628 
2629 static CPAccessResult gt_ptimer_access(CPUARMState *env, const ARMCPRegInfo *ri,
2630                                        bool isread)
2631 {
2632     return gt_timer_access(env, GTIMER_PHYS, isread);
2633 }
2634 
2635 static CPAccessResult gt_vtimer_access(CPUARMState *env, const ARMCPRegInfo *ri,
2636                                        bool isread)
2637 {
2638     return gt_timer_access(env, GTIMER_VIRT, isread);
2639 }
2640 
2641 static CPAccessResult gt_stimer_access(CPUARMState *env,
2642                                        const ARMCPRegInfo *ri,
2643                                        bool isread)
2644 {
2645     /* The AArch64 register view of the secure physical timer is
2646      * always accessible from EL3, and configurably accessible from
2647      * Secure EL1.
2648      */
2649     switch (arm_current_el(env)) {
2650     case 1:
2651         if (!arm_is_secure(env)) {
2652             return CP_ACCESS_TRAP;
2653         }
2654         if (!(env->cp15.scr_el3 & SCR_ST)) {
2655             return CP_ACCESS_TRAP_EL3;
2656         }
2657         return CP_ACCESS_OK;
2658     case 0:
2659     case 2:
2660         return CP_ACCESS_TRAP;
2661     case 3:
2662         return CP_ACCESS_OK;
2663     default:
2664         g_assert_not_reached();
2665     }
2666 }
2667 
2668 static uint64_t gt_get_countervalue(CPUARMState *env)
2669 {
2670     ARMCPU *cpu = env_archcpu(env);
2671 
2672     return qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) / gt_cntfrq_period_ns(cpu);
2673 }
2674 
2675 static void gt_recalc_timer(ARMCPU *cpu, int timeridx)
2676 {
2677     ARMGenericTimer *gt = &cpu->env.cp15.c14_timer[timeridx];
2678 
2679     if (gt->ctl & 1) {
2680         /* Timer enabled: calculate and set current ISTATUS, irq, and
2681          * reset timer to when ISTATUS next has to change
2682          */
2683         uint64_t offset = timeridx == GTIMER_VIRT ?
2684                                       cpu->env.cp15.cntvoff_el2 : 0;
2685         uint64_t count = gt_get_countervalue(&cpu->env);
2686         /* Note that this must be unsigned 64 bit arithmetic: */
2687         int istatus = count - offset >= gt->cval;
2688         uint64_t nexttick;
2689         int irqstate;
2690 
2691         gt->ctl = deposit32(gt->ctl, 2, 1, istatus);
2692 
2693         irqstate = (istatus && !(gt->ctl & 2));
2694         qemu_set_irq(cpu->gt_timer_outputs[timeridx], irqstate);
2695 
2696         if (istatus) {
2697             /* Next transition is when count rolls back over to zero */
2698             nexttick = UINT64_MAX;
2699         } else {
2700             /* Next transition is when we hit cval */
2701             nexttick = gt->cval + offset;
2702         }
2703         /* Note that the desired next expiry time might be beyond the
2704          * signed-64-bit range of a QEMUTimer -- in this case we just
2705          * set the timer for as far in the future as possible. When the
2706          * timer expires we will reset the timer for any remaining period.
2707          */
2708         if (nexttick > INT64_MAX / gt_cntfrq_period_ns(cpu)) {
2709             timer_mod_ns(cpu->gt_timer[timeridx], INT64_MAX);
2710         } else {
2711             timer_mod(cpu->gt_timer[timeridx], nexttick);
2712         }
2713         trace_arm_gt_recalc(timeridx, irqstate, nexttick);
2714     } else {
2715         /* Timer disabled: ISTATUS and timer output always clear */
2716         gt->ctl &= ~4;
2717         qemu_set_irq(cpu->gt_timer_outputs[timeridx], 0);
2718         timer_del(cpu->gt_timer[timeridx]);
2719         trace_arm_gt_recalc_disabled(timeridx);
2720     }
2721 }
2722 
2723 static void gt_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri,
2724                            int timeridx)
2725 {
2726     ARMCPU *cpu = env_archcpu(env);
2727 
2728     timer_del(cpu->gt_timer[timeridx]);
2729 }
2730 
2731 static uint64_t gt_cnt_read(CPUARMState *env, const ARMCPRegInfo *ri)
2732 {
2733     return gt_get_countervalue(env);
2734 }
2735 
2736 static uint64_t gt_virt_cnt_offset(CPUARMState *env)
2737 {
2738     uint64_t hcr;
2739 
2740     switch (arm_current_el(env)) {
2741     case 2:
2742         hcr = arm_hcr_el2_eff(env);
2743         if (hcr & HCR_E2H) {
2744             return 0;
2745         }
2746         break;
2747     case 0:
2748         hcr = arm_hcr_el2_eff(env);
2749         if ((hcr & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)) {
2750             return 0;
2751         }
2752         break;
2753     }
2754 
2755     return env->cp15.cntvoff_el2;
2756 }
2757 
2758 static uint64_t gt_virt_cnt_read(CPUARMState *env, const ARMCPRegInfo *ri)
2759 {
2760     return gt_get_countervalue(env) - gt_virt_cnt_offset(env);
2761 }
2762 
2763 static void gt_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
2764                           int timeridx,
2765                           uint64_t value)
2766 {
2767     trace_arm_gt_cval_write(timeridx, value);
2768     env->cp15.c14_timer[timeridx].cval = value;
2769     gt_recalc_timer(env_archcpu(env), timeridx);
2770 }
2771 
2772 static uint64_t gt_tval_read(CPUARMState *env, const ARMCPRegInfo *ri,
2773                              int timeridx)
2774 {
2775     uint64_t offset = 0;
2776 
2777     switch (timeridx) {
2778     case GTIMER_VIRT:
2779     case GTIMER_HYPVIRT:
2780         offset = gt_virt_cnt_offset(env);
2781         break;
2782     }
2783 
2784     return (uint32_t)(env->cp15.c14_timer[timeridx].cval -
2785                       (gt_get_countervalue(env) - offset));
2786 }
2787 
2788 static void gt_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
2789                           int timeridx,
2790                           uint64_t value)
2791 {
2792     uint64_t offset = 0;
2793 
2794     switch (timeridx) {
2795     case GTIMER_VIRT:
2796     case GTIMER_HYPVIRT:
2797         offset = gt_virt_cnt_offset(env);
2798         break;
2799     }
2800 
2801     trace_arm_gt_tval_write(timeridx, value);
2802     env->cp15.c14_timer[timeridx].cval = gt_get_countervalue(env) - offset +
2803                                          sextract64(value, 0, 32);
2804     gt_recalc_timer(env_archcpu(env), timeridx);
2805 }
2806 
2807 static void gt_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
2808                          int timeridx,
2809                          uint64_t value)
2810 {
2811     ARMCPU *cpu = env_archcpu(env);
2812     uint32_t oldval = env->cp15.c14_timer[timeridx].ctl;
2813 
2814     trace_arm_gt_ctl_write(timeridx, value);
2815     env->cp15.c14_timer[timeridx].ctl = deposit64(oldval, 0, 2, value);
2816     if ((oldval ^ value) & 1) {
2817         /* Enable toggled */
2818         gt_recalc_timer(cpu, timeridx);
2819     } else if ((oldval ^ value) & 2) {
2820         /* IMASK toggled: don't need to recalculate,
2821          * just set the interrupt line based on ISTATUS
2822          */
2823         int irqstate = (oldval & 4) && !(value & 2);
2824 
2825         trace_arm_gt_imask_toggle(timeridx, irqstate);
2826         qemu_set_irq(cpu->gt_timer_outputs[timeridx], irqstate);
2827     }
2828 }
2829 
2830 static void gt_phys_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri)
2831 {
2832     gt_timer_reset(env, ri, GTIMER_PHYS);
2833 }
2834 
2835 static void gt_phys_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
2836                                uint64_t value)
2837 {
2838     gt_cval_write(env, ri, GTIMER_PHYS, value);
2839 }
2840 
2841 static uint64_t gt_phys_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
2842 {
2843     return gt_tval_read(env, ri, GTIMER_PHYS);
2844 }
2845 
2846 static void gt_phys_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
2847                                uint64_t value)
2848 {
2849     gt_tval_write(env, ri, GTIMER_PHYS, value);
2850 }
2851 
2852 static void gt_phys_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
2853                               uint64_t value)
2854 {
2855     gt_ctl_write(env, ri, GTIMER_PHYS, value);
2856 }
2857 
2858 static int gt_phys_redir_timeridx(CPUARMState *env)
2859 {
2860     switch (arm_mmu_idx(env)) {
2861     case ARMMMUIdx_E20_0:
2862     case ARMMMUIdx_E20_2:
2863     case ARMMMUIdx_E20_2_PAN:
2864         return GTIMER_HYP;
2865     default:
2866         return GTIMER_PHYS;
2867     }
2868 }
2869 
2870 static int gt_virt_redir_timeridx(CPUARMState *env)
2871 {
2872     switch (arm_mmu_idx(env)) {
2873     case ARMMMUIdx_E20_0:
2874     case ARMMMUIdx_E20_2:
2875     case ARMMMUIdx_E20_2_PAN:
2876         return GTIMER_HYPVIRT;
2877     default:
2878         return GTIMER_VIRT;
2879     }
2880 }
2881 
2882 static uint64_t gt_phys_redir_cval_read(CPUARMState *env,
2883                                         const ARMCPRegInfo *ri)
2884 {
2885     int timeridx = gt_phys_redir_timeridx(env);
2886     return env->cp15.c14_timer[timeridx].cval;
2887 }
2888 
2889 static void gt_phys_redir_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
2890                                      uint64_t value)
2891 {
2892     int timeridx = gt_phys_redir_timeridx(env);
2893     gt_cval_write(env, ri, timeridx, value);
2894 }
2895 
2896 static uint64_t gt_phys_redir_tval_read(CPUARMState *env,
2897                                         const ARMCPRegInfo *ri)
2898 {
2899     int timeridx = gt_phys_redir_timeridx(env);
2900     return gt_tval_read(env, ri, timeridx);
2901 }
2902 
2903 static void gt_phys_redir_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
2904                                      uint64_t value)
2905 {
2906     int timeridx = gt_phys_redir_timeridx(env);
2907     gt_tval_write(env, ri, timeridx, value);
2908 }
2909 
2910 static uint64_t gt_phys_redir_ctl_read(CPUARMState *env,
2911                                        const ARMCPRegInfo *ri)
2912 {
2913     int timeridx = gt_phys_redir_timeridx(env);
2914     return env->cp15.c14_timer[timeridx].ctl;
2915 }
2916 
2917 static void gt_phys_redir_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
2918                                     uint64_t value)
2919 {
2920     int timeridx = gt_phys_redir_timeridx(env);
2921     gt_ctl_write(env, ri, timeridx, value);
2922 }
2923 
2924 static void gt_virt_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri)
2925 {
2926     gt_timer_reset(env, ri, GTIMER_VIRT);
2927 }
2928 
2929 static void gt_virt_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
2930                                uint64_t value)
2931 {
2932     gt_cval_write(env, ri, GTIMER_VIRT, value);
2933 }
2934 
2935 static uint64_t gt_virt_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
2936 {
2937     return gt_tval_read(env, ri, GTIMER_VIRT);
2938 }
2939 
2940 static void gt_virt_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
2941                                uint64_t value)
2942 {
2943     gt_tval_write(env, ri, GTIMER_VIRT, value);
2944 }
2945 
2946 static void gt_virt_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
2947                               uint64_t value)
2948 {
2949     gt_ctl_write(env, ri, GTIMER_VIRT, value);
2950 }
2951 
2952 static void gt_cntvoff_write(CPUARMState *env, const ARMCPRegInfo *ri,
2953                               uint64_t value)
2954 {
2955     ARMCPU *cpu = env_archcpu(env);
2956 
2957     trace_arm_gt_cntvoff_write(value);
2958     raw_write(env, ri, value);
2959     gt_recalc_timer(cpu, GTIMER_VIRT);
2960 }
2961 
2962 static uint64_t gt_virt_redir_cval_read(CPUARMState *env,
2963                                         const ARMCPRegInfo *ri)
2964 {
2965     int timeridx = gt_virt_redir_timeridx(env);
2966     return env->cp15.c14_timer[timeridx].cval;
2967 }
2968 
2969 static void gt_virt_redir_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
2970                                      uint64_t value)
2971 {
2972     int timeridx = gt_virt_redir_timeridx(env);
2973     gt_cval_write(env, ri, timeridx, value);
2974 }
2975 
2976 static uint64_t gt_virt_redir_tval_read(CPUARMState *env,
2977                                         const ARMCPRegInfo *ri)
2978 {
2979     int timeridx = gt_virt_redir_timeridx(env);
2980     return gt_tval_read(env, ri, timeridx);
2981 }
2982 
2983 static void gt_virt_redir_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
2984                                      uint64_t value)
2985 {
2986     int timeridx = gt_virt_redir_timeridx(env);
2987     gt_tval_write(env, ri, timeridx, value);
2988 }
2989 
2990 static uint64_t gt_virt_redir_ctl_read(CPUARMState *env,
2991                                        const ARMCPRegInfo *ri)
2992 {
2993     int timeridx = gt_virt_redir_timeridx(env);
2994     return env->cp15.c14_timer[timeridx].ctl;
2995 }
2996 
2997 static void gt_virt_redir_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
2998                                     uint64_t value)
2999 {
3000     int timeridx = gt_virt_redir_timeridx(env);
3001     gt_ctl_write(env, ri, timeridx, value);
3002 }
3003 
3004 static void gt_hyp_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri)
3005 {
3006     gt_timer_reset(env, ri, GTIMER_HYP);
3007 }
3008 
3009 static void gt_hyp_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
3010                               uint64_t value)
3011 {
3012     gt_cval_write(env, ri, GTIMER_HYP, value);
3013 }
3014 
3015 static uint64_t gt_hyp_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
3016 {
3017     return gt_tval_read(env, ri, GTIMER_HYP);
3018 }
3019 
3020 static void gt_hyp_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
3021                               uint64_t value)
3022 {
3023     gt_tval_write(env, ri, GTIMER_HYP, value);
3024 }
3025 
3026 static void gt_hyp_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
3027                               uint64_t value)
3028 {
3029     gt_ctl_write(env, ri, GTIMER_HYP, value);
3030 }
3031 
3032 static void gt_sec_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri)
3033 {
3034     gt_timer_reset(env, ri, GTIMER_SEC);
3035 }
3036 
3037 static void gt_sec_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
3038                               uint64_t value)
3039 {
3040     gt_cval_write(env, ri, GTIMER_SEC, value);
3041 }
3042 
3043 static uint64_t gt_sec_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
3044 {
3045     return gt_tval_read(env, ri, GTIMER_SEC);
3046 }
3047 
3048 static void gt_sec_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
3049                               uint64_t value)
3050 {
3051     gt_tval_write(env, ri, GTIMER_SEC, value);
3052 }
3053 
3054 static void gt_sec_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
3055                               uint64_t value)
3056 {
3057     gt_ctl_write(env, ri, GTIMER_SEC, value);
3058 }
3059 
3060 static void gt_hv_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri)
3061 {
3062     gt_timer_reset(env, ri, GTIMER_HYPVIRT);
3063 }
3064 
3065 static void gt_hv_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
3066                              uint64_t value)
3067 {
3068     gt_cval_write(env, ri, GTIMER_HYPVIRT, value);
3069 }
3070 
3071 static uint64_t gt_hv_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
3072 {
3073     return gt_tval_read(env, ri, GTIMER_HYPVIRT);
3074 }
3075 
3076 static void gt_hv_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
3077                              uint64_t value)
3078 {
3079     gt_tval_write(env, ri, GTIMER_HYPVIRT, value);
3080 }
3081 
3082 static void gt_hv_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
3083                             uint64_t value)
3084 {
3085     gt_ctl_write(env, ri, GTIMER_HYPVIRT, value);
3086 }
3087 
3088 void arm_gt_ptimer_cb(void *opaque)
3089 {
3090     ARMCPU *cpu = opaque;
3091 
3092     gt_recalc_timer(cpu, GTIMER_PHYS);
3093 }
3094 
3095 void arm_gt_vtimer_cb(void *opaque)
3096 {
3097     ARMCPU *cpu = opaque;
3098 
3099     gt_recalc_timer(cpu, GTIMER_VIRT);
3100 }
3101 
3102 void arm_gt_htimer_cb(void *opaque)
3103 {
3104     ARMCPU *cpu = opaque;
3105 
3106     gt_recalc_timer(cpu, GTIMER_HYP);
3107 }
3108 
3109 void arm_gt_stimer_cb(void *opaque)
3110 {
3111     ARMCPU *cpu = opaque;
3112 
3113     gt_recalc_timer(cpu, GTIMER_SEC);
3114 }
3115 
3116 void arm_gt_hvtimer_cb(void *opaque)
3117 {
3118     ARMCPU *cpu = opaque;
3119 
3120     gt_recalc_timer(cpu, GTIMER_HYPVIRT);
3121 }
3122 
3123 static void arm_gt_cntfrq_reset(CPUARMState *env, const ARMCPRegInfo *opaque)
3124 {
3125     ARMCPU *cpu = env_archcpu(env);
3126 
3127     cpu->env.cp15.c14_cntfrq = cpu->gt_cntfrq_hz;
3128 }
3129 
3130 static const ARMCPRegInfo generic_timer_cp_reginfo[] = {
3131     /* Note that CNTFRQ is purely reads-as-written for the benefit
3132      * of software; writing it doesn't actually change the timer frequency.
3133      * Our reset value matches the fixed frequency we implement the timer at.
3134      */
3135     { .name = "CNTFRQ", .cp = 15, .crn = 14, .crm = 0, .opc1 = 0, .opc2 = 0,
3136       .type = ARM_CP_ALIAS,
3137       .access = PL1_RW | PL0_R, .accessfn = gt_cntfrq_access,
3138       .fieldoffset = offsetoflow32(CPUARMState, cp15.c14_cntfrq),
3139     },
3140     { .name = "CNTFRQ_EL0", .state = ARM_CP_STATE_AA64,
3141       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 0, .opc2 = 0,
3142       .access = PL1_RW | PL0_R, .accessfn = gt_cntfrq_access,
3143       .fieldoffset = offsetof(CPUARMState, cp15.c14_cntfrq),
3144       .resetfn = arm_gt_cntfrq_reset,
3145     },
3146     /* overall control: mostly access permissions */
3147     { .name = "CNTKCTL", .state = ARM_CP_STATE_BOTH,
3148       .opc0 = 3, .opc1 = 0, .crn = 14, .crm = 1, .opc2 = 0,
3149       .access = PL1_RW,
3150       .fieldoffset = offsetof(CPUARMState, cp15.c14_cntkctl),
3151       .resetvalue = 0,
3152     },
3153     /* per-timer control */
3154     { .name = "CNTP_CTL", .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 1,
3155       .secure = ARM_CP_SECSTATE_NS,
3156       .type = ARM_CP_IO | ARM_CP_ALIAS, .access = PL0_RW,
3157       .accessfn = gt_ptimer_access,
3158       .fieldoffset = offsetoflow32(CPUARMState,
3159                                    cp15.c14_timer[GTIMER_PHYS].ctl),
3160       .readfn = gt_phys_redir_ctl_read, .raw_readfn = raw_read,
3161       .writefn = gt_phys_redir_ctl_write, .raw_writefn = raw_write,
3162     },
3163     { .name = "CNTP_CTL_S",
3164       .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 1,
3165       .secure = ARM_CP_SECSTATE_S,
3166       .type = ARM_CP_IO | ARM_CP_ALIAS, .access = PL0_RW,
3167       .accessfn = gt_ptimer_access,
3168       .fieldoffset = offsetoflow32(CPUARMState,
3169                                    cp15.c14_timer[GTIMER_SEC].ctl),
3170       .writefn = gt_sec_ctl_write, .raw_writefn = raw_write,
3171     },
3172     { .name = "CNTP_CTL_EL0", .state = ARM_CP_STATE_AA64,
3173       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 2, .opc2 = 1,
3174       .type = ARM_CP_IO, .access = PL0_RW,
3175       .accessfn = gt_ptimer_access,
3176       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].ctl),
3177       .resetvalue = 0,
3178       .readfn = gt_phys_redir_ctl_read, .raw_readfn = raw_read,
3179       .writefn = gt_phys_redir_ctl_write, .raw_writefn = raw_write,
3180     },
3181     { .name = "CNTV_CTL", .cp = 15, .crn = 14, .crm = 3, .opc1 = 0, .opc2 = 1,
3182       .type = ARM_CP_IO | ARM_CP_ALIAS, .access = PL0_RW,
3183       .accessfn = gt_vtimer_access,
3184       .fieldoffset = offsetoflow32(CPUARMState,
3185                                    cp15.c14_timer[GTIMER_VIRT].ctl),
3186       .readfn = gt_virt_redir_ctl_read, .raw_readfn = raw_read,
3187       .writefn = gt_virt_redir_ctl_write, .raw_writefn = raw_write,
3188     },
3189     { .name = "CNTV_CTL_EL0", .state = ARM_CP_STATE_AA64,
3190       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 1,
3191       .type = ARM_CP_IO, .access = PL0_RW,
3192       .accessfn = gt_vtimer_access,
3193       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].ctl),
3194       .resetvalue = 0,
3195       .readfn = gt_virt_redir_ctl_read, .raw_readfn = raw_read,
3196       .writefn = gt_virt_redir_ctl_write, .raw_writefn = raw_write,
3197     },
3198     /* TimerValue views: a 32 bit downcounting view of the underlying state */
3199     { .name = "CNTP_TVAL", .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 0,
3200       .secure = ARM_CP_SECSTATE_NS,
3201       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL0_RW,
3202       .accessfn = gt_ptimer_access,
3203       .readfn = gt_phys_redir_tval_read, .writefn = gt_phys_redir_tval_write,
3204     },
3205     { .name = "CNTP_TVAL_S",
3206       .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 0,
3207       .secure = ARM_CP_SECSTATE_S,
3208       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL0_RW,
3209       .accessfn = gt_ptimer_access,
3210       .readfn = gt_sec_tval_read, .writefn = gt_sec_tval_write,
3211     },
3212     { .name = "CNTP_TVAL_EL0", .state = ARM_CP_STATE_AA64,
3213       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 2, .opc2 = 0,
3214       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL0_RW,
3215       .accessfn = gt_ptimer_access, .resetfn = gt_phys_timer_reset,
3216       .readfn = gt_phys_redir_tval_read, .writefn = gt_phys_redir_tval_write,
3217     },
3218     { .name = "CNTV_TVAL", .cp = 15, .crn = 14, .crm = 3, .opc1 = 0, .opc2 = 0,
3219       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL0_RW,
3220       .accessfn = gt_vtimer_access,
3221       .readfn = gt_virt_redir_tval_read, .writefn = gt_virt_redir_tval_write,
3222     },
3223     { .name = "CNTV_TVAL_EL0", .state = ARM_CP_STATE_AA64,
3224       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 0,
3225       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL0_RW,
3226       .accessfn = gt_vtimer_access, .resetfn = gt_virt_timer_reset,
3227       .readfn = gt_virt_redir_tval_read, .writefn = gt_virt_redir_tval_write,
3228     },
3229     /* The counter itself */
3230     { .name = "CNTPCT", .cp = 15, .crm = 14, .opc1 = 0,
3231       .access = PL0_R, .type = ARM_CP_64BIT | ARM_CP_NO_RAW | ARM_CP_IO,
3232       .accessfn = gt_pct_access,
3233       .readfn = gt_cnt_read, .resetfn = arm_cp_reset_ignore,
3234     },
3235     { .name = "CNTPCT_EL0", .state = ARM_CP_STATE_AA64,
3236       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 0, .opc2 = 1,
3237       .access = PL0_R, .type = ARM_CP_NO_RAW | ARM_CP_IO,
3238       .accessfn = gt_pct_access, .readfn = gt_cnt_read,
3239     },
3240     { .name = "CNTVCT", .cp = 15, .crm = 14, .opc1 = 1,
3241       .access = PL0_R, .type = ARM_CP_64BIT | ARM_CP_NO_RAW | ARM_CP_IO,
3242       .accessfn = gt_vct_access,
3243       .readfn = gt_virt_cnt_read, .resetfn = arm_cp_reset_ignore,
3244     },
3245     { .name = "CNTVCT_EL0", .state = ARM_CP_STATE_AA64,
3246       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 0, .opc2 = 2,
3247       .access = PL0_R, .type = ARM_CP_NO_RAW | ARM_CP_IO,
3248       .accessfn = gt_vct_access, .readfn = gt_virt_cnt_read,
3249     },
3250     /* Comparison value, indicating when the timer goes off */
3251     { .name = "CNTP_CVAL", .cp = 15, .crm = 14, .opc1 = 2,
3252       .secure = ARM_CP_SECSTATE_NS,
3253       .access = PL0_RW,
3254       .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_ALIAS,
3255       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].cval),
3256       .accessfn = gt_ptimer_access,
3257       .readfn = gt_phys_redir_cval_read, .raw_readfn = raw_read,
3258       .writefn = gt_phys_redir_cval_write, .raw_writefn = raw_write,
3259     },
3260     { .name = "CNTP_CVAL_S", .cp = 15, .crm = 14, .opc1 = 2,
3261       .secure = ARM_CP_SECSTATE_S,
3262       .access = PL0_RW,
3263       .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_ALIAS,
3264       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_SEC].cval),
3265       .accessfn = gt_ptimer_access,
3266       .writefn = gt_sec_cval_write, .raw_writefn = raw_write,
3267     },
3268     { .name = "CNTP_CVAL_EL0", .state = ARM_CP_STATE_AA64,
3269       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 2, .opc2 = 2,
3270       .access = PL0_RW,
3271       .type = ARM_CP_IO,
3272       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].cval),
3273       .resetvalue = 0, .accessfn = gt_ptimer_access,
3274       .readfn = gt_phys_redir_cval_read, .raw_readfn = raw_read,
3275       .writefn = gt_phys_redir_cval_write, .raw_writefn = raw_write,
3276     },
3277     { .name = "CNTV_CVAL", .cp = 15, .crm = 14, .opc1 = 3,
3278       .access = PL0_RW,
3279       .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_ALIAS,
3280       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].cval),
3281       .accessfn = gt_vtimer_access,
3282       .readfn = gt_virt_redir_cval_read, .raw_readfn = raw_read,
3283       .writefn = gt_virt_redir_cval_write, .raw_writefn = raw_write,
3284     },
3285     { .name = "CNTV_CVAL_EL0", .state = ARM_CP_STATE_AA64,
3286       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 2,
3287       .access = PL0_RW,
3288       .type = ARM_CP_IO,
3289       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].cval),
3290       .resetvalue = 0, .accessfn = gt_vtimer_access,
3291       .readfn = gt_virt_redir_cval_read, .raw_readfn = raw_read,
3292       .writefn = gt_virt_redir_cval_write, .raw_writefn = raw_write,
3293     },
3294     /* Secure timer -- this is actually restricted to only EL3
3295      * and configurably Secure-EL1 via the accessfn.
3296      */
3297     { .name = "CNTPS_TVAL_EL1", .state = ARM_CP_STATE_AA64,
3298       .opc0 = 3, .opc1 = 7, .crn = 14, .crm = 2, .opc2 = 0,
3299       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW,
3300       .accessfn = gt_stimer_access,
3301       .readfn = gt_sec_tval_read,
3302       .writefn = gt_sec_tval_write,
3303       .resetfn = gt_sec_timer_reset,
3304     },
3305     { .name = "CNTPS_CTL_EL1", .state = ARM_CP_STATE_AA64,
3306       .opc0 = 3, .opc1 = 7, .crn = 14, .crm = 2, .opc2 = 1,
3307       .type = ARM_CP_IO, .access = PL1_RW,
3308       .accessfn = gt_stimer_access,
3309       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_SEC].ctl),
3310       .resetvalue = 0,
3311       .writefn = gt_sec_ctl_write, .raw_writefn = raw_write,
3312     },
3313     { .name = "CNTPS_CVAL_EL1", .state = ARM_CP_STATE_AA64,
3314       .opc0 = 3, .opc1 = 7, .crn = 14, .crm = 2, .opc2 = 2,
3315       .type = ARM_CP_IO, .access = PL1_RW,
3316       .accessfn = gt_stimer_access,
3317       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_SEC].cval),
3318       .writefn = gt_sec_cval_write, .raw_writefn = raw_write,
3319     },
3320     REGINFO_SENTINEL
3321 };
3322 
3323 static CPAccessResult e2h_access(CPUARMState *env, const ARMCPRegInfo *ri,
3324                                  bool isread)
3325 {
3326     if (!(arm_hcr_el2_eff(env) & HCR_E2H)) {
3327         return CP_ACCESS_TRAP;
3328     }
3329     return CP_ACCESS_OK;
3330 }
3331 
3332 #else
3333 
3334 /* In user-mode most of the generic timer registers are inaccessible
3335  * however modern kernels (4.12+) allow access to cntvct_el0
3336  */
3337 
3338 static uint64_t gt_virt_cnt_read(CPUARMState *env, const ARMCPRegInfo *ri)
3339 {
3340     ARMCPU *cpu = env_archcpu(env);
3341 
3342     /* Currently we have no support for QEMUTimer in linux-user so we
3343      * can't call gt_get_countervalue(env), instead we directly
3344      * call the lower level functions.
3345      */
3346     return cpu_get_clock() / gt_cntfrq_period_ns(cpu);
3347 }
3348 
3349 static const ARMCPRegInfo generic_timer_cp_reginfo[] = {
3350     { .name = "CNTFRQ_EL0", .state = ARM_CP_STATE_AA64,
3351       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 0, .opc2 = 0,
3352       .type = ARM_CP_CONST, .access = PL0_R /* no PL1_RW in linux-user */,
3353       .fieldoffset = offsetof(CPUARMState, cp15.c14_cntfrq),
3354       .resetvalue = NANOSECONDS_PER_SECOND / GTIMER_SCALE,
3355     },
3356     { .name = "CNTVCT_EL0", .state = ARM_CP_STATE_AA64,
3357       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 0, .opc2 = 2,
3358       .access = PL0_R, .type = ARM_CP_NO_RAW | ARM_CP_IO,
3359       .readfn = gt_virt_cnt_read,
3360     },
3361     REGINFO_SENTINEL
3362 };
3363 
3364 #endif
3365 
3366 static void par_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
3367 {
3368     if (arm_feature(env, ARM_FEATURE_LPAE)) {
3369         raw_write(env, ri, value);
3370     } else if (arm_feature(env, ARM_FEATURE_V7)) {
3371         raw_write(env, ri, value & 0xfffff6ff);
3372     } else {
3373         raw_write(env, ri, value & 0xfffff1ff);
3374     }
3375 }
3376 
3377 #ifndef CONFIG_USER_ONLY
3378 /* get_phys_addr() isn't present for user-mode-only targets */
3379 
3380 static CPAccessResult ats_access(CPUARMState *env, const ARMCPRegInfo *ri,
3381                                  bool isread)
3382 {
3383     if (ri->opc2 & 4) {
3384         /* The ATS12NSO* operations must trap to EL3 if executed in
3385          * Secure EL1 (which can only happen if EL3 is AArch64).
3386          * They are simply UNDEF if executed from NS EL1.
3387          * They function normally from EL2 or EL3.
3388          */
3389         if (arm_current_el(env) == 1) {
3390             if (arm_is_secure_below_el3(env)) {
3391                 return CP_ACCESS_TRAP_UNCATEGORIZED_EL3;
3392             }
3393             return CP_ACCESS_TRAP_UNCATEGORIZED;
3394         }
3395     }
3396     return CP_ACCESS_OK;
3397 }
3398 
3399 #ifdef CONFIG_TCG
3400 static uint64_t do_ats_write(CPUARMState *env, uint64_t value,
3401                              MMUAccessType access_type, ARMMMUIdx mmu_idx)
3402 {
3403     hwaddr phys_addr;
3404     target_ulong page_size;
3405     int prot;
3406     bool ret;
3407     uint64_t par64;
3408     bool format64 = false;
3409     MemTxAttrs attrs = {};
3410     ARMMMUFaultInfo fi = {};
3411     ARMCacheAttrs cacheattrs = {};
3412 
3413     ret = get_phys_addr(env, value, access_type, mmu_idx, &phys_addr, &attrs,
3414                         &prot, &page_size, &fi, &cacheattrs);
3415 
3416     if (ret) {
3417         /*
3418          * Some kinds of translation fault must cause exceptions rather
3419          * than being reported in the PAR.
3420          */
3421         int current_el = arm_current_el(env);
3422         int target_el;
3423         uint32_t syn, fsr, fsc;
3424         bool take_exc = false;
3425 
3426         if (fi.s1ptw && current_el == 1 && !arm_is_secure(env)
3427             && arm_mmu_idx_is_stage1_of_2(mmu_idx)) {
3428             /*
3429              * Synchronous stage 2 fault on an access made as part of the
3430              * translation table walk for AT S1E0* or AT S1E1* insn
3431              * executed from NS EL1. If this is a synchronous external abort
3432              * and SCR_EL3.EA == 1, then we take a synchronous external abort
3433              * to EL3. Otherwise the fault is taken as an exception to EL2,
3434              * and HPFAR_EL2 holds the faulting IPA.
3435              */
3436             if (fi.type == ARMFault_SyncExternalOnWalk &&
3437                 (env->cp15.scr_el3 & SCR_EA)) {
3438                 target_el = 3;
3439             } else {
3440                 env->cp15.hpfar_el2 = extract64(fi.s2addr, 12, 47) << 4;
3441                 target_el = 2;
3442             }
3443             take_exc = true;
3444         } else if (fi.type == ARMFault_SyncExternalOnWalk) {
3445             /*
3446              * Synchronous external aborts during a translation table walk
3447              * are taken as Data Abort exceptions.
3448              */
3449             if (fi.stage2) {
3450                 if (current_el == 3) {
3451                     target_el = 3;
3452                 } else {
3453                     target_el = 2;
3454                 }
3455             } else {
3456                 target_el = exception_target_el(env);
3457             }
3458             take_exc = true;
3459         }
3460 
3461         if (take_exc) {
3462             /* Construct FSR and FSC using same logic as arm_deliver_fault() */
3463             if (target_el == 2 || arm_el_is_aa64(env, target_el) ||
3464                 arm_s1_regime_using_lpae_format(env, mmu_idx)) {
3465                 fsr = arm_fi_to_lfsc(&fi);
3466                 fsc = extract32(fsr, 0, 6);
3467             } else {
3468                 fsr = arm_fi_to_sfsc(&fi);
3469                 fsc = 0x3f;
3470             }
3471             /*
3472              * Report exception with ESR indicating a fault due to a
3473              * translation table walk for a cache maintenance instruction.
3474              */
3475             syn = syn_data_abort_no_iss(current_el == target_el, 0,
3476                                         fi.ea, 1, fi.s1ptw, 1, fsc);
3477             env->exception.vaddress = value;
3478             env->exception.fsr = fsr;
3479             raise_exception(env, EXCP_DATA_ABORT, syn, target_el);
3480         }
3481     }
3482 
3483     if (is_a64(env)) {
3484         format64 = true;
3485     } else if (arm_feature(env, ARM_FEATURE_LPAE)) {
3486         /*
3487          * ATS1Cxx:
3488          * * TTBCR.EAE determines whether the result is returned using the
3489          *   32-bit or the 64-bit PAR format
3490          * * Instructions executed in Hyp mode always use the 64bit format
3491          *
3492          * ATS1S2NSOxx uses the 64bit format if any of the following is true:
3493          * * The Non-secure TTBCR.EAE bit is set to 1
3494          * * The implementation includes EL2, and the value of HCR.VM is 1
3495          *
3496          * (Note that HCR.DC makes HCR.VM behave as if it is 1.)
3497          *
3498          * ATS1Hx always uses the 64bit format.
3499          */
3500         format64 = arm_s1_regime_using_lpae_format(env, mmu_idx);
3501 
3502         if (arm_feature(env, ARM_FEATURE_EL2)) {
3503             if (mmu_idx == ARMMMUIdx_E10_0 ||
3504                 mmu_idx == ARMMMUIdx_E10_1 ||
3505                 mmu_idx == ARMMMUIdx_E10_1_PAN) {
3506                 format64 |= env->cp15.hcr_el2 & (HCR_VM | HCR_DC);
3507             } else {
3508                 format64 |= arm_current_el(env) == 2;
3509             }
3510         }
3511     }
3512 
3513     if (format64) {
3514         /* Create a 64-bit PAR */
3515         par64 = (1 << 11); /* LPAE bit always set */
3516         if (!ret) {
3517             par64 |= phys_addr & ~0xfffULL;
3518             if (!attrs.secure) {
3519                 par64 |= (1 << 9); /* NS */
3520             }
3521             par64 |= (uint64_t)cacheattrs.attrs << 56; /* ATTR */
3522             par64 |= cacheattrs.shareability << 7; /* SH */
3523         } else {
3524             uint32_t fsr = arm_fi_to_lfsc(&fi);
3525 
3526             par64 |= 1; /* F */
3527             par64 |= (fsr & 0x3f) << 1; /* FS */
3528             if (fi.stage2) {
3529                 par64 |= (1 << 9); /* S */
3530             }
3531             if (fi.s1ptw) {
3532                 par64 |= (1 << 8); /* PTW */
3533             }
3534         }
3535     } else {
3536         /* fsr is a DFSR/IFSR value for the short descriptor
3537          * translation table format (with WnR always clear).
3538          * Convert it to a 32-bit PAR.
3539          */
3540         if (!ret) {
3541             /* We do not set any attribute bits in the PAR */
3542             if (page_size == (1 << 24)
3543                 && arm_feature(env, ARM_FEATURE_V7)) {
3544                 par64 = (phys_addr & 0xff000000) | (1 << 1);
3545             } else {
3546                 par64 = phys_addr & 0xfffff000;
3547             }
3548             if (!attrs.secure) {
3549                 par64 |= (1 << 9); /* NS */
3550             }
3551         } else {
3552             uint32_t fsr = arm_fi_to_sfsc(&fi);
3553 
3554             par64 = ((fsr & (1 << 10)) >> 5) | ((fsr & (1 << 12)) >> 6) |
3555                     ((fsr & 0xf) << 1) | 1;
3556         }
3557     }
3558     return par64;
3559 }
3560 #endif /* CONFIG_TCG */
3561 
3562 static void ats_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
3563 {
3564 #ifdef CONFIG_TCG
3565     MMUAccessType access_type = ri->opc2 & 1 ? MMU_DATA_STORE : MMU_DATA_LOAD;
3566     uint64_t par64;
3567     ARMMMUIdx mmu_idx;
3568     int el = arm_current_el(env);
3569     bool secure = arm_is_secure_below_el3(env);
3570 
3571     switch (ri->opc2 & 6) {
3572     case 0:
3573         /* stage 1 current state PL1: ATS1CPR, ATS1CPW, ATS1CPRP, ATS1CPWP */
3574         switch (el) {
3575         case 3:
3576             mmu_idx = ARMMMUIdx_SE3;
3577             break;
3578         case 2:
3579             g_assert(!secure);  /* TODO: ARMv8.4-SecEL2 */
3580             /* fall through */
3581         case 1:
3582             if (ri->crm == 9 && (env->uncached_cpsr & CPSR_PAN)) {
3583                 mmu_idx = (secure ? ARMMMUIdx_SE10_1_PAN
3584                            : ARMMMUIdx_Stage1_E1_PAN);
3585             } else {
3586                 mmu_idx = secure ? ARMMMUIdx_SE10_1 : ARMMMUIdx_Stage1_E1;
3587             }
3588             break;
3589         default:
3590             g_assert_not_reached();
3591         }
3592         break;
3593     case 2:
3594         /* stage 1 current state PL0: ATS1CUR, ATS1CUW */
3595         switch (el) {
3596         case 3:
3597             mmu_idx = ARMMMUIdx_SE10_0;
3598             break;
3599         case 2:
3600             mmu_idx = ARMMMUIdx_Stage1_E0;
3601             break;
3602         case 1:
3603             mmu_idx = secure ? ARMMMUIdx_SE10_0 : ARMMMUIdx_Stage1_E0;
3604             break;
3605         default:
3606             g_assert_not_reached();
3607         }
3608         break;
3609     case 4:
3610         /* stage 1+2 NonSecure PL1: ATS12NSOPR, ATS12NSOPW */
3611         mmu_idx = ARMMMUIdx_E10_1;
3612         break;
3613     case 6:
3614         /* stage 1+2 NonSecure PL0: ATS12NSOUR, ATS12NSOUW */
3615         mmu_idx = ARMMMUIdx_E10_0;
3616         break;
3617     default:
3618         g_assert_not_reached();
3619     }
3620 
3621     par64 = do_ats_write(env, value, access_type, mmu_idx);
3622 
3623     A32_BANKED_CURRENT_REG_SET(env, par, par64);
3624 #else
3625     /* Handled by hardware accelerator. */
3626     g_assert_not_reached();
3627 #endif /* CONFIG_TCG */
3628 }
3629 
3630 static void ats1h_write(CPUARMState *env, const ARMCPRegInfo *ri,
3631                         uint64_t value)
3632 {
3633 #ifdef CONFIG_TCG
3634     MMUAccessType access_type = ri->opc2 & 1 ? MMU_DATA_STORE : MMU_DATA_LOAD;
3635     uint64_t par64;
3636 
3637     par64 = do_ats_write(env, value, access_type, ARMMMUIdx_E2);
3638 
3639     A32_BANKED_CURRENT_REG_SET(env, par, par64);
3640 #else
3641     /* Handled by hardware accelerator. */
3642     g_assert_not_reached();
3643 #endif /* CONFIG_TCG */
3644 }
3645 
3646 static CPAccessResult at_s1e2_access(CPUARMState *env, const ARMCPRegInfo *ri,
3647                                      bool isread)
3648 {
3649     if (arm_current_el(env) == 3 && !(env->cp15.scr_el3 & SCR_NS)) {
3650         return CP_ACCESS_TRAP;
3651     }
3652     return CP_ACCESS_OK;
3653 }
3654 
3655 static void ats_write64(CPUARMState *env, const ARMCPRegInfo *ri,
3656                         uint64_t value)
3657 {
3658 #ifdef CONFIG_TCG
3659     MMUAccessType access_type = ri->opc2 & 1 ? MMU_DATA_STORE : MMU_DATA_LOAD;
3660     ARMMMUIdx mmu_idx;
3661     int secure = arm_is_secure_below_el3(env);
3662 
3663     switch (ri->opc2 & 6) {
3664     case 0:
3665         switch (ri->opc1) {
3666         case 0: /* AT S1E1R, AT S1E1W, AT S1E1RP, AT S1E1WP */
3667             if (ri->crm == 9 && (env->pstate & PSTATE_PAN)) {
3668                 mmu_idx = (secure ? ARMMMUIdx_SE10_1_PAN
3669                            : ARMMMUIdx_Stage1_E1_PAN);
3670             } else {
3671                 mmu_idx = secure ? ARMMMUIdx_SE10_1 : ARMMMUIdx_Stage1_E1;
3672             }
3673             break;
3674         case 4: /* AT S1E2R, AT S1E2W */
3675             mmu_idx = ARMMMUIdx_E2;
3676             break;
3677         case 6: /* AT S1E3R, AT S1E3W */
3678             mmu_idx = ARMMMUIdx_SE3;
3679             break;
3680         default:
3681             g_assert_not_reached();
3682         }
3683         break;
3684     case 2: /* AT S1E0R, AT S1E0W */
3685         mmu_idx = secure ? ARMMMUIdx_SE10_0 : ARMMMUIdx_Stage1_E0;
3686         break;
3687     case 4: /* AT S12E1R, AT S12E1W */
3688         mmu_idx = secure ? ARMMMUIdx_SE10_1 : ARMMMUIdx_E10_1;
3689         break;
3690     case 6: /* AT S12E0R, AT S12E0W */
3691         mmu_idx = secure ? ARMMMUIdx_SE10_0 : ARMMMUIdx_E10_0;
3692         break;
3693     default:
3694         g_assert_not_reached();
3695     }
3696 
3697     env->cp15.par_el[1] = do_ats_write(env, value, access_type, mmu_idx);
3698 #else
3699     /* Handled by hardware accelerator. */
3700     g_assert_not_reached();
3701 #endif /* CONFIG_TCG */
3702 }
3703 #endif
3704 
3705 static const ARMCPRegInfo vapa_cp_reginfo[] = {
3706     { .name = "PAR", .cp = 15, .crn = 7, .crm = 4, .opc1 = 0, .opc2 = 0,
3707       .access = PL1_RW, .resetvalue = 0,
3708       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.par_s),
3709                              offsetoflow32(CPUARMState, cp15.par_ns) },
3710       .writefn = par_write },
3711 #ifndef CONFIG_USER_ONLY
3712     /* This underdecoding is safe because the reginfo is NO_RAW. */
3713     { .name = "ATS", .cp = 15, .crn = 7, .crm = 8, .opc1 = 0, .opc2 = CP_ANY,
3714       .access = PL1_W, .accessfn = ats_access,
3715       .writefn = ats_write, .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC },
3716 #endif
3717     REGINFO_SENTINEL
3718 };
3719 
3720 /* Return basic MPU access permission bits.  */
3721 static uint32_t simple_mpu_ap_bits(uint32_t val)
3722 {
3723     uint32_t ret;
3724     uint32_t mask;
3725     int i;
3726     ret = 0;
3727     mask = 3;
3728     for (i = 0; i < 16; i += 2) {
3729         ret |= (val >> i) & mask;
3730         mask <<= 2;
3731     }
3732     return ret;
3733 }
3734 
3735 /* Pad basic MPU access permission bits to extended format.  */
3736 static uint32_t extended_mpu_ap_bits(uint32_t val)
3737 {
3738     uint32_t ret;
3739     uint32_t mask;
3740     int i;
3741     ret = 0;
3742     mask = 3;
3743     for (i = 0; i < 16; i += 2) {
3744         ret |= (val & mask) << i;
3745         mask <<= 2;
3746     }
3747     return ret;
3748 }
3749 
3750 static void pmsav5_data_ap_write(CPUARMState *env, const ARMCPRegInfo *ri,
3751                                  uint64_t value)
3752 {
3753     env->cp15.pmsav5_data_ap = extended_mpu_ap_bits(value);
3754 }
3755 
3756 static uint64_t pmsav5_data_ap_read(CPUARMState *env, const ARMCPRegInfo *ri)
3757 {
3758     return simple_mpu_ap_bits(env->cp15.pmsav5_data_ap);
3759 }
3760 
3761 static void pmsav5_insn_ap_write(CPUARMState *env, const ARMCPRegInfo *ri,
3762                                  uint64_t value)
3763 {
3764     env->cp15.pmsav5_insn_ap = extended_mpu_ap_bits(value);
3765 }
3766 
3767 static uint64_t pmsav5_insn_ap_read(CPUARMState *env, const ARMCPRegInfo *ri)
3768 {
3769     return simple_mpu_ap_bits(env->cp15.pmsav5_insn_ap);
3770 }
3771 
3772 static uint64_t pmsav7_read(CPUARMState *env, const ARMCPRegInfo *ri)
3773 {
3774     uint32_t *u32p = *(uint32_t **)raw_ptr(env, ri);
3775 
3776     if (!u32p) {
3777         return 0;
3778     }
3779 
3780     u32p += env->pmsav7.rnr[M_REG_NS];
3781     return *u32p;
3782 }
3783 
3784 static void pmsav7_write(CPUARMState *env, const ARMCPRegInfo *ri,
3785                          uint64_t value)
3786 {
3787     ARMCPU *cpu = env_archcpu(env);
3788     uint32_t *u32p = *(uint32_t **)raw_ptr(env, ri);
3789 
3790     if (!u32p) {
3791         return;
3792     }
3793 
3794     u32p += env->pmsav7.rnr[M_REG_NS];
3795     tlb_flush(CPU(cpu)); /* Mappings may have changed - purge! */
3796     *u32p = value;
3797 }
3798 
3799 static void pmsav7_rgnr_write(CPUARMState *env, const ARMCPRegInfo *ri,
3800                               uint64_t value)
3801 {
3802     ARMCPU *cpu = env_archcpu(env);
3803     uint32_t nrgs = cpu->pmsav7_dregion;
3804 
3805     if (value >= nrgs) {
3806         qemu_log_mask(LOG_GUEST_ERROR,
3807                       "PMSAv7 RGNR write >= # supported regions, %" PRIu32
3808                       " > %" PRIu32 "\n", (uint32_t)value, nrgs);
3809         return;
3810     }
3811 
3812     raw_write(env, ri, value);
3813 }
3814 
3815 static const ARMCPRegInfo pmsav7_cp_reginfo[] = {
3816     /* Reset for all these registers is handled in arm_cpu_reset(),
3817      * because the PMSAv7 is also used by M-profile CPUs, which do
3818      * not register cpregs but still need the state to be reset.
3819      */
3820     { .name = "DRBAR", .cp = 15, .crn = 6, .opc1 = 0, .crm = 1, .opc2 = 0,
3821       .access = PL1_RW, .type = ARM_CP_NO_RAW,
3822       .fieldoffset = offsetof(CPUARMState, pmsav7.drbar),
3823       .readfn = pmsav7_read, .writefn = pmsav7_write,
3824       .resetfn = arm_cp_reset_ignore },
3825     { .name = "DRSR", .cp = 15, .crn = 6, .opc1 = 0, .crm = 1, .opc2 = 2,
3826       .access = PL1_RW, .type = ARM_CP_NO_RAW,
3827       .fieldoffset = offsetof(CPUARMState, pmsav7.drsr),
3828       .readfn = pmsav7_read, .writefn = pmsav7_write,
3829       .resetfn = arm_cp_reset_ignore },
3830     { .name = "DRACR", .cp = 15, .crn = 6, .opc1 = 0, .crm = 1, .opc2 = 4,
3831       .access = PL1_RW, .type = ARM_CP_NO_RAW,
3832       .fieldoffset = offsetof(CPUARMState, pmsav7.dracr),
3833       .readfn = pmsav7_read, .writefn = pmsav7_write,
3834       .resetfn = arm_cp_reset_ignore },
3835     { .name = "RGNR", .cp = 15, .crn = 6, .opc1 = 0, .crm = 2, .opc2 = 0,
3836       .access = PL1_RW,
3837       .fieldoffset = offsetof(CPUARMState, pmsav7.rnr[M_REG_NS]),
3838       .writefn = pmsav7_rgnr_write,
3839       .resetfn = arm_cp_reset_ignore },
3840     REGINFO_SENTINEL
3841 };
3842 
3843 static const ARMCPRegInfo pmsav5_cp_reginfo[] = {
3844     { .name = "DATA_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 0,
3845       .access = PL1_RW, .type = ARM_CP_ALIAS,
3846       .fieldoffset = offsetof(CPUARMState, cp15.pmsav5_data_ap),
3847       .readfn = pmsav5_data_ap_read, .writefn = pmsav5_data_ap_write, },
3848     { .name = "INSN_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 1,
3849       .access = PL1_RW, .type = ARM_CP_ALIAS,
3850       .fieldoffset = offsetof(CPUARMState, cp15.pmsav5_insn_ap),
3851       .readfn = pmsav5_insn_ap_read, .writefn = pmsav5_insn_ap_write, },
3852     { .name = "DATA_EXT_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 2,
3853       .access = PL1_RW,
3854       .fieldoffset = offsetof(CPUARMState, cp15.pmsav5_data_ap),
3855       .resetvalue = 0, },
3856     { .name = "INSN_EXT_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 3,
3857       .access = PL1_RW,
3858       .fieldoffset = offsetof(CPUARMState, cp15.pmsav5_insn_ap),
3859       .resetvalue = 0, },
3860     { .name = "DCACHE_CFG", .cp = 15, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 0,
3861       .access = PL1_RW,
3862       .fieldoffset = offsetof(CPUARMState, cp15.c2_data), .resetvalue = 0, },
3863     { .name = "ICACHE_CFG", .cp = 15, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 1,
3864       .access = PL1_RW,
3865       .fieldoffset = offsetof(CPUARMState, cp15.c2_insn), .resetvalue = 0, },
3866     /* Protection region base and size registers */
3867     { .name = "946_PRBS0", .cp = 15, .crn = 6, .crm = 0, .opc1 = 0,
3868       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
3869       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[0]) },
3870     { .name = "946_PRBS1", .cp = 15, .crn = 6, .crm = 1, .opc1 = 0,
3871       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
3872       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[1]) },
3873     { .name = "946_PRBS2", .cp = 15, .crn = 6, .crm = 2, .opc1 = 0,
3874       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
3875       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[2]) },
3876     { .name = "946_PRBS3", .cp = 15, .crn = 6, .crm = 3, .opc1 = 0,
3877       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
3878       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[3]) },
3879     { .name = "946_PRBS4", .cp = 15, .crn = 6, .crm = 4, .opc1 = 0,
3880       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
3881       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[4]) },
3882     { .name = "946_PRBS5", .cp = 15, .crn = 6, .crm = 5, .opc1 = 0,
3883       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
3884       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[5]) },
3885     { .name = "946_PRBS6", .cp = 15, .crn = 6, .crm = 6, .opc1 = 0,
3886       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
3887       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[6]) },
3888     { .name = "946_PRBS7", .cp = 15, .crn = 6, .crm = 7, .opc1 = 0,
3889       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
3890       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[7]) },
3891     REGINFO_SENTINEL
3892 };
3893 
3894 static void vmsa_ttbcr_raw_write(CPUARMState *env, const ARMCPRegInfo *ri,
3895                                  uint64_t value)
3896 {
3897     TCR *tcr = raw_ptr(env, ri);
3898     int maskshift = extract32(value, 0, 3);
3899 
3900     if (!arm_feature(env, ARM_FEATURE_V8)) {
3901         if (arm_feature(env, ARM_FEATURE_LPAE) && (value & TTBCR_EAE)) {
3902             /* Pre ARMv8 bits [21:19], [15:14] and [6:3] are UNK/SBZP when
3903              * using Long-desciptor translation table format */
3904             value &= ~((7 << 19) | (3 << 14) | (0xf << 3));
3905         } else if (arm_feature(env, ARM_FEATURE_EL3)) {
3906             /* In an implementation that includes the Security Extensions
3907              * TTBCR has additional fields PD0 [4] and PD1 [5] for
3908              * Short-descriptor translation table format.
3909              */
3910             value &= TTBCR_PD1 | TTBCR_PD0 | TTBCR_N;
3911         } else {
3912             value &= TTBCR_N;
3913         }
3914     }
3915 
3916     /* Update the masks corresponding to the TCR bank being written
3917      * Note that we always calculate mask and base_mask, but
3918      * they are only used for short-descriptor tables (ie if EAE is 0);
3919      * for long-descriptor tables the TCR fields are used differently
3920      * and the mask and base_mask values are meaningless.
3921      */
3922     tcr->raw_tcr = value;
3923     tcr->mask = ~(((uint32_t)0xffffffffu) >> maskshift);
3924     tcr->base_mask = ~((uint32_t)0x3fffu >> maskshift);
3925 }
3926 
3927 static void vmsa_ttbcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
3928                              uint64_t value)
3929 {
3930     ARMCPU *cpu = env_archcpu(env);
3931     TCR *tcr = raw_ptr(env, ri);
3932 
3933     if (arm_feature(env, ARM_FEATURE_LPAE)) {
3934         /* With LPAE the TTBCR could result in a change of ASID
3935          * via the TTBCR.A1 bit, so do a TLB flush.
3936          */
3937         tlb_flush(CPU(cpu));
3938     }
3939     /* Preserve the high half of TCR_EL1, set via TTBCR2.  */
3940     value = deposit64(tcr->raw_tcr, 0, 32, value);
3941     vmsa_ttbcr_raw_write(env, ri, value);
3942 }
3943 
3944 static void vmsa_ttbcr_reset(CPUARMState *env, const ARMCPRegInfo *ri)
3945 {
3946     TCR *tcr = raw_ptr(env, ri);
3947 
3948     /* Reset both the TCR as well as the masks corresponding to the bank of
3949      * the TCR being reset.
3950      */
3951     tcr->raw_tcr = 0;
3952     tcr->mask = 0;
3953     tcr->base_mask = 0xffffc000u;
3954 }
3955 
3956 static void vmsa_tcr_el12_write(CPUARMState *env, const ARMCPRegInfo *ri,
3957                                uint64_t value)
3958 {
3959     ARMCPU *cpu = env_archcpu(env);
3960     TCR *tcr = raw_ptr(env, ri);
3961 
3962     /* For AArch64 the A1 bit could result in a change of ASID, so TLB flush. */
3963     tlb_flush(CPU(cpu));
3964     tcr->raw_tcr = value;
3965 }
3966 
3967 static void vmsa_ttbr_write(CPUARMState *env, const ARMCPRegInfo *ri,
3968                             uint64_t value)
3969 {
3970     /* If the ASID changes (with a 64-bit write), we must flush the TLB.  */
3971     if (cpreg_field_is_64bit(ri) &&
3972         extract64(raw_read(env, ri) ^ value, 48, 16) != 0) {
3973         ARMCPU *cpu = env_archcpu(env);
3974         tlb_flush(CPU(cpu));
3975     }
3976     raw_write(env, ri, value);
3977 }
3978 
3979 static void vmsa_tcr_ttbr_el2_write(CPUARMState *env, const ARMCPRegInfo *ri,
3980                                     uint64_t value)
3981 {
3982     /*
3983      * If we are running with E2&0 regime, then an ASID is active.
3984      * Flush if that might be changing.  Note we're not checking
3985      * TCR_EL2.A1 to know if this is really the TTBRx_EL2 that
3986      * holds the active ASID, only checking the field that might.
3987      */
3988     if (extract64(raw_read(env, ri) ^ value, 48, 16) &&
3989         (arm_hcr_el2_eff(env) & HCR_E2H)) {
3990         tlb_flush_by_mmuidx(env_cpu(env),
3991                             ARMMMUIdxBit_E20_2 |
3992                             ARMMMUIdxBit_E20_2_PAN |
3993                             ARMMMUIdxBit_E20_0);
3994     }
3995     raw_write(env, ri, value);
3996 }
3997 
3998 static void vttbr_write(CPUARMState *env, const ARMCPRegInfo *ri,
3999                         uint64_t value)
4000 {
4001     ARMCPU *cpu = env_archcpu(env);
4002     CPUState *cs = CPU(cpu);
4003 
4004     /*
4005      * A change in VMID to the stage2 page table (Stage2) invalidates
4006      * the combined stage 1&2 tlbs (EL10_1 and EL10_0).
4007      */
4008     if (raw_read(env, ri) != value) {
4009         tlb_flush_by_mmuidx(cs,
4010                             ARMMMUIdxBit_E10_1 |
4011                             ARMMMUIdxBit_E10_1_PAN |
4012                             ARMMMUIdxBit_E10_0);
4013         raw_write(env, ri, value);
4014     }
4015 }
4016 
4017 static const ARMCPRegInfo vmsa_pmsa_cp_reginfo[] = {
4018     { .name = "DFSR", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 0,
4019       .access = PL1_RW, .accessfn = access_tvm_trvm, .type = ARM_CP_ALIAS,
4020       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.dfsr_s),
4021                              offsetoflow32(CPUARMState, cp15.dfsr_ns) }, },
4022     { .name = "IFSR", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 1,
4023       .access = PL1_RW, .accessfn = access_tvm_trvm, .resetvalue = 0,
4024       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.ifsr_s),
4025                              offsetoflow32(CPUARMState, cp15.ifsr_ns) } },
4026     { .name = "DFAR", .cp = 15, .opc1 = 0, .crn = 6, .crm = 0, .opc2 = 0,
4027       .access = PL1_RW, .accessfn = access_tvm_trvm, .resetvalue = 0,
4028       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.dfar_s),
4029                              offsetof(CPUARMState, cp15.dfar_ns) } },
4030     { .name = "FAR_EL1", .state = ARM_CP_STATE_AA64,
4031       .opc0 = 3, .crn = 6, .crm = 0, .opc1 = 0, .opc2 = 0,
4032       .access = PL1_RW, .accessfn = access_tvm_trvm,
4033       .fieldoffset = offsetof(CPUARMState, cp15.far_el[1]),
4034       .resetvalue = 0, },
4035     REGINFO_SENTINEL
4036 };
4037 
4038 static const ARMCPRegInfo vmsa_cp_reginfo[] = {
4039     { .name = "ESR_EL1", .state = ARM_CP_STATE_AA64,
4040       .opc0 = 3, .crn = 5, .crm = 2, .opc1 = 0, .opc2 = 0,
4041       .access = PL1_RW, .accessfn = access_tvm_trvm,
4042       .fieldoffset = offsetof(CPUARMState, cp15.esr_el[1]), .resetvalue = 0, },
4043     { .name = "TTBR0_EL1", .state = ARM_CP_STATE_BOTH,
4044       .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 0, .opc2 = 0,
4045       .access = PL1_RW, .accessfn = access_tvm_trvm,
4046       .writefn = vmsa_ttbr_write, .resetvalue = 0,
4047       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr0_s),
4048                              offsetof(CPUARMState, cp15.ttbr0_ns) } },
4049     { .name = "TTBR1_EL1", .state = ARM_CP_STATE_BOTH,
4050       .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 0, .opc2 = 1,
4051       .access = PL1_RW, .accessfn = access_tvm_trvm,
4052       .writefn = vmsa_ttbr_write, .resetvalue = 0,
4053       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr1_s),
4054                              offsetof(CPUARMState, cp15.ttbr1_ns) } },
4055     { .name = "TCR_EL1", .state = ARM_CP_STATE_AA64,
4056       .opc0 = 3, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 2,
4057       .access = PL1_RW, .accessfn = access_tvm_trvm,
4058       .writefn = vmsa_tcr_el12_write,
4059       .resetfn = vmsa_ttbcr_reset, .raw_writefn = raw_write,
4060       .fieldoffset = offsetof(CPUARMState, cp15.tcr_el[1]) },
4061     { .name = "TTBCR", .cp = 15, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 2,
4062       .access = PL1_RW, .accessfn = access_tvm_trvm,
4063       .type = ARM_CP_ALIAS, .writefn = vmsa_ttbcr_write,
4064       .raw_writefn = vmsa_ttbcr_raw_write,
4065       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.tcr_el[3]),
4066                              offsetoflow32(CPUARMState, cp15.tcr_el[1])} },
4067     REGINFO_SENTINEL
4068 };
4069 
4070 /* Note that unlike TTBCR, writing to TTBCR2 does not require flushing
4071  * qemu tlbs nor adjusting cached masks.
4072  */
4073 static const ARMCPRegInfo ttbcr2_reginfo = {
4074     .name = "TTBCR2", .cp = 15, .opc1 = 0, .crn = 2, .crm = 0, .opc2 = 3,
4075     .access = PL1_RW, .accessfn = access_tvm_trvm,
4076     .type = ARM_CP_ALIAS,
4077     .bank_fieldoffsets = { offsetofhigh32(CPUARMState, cp15.tcr_el[3]),
4078                            offsetofhigh32(CPUARMState, cp15.tcr_el[1]) },
4079 };
4080 
4081 static void omap_ticonfig_write(CPUARMState *env, const ARMCPRegInfo *ri,
4082                                 uint64_t value)
4083 {
4084     env->cp15.c15_ticonfig = value & 0xe7;
4085     /* The OS_TYPE bit in this register changes the reported CPUID! */
4086     env->cp15.c0_cpuid = (value & (1 << 5)) ?
4087         ARM_CPUID_TI915T : ARM_CPUID_TI925T;
4088 }
4089 
4090 static void omap_threadid_write(CPUARMState *env, const ARMCPRegInfo *ri,
4091                                 uint64_t value)
4092 {
4093     env->cp15.c15_threadid = value & 0xffff;
4094 }
4095 
4096 static void omap_wfi_write(CPUARMState *env, const ARMCPRegInfo *ri,
4097                            uint64_t value)
4098 {
4099     /* Wait-for-interrupt (deprecated) */
4100     cpu_interrupt(env_cpu(env), CPU_INTERRUPT_HALT);
4101 }
4102 
4103 static void omap_cachemaint_write(CPUARMState *env, const ARMCPRegInfo *ri,
4104                                   uint64_t value)
4105 {
4106     /* On OMAP there are registers indicating the max/min index of dcache lines
4107      * containing a dirty line; cache flush operations have to reset these.
4108      */
4109     env->cp15.c15_i_max = 0x000;
4110     env->cp15.c15_i_min = 0xff0;
4111 }
4112 
4113 static const ARMCPRegInfo omap_cp_reginfo[] = {
4114     { .name = "DFSR", .cp = 15, .crn = 5, .crm = CP_ANY,
4115       .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_OVERRIDE,
4116       .fieldoffset = offsetoflow32(CPUARMState, cp15.esr_el[1]),
4117       .resetvalue = 0, },
4118     { .name = "", .cp = 15, .crn = 15, .crm = 0, .opc1 = 0, .opc2 = 0,
4119       .access = PL1_RW, .type = ARM_CP_NOP },
4120     { .name = "TICONFIG", .cp = 15, .crn = 15, .crm = 1, .opc1 = 0, .opc2 = 0,
4121       .access = PL1_RW,
4122       .fieldoffset = offsetof(CPUARMState, cp15.c15_ticonfig), .resetvalue = 0,
4123       .writefn = omap_ticonfig_write },
4124     { .name = "IMAX", .cp = 15, .crn = 15, .crm = 2, .opc1 = 0, .opc2 = 0,
4125       .access = PL1_RW,
4126       .fieldoffset = offsetof(CPUARMState, cp15.c15_i_max), .resetvalue = 0, },
4127     { .name = "IMIN", .cp = 15, .crn = 15, .crm = 3, .opc1 = 0, .opc2 = 0,
4128       .access = PL1_RW, .resetvalue = 0xff0,
4129       .fieldoffset = offsetof(CPUARMState, cp15.c15_i_min) },
4130     { .name = "THREADID", .cp = 15, .crn = 15, .crm = 4, .opc1 = 0, .opc2 = 0,
4131       .access = PL1_RW,
4132       .fieldoffset = offsetof(CPUARMState, cp15.c15_threadid), .resetvalue = 0,
4133       .writefn = omap_threadid_write },
4134     { .name = "TI925T_STATUS", .cp = 15, .crn = 15,
4135       .crm = 8, .opc1 = 0, .opc2 = 0, .access = PL1_RW,
4136       .type = ARM_CP_NO_RAW,
4137       .readfn = arm_cp_read_zero, .writefn = omap_wfi_write, },
4138     /* TODO: Peripheral port remap register:
4139      * On OMAP2 mcr p15, 0, rn, c15, c2, 4 sets up the interrupt controller
4140      * base address at $rn & ~0xfff and map size of 0x200 << ($rn & 0xfff),
4141      * when MMU is off.
4142      */
4143     { .name = "OMAP_CACHEMAINT", .cp = 15, .crn = 7, .crm = CP_ANY,
4144       .opc1 = 0, .opc2 = CP_ANY, .access = PL1_W,
4145       .type = ARM_CP_OVERRIDE | ARM_CP_NO_RAW,
4146       .writefn = omap_cachemaint_write },
4147     { .name = "C9", .cp = 15, .crn = 9,
4148       .crm = CP_ANY, .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW,
4149       .type = ARM_CP_CONST | ARM_CP_OVERRIDE, .resetvalue = 0 },
4150     REGINFO_SENTINEL
4151 };
4152 
4153 static void xscale_cpar_write(CPUARMState *env, const ARMCPRegInfo *ri,
4154                               uint64_t value)
4155 {
4156     env->cp15.c15_cpar = value & 0x3fff;
4157 }
4158 
4159 static const ARMCPRegInfo xscale_cp_reginfo[] = {
4160     { .name = "XSCALE_CPAR",
4161       .cp = 15, .crn = 15, .crm = 1, .opc1 = 0, .opc2 = 0, .access = PL1_RW,
4162       .fieldoffset = offsetof(CPUARMState, cp15.c15_cpar), .resetvalue = 0,
4163       .writefn = xscale_cpar_write, },
4164     { .name = "XSCALE_AUXCR",
4165       .cp = 15, .crn = 1, .crm = 0, .opc1 = 0, .opc2 = 1, .access = PL1_RW,
4166       .fieldoffset = offsetof(CPUARMState, cp15.c1_xscaleauxcr),
4167       .resetvalue = 0, },
4168     /* XScale specific cache-lockdown: since we have no cache we NOP these
4169      * and hope the guest does not really rely on cache behaviour.
4170      */
4171     { .name = "XSCALE_LOCK_ICACHE_LINE",
4172       .cp = 15, .opc1 = 0, .crn = 9, .crm = 1, .opc2 = 0,
4173       .access = PL1_W, .type = ARM_CP_NOP },
4174     { .name = "XSCALE_UNLOCK_ICACHE",
4175       .cp = 15, .opc1 = 0, .crn = 9, .crm = 1, .opc2 = 1,
4176       .access = PL1_W, .type = ARM_CP_NOP },
4177     { .name = "XSCALE_DCACHE_LOCK",
4178       .cp = 15, .opc1 = 0, .crn = 9, .crm = 2, .opc2 = 0,
4179       .access = PL1_RW, .type = ARM_CP_NOP },
4180     { .name = "XSCALE_UNLOCK_DCACHE",
4181       .cp = 15, .opc1 = 0, .crn = 9, .crm = 2, .opc2 = 1,
4182       .access = PL1_W, .type = ARM_CP_NOP },
4183     REGINFO_SENTINEL
4184 };
4185 
4186 static const ARMCPRegInfo dummy_c15_cp_reginfo[] = {
4187     /* RAZ/WI the whole crn=15 space, when we don't have a more specific
4188      * implementation of this implementation-defined space.
4189      * Ideally this should eventually disappear in favour of actually
4190      * implementing the correct behaviour for all cores.
4191      */
4192     { .name = "C15_IMPDEF", .cp = 15, .crn = 15,
4193       .crm = CP_ANY, .opc1 = CP_ANY, .opc2 = CP_ANY,
4194       .access = PL1_RW,
4195       .type = ARM_CP_CONST | ARM_CP_NO_RAW | ARM_CP_OVERRIDE,
4196       .resetvalue = 0 },
4197     REGINFO_SENTINEL
4198 };
4199 
4200 static const ARMCPRegInfo cache_dirty_status_cp_reginfo[] = {
4201     /* Cache status: RAZ because we have no cache so it's always clean */
4202     { .name = "CDSR", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 6,
4203       .access = PL1_R, .type = ARM_CP_CONST | ARM_CP_NO_RAW,
4204       .resetvalue = 0 },
4205     REGINFO_SENTINEL
4206 };
4207 
4208 static const ARMCPRegInfo cache_block_ops_cp_reginfo[] = {
4209     /* We never have a a block transfer operation in progress */
4210     { .name = "BXSR", .cp = 15, .crn = 7, .crm = 12, .opc1 = 0, .opc2 = 4,
4211       .access = PL0_R, .type = ARM_CP_CONST | ARM_CP_NO_RAW,
4212       .resetvalue = 0 },
4213     /* The cache ops themselves: these all NOP for QEMU */
4214     { .name = "IICR", .cp = 15, .crm = 5, .opc1 = 0,
4215       .access = PL1_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
4216     { .name = "IDCR", .cp = 15, .crm = 6, .opc1 = 0,
4217       .access = PL1_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
4218     { .name = "CDCR", .cp = 15, .crm = 12, .opc1 = 0,
4219       .access = PL0_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
4220     { .name = "PIR", .cp = 15, .crm = 12, .opc1 = 1,
4221       .access = PL0_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
4222     { .name = "PDR", .cp = 15, .crm = 12, .opc1 = 2,
4223       .access = PL0_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
4224     { .name = "CIDCR", .cp = 15, .crm = 14, .opc1 = 0,
4225       .access = PL1_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
4226     REGINFO_SENTINEL
4227 };
4228 
4229 static const ARMCPRegInfo cache_test_clean_cp_reginfo[] = {
4230     /* The cache test-and-clean instructions always return (1 << 30)
4231      * to indicate that there are no dirty cache lines.
4232      */
4233     { .name = "TC_DCACHE", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 3,
4234       .access = PL0_R, .type = ARM_CP_CONST | ARM_CP_NO_RAW,
4235       .resetvalue = (1 << 30) },
4236     { .name = "TCI_DCACHE", .cp = 15, .crn = 7, .crm = 14, .opc1 = 0, .opc2 = 3,
4237       .access = PL0_R, .type = ARM_CP_CONST | ARM_CP_NO_RAW,
4238       .resetvalue = (1 << 30) },
4239     REGINFO_SENTINEL
4240 };
4241 
4242 static const ARMCPRegInfo strongarm_cp_reginfo[] = {
4243     /* Ignore ReadBuffer accesses */
4244     { .name = "C9_READBUFFER", .cp = 15, .crn = 9,
4245       .crm = CP_ANY, .opc1 = CP_ANY, .opc2 = CP_ANY,
4246       .access = PL1_RW, .resetvalue = 0,
4247       .type = ARM_CP_CONST | ARM_CP_OVERRIDE | ARM_CP_NO_RAW },
4248     REGINFO_SENTINEL
4249 };
4250 
4251 static uint64_t midr_read(CPUARMState *env, const ARMCPRegInfo *ri)
4252 {
4253     ARMCPU *cpu = env_archcpu(env);
4254     unsigned int cur_el = arm_current_el(env);
4255     bool secure = arm_is_secure(env);
4256 
4257     if (arm_feature(&cpu->env, ARM_FEATURE_EL2) && !secure && cur_el == 1) {
4258         return env->cp15.vpidr_el2;
4259     }
4260     return raw_read(env, ri);
4261 }
4262 
4263 static uint64_t mpidr_read_val(CPUARMState *env)
4264 {
4265     ARMCPU *cpu = env_archcpu(env);
4266     uint64_t mpidr = cpu->mp_affinity;
4267 
4268     if (arm_feature(env, ARM_FEATURE_V7MP)) {
4269         mpidr |= (1U << 31);
4270         /* Cores which are uniprocessor (non-coherent)
4271          * but still implement the MP extensions set
4272          * bit 30. (For instance, Cortex-R5).
4273          */
4274         if (cpu->mp_is_up) {
4275             mpidr |= (1u << 30);
4276         }
4277     }
4278     return mpidr;
4279 }
4280 
4281 static uint64_t mpidr_read(CPUARMState *env, const ARMCPRegInfo *ri)
4282 {
4283     unsigned int cur_el = arm_current_el(env);
4284     bool secure = arm_is_secure(env);
4285 
4286     if (arm_feature(env, ARM_FEATURE_EL2) && !secure && cur_el == 1) {
4287         return env->cp15.vmpidr_el2;
4288     }
4289     return mpidr_read_val(env);
4290 }
4291 
4292 static const ARMCPRegInfo lpae_cp_reginfo[] = {
4293     /* NOP AMAIR0/1 */
4294     { .name = "AMAIR0", .state = ARM_CP_STATE_BOTH,
4295       .opc0 = 3, .crn = 10, .crm = 3, .opc1 = 0, .opc2 = 0,
4296       .access = PL1_RW, .accessfn = access_tvm_trvm,
4297       .type = ARM_CP_CONST, .resetvalue = 0 },
4298     /* AMAIR1 is mapped to AMAIR_EL1[63:32] */
4299     { .name = "AMAIR1", .cp = 15, .crn = 10, .crm = 3, .opc1 = 0, .opc2 = 1,
4300       .access = PL1_RW, .accessfn = access_tvm_trvm,
4301       .type = ARM_CP_CONST, .resetvalue = 0 },
4302     { .name = "PAR", .cp = 15, .crm = 7, .opc1 = 0,
4303       .access = PL1_RW, .type = ARM_CP_64BIT, .resetvalue = 0,
4304       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.par_s),
4305                              offsetof(CPUARMState, cp15.par_ns)} },
4306     { .name = "TTBR0", .cp = 15, .crm = 2, .opc1 = 0,
4307       .access = PL1_RW, .accessfn = access_tvm_trvm,
4308       .type = ARM_CP_64BIT | ARM_CP_ALIAS,
4309       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr0_s),
4310                              offsetof(CPUARMState, cp15.ttbr0_ns) },
4311       .writefn = vmsa_ttbr_write, },
4312     { .name = "TTBR1", .cp = 15, .crm = 2, .opc1 = 1,
4313       .access = PL1_RW, .accessfn = access_tvm_trvm,
4314       .type = ARM_CP_64BIT | ARM_CP_ALIAS,
4315       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr1_s),
4316                              offsetof(CPUARMState, cp15.ttbr1_ns) },
4317       .writefn = vmsa_ttbr_write, },
4318     REGINFO_SENTINEL
4319 };
4320 
4321 static uint64_t aa64_fpcr_read(CPUARMState *env, const ARMCPRegInfo *ri)
4322 {
4323     return vfp_get_fpcr(env);
4324 }
4325 
4326 static void aa64_fpcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
4327                             uint64_t value)
4328 {
4329     vfp_set_fpcr(env, value);
4330 }
4331 
4332 static uint64_t aa64_fpsr_read(CPUARMState *env, const ARMCPRegInfo *ri)
4333 {
4334     return vfp_get_fpsr(env);
4335 }
4336 
4337 static void aa64_fpsr_write(CPUARMState *env, const ARMCPRegInfo *ri,
4338                             uint64_t value)
4339 {
4340     vfp_set_fpsr(env, value);
4341 }
4342 
4343 static CPAccessResult aa64_daif_access(CPUARMState *env, const ARMCPRegInfo *ri,
4344                                        bool isread)
4345 {
4346     if (arm_current_el(env) == 0 && !(arm_sctlr(env, 0) & SCTLR_UMA)) {
4347         return CP_ACCESS_TRAP;
4348     }
4349     return CP_ACCESS_OK;
4350 }
4351 
4352 static void aa64_daif_write(CPUARMState *env, const ARMCPRegInfo *ri,
4353                             uint64_t value)
4354 {
4355     env->daif = value & PSTATE_DAIF;
4356 }
4357 
4358 static uint64_t aa64_pan_read(CPUARMState *env, const ARMCPRegInfo *ri)
4359 {
4360     return env->pstate & PSTATE_PAN;
4361 }
4362 
4363 static void aa64_pan_write(CPUARMState *env, const ARMCPRegInfo *ri,
4364                            uint64_t value)
4365 {
4366     env->pstate = (env->pstate & ~PSTATE_PAN) | (value & PSTATE_PAN);
4367 }
4368 
4369 static const ARMCPRegInfo pan_reginfo = {
4370     .name = "PAN", .state = ARM_CP_STATE_AA64,
4371     .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 2, .opc2 = 3,
4372     .type = ARM_CP_NO_RAW, .access = PL1_RW,
4373     .readfn = aa64_pan_read, .writefn = aa64_pan_write
4374 };
4375 
4376 static uint64_t aa64_uao_read(CPUARMState *env, const ARMCPRegInfo *ri)
4377 {
4378     return env->pstate & PSTATE_UAO;
4379 }
4380 
4381 static void aa64_uao_write(CPUARMState *env, const ARMCPRegInfo *ri,
4382                            uint64_t value)
4383 {
4384     env->pstate = (env->pstate & ~PSTATE_UAO) | (value & PSTATE_UAO);
4385 }
4386 
4387 static const ARMCPRegInfo uao_reginfo = {
4388     .name = "UAO", .state = ARM_CP_STATE_AA64,
4389     .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 2, .opc2 = 4,
4390     .type = ARM_CP_NO_RAW, .access = PL1_RW,
4391     .readfn = aa64_uao_read, .writefn = aa64_uao_write
4392 };
4393 
4394 static CPAccessResult aa64_cacheop_poc_access(CPUARMState *env,
4395                                               const ARMCPRegInfo *ri,
4396                                               bool isread)
4397 {
4398     /* Cache invalidate/clean to Point of Coherency or Persistence...  */
4399     switch (arm_current_el(env)) {
4400     case 0:
4401         /* ... EL0 must UNDEF unless SCTLR_EL1.UCI is set.  */
4402         if (!(arm_sctlr(env, 0) & SCTLR_UCI)) {
4403             return CP_ACCESS_TRAP;
4404         }
4405         /* fall through */
4406     case 1:
4407         /* ... EL1 must trap to EL2 if HCR_EL2.TPCP is set.  */
4408         if (arm_hcr_el2_eff(env) & HCR_TPCP) {
4409             return CP_ACCESS_TRAP_EL2;
4410         }
4411         break;
4412     }
4413     return CP_ACCESS_OK;
4414 }
4415 
4416 static CPAccessResult aa64_cacheop_pou_access(CPUARMState *env,
4417                                               const ARMCPRegInfo *ri,
4418                                               bool isread)
4419 {
4420     /* Cache invalidate/clean to Point of Unification... */
4421     switch (arm_current_el(env)) {
4422     case 0:
4423         /* ... EL0 must UNDEF unless SCTLR_EL1.UCI is set.  */
4424         if (!(arm_sctlr(env, 0) & SCTLR_UCI)) {
4425             return CP_ACCESS_TRAP;
4426         }
4427         /* fall through */
4428     case 1:
4429         /* ... EL1 must trap to EL2 if HCR_EL2.TPU is set.  */
4430         if (arm_hcr_el2_eff(env) & HCR_TPU) {
4431             return CP_ACCESS_TRAP_EL2;
4432         }
4433         break;
4434     }
4435     return CP_ACCESS_OK;
4436 }
4437 
4438 /* See: D4.7.2 TLB maintenance requirements and the TLB maintenance instructions
4439  * Page D4-1736 (DDI0487A.b)
4440  */
4441 
4442 static int vae1_tlbmask(CPUARMState *env)
4443 {
4444     /* Since we exclude secure first, we may read HCR_EL2 directly. */
4445     if (arm_is_secure_below_el3(env)) {
4446         return ARMMMUIdxBit_SE10_1 |
4447                ARMMMUIdxBit_SE10_1_PAN |
4448                ARMMMUIdxBit_SE10_0;
4449     } else if ((env->cp15.hcr_el2 & (HCR_E2H | HCR_TGE))
4450                == (HCR_E2H | HCR_TGE)) {
4451         return ARMMMUIdxBit_E20_2 |
4452                ARMMMUIdxBit_E20_2_PAN |
4453                ARMMMUIdxBit_E20_0;
4454     } else {
4455         return ARMMMUIdxBit_E10_1 |
4456                ARMMMUIdxBit_E10_1_PAN |
4457                ARMMMUIdxBit_E10_0;
4458     }
4459 }
4460 
4461 /* Return 56 if TBI is enabled, 64 otherwise. */
4462 static int tlbbits_for_regime(CPUARMState *env, ARMMMUIdx mmu_idx,
4463                               uint64_t addr)
4464 {
4465     uint64_t tcr = regime_tcr(env, mmu_idx)->raw_tcr;
4466     int tbi = aa64_va_parameter_tbi(tcr, mmu_idx);
4467     int select = extract64(addr, 55, 1);
4468 
4469     return (tbi >> select) & 1 ? 56 : 64;
4470 }
4471 
4472 static int vae1_tlbbits(CPUARMState *env, uint64_t addr)
4473 {
4474     ARMMMUIdx mmu_idx;
4475 
4476     /* Only the regime of the mmu_idx below is significant. */
4477     if (arm_is_secure_below_el3(env)) {
4478         mmu_idx = ARMMMUIdx_SE10_0;
4479     } else if ((env->cp15.hcr_el2 & (HCR_E2H | HCR_TGE))
4480                == (HCR_E2H | HCR_TGE)) {
4481         mmu_idx = ARMMMUIdx_E20_0;
4482     } else {
4483         mmu_idx = ARMMMUIdx_E10_0;
4484     }
4485     return tlbbits_for_regime(env, mmu_idx, addr);
4486 }
4487 
4488 static void tlbi_aa64_vmalle1is_write(CPUARMState *env, const ARMCPRegInfo *ri,
4489                                       uint64_t value)
4490 {
4491     CPUState *cs = env_cpu(env);
4492     int mask = vae1_tlbmask(env);
4493 
4494     tlb_flush_by_mmuidx_all_cpus_synced(cs, mask);
4495 }
4496 
4497 static void tlbi_aa64_vmalle1_write(CPUARMState *env, const ARMCPRegInfo *ri,
4498                                     uint64_t value)
4499 {
4500     CPUState *cs = env_cpu(env);
4501     int mask = vae1_tlbmask(env);
4502 
4503     if (tlb_force_broadcast(env)) {
4504         tlb_flush_by_mmuidx_all_cpus_synced(cs, mask);
4505     } else {
4506         tlb_flush_by_mmuidx(cs, mask);
4507     }
4508 }
4509 
4510 static int alle1_tlbmask(CPUARMState *env)
4511 {
4512     /*
4513      * Note that the 'ALL' scope must invalidate both stage 1 and
4514      * stage 2 translations, whereas most other scopes only invalidate
4515      * stage 1 translations.
4516      */
4517     if (arm_is_secure_below_el3(env)) {
4518         return ARMMMUIdxBit_SE10_1 |
4519                ARMMMUIdxBit_SE10_1_PAN |
4520                ARMMMUIdxBit_SE10_0;
4521     } else {
4522         return ARMMMUIdxBit_E10_1 |
4523                ARMMMUIdxBit_E10_1_PAN |
4524                ARMMMUIdxBit_E10_0;
4525     }
4526 }
4527 
4528 static int e2_tlbmask(CPUARMState *env)
4529 {
4530     /* TODO: ARMv8.4-SecEL2 */
4531     return ARMMMUIdxBit_E20_0 |
4532            ARMMMUIdxBit_E20_2 |
4533            ARMMMUIdxBit_E20_2_PAN |
4534            ARMMMUIdxBit_E2;
4535 }
4536 
4537 static void tlbi_aa64_alle1_write(CPUARMState *env, const ARMCPRegInfo *ri,
4538                                   uint64_t value)
4539 {
4540     CPUState *cs = env_cpu(env);
4541     int mask = alle1_tlbmask(env);
4542 
4543     tlb_flush_by_mmuidx(cs, mask);
4544 }
4545 
4546 static void tlbi_aa64_alle2_write(CPUARMState *env, const ARMCPRegInfo *ri,
4547                                   uint64_t value)
4548 {
4549     CPUState *cs = env_cpu(env);
4550     int mask = e2_tlbmask(env);
4551 
4552     tlb_flush_by_mmuidx(cs, mask);
4553 }
4554 
4555 static void tlbi_aa64_alle3_write(CPUARMState *env, const ARMCPRegInfo *ri,
4556                                   uint64_t value)
4557 {
4558     ARMCPU *cpu = env_archcpu(env);
4559     CPUState *cs = CPU(cpu);
4560 
4561     tlb_flush_by_mmuidx(cs, ARMMMUIdxBit_SE3);
4562 }
4563 
4564 static void tlbi_aa64_alle1is_write(CPUARMState *env, const ARMCPRegInfo *ri,
4565                                     uint64_t value)
4566 {
4567     CPUState *cs = env_cpu(env);
4568     int mask = alle1_tlbmask(env);
4569 
4570     tlb_flush_by_mmuidx_all_cpus_synced(cs, mask);
4571 }
4572 
4573 static void tlbi_aa64_alle2is_write(CPUARMState *env, const ARMCPRegInfo *ri,
4574                                     uint64_t value)
4575 {
4576     CPUState *cs = env_cpu(env);
4577     int mask = e2_tlbmask(env);
4578 
4579     tlb_flush_by_mmuidx_all_cpus_synced(cs, mask);
4580 }
4581 
4582 static void tlbi_aa64_alle3is_write(CPUARMState *env, const ARMCPRegInfo *ri,
4583                                     uint64_t value)
4584 {
4585     CPUState *cs = env_cpu(env);
4586 
4587     tlb_flush_by_mmuidx_all_cpus_synced(cs, ARMMMUIdxBit_SE3);
4588 }
4589 
4590 static void tlbi_aa64_vae2_write(CPUARMState *env, const ARMCPRegInfo *ri,
4591                                  uint64_t value)
4592 {
4593     /* Invalidate by VA, EL2
4594      * Currently handles both VAE2 and VALE2, since we don't support
4595      * flush-last-level-only.
4596      */
4597     CPUState *cs = env_cpu(env);
4598     int mask = e2_tlbmask(env);
4599     uint64_t pageaddr = sextract64(value << 12, 0, 56);
4600 
4601     tlb_flush_page_by_mmuidx(cs, pageaddr, mask);
4602 }
4603 
4604 static void tlbi_aa64_vae3_write(CPUARMState *env, const ARMCPRegInfo *ri,
4605                                  uint64_t value)
4606 {
4607     /* Invalidate by VA, EL3
4608      * Currently handles both VAE3 and VALE3, since we don't support
4609      * flush-last-level-only.
4610      */
4611     ARMCPU *cpu = env_archcpu(env);
4612     CPUState *cs = CPU(cpu);
4613     uint64_t pageaddr = sextract64(value << 12, 0, 56);
4614 
4615     tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdxBit_SE3);
4616 }
4617 
4618 static void tlbi_aa64_vae1is_write(CPUARMState *env, const ARMCPRegInfo *ri,
4619                                    uint64_t value)
4620 {
4621     CPUState *cs = env_cpu(env);
4622     int mask = vae1_tlbmask(env);
4623     uint64_t pageaddr = sextract64(value << 12, 0, 56);
4624     int bits = vae1_tlbbits(env, pageaddr);
4625 
4626     tlb_flush_page_bits_by_mmuidx_all_cpus_synced(cs, pageaddr, mask, bits);
4627 }
4628 
4629 static void tlbi_aa64_vae1_write(CPUARMState *env, const ARMCPRegInfo *ri,
4630                                  uint64_t value)
4631 {
4632     /* Invalidate by VA, EL1&0 (AArch64 version).
4633      * Currently handles all of VAE1, VAAE1, VAALE1 and VALE1,
4634      * since we don't support flush-for-specific-ASID-only or
4635      * flush-last-level-only.
4636      */
4637     CPUState *cs = env_cpu(env);
4638     int mask = vae1_tlbmask(env);
4639     uint64_t pageaddr = sextract64(value << 12, 0, 56);
4640     int bits = vae1_tlbbits(env, pageaddr);
4641 
4642     if (tlb_force_broadcast(env)) {
4643         tlb_flush_page_bits_by_mmuidx_all_cpus_synced(cs, pageaddr, mask, bits);
4644     } else {
4645         tlb_flush_page_bits_by_mmuidx(cs, pageaddr, mask, bits);
4646     }
4647 }
4648 
4649 static void tlbi_aa64_vae2is_write(CPUARMState *env, const ARMCPRegInfo *ri,
4650                                    uint64_t value)
4651 {
4652     CPUState *cs = env_cpu(env);
4653     uint64_t pageaddr = sextract64(value << 12, 0, 56);
4654     int bits = tlbbits_for_regime(env, ARMMMUIdx_E2, pageaddr);
4655 
4656     tlb_flush_page_bits_by_mmuidx_all_cpus_synced(cs, pageaddr,
4657                                                   ARMMMUIdxBit_E2, bits);
4658 }
4659 
4660 static void tlbi_aa64_vae3is_write(CPUARMState *env, const ARMCPRegInfo *ri,
4661                                    uint64_t value)
4662 {
4663     CPUState *cs = env_cpu(env);
4664     uint64_t pageaddr = sextract64(value << 12, 0, 56);
4665     int bits = tlbbits_for_regime(env, ARMMMUIdx_SE3, pageaddr);
4666 
4667     tlb_flush_page_bits_by_mmuidx_all_cpus_synced(cs, pageaddr,
4668                                                   ARMMMUIdxBit_SE3, bits);
4669 }
4670 
4671 static CPAccessResult aa64_zva_access(CPUARMState *env, const ARMCPRegInfo *ri,
4672                                       bool isread)
4673 {
4674     int cur_el = arm_current_el(env);
4675 
4676     if (cur_el < 2) {
4677         uint64_t hcr = arm_hcr_el2_eff(env);
4678 
4679         if (cur_el == 0) {
4680             if ((hcr & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)) {
4681                 if (!(env->cp15.sctlr_el[2] & SCTLR_DZE)) {
4682                     return CP_ACCESS_TRAP_EL2;
4683                 }
4684             } else {
4685                 if (!(env->cp15.sctlr_el[1] & SCTLR_DZE)) {
4686                     return CP_ACCESS_TRAP;
4687                 }
4688                 if (hcr & HCR_TDZ) {
4689                     return CP_ACCESS_TRAP_EL2;
4690                 }
4691             }
4692         } else if (hcr & HCR_TDZ) {
4693             return CP_ACCESS_TRAP_EL2;
4694         }
4695     }
4696     return CP_ACCESS_OK;
4697 }
4698 
4699 static uint64_t aa64_dczid_read(CPUARMState *env, const ARMCPRegInfo *ri)
4700 {
4701     ARMCPU *cpu = env_archcpu(env);
4702     int dzp_bit = 1 << 4;
4703 
4704     /* DZP indicates whether DC ZVA access is allowed */
4705     if (aa64_zva_access(env, NULL, false) == CP_ACCESS_OK) {
4706         dzp_bit = 0;
4707     }
4708     return cpu->dcz_blocksize | dzp_bit;
4709 }
4710 
4711 static CPAccessResult sp_el0_access(CPUARMState *env, const ARMCPRegInfo *ri,
4712                                     bool isread)
4713 {
4714     if (!(env->pstate & PSTATE_SP)) {
4715         /* Access to SP_EL0 is undefined if it's being used as
4716          * the stack pointer.
4717          */
4718         return CP_ACCESS_TRAP_UNCATEGORIZED;
4719     }
4720     return CP_ACCESS_OK;
4721 }
4722 
4723 static uint64_t spsel_read(CPUARMState *env, const ARMCPRegInfo *ri)
4724 {
4725     return env->pstate & PSTATE_SP;
4726 }
4727 
4728 static void spsel_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t val)
4729 {
4730     update_spsel(env, val);
4731 }
4732 
4733 static void sctlr_write(CPUARMState *env, const ARMCPRegInfo *ri,
4734                         uint64_t value)
4735 {
4736     ARMCPU *cpu = env_archcpu(env);
4737 
4738     if (arm_feature(env, ARM_FEATURE_PMSA) && !cpu->has_mpu) {
4739         /* M bit is RAZ/WI for PMSA with no MPU implemented */
4740         value &= ~SCTLR_M;
4741     }
4742 
4743     /* ??? Lots of these bits are not implemented.  */
4744 
4745     if (ri->state == ARM_CP_STATE_AA64 && !cpu_isar_feature(aa64_mte, cpu)) {
4746         if (ri->opc1 == 6) { /* SCTLR_EL3 */
4747             value &= ~(SCTLR_ITFSB | SCTLR_TCF | SCTLR_ATA);
4748         } else {
4749             value &= ~(SCTLR_ITFSB | SCTLR_TCF0 | SCTLR_TCF |
4750                        SCTLR_ATA0 | SCTLR_ATA);
4751         }
4752     }
4753 
4754     if (raw_read(env, ri) == value) {
4755         /* Skip the TLB flush if nothing actually changed; Linux likes
4756          * to do a lot of pointless SCTLR writes.
4757          */
4758         return;
4759     }
4760 
4761     raw_write(env, ri, value);
4762 
4763     /* This may enable/disable the MMU, so do a TLB flush.  */
4764     tlb_flush(CPU(cpu));
4765 
4766     if (ri->type & ARM_CP_SUPPRESS_TB_END) {
4767         /*
4768          * Normally we would always end the TB on an SCTLR write; see the
4769          * comment in ARMCPRegInfo sctlr initialization below for why Xscale
4770          * is special.  Setting ARM_CP_SUPPRESS_TB_END also stops the rebuild
4771          * of hflags from the translator, so do it here.
4772          */
4773         arm_rebuild_hflags(env);
4774     }
4775 }
4776 
4777 static CPAccessResult fpexc32_access(CPUARMState *env, const ARMCPRegInfo *ri,
4778                                      bool isread)
4779 {
4780     if ((env->cp15.cptr_el[2] & CPTR_TFP) && arm_current_el(env) == 2) {
4781         return CP_ACCESS_TRAP_FP_EL2;
4782     }
4783     if (env->cp15.cptr_el[3] & CPTR_TFP) {
4784         return CP_ACCESS_TRAP_FP_EL3;
4785     }
4786     return CP_ACCESS_OK;
4787 }
4788 
4789 static void sdcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
4790                        uint64_t value)
4791 {
4792     env->cp15.mdcr_el3 = value & SDCR_VALID_MASK;
4793 }
4794 
4795 static const ARMCPRegInfo v8_cp_reginfo[] = {
4796     /* Minimal set of EL0-visible registers. This will need to be expanded
4797      * significantly for system emulation of AArch64 CPUs.
4798      */
4799     { .name = "NZCV", .state = ARM_CP_STATE_AA64,
4800       .opc0 = 3, .opc1 = 3, .opc2 = 0, .crn = 4, .crm = 2,
4801       .access = PL0_RW, .type = ARM_CP_NZCV },
4802     { .name = "DAIF", .state = ARM_CP_STATE_AA64,
4803       .opc0 = 3, .opc1 = 3, .opc2 = 1, .crn = 4, .crm = 2,
4804       .type = ARM_CP_NO_RAW,
4805       .access = PL0_RW, .accessfn = aa64_daif_access,
4806       .fieldoffset = offsetof(CPUARMState, daif),
4807       .writefn = aa64_daif_write, .resetfn = arm_cp_reset_ignore },
4808     { .name = "FPCR", .state = ARM_CP_STATE_AA64,
4809       .opc0 = 3, .opc1 = 3, .opc2 = 0, .crn = 4, .crm = 4,
4810       .access = PL0_RW, .type = ARM_CP_FPU | ARM_CP_SUPPRESS_TB_END,
4811       .readfn = aa64_fpcr_read, .writefn = aa64_fpcr_write },
4812     { .name = "FPSR", .state = ARM_CP_STATE_AA64,
4813       .opc0 = 3, .opc1 = 3, .opc2 = 1, .crn = 4, .crm = 4,
4814       .access = PL0_RW, .type = ARM_CP_FPU | ARM_CP_SUPPRESS_TB_END,
4815       .readfn = aa64_fpsr_read, .writefn = aa64_fpsr_write },
4816     { .name = "DCZID_EL0", .state = ARM_CP_STATE_AA64,
4817       .opc0 = 3, .opc1 = 3, .opc2 = 7, .crn = 0, .crm = 0,
4818       .access = PL0_R, .type = ARM_CP_NO_RAW,
4819       .readfn = aa64_dczid_read },
4820     { .name = "DC_ZVA", .state = ARM_CP_STATE_AA64,
4821       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 4, .opc2 = 1,
4822       .access = PL0_W, .type = ARM_CP_DC_ZVA,
4823 #ifndef CONFIG_USER_ONLY
4824       /* Avoid overhead of an access check that always passes in user-mode */
4825       .accessfn = aa64_zva_access,
4826 #endif
4827     },
4828     { .name = "CURRENTEL", .state = ARM_CP_STATE_AA64,
4829       .opc0 = 3, .opc1 = 0, .opc2 = 2, .crn = 4, .crm = 2,
4830       .access = PL1_R, .type = ARM_CP_CURRENTEL },
4831     /* Cache ops: all NOPs since we don't emulate caches */
4832     { .name = "IC_IALLUIS", .state = ARM_CP_STATE_AA64,
4833       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 1, .opc2 = 0,
4834       .access = PL1_W, .type = ARM_CP_NOP,
4835       .accessfn = aa64_cacheop_pou_access },
4836     { .name = "IC_IALLU", .state = ARM_CP_STATE_AA64,
4837       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 0,
4838       .access = PL1_W, .type = ARM_CP_NOP,
4839       .accessfn = aa64_cacheop_pou_access },
4840     { .name = "IC_IVAU", .state = ARM_CP_STATE_AA64,
4841       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 5, .opc2 = 1,
4842       .access = PL0_W, .type = ARM_CP_NOP,
4843       .accessfn = aa64_cacheop_pou_access },
4844     { .name = "DC_IVAC", .state = ARM_CP_STATE_AA64,
4845       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 1,
4846       .access = PL1_W, .accessfn = aa64_cacheop_poc_access,
4847       .type = ARM_CP_NOP },
4848     { .name = "DC_ISW", .state = ARM_CP_STATE_AA64,
4849       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 2,
4850       .access = PL1_W, .accessfn = access_tsw, .type = ARM_CP_NOP },
4851     { .name = "DC_CVAC", .state = ARM_CP_STATE_AA64,
4852       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 10, .opc2 = 1,
4853       .access = PL0_W, .type = ARM_CP_NOP,
4854       .accessfn = aa64_cacheop_poc_access },
4855     { .name = "DC_CSW", .state = ARM_CP_STATE_AA64,
4856       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 10, .opc2 = 2,
4857       .access = PL1_W, .accessfn = access_tsw, .type = ARM_CP_NOP },
4858     { .name = "DC_CVAU", .state = ARM_CP_STATE_AA64,
4859       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 11, .opc2 = 1,
4860       .access = PL0_W, .type = ARM_CP_NOP,
4861       .accessfn = aa64_cacheop_pou_access },
4862     { .name = "DC_CIVAC", .state = ARM_CP_STATE_AA64,
4863       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 14, .opc2 = 1,
4864       .access = PL0_W, .type = ARM_CP_NOP,
4865       .accessfn = aa64_cacheop_poc_access },
4866     { .name = "DC_CISW", .state = ARM_CP_STATE_AA64,
4867       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 14, .opc2 = 2,
4868       .access = PL1_W, .accessfn = access_tsw, .type = ARM_CP_NOP },
4869     /* TLBI operations */
4870     { .name = "TLBI_VMALLE1IS", .state = ARM_CP_STATE_AA64,
4871       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 0,
4872       .access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
4873       .writefn = tlbi_aa64_vmalle1is_write },
4874     { .name = "TLBI_VAE1IS", .state = ARM_CP_STATE_AA64,
4875       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 1,
4876       .access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
4877       .writefn = tlbi_aa64_vae1is_write },
4878     { .name = "TLBI_ASIDE1IS", .state = ARM_CP_STATE_AA64,
4879       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 2,
4880       .access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
4881       .writefn = tlbi_aa64_vmalle1is_write },
4882     { .name = "TLBI_VAAE1IS", .state = ARM_CP_STATE_AA64,
4883       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 3,
4884       .access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
4885       .writefn = tlbi_aa64_vae1is_write },
4886     { .name = "TLBI_VALE1IS", .state = ARM_CP_STATE_AA64,
4887       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 5,
4888       .access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
4889       .writefn = tlbi_aa64_vae1is_write },
4890     { .name = "TLBI_VAALE1IS", .state = ARM_CP_STATE_AA64,
4891       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 7,
4892       .access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
4893       .writefn = tlbi_aa64_vae1is_write },
4894     { .name = "TLBI_VMALLE1", .state = ARM_CP_STATE_AA64,
4895       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 0,
4896       .access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
4897       .writefn = tlbi_aa64_vmalle1_write },
4898     { .name = "TLBI_VAE1", .state = ARM_CP_STATE_AA64,
4899       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 1,
4900       .access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
4901       .writefn = tlbi_aa64_vae1_write },
4902     { .name = "TLBI_ASIDE1", .state = ARM_CP_STATE_AA64,
4903       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 2,
4904       .access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
4905       .writefn = tlbi_aa64_vmalle1_write },
4906     { .name = "TLBI_VAAE1", .state = ARM_CP_STATE_AA64,
4907       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 3,
4908       .access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
4909       .writefn = tlbi_aa64_vae1_write },
4910     { .name = "TLBI_VALE1", .state = ARM_CP_STATE_AA64,
4911       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 5,
4912       .access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
4913       .writefn = tlbi_aa64_vae1_write },
4914     { .name = "TLBI_VAALE1", .state = ARM_CP_STATE_AA64,
4915       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 7,
4916       .access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
4917       .writefn = tlbi_aa64_vae1_write },
4918     { .name = "TLBI_IPAS2E1IS", .state = ARM_CP_STATE_AA64,
4919       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 1,
4920       .access = PL2_W, .type = ARM_CP_NOP },
4921     { .name = "TLBI_IPAS2LE1IS", .state = ARM_CP_STATE_AA64,
4922       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 5,
4923       .access = PL2_W, .type = ARM_CP_NOP },
4924     { .name = "TLBI_ALLE1IS", .state = ARM_CP_STATE_AA64,
4925       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 4,
4926       .access = PL2_W, .type = ARM_CP_NO_RAW,
4927       .writefn = tlbi_aa64_alle1is_write },
4928     { .name = "TLBI_VMALLS12E1IS", .state = ARM_CP_STATE_AA64,
4929       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 6,
4930       .access = PL2_W, .type = ARM_CP_NO_RAW,
4931       .writefn = tlbi_aa64_alle1is_write },
4932     { .name = "TLBI_IPAS2E1", .state = ARM_CP_STATE_AA64,
4933       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 1,
4934       .access = PL2_W, .type = ARM_CP_NOP },
4935     { .name = "TLBI_IPAS2LE1", .state = ARM_CP_STATE_AA64,
4936       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 5,
4937       .access = PL2_W, .type = ARM_CP_NOP },
4938     { .name = "TLBI_ALLE1", .state = ARM_CP_STATE_AA64,
4939       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 4,
4940       .access = PL2_W, .type = ARM_CP_NO_RAW,
4941       .writefn = tlbi_aa64_alle1_write },
4942     { .name = "TLBI_VMALLS12E1", .state = ARM_CP_STATE_AA64,
4943       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 6,
4944       .access = PL2_W, .type = ARM_CP_NO_RAW,
4945       .writefn = tlbi_aa64_alle1is_write },
4946 #ifndef CONFIG_USER_ONLY
4947     /* 64 bit address translation operations */
4948     { .name = "AT_S1E1R", .state = ARM_CP_STATE_AA64,
4949       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 0,
4950       .access = PL1_W, .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC,
4951       .writefn = ats_write64 },
4952     { .name = "AT_S1E1W", .state = ARM_CP_STATE_AA64,
4953       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 1,
4954       .access = PL1_W, .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC,
4955       .writefn = ats_write64 },
4956     { .name = "AT_S1E0R", .state = ARM_CP_STATE_AA64,
4957       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 2,
4958       .access = PL1_W, .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC,
4959       .writefn = ats_write64 },
4960     { .name = "AT_S1E0W", .state = ARM_CP_STATE_AA64,
4961       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 3,
4962       .access = PL1_W, .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC,
4963       .writefn = ats_write64 },
4964     { .name = "AT_S12E1R", .state = ARM_CP_STATE_AA64,
4965       .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 4,
4966       .access = PL2_W, .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC,
4967       .writefn = ats_write64 },
4968     { .name = "AT_S12E1W", .state = ARM_CP_STATE_AA64,
4969       .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 5,
4970       .access = PL2_W, .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC,
4971       .writefn = ats_write64 },
4972     { .name = "AT_S12E0R", .state = ARM_CP_STATE_AA64,
4973       .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 6,
4974       .access = PL2_W, .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC,
4975       .writefn = ats_write64 },
4976     { .name = "AT_S12E0W", .state = ARM_CP_STATE_AA64,
4977       .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 7,
4978       .access = PL2_W, .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC,
4979       .writefn = ats_write64 },
4980     /* AT S1E2* are elsewhere as they UNDEF from EL3 if EL2 is not present */
4981     { .name = "AT_S1E3R", .state = ARM_CP_STATE_AA64,
4982       .opc0 = 1, .opc1 = 6, .crn = 7, .crm = 8, .opc2 = 0,
4983       .access = PL3_W, .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC,
4984       .writefn = ats_write64 },
4985     { .name = "AT_S1E3W", .state = ARM_CP_STATE_AA64,
4986       .opc0 = 1, .opc1 = 6, .crn = 7, .crm = 8, .opc2 = 1,
4987       .access = PL3_W, .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC,
4988       .writefn = ats_write64 },
4989     { .name = "PAR_EL1", .state = ARM_CP_STATE_AA64,
4990       .type = ARM_CP_ALIAS,
4991       .opc0 = 3, .opc1 = 0, .crn = 7, .crm = 4, .opc2 = 0,
4992       .access = PL1_RW, .resetvalue = 0,
4993       .fieldoffset = offsetof(CPUARMState, cp15.par_el[1]),
4994       .writefn = par_write },
4995 #endif
4996     /* TLB invalidate last level of translation table walk */
4997     { .name = "TLBIMVALIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 5,
4998       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlb,
4999       .writefn = tlbimva_is_write },
5000     { .name = "TLBIMVAALIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 7,
5001       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlb,
5002       .writefn = tlbimvaa_is_write },
5003     { .name = "TLBIMVAL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 5,
5004       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlb,
5005       .writefn = tlbimva_write },
5006     { .name = "TLBIMVAAL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 7,
5007       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlb,
5008       .writefn = tlbimvaa_write },
5009     { .name = "TLBIMVALH", .cp = 15, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 5,
5010       .type = ARM_CP_NO_RAW, .access = PL2_W,
5011       .writefn = tlbimva_hyp_write },
5012     { .name = "TLBIMVALHIS",
5013       .cp = 15, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 5,
5014       .type = ARM_CP_NO_RAW, .access = PL2_W,
5015       .writefn = tlbimva_hyp_is_write },
5016     { .name = "TLBIIPAS2",
5017       .cp = 15, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 1,
5018       .type = ARM_CP_NOP, .access = PL2_W },
5019     { .name = "TLBIIPAS2IS",
5020       .cp = 15, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 1,
5021       .type = ARM_CP_NOP, .access = PL2_W },
5022     { .name = "TLBIIPAS2L",
5023       .cp = 15, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 5,
5024       .type = ARM_CP_NOP, .access = PL2_W },
5025     { .name = "TLBIIPAS2LIS",
5026       .cp = 15, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 5,
5027       .type = ARM_CP_NOP, .access = PL2_W },
5028     /* 32 bit cache operations */
5029     { .name = "ICIALLUIS", .cp = 15, .opc1 = 0, .crn = 7, .crm = 1, .opc2 = 0,
5030       .type = ARM_CP_NOP, .access = PL1_W, .accessfn = aa64_cacheop_pou_access },
5031     { .name = "BPIALLUIS", .cp = 15, .opc1 = 0, .crn = 7, .crm = 1, .opc2 = 6,
5032       .type = ARM_CP_NOP, .access = PL1_W },
5033     { .name = "ICIALLU", .cp = 15, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 0,
5034       .type = ARM_CP_NOP, .access = PL1_W, .accessfn = aa64_cacheop_pou_access },
5035     { .name = "ICIMVAU", .cp = 15, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 1,
5036       .type = ARM_CP_NOP, .access = PL1_W, .accessfn = aa64_cacheop_pou_access },
5037     { .name = "BPIALL", .cp = 15, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 6,
5038       .type = ARM_CP_NOP, .access = PL1_W },
5039     { .name = "BPIMVA", .cp = 15, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 7,
5040       .type = ARM_CP_NOP, .access = PL1_W },
5041     { .name = "DCIMVAC", .cp = 15, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 1,
5042       .type = ARM_CP_NOP, .access = PL1_W, .accessfn = aa64_cacheop_poc_access },
5043     { .name = "DCISW", .cp = 15, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 2,
5044       .type = ARM_CP_NOP, .access = PL1_W, .accessfn = access_tsw },
5045     { .name = "DCCMVAC", .cp = 15, .opc1 = 0, .crn = 7, .crm = 10, .opc2 = 1,
5046       .type = ARM_CP_NOP, .access = PL1_W, .accessfn = aa64_cacheop_poc_access },
5047     { .name = "DCCSW", .cp = 15, .opc1 = 0, .crn = 7, .crm = 10, .opc2 = 2,
5048       .type = ARM_CP_NOP, .access = PL1_W, .accessfn = access_tsw },
5049     { .name = "DCCMVAU", .cp = 15, .opc1 = 0, .crn = 7, .crm = 11, .opc2 = 1,
5050       .type = ARM_CP_NOP, .access = PL1_W, .accessfn = aa64_cacheop_pou_access },
5051     { .name = "DCCIMVAC", .cp = 15, .opc1 = 0, .crn = 7, .crm = 14, .opc2 = 1,
5052       .type = ARM_CP_NOP, .access = PL1_W, .accessfn = aa64_cacheop_poc_access },
5053     { .name = "DCCISW", .cp = 15, .opc1 = 0, .crn = 7, .crm = 14, .opc2 = 2,
5054       .type = ARM_CP_NOP, .access = PL1_W, .accessfn = access_tsw },
5055     /* MMU Domain access control / MPU write buffer control */
5056     { .name = "DACR", .cp = 15, .opc1 = 0, .crn = 3, .crm = 0, .opc2 = 0,
5057       .access = PL1_RW, .accessfn = access_tvm_trvm, .resetvalue = 0,
5058       .writefn = dacr_write, .raw_writefn = raw_write,
5059       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.dacr_s),
5060                              offsetoflow32(CPUARMState, cp15.dacr_ns) } },
5061     { .name = "ELR_EL1", .state = ARM_CP_STATE_AA64,
5062       .type = ARM_CP_ALIAS,
5063       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 0, .opc2 = 1,
5064       .access = PL1_RW,
5065       .fieldoffset = offsetof(CPUARMState, elr_el[1]) },
5066     { .name = "SPSR_EL1", .state = ARM_CP_STATE_AA64,
5067       .type = ARM_CP_ALIAS,
5068       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 0, .opc2 = 0,
5069       .access = PL1_RW,
5070       .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_SVC]) },
5071     /* We rely on the access checks not allowing the guest to write to the
5072      * state field when SPSel indicates that it's being used as the stack
5073      * pointer.
5074      */
5075     { .name = "SP_EL0", .state = ARM_CP_STATE_AA64,
5076       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 1, .opc2 = 0,
5077       .access = PL1_RW, .accessfn = sp_el0_access,
5078       .type = ARM_CP_ALIAS,
5079       .fieldoffset = offsetof(CPUARMState, sp_el[0]) },
5080     { .name = "SP_EL1", .state = ARM_CP_STATE_AA64,
5081       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 1, .opc2 = 0,
5082       .access = PL2_RW, .type = ARM_CP_ALIAS,
5083       .fieldoffset = offsetof(CPUARMState, sp_el[1]) },
5084     { .name = "SPSel", .state = ARM_CP_STATE_AA64,
5085       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 2, .opc2 = 0,
5086       .type = ARM_CP_NO_RAW,
5087       .access = PL1_RW, .readfn = spsel_read, .writefn = spsel_write },
5088     { .name = "FPEXC32_EL2", .state = ARM_CP_STATE_AA64,
5089       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 3, .opc2 = 0,
5090       .type = ARM_CP_ALIAS,
5091       .fieldoffset = offsetof(CPUARMState, vfp.xregs[ARM_VFP_FPEXC]),
5092       .access = PL2_RW, .accessfn = fpexc32_access },
5093     { .name = "DACR32_EL2", .state = ARM_CP_STATE_AA64,
5094       .opc0 = 3, .opc1 = 4, .crn = 3, .crm = 0, .opc2 = 0,
5095       .access = PL2_RW, .resetvalue = 0,
5096       .writefn = dacr_write, .raw_writefn = raw_write,
5097       .fieldoffset = offsetof(CPUARMState, cp15.dacr32_el2) },
5098     { .name = "IFSR32_EL2", .state = ARM_CP_STATE_AA64,
5099       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 0, .opc2 = 1,
5100       .access = PL2_RW, .resetvalue = 0,
5101       .fieldoffset = offsetof(CPUARMState, cp15.ifsr32_el2) },
5102     { .name = "SPSR_IRQ", .state = ARM_CP_STATE_AA64,
5103       .type = ARM_CP_ALIAS,
5104       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 3, .opc2 = 0,
5105       .access = PL2_RW,
5106       .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_IRQ]) },
5107     { .name = "SPSR_ABT", .state = ARM_CP_STATE_AA64,
5108       .type = ARM_CP_ALIAS,
5109       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 3, .opc2 = 1,
5110       .access = PL2_RW,
5111       .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_ABT]) },
5112     { .name = "SPSR_UND", .state = ARM_CP_STATE_AA64,
5113       .type = ARM_CP_ALIAS,
5114       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 3, .opc2 = 2,
5115       .access = PL2_RW,
5116       .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_UND]) },
5117     { .name = "SPSR_FIQ", .state = ARM_CP_STATE_AA64,
5118       .type = ARM_CP_ALIAS,
5119       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 3, .opc2 = 3,
5120       .access = PL2_RW,
5121       .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_FIQ]) },
5122     { .name = "MDCR_EL3", .state = ARM_CP_STATE_AA64,
5123       .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 3, .opc2 = 1,
5124       .resetvalue = 0,
5125       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.mdcr_el3) },
5126     { .name = "SDCR", .type = ARM_CP_ALIAS,
5127       .cp = 15, .opc1 = 0, .crn = 1, .crm = 3, .opc2 = 1,
5128       .access = PL1_RW, .accessfn = access_trap_aa32s_el1,
5129       .writefn = sdcr_write,
5130       .fieldoffset = offsetoflow32(CPUARMState, cp15.mdcr_el3) },
5131     REGINFO_SENTINEL
5132 };
5133 
5134 /* Used to describe the behaviour of EL2 regs when EL2 does not exist.  */
5135 static const ARMCPRegInfo el3_no_el2_cp_reginfo[] = {
5136     { .name = "VBAR_EL2", .state = ARM_CP_STATE_BOTH,
5137       .opc0 = 3, .opc1 = 4, .crn = 12, .crm = 0, .opc2 = 0,
5138       .access = PL2_RW,
5139       .readfn = arm_cp_read_zero, .writefn = arm_cp_write_ignore },
5140     { .name = "HCR_EL2", .state = ARM_CP_STATE_BOTH,
5141       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 0,
5142       .access = PL2_RW,
5143       .type = ARM_CP_CONST, .resetvalue = 0 },
5144     { .name = "HACR_EL2", .state = ARM_CP_STATE_BOTH,
5145       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 7,
5146       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
5147     { .name = "ESR_EL2", .state = ARM_CP_STATE_BOTH,
5148       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 2, .opc2 = 0,
5149       .access = PL2_RW,
5150       .type = ARM_CP_CONST, .resetvalue = 0 },
5151     { .name = "CPTR_EL2", .state = ARM_CP_STATE_BOTH,
5152       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 2,
5153       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
5154     { .name = "MAIR_EL2", .state = ARM_CP_STATE_BOTH,
5155       .opc0 = 3, .opc1 = 4, .crn = 10, .crm = 2, .opc2 = 0,
5156       .access = PL2_RW, .type = ARM_CP_CONST,
5157       .resetvalue = 0 },
5158     { .name = "HMAIR1", .state = ARM_CP_STATE_AA32,
5159       .cp = 15, .opc1 = 4, .crn = 10, .crm = 2, .opc2 = 1,
5160       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
5161     { .name = "AMAIR_EL2", .state = ARM_CP_STATE_BOTH,
5162       .opc0 = 3, .opc1 = 4, .crn = 10, .crm = 3, .opc2 = 0,
5163       .access = PL2_RW, .type = ARM_CP_CONST,
5164       .resetvalue = 0 },
5165     { .name = "HAMAIR1", .state = ARM_CP_STATE_AA32,
5166       .cp = 15, .opc1 = 4, .crn = 10, .crm = 3, .opc2 = 1,
5167       .access = PL2_RW, .type = ARM_CP_CONST,
5168       .resetvalue = 0 },
5169     { .name = "AFSR0_EL2", .state = ARM_CP_STATE_BOTH,
5170       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 1, .opc2 = 0,
5171       .access = PL2_RW, .type = ARM_CP_CONST,
5172       .resetvalue = 0 },
5173     { .name = "AFSR1_EL2", .state = ARM_CP_STATE_BOTH,
5174       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 1, .opc2 = 1,
5175       .access = PL2_RW, .type = ARM_CP_CONST,
5176       .resetvalue = 0 },
5177     { .name = "TCR_EL2", .state = ARM_CP_STATE_BOTH,
5178       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 2,
5179       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
5180     { .name = "VTCR_EL2", .state = ARM_CP_STATE_BOTH,
5181       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 2,
5182       .access = PL2_RW, .accessfn = access_el3_aa32ns,
5183       .type = ARM_CP_CONST, .resetvalue = 0 },
5184     { .name = "VTTBR", .state = ARM_CP_STATE_AA32,
5185       .cp = 15, .opc1 = 6, .crm = 2,
5186       .access = PL2_RW, .accessfn = access_el3_aa32ns,
5187       .type = ARM_CP_CONST | ARM_CP_64BIT, .resetvalue = 0 },
5188     { .name = "VTTBR_EL2", .state = ARM_CP_STATE_AA64,
5189       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 0,
5190       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
5191     { .name = "SCTLR_EL2", .state = ARM_CP_STATE_BOTH,
5192       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 0, .opc2 = 0,
5193       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
5194     { .name = "TPIDR_EL2", .state = ARM_CP_STATE_BOTH,
5195       .opc0 = 3, .opc1 = 4, .crn = 13, .crm = 0, .opc2 = 2,
5196       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
5197     { .name = "TTBR0_EL2", .state = ARM_CP_STATE_AA64,
5198       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 0,
5199       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
5200     { .name = "HTTBR", .cp = 15, .opc1 = 4, .crm = 2,
5201       .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_CONST,
5202       .resetvalue = 0 },
5203     { .name = "CNTHCTL_EL2", .state = ARM_CP_STATE_BOTH,
5204       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 1, .opc2 = 0,
5205       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
5206     { .name = "CNTVOFF_EL2", .state = ARM_CP_STATE_AA64,
5207       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 0, .opc2 = 3,
5208       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
5209     { .name = "CNTVOFF", .cp = 15, .opc1 = 4, .crm = 14,
5210       .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_CONST,
5211       .resetvalue = 0 },
5212     { .name = "CNTHP_CVAL_EL2", .state = ARM_CP_STATE_AA64,
5213       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 2,
5214       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
5215     { .name = "CNTHP_CVAL", .cp = 15, .opc1 = 6, .crm = 14,
5216       .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_CONST,
5217       .resetvalue = 0 },
5218     { .name = "CNTHP_TVAL_EL2", .state = ARM_CP_STATE_BOTH,
5219       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 0,
5220       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
5221     { .name = "CNTHP_CTL_EL2", .state = ARM_CP_STATE_BOTH,
5222       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 1,
5223       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
5224     { .name = "MDCR_EL2", .state = ARM_CP_STATE_BOTH,
5225       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 1,
5226       .access = PL2_RW, .accessfn = access_tda,
5227       .type = ARM_CP_CONST, .resetvalue = 0 },
5228     { .name = "HPFAR_EL2", .state = ARM_CP_STATE_BOTH,
5229       .opc0 = 3, .opc1 = 4, .crn = 6, .crm = 0, .opc2 = 4,
5230       .access = PL2_RW, .accessfn = access_el3_aa32ns,
5231       .type = ARM_CP_CONST, .resetvalue = 0 },
5232     { .name = "HSTR_EL2", .state = ARM_CP_STATE_BOTH,
5233       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 3,
5234       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
5235     { .name = "FAR_EL2", .state = ARM_CP_STATE_BOTH,
5236       .opc0 = 3, .opc1 = 4, .crn = 6, .crm = 0, .opc2 = 0,
5237       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
5238     { .name = "HIFAR", .state = ARM_CP_STATE_AA32,
5239       .type = ARM_CP_CONST,
5240       .cp = 15, .opc1 = 4, .crn = 6, .crm = 0, .opc2 = 2,
5241       .access = PL2_RW, .resetvalue = 0 },
5242     REGINFO_SENTINEL
5243 };
5244 
5245 /* Ditto, but for registers which exist in ARMv8 but not v7 */
5246 static const ARMCPRegInfo el3_no_el2_v8_cp_reginfo[] = {
5247     { .name = "HCR2", .state = ARM_CP_STATE_AA32,
5248       .cp = 15, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 4,
5249       .access = PL2_RW,
5250       .type = ARM_CP_CONST, .resetvalue = 0 },
5251     REGINFO_SENTINEL
5252 };
5253 
5254 static void do_hcr_write(CPUARMState *env, uint64_t value, uint64_t valid_mask)
5255 {
5256     ARMCPU *cpu = env_archcpu(env);
5257 
5258     if (arm_feature(env, ARM_FEATURE_V8)) {
5259         valid_mask |= MAKE_64BIT_MASK(0, 34);  /* ARMv8.0 */
5260     } else {
5261         valid_mask |= MAKE_64BIT_MASK(0, 28);  /* ARMv7VE */
5262     }
5263 
5264     if (arm_feature(env, ARM_FEATURE_EL3)) {
5265         valid_mask &= ~HCR_HCD;
5266     } else if (cpu->psci_conduit != QEMU_PSCI_CONDUIT_SMC) {
5267         /* Architecturally HCR.TSC is RES0 if EL3 is not implemented.
5268          * However, if we're using the SMC PSCI conduit then QEMU is
5269          * effectively acting like EL3 firmware and so the guest at
5270          * EL2 should retain the ability to prevent EL1 from being
5271          * able to make SMC calls into the ersatz firmware, so in
5272          * that case HCR.TSC should be read/write.
5273          */
5274         valid_mask &= ~HCR_TSC;
5275     }
5276 
5277     if (arm_feature(env, ARM_FEATURE_AARCH64)) {
5278         if (cpu_isar_feature(aa64_vh, cpu)) {
5279             valid_mask |= HCR_E2H;
5280         }
5281         if (cpu_isar_feature(aa64_lor, cpu)) {
5282             valid_mask |= HCR_TLOR;
5283         }
5284         if (cpu_isar_feature(aa64_pauth, cpu)) {
5285             valid_mask |= HCR_API | HCR_APK;
5286         }
5287         if (cpu_isar_feature(aa64_mte, cpu)) {
5288             valid_mask |= HCR_ATA | HCR_DCT | HCR_TID5;
5289         }
5290     }
5291 
5292     /* Clear RES0 bits.  */
5293     value &= valid_mask;
5294 
5295     /*
5296      * These bits change the MMU setup:
5297      * HCR_VM enables stage 2 translation
5298      * HCR_PTW forbids certain page-table setups
5299      * HCR_DC disables stage1 and enables stage2 translation
5300      * HCR_DCT enables tagging on (disabled) stage1 translation
5301      */
5302     if ((env->cp15.hcr_el2 ^ value) & (HCR_VM | HCR_PTW | HCR_DC | HCR_DCT)) {
5303         tlb_flush(CPU(cpu));
5304     }
5305     env->cp15.hcr_el2 = value;
5306 
5307     /*
5308      * Updates to VI and VF require us to update the status of
5309      * virtual interrupts, which are the logical OR of these bits
5310      * and the state of the input lines from the GIC. (This requires
5311      * that we have the iothread lock, which is done by marking the
5312      * reginfo structs as ARM_CP_IO.)
5313      * Note that if a write to HCR pends a VIRQ or VFIQ it is never
5314      * possible for it to be taken immediately, because VIRQ and
5315      * VFIQ are masked unless running at EL0 or EL1, and HCR
5316      * can only be written at EL2.
5317      */
5318     g_assert(qemu_mutex_iothread_locked());
5319     arm_cpu_update_virq(cpu);
5320     arm_cpu_update_vfiq(cpu);
5321 }
5322 
5323 static void hcr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
5324 {
5325     do_hcr_write(env, value, 0);
5326 }
5327 
5328 static void hcr_writehigh(CPUARMState *env, const ARMCPRegInfo *ri,
5329                           uint64_t value)
5330 {
5331     /* Handle HCR2 write, i.e. write to high half of HCR_EL2 */
5332     value = deposit64(env->cp15.hcr_el2, 32, 32, value);
5333     do_hcr_write(env, value, MAKE_64BIT_MASK(0, 32));
5334 }
5335 
5336 static void hcr_writelow(CPUARMState *env, const ARMCPRegInfo *ri,
5337                          uint64_t value)
5338 {
5339     /* Handle HCR write, i.e. write to low half of HCR_EL2 */
5340     value = deposit64(env->cp15.hcr_el2, 0, 32, value);
5341     do_hcr_write(env, value, MAKE_64BIT_MASK(32, 32));
5342 }
5343 
5344 /*
5345  * Return the effective value of HCR_EL2.
5346  * Bits that are not included here:
5347  * RW       (read from SCR_EL3.RW as needed)
5348  */
5349 uint64_t arm_hcr_el2_eff(CPUARMState *env)
5350 {
5351     uint64_t ret = env->cp15.hcr_el2;
5352 
5353     if (arm_is_secure_below_el3(env)) {
5354         /*
5355          * "This register has no effect if EL2 is not enabled in the
5356          * current Security state".  This is ARMv8.4-SecEL2 speak for
5357          * !(SCR_EL3.NS==1 || SCR_EL3.EEL2==1).
5358          *
5359          * Prior to that, the language was "In an implementation that
5360          * includes EL3, when the value of SCR_EL3.NS is 0 the PE behaves
5361          * as if this field is 0 for all purposes other than a direct
5362          * read or write access of HCR_EL2".  With lots of enumeration
5363          * on a per-field basis.  In current QEMU, this is condition
5364          * is arm_is_secure_below_el3.
5365          *
5366          * Since the v8.4 language applies to the entire register, and
5367          * appears to be backward compatible, use that.
5368          */
5369         return 0;
5370     }
5371 
5372     /*
5373      * For a cpu that supports both aarch64 and aarch32, we can set bits
5374      * in HCR_EL2 (e.g. via EL3) that are RES0 when we enter EL2 as aa32.
5375      * Ignore all of the bits in HCR+HCR2 that are not valid for aarch32.
5376      */
5377     if (!arm_el_is_aa64(env, 2)) {
5378         uint64_t aa32_valid;
5379 
5380         /*
5381          * These bits are up-to-date as of ARMv8.6.
5382          * For HCR, it's easiest to list just the 2 bits that are invalid.
5383          * For HCR2, list those that are valid.
5384          */
5385         aa32_valid = MAKE_64BIT_MASK(0, 32) & ~(HCR_RW | HCR_TDZ);
5386         aa32_valid |= (HCR_CD | HCR_ID | HCR_TERR | HCR_TEA | HCR_MIOCNCE |
5387                        HCR_TID4 | HCR_TICAB | HCR_TOCU | HCR_TTLBIS);
5388         ret &= aa32_valid;
5389     }
5390 
5391     if (ret & HCR_TGE) {
5392         /* These bits are up-to-date as of ARMv8.6.  */
5393         if (ret & HCR_E2H) {
5394             ret &= ~(HCR_VM | HCR_FMO | HCR_IMO | HCR_AMO |
5395                      HCR_BSU_MASK | HCR_DC | HCR_TWI | HCR_TWE |
5396                      HCR_TID0 | HCR_TID2 | HCR_TPCP | HCR_TPU |
5397                      HCR_TDZ | HCR_CD | HCR_ID | HCR_MIOCNCE |
5398                      HCR_TID4 | HCR_TICAB | HCR_TOCU | HCR_ENSCXT |
5399                      HCR_TTLBIS | HCR_TTLBOS | HCR_TID5);
5400         } else {
5401             ret |= HCR_FMO | HCR_IMO | HCR_AMO;
5402         }
5403         ret &= ~(HCR_SWIO | HCR_PTW | HCR_VF | HCR_VI | HCR_VSE |
5404                  HCR_FB | HCR_TID1 | HCR_TID3 | HCR_TSC | HCR_TACR |
5405                  HCR_TSW | HCR_TTLB | HCR_TVM | HCR_HCD | HCR_TRVM |
5406                  HCR_TLOR);
5407     }
5408 
5409     return ret;
5410 }
5411 
5412 static void cptr_el2_write(CPUARMState *env, const ARMCPRegInfo *ri,
5413                            uint64_t value)
5414 {
5415     /*
5416      * For A-profile AArch32 EL3, if NSACR.CP10
5417      * is 0 then HCPTR.{TCP11,TCP10} ignore writes and read as 1.
5418      */
5419     if (arm_feature(env, ARM_FEATURE_EL3) && !arm_el_is_aa64(env, 3) &&
5420         !arm_is_secure(env) && !extract32(env->cp15.nsacr, 10, 1)) {
5421         value &= ~(0x3 << 10);
5422         value |= env->cp15.cptr_el[2] & (0x3 << 10);
5423     }
5424     env->cp15.cptr_el[2] = value;
5425 }
5426 
5427 static uint64_t cptr_el2_read(CPUARMState *env, const ARMCPRegInfo *ri)
5428 {
5429     /*
5430      * For A-profile AArch32 EL3, if NSACR.CP10
5431      * is 0 then HCPTR.{TCP11,TCP10} ignore writes and read as 1.
5432      */
5433     uint64_t value = env->cp15.cptr_el[2];
5434 
5435     if (arm_feature(env, ARM_FEATURE_EL3) && !arm_el_is_aa64(env, 3) &&
5436         !arm_is_secure(env) && !extract32(env->cp15.nsacr, 10, 1)) {
5437         value |= 0x3 << 10;
5438     }
5439     return value;
5440 }
5441 
5442 static const ARMCPRegInfo el2_cp_reginfo[] = {
5443     { .name = "HCR_EL2", .state = ARM_CP_STATE_AA64,
5444       .type = ARM_CP_IO,
5445       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 0,
5446       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.hcr_el2),
5447       .writefn = hcr_write },
5448     { .name = "HCR", .state = ARM_CP_STATE_AA32,
5449       .type = ARM_CP_ALIAS | ARM_CP_IO,
5450       .cp = 15, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 0,
5451       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.hcr_el2),
5452       .writefn = hcr_writelow },
5453     { .name = "HACR_EL2", .state = ARM_CP_STATE_BOTH,
5454       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 7,
5455       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
5456     { .name = "ELR_EL2", .state = ARM_CP_STATE_AA64,
5457       .type = ARM_CP_ALIAS,
5458       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 0, .opc2 = 1,
5459       .access = PL2_RW,
5460       .fieldoffset = offsetof(CPUARMState, elr_el[2]) },
5461     { .name = "ESR_EL2", .state = ARM_CP_STATE_BOTH,
5462       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 2, .opc2 = 0,
5463       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.esr_el[2]) },
5464     { .name = "FAR_EL2", .state = ARM_CP_STATE_BOTH,
5465       .opc0 = 3, .opc1 = 4, .crn = 6, .crm = 0, .opc2 = 0,
5466       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.far_el[2]) },
5467     { .name = "HIFAR", .state = ARM_CP_STATE_AA32,
5468       .type = ARM_CP_ALIAS,
5469       .cp = 15, .opc1 = 4, .crn = 6, .crm = 0, .opc2 = 2,
5470       .access = PL2_RW,
5471       .fieldoffset = offsetofhigh32(CPUARMState, cp15.far_el[2]) },
5472     { .name = "SPSR_EL2", .state = ARM_CP_STATE_AA64,
5473       .type = ARM_CP_ALIAS,
5474       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 0, .opc2 = 0,
5475       .access = PL2_RW,
5476       .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_HYP]) },
5477     { .name = "VBAR_EL2", .state = ARM_CP_STATE_BOTH,
5478       .opc0 = 3, .opc1 = 4, .crn = 12, .crm = 0, .opc2 = 0,
5479       .access = PL2_RW, .writefn = vbar_write,
5480       .fieldoffset = offsetof(CPUARMState, cp15.vbar_el[2]),
5481       .resetvalue = 0 },
5482     { .name = "SP_EL2", .state = ARM_CP_STATE_AA64,
5483       .opc0 = 3, .opc1 = 6, .crn = 4, .crm = 1, .opc2 = 0,
5484       .access = PL3_RW, .type = ARM_CP_ALIAS,
5485       .fieldoffset = offsetof(CPUARMState, sp_el[2]) },
5486     { .name = "CPTR_EL2", .state = ARM_CP_STATE_BOTH,
5487       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 2,
5488       .access = PL2_RW, .accessfn = cptr_access, .resetvalue = 0,
5489       .fieldoffset = offsetof(CPUARMState, cp15.cptr_el[2]),
5490       .readfn = cptr_el2_read, .writefn = cptr_el2_write },
5491     { .name = "MAIR_EL2", .state = ARM_CP_STATE_BOTH,
5492       .opc0 = 3, .opc1 = 4, .crn = 10, .crm = 2, .opc2 = 0,
5493       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.mair_el[2]),
5494       .resetvalue = 0 },
5495     { .name = "HMAIR1", .state = ARM_CP_STATE_AA32,
5496       .cp = 15, .opc1 = 4, .crn = 10, .crm = 2, .opc2 = 1,
5497       .access = PL2_RW, .type = ARM_CP_ALIAS,
5498       .fieldoffset = offsetofhigh32(CPUARMState, cp15.mair_el[2]) },
5499     { .name = "AMAIR_EL2", .state = ARM_CP_STATE_BOTH,
5500       .opc0 = 3, .opc1 = 4, .crn = 10, .crm = 3, .opc2 = 0,
5501       .access = PL2_RW, .type = ARM_CP_CONST,
5502       .resetvalue = 0 },
5503     /* HAMAIR1 is mapped to AMAIR_EL2[63:32] */
5504     { .name = "HAMAIR1", .state = ARM_CP_STATE_AA32,
5505       .cp = 15, .opc1 = 4, .crn = 10, .crm = 3, .opc2 = 1,
5506       .access = PL2_RW, .type = ARM_CP_CONST,
5507       .resetvalue = 0 },
5508     { .name = "AFSR0_EL2", .state = ARM_CP_STATE_BOTH,
5509       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 1, .opc2 = 0,
5510       .access = PL2_RW, .type = ARM_CP_CONST,
5511       .resetvalue = 0 },
5512     { .name = "AFSR1_EL2", .state = ARM_CP_STATE_BOTH,
5513       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 1, .opc2 = 1,
5514       .access = PL2_RW, .type = ARM_CP_CONST,
5515       .resetvalue = 0 },
5516     { .name = "TCR_EL2", .state = ARM_CP_STATE_BOTH,
5517       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 2,
5518       .access = PL2_RW, .writefn = vmsa_tcr_el12_write,
5519       /* no .raw_writefn or .resetfn needed as we never use mask/base_mask */
5520       .fieldoffset = offsetof(CPUARMState, cp15.tcr_el[2]) },
5521     { .name = "VTCR", .state = ARM_CP_STATE_AA32,
5522       .cp = 15, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 2,
5523       .type = ARM_CP_ALIAS,
5524       .access = PL2_RW, .accessfn = access_el3_aa32ns,
5525       .fieldoffset = offsetof(CPUARMState, cp15.vtcr_el2) },
5526     { .name = "VTCR_EL2", .state = ARM_CP_STATE_AA64,
5527       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 2,
5528       .access = PL2_RW,
5529       /* no .writefn needed as this can't cause an ASID change;
5530        * no .raw_writefn or .resetfn needed as we never use mask/base_mask
5531        */
5532       .fieldoffset = offsetof(CPUARMState, cp15.vtcr_el2) },
5533     { .name = "VTTBR", .state = ARM_CP_STATE_AA32,
5534       .cp = 15, .opc1 = 6, .crm = 2,
5535       .type = ARM_CP_64BIT | ARM_CP_ALIAS,
5536       .access = PL2_RW, .accessfn = access_el3_aa32ns,
5537       .fieldoffset = offsetof(CPUARMState, cp15.vttbr_el2),
5538       .writefn = vttbr_write },
5539     { .name = "VTTBR_EL2", .state = ARM_CP_STATE_AA64,
5540       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 0,
5541       .access = PL2_RW, .writefn = vttbr_write,
5542       .fieldoffset = offsetof(CPUARMState, cp15.vttbr_el2) },
5543     { .name = "SCTLR_EL2", .state = ARM_CP_STATE_BOTH,
5544       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 0, .opc2 = 0,
5545       .access = PL2_RW, .raw_writefn = raw_write, .writefn = sctlr_write,
5546       .fieldoffset = offsetof(CPUARMState, cp15.sctlr_el[2]) },
5547     { .name = "TPIDR_EL2", .state = ARM_CP_STATE_BOTH,
5548       .opc0 = 3, .opc1 = 4, .crn = 13, .crm = 0, .opc2 = 2,
5549       .access = PL2_RW, .resetvalue = 0,
5550       .fieldoffset = offsetof(CPUARMState, cp15.tpidr_el[2]) },
5551     { .name = "TTBR0_EL2", .state = ARM_CP_STATE_AA64,
5552       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 0,
5553       .access = PL2_RW, .resetvalue = 0, .writefn = vmsa_tcr_ttbr_el2_write,
5554       .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el[2]) },
5555     { .name = "HTTBR", .cp = 15, .opc1 = 4, .crm = 2,
5556       .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_ALIAS,
5557       .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el[2]) },
5558     { .name = "TLBIALLNSNH",
5559       .cp = 15, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 4,
5560       .type = ARM_CP_NO_RAW, .access = PL2_W,
5561       .writefn = tlbiall_nsnh_write },
5562     { .name = "TLBIALLNSNHIS",
5563       .cp = 15, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 4,
5564       .type = ARM_CP_NO_RAW, .access = PL2_W,
5565       .writefn = tlbiall_nsnh_is_write },
5566     { .name = "TLBIALLH", .cp = 15, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 0,
5567       .type = ARM_CP_NO_RAW, .access = PL2_W,
5568       .writefn = tlbiall_hyp_write },
5569     { .name = "TLBIALLHIS", .cp = 15, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 0,
5570       .type = ARM_CP_NO_RAW, .access = PL2_W,
5571       .writefn = tlbiall_hyp_is_write },
5572     { .name = "TLBIMVAH", .cp = 15, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 1,
5573       .type = ARM_CP_NO_RAW, .access = PL2_W,
5574       .writefn = tlbimva_hyp_write },
5575     { .name = "TLBIMVAHIS", .cp = 15, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 1,
5576       .type = ARM_CP_NO_RAW, .access = PL2_W,
5577       .writefn = tlbimva_hyp_is_write },
5578     { .name = "TLBI_ALLE2", .state = ARM_CP_STATE_AA64,
5579       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 0,
5580       .type = ARM_CP_NO_RAW, .access = PL2_W,
5581       .writefn = tlbi_aa64_alle2_write },
5582     { .name = "TLBI_VAE2", .state = ARM_CP_STATE_AA64,
5583       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 1,
5584       .type = ARM_CP_NO_RAW, .access = PL2_W,
5585       .writefn = tlbi_aa64_vae2_write },
5586     { .name = "TLBI_VALE2", .state = ARM_CP_STATE_AA64,
5587       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 5,
5588       .access = PL2_W, .type = ARM_CP_NO_RAW,
5589       .writefn = tlbi_aa64_vae2_write },
5590     { .name = "TLBI_ALLE2IS", .state = ARM_CP_STATE_AA64,
5591       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 0,
5592       .access = PL2_W, .type = ARM_CP_NO_RAW,
5593       .writefn = tlbi_aa64_alle2is_write },
5594     { .name = "TLBI_VAE2IS", .state = ARM_CP_STATE_AA64,
5595       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 1,
5596       .type = ARM_CP_NO_RAW, .access = PL2_W,
5597       .writefn = tlbi_aa64_vae2is_write },
5598     { .name = "TLBI_VALE2IS", .state = ARM_CP_STATE_AA64,
5599       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 5,
5600       .access = PL2_W, .type = ARM_CP_NO_RAW,
5601       .writefn = tlbi_aa64_vae2is_write },
5602 #ifndef CONFIG_USER_ONLY
5603     /* Unlike the other EL2-related AT operations, these must
5604      * UNDEF from EL3 if EL2 is not implemented, which is why we
5605      * define them here rather than with the rest of the AT ops.
5606      */
5607     { .name = "AT_S1E2R", .state = ARM_CP_STATE_AA64,
5608       .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 0,
5609       .access = PL2_W, .accessfn = at_s1e2_access,
5610       .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC, .writefn = ats_write64 },
5611     { .name = "AT_S1E2W", .state = ARM_CP_STATE_AA64,
5612       .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 1,
5613       .access = PL2_W, .accessfn = at_s1e2_access,
5614       .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC, .writefn = ats_write64 },
5615     /* The AArch32 ATS1H* operations are CONSTRAINED UNPREDICTABLE
5616      * if EL2 is not implemented; we choose to UNDEF. Behaviour at EL3
5617      * with SCR.NS == 0 outside Monitor mode is UNPREDICTABLE; we choose
5618      * to behave as if SCR.NS was 1.
5619      */
5620     { .name = "ATS1HR", .cp = 15, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 0,
5621       .access = PL2_W,
5622       .writefn = ats1h_write, .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC },
5623     { .name = "ATS1HW", .cp = 15, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 1,
5624       .access = PL2_W,
5625       .writefn = ats1h_write, .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC },
5626     { .name = "CNTHCTL_EL2", .state = ARM_CP_STATE_BOTH,
5627       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 1, .opc2 = 0,
5628       /* ARMv7 requires bit 0 and 1 to reset to 1. ARMv8 defines the
5629        * reset values as IMPDEF. We choose to reset to 3 to comply with
5630        * both ARMv7 and ARMv8.
5631        */
5632       .access = PL2_RW, .resetvalue = 3,
5633       .fieldoffset = offsetof(CPUARMState, cp15.cnthctl_el2) },
5634     { .name = "CNTVOFF_EL2", .state = ARM_CP_STATE_AA64,
5635       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 0, .opc2 = 3,
5636       .access = PL2_RW, .type = ARM_CP_IO, .resetvalue = 0,
5637       .writefn = gt_cntvoff_write,
5638       .fieldoffset = offsetof(CPUARMState, cp15.cntvoff_el2) },
5639     { .name = "CNTVOFF", .cp = 15, .opc1 = 4, .crm = 14,
5640       .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_ALIAS | ARM_CP_IO,
5641       .writefn = gt_cntvoff_write,
5642       .fieldoffset = offsetof(CPUARMState, cp15.cntvoff_el2) },
5643     { .name = "CNTHP_CVAL_EL2", .state = ARM_CP_STATE_AA64,
5644       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 2,
5645       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_HYP].cval),
5646       .type = ARM_CP_IO, .access = PL2_RW,
5647       .writefn = gt_hyp_cval_write, .raw_writefn = raw_write },
5648     { .name = "CNTHP_CVAL", .cp = 15, .opc1 = 6, .crm = 14,
5649       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_HYP].cval),
5650       .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_IO,
5651       .writefn = gt_hyp_cval_write, .raw_writefn = raw_write },
5652     { .name = "CNTHP_TVAL_EL2", .state = ARM_CP_STATE_BOTH,
5653       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 0,
5654       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL2_RW,
5655       .resetfn = gt_hyp_timer_reset,
5656       .readfn = gt_hyp_tval_read, .writefn = gt_hyp_tval_write },
5657     { .name = "CNTHP_CTL_EL2", .state = ARM_CP_STATE_BOTH,
5658       .type = ARM_CP_IO,
5659       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 1,
5660       .access = PL2_RW,
5661       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_HYP].ctl),
5662       .resetvalue = 0,
5663       .writefn = gt_hyp_ctl_write, .raw_writefn = raw_write },
5664 #endif
5665     /* The only field of MDCR_EL2 that has a defined architectural reset value
5666      * is MDCR_EL2.HPMN which should reset to the value of PMCR_EL0.N; but we
5667      * don't implement any PMU event counters, so using zero as a reset
5668      * value for MDCR_EL2 is okay
5669      */
5670     { .name = "MDCR_EL2", .state = ARM_CP_STATE_BOTH,
5671       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 1,
5672       .access = PL2_RW, .resetvalue = 0,
5673       .fieldoffset = offsetof(CPUARMState, cp15.mdcr_el2), },
5674     { .name = "HPFAR", .state = ARM_CP_STATE_AA32,
5675       .cp = 15, .opc1 = 4, .crn = 6, .crm = 0, .opc2 = 4,
5676       .access = PL2_RW, .accessfn = access_el3_aa32ns,
5677       .fieldoffset = offsetof(CPUARMState, cp15.hpfar_el2) },
5678     { .name = "HPFAR_EL2", .state = ARM_CP_STATE_AA64,
5679       .opc0 = 3, .opc1 = 4, .crn = 6, .crm = 0, .opc2 = 4,
5680       .access = PL2_RW,
5681       .fieldoffset = offsetof(CPUARMState, cp15.hpfar_el2) },
5682     { .name = "HSTR_EL2", .state = ARM_CP_STATE_BOTH,
5683       .cp = 15, .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 3,
5684       .access = PL2_RW,
5685       .fieldoffset = offsetof(CPUARMState, cp15.hstr_el2) },
5686     REGINFO_SENTINEL
5687 };
5688 
5689 static const ARMCPRegInfo el2_v8_cp_reginfo[] = {
5690     { .name = "HCR2", .state = ARM_CP_STATE_AA32,
5691       .type = ARM_CP_ALIAS | ARM_CP_IO,
5692       .cp = 15, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 4,
5693       .access = PL2_RW,
5694       .fieldoffset = offsetofhigh32(CPUARMState, cp15.hcr_el2),
5695       .writefn = hcr_writehigh },
5696     REGINFO_SENTINEL
5697 };
5698 
5699 static CPAccessResult nsacr_access(CPUARMState *env, const ARMCPRegInfo *ri,
5700                                    bool isread)
5701 {
5702     /* The NSACR is RW at EL3, and RO for NS EL1 and NS EL2.
5703      * At Secure EL1 it traps to EL3.
5704      */
5705     if (arm_current_el(env) == 3) {
5706         return CP_ACCESS_OK;
5707     }
5708     if (arm_is_secure_below_el3(env)) {
5709         return CP_ACCESS_TRAP_EL3;
5710     }
5711     /* Accesses from EL1 NS and EL2 NS are UNDEF for write but allow reads. */
5712     if (isread) {
5713         return CP_ACCESS_OK;
5714     }
5715     return CP_ACCESS_TRAP_UNCATEGORIZED;
5716 }
5717 
5718 static const ARMCPRegInfo el3_cp_reginfo[] = {
5719     { .name = "SCR_EL3", .state = ARM_CP_STATE_AA64,
5720       .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 1, .opc2 = 0,
5721       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.scr_el3),
5722       .resetvalue = 0, .writefn = scr_write },
5723     { .name = "SCR",  .type = ARM_CP_ALIAS | ARM_CP_NEWEL,
5724       .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 0,
5725       .access = PL1_RW, .accessfn = access_trap_aa32s_el1,
5726       .fieldoffset = offsetoflow32(CPUARMState, cp15.scr_el3),
5727       .writefn = scr_write },
5728     { .name = "SDER32_EL3", .state = ARM_CP_STATE_AA64,
5729       .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 1, .opc2 = 1,
5730       .access = PL3_RW, .resetvalue = 0,
5731       .fieldoffset = offsetof(CPUARMState, cp15.sder) },
5732     { .name = "SDER",
5733       .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 1,
5734       .access = PL3_RW, .resetvalue = 0,
5735       .fieldoffset = offsetoflow32(CPUARMState, cp15.sder) },
5736     { .name = "MVBAR", .cp = 15, .opc1 = 0, .crn = 12, .crm = 0, .opc2 = 1,
5737       .access = PL1_RW, .accessfn = access_trap_aa32s_el1,
5738       .writefn = vbar_write, .resetvalue = 0,
5739       .fieldoffset = offsetof(CPUARMState, cp15.mvbar) },
5740     { .name = "TTBR0_EL3", .state = ARM_CP_STATE_AA64,
5741       .opc0 = 3, .opc1 = 6, .crn = 2, .crm = 0, .opc2 = 0,
5742       .access = PL3_RW, .resetvalue = 0,
5743       .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el[3]) },
5744     { .name = "TCR_EL3", .state = ARM_CP_STATE_AA64,
5745       .opc0 = 3, .opc1 = 6, .crn = 2, .crm = 0, .opc2 = 2,
5746       .access = PL3_RW,
5747       /* no .writefn needed as this can't cause an ASID change;
5748        * we must provide a .raw_writefn and .resetfn because we handle
5749        * reset and migration for the AArch32 TTBCR(S), which might be
5750        * using mask and base_mask.
5751        */
5752       .resetfn = vmsa_ttbcr_reset, .raw_writefn = vmsa_ttbcr_raw_write,
5753       .fieldoffset = offsetof(CPUARMState, cp15.tcr_el[3]) },
5754     { .name = "ELR_EL3", .state = ARM_CP_STATE_AA64,
5755       .type = ARM_CP_ALIAS,
5756       .opc0 = 3, .opc1 = 6, .crn = 4, .crm = 0, .opc2 = 1,
5757       .access = PL3_RW,
5758       .fieldoffset = offsetof(CPUARMState, elr_el[3]) },
5759     { .name = "ESR_EL3", .state = ARM_CP_STATE_AA64,
5760       .opc0 = 3, .opc1 = 6, .crn = 5, .crm = 2, .opc2 = 0,
5761       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.esr_el[3]) },
5762     { .name = "FAR_EL3", .state = ARM_CP_STATE_AA64,
5763       .opc0 = 3, .opc1 = 6, .crn = 6, .crm = 0, .opc2 = 0,
5764       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.far_el[3]) },
5765     { .name = "SPSR_EL3", .state = ARM_CP_STATE_AA64,
5766       .type = ARM_CP_ALIAS,
5767       .opc0 = 3, .opc1 = 6, .crn = 4, .crm = 0, .opc2 = 0,
5768       .access = PL3_RW,
5769       .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_MON]) },
5770     { .name = "VBAR_EL3", .state = ARM_CP_STATE_AA64,
5771       .opc0 = 3, .opc1 = 6, .crn = 12, .crm = 0, .opc2 = 0,
5772       .access = PL3_RW, .writefn = vbar_write,
5773       .fieldoffset = offsetof(CPUARMState, cp15.vbar_el[3]),
5774       .resetvalue = 0 },
5775     { .name = "CPTR_EL3", .state = ARM_CP_STATE_AA64,
5776       .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 1, .opc2 = 2,
5777       .access = PL3_RW, .accessfn = cptr_access, .resetvalue = 0,
5778       .fieldoffset = offsetof(CPUARMState, cp15.cptr_el[3]) },
5779     { .name = "TPIDR_EL3", .state = ARM_CP_STATE_AA64,
5780       .opc0 = 3, .opc1 = 6, .crn = 13, .crm = 0, .opc2 = 2,
5781       .access = PL3_RW, .resetvalue = 0,
5782       .fieldoffset = offsetof(CPUARMState, cp15.tpidr_el[3]) },
5783     { .name = "AMAIR_EL3", .state = ARM_CP_STATE_AA64,
5784       .opc0 = 3, .opc1 = 6, .crn = 10, .crm = 3, .opc2 = 0,
5785       .access = PL3_RW, .type = ARM_CP_CONST,
5786       .resetvalue = 0 },
5787     { .name = "AFSR0_EL3", .state = ARM_CP_STATE_BOTH,
5788       .opc0 = 3, .opc1 = 6, .crn = 5, .crm = 1, .opc2 = 0,
5789       .access = PL3_RW, .type = ARM_CP_CONST,
5790       .resetvalue = 0 },
5791     { .name = "AFSR1_EL3", .state = ARM_CP_STATE_BOTH,
5792       .opc0 = 3, .opc1 = 6, .crn = 5, .crm = 1, .opc2 = 1,
5793       .access = PL3_RW, .type = ARM_CP_CONST,
5794       .resetvalue = 0 },
5795     { .name = "TLBI_ALLE3IS", .state = ARM_CP_STATE_AA64,
5796       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 3, .opc2 = 0,
5797       .access = PL3_W, .type = ARM_CP_NO_RAW,
5798       .writefn = tlbi_aa64_alle3is_write },
5799     { .name = "TLBI_VAE3IS", .state = ARM_CP_STATE_AA64,
5800       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 3, .opc2 = 1,
5801       .access = PL3_W, .type = ARM_CP_NO_RAW,
5802       .writefn = tlbi_aa64_vae3is_write },
5803     { .name = "TLBI_VALE3IS", .state = ARM_CP_STATE_AA64,
5804       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 3, .opc2 = 5,
5805       .access = PL3_W, .type = ARM_CP_NO_RAW,
5806       .writefn = tlbi_aa64_vae3is_write },
5807     { .name = "TLBI_ALLE3", .state = ARM_CP_STATE_AA64,
5808       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 7, .opc2 = 0,
5809       .access = PL3_W, .type = ARM_CP_NO_RAW,
5810       .writefn = tlbi_aa64_alle3_write },
5811     { .name = "TLBI_VAE3", .state = ARM_CP_STATE_AA64,
5812       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 7, .opc2 = 1,
5813       .access = PL3_W, .type = ARM_CP_NO_RAW,
5814       .writefn = tlbi_aa64_vae3_write },
5815     { .name = "TLBI_VALE3", .state = ARM_CP_STATE_AA64,
5816       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 7, .opc2 = 5,
5817       .access = PL3_W, .type = ARM_CP_NO_RAW,
5818       .writefn = tlbi_aa64_vae3_write },
5819     REGINFO_SENTINEL
5820 };
5821 
5822 #ifndef CONFIG_USER_ONLY
5823 /* Test if system register redirection is to occur in the current state.  */
5824 static bool redirect_for_e2h(CPUARMState *env)
5825 {
5826     return arm_current_el(env) == 2 && (arm_hcr_el2_eff(env) & HCR_E2H);
5827 }
5828 
5829 static uint64_t el2_e2h_read(CPUARMState *env, const ARMCPRegInfo *ri)
5830 {
5831     CPReadFn *readfn;
5832 
5833     if (redirect_for_e2h(env)) {
5834         /* Switch to the saved EL2 version of the register.  */
5835         ri = ri->opaque;
5836         readfn = ri->readfn;
5837     } else {
5838         readfn = ri->orig_readfn;
5839     }
5840     if (readfn == NULL) {
5841         readfn = raw_read;
5842     }
5843     return readfn(env, ri);
5844 }
5845 
5846 static void el2_e2h_write(CPUARMState *env, const ARMCPRegInfo *ri,
5847                           uint64_t value)
5848 {
5849     CPWriteFn *writefn;
5850 
5851     if (redirect_for_e2h(env)) {
5852         /* Switch to the saved EL2 version of the register.  */
5853         ri = ri->opaque;
5854         writefn = ri->writefn;
5855     } else {
5856         writefn = ri->orig_writefn;
5857     }
5858     if (writefn == NULL) {
5859         writefn = raw_write;
5860     }
5861     writefn(env, ri, value);
5862 }
5863 
5864 static void define_arm_vh_e2h_redirects_aliases(ARMCPU *cpu)
5865 {
5866     struct E2HAlias {
5867         uint32_t src_key, dst_key, new_key;
5868         const char *src_name, *dst_name, *new_name;
5869         bool (*feature)(const ARMISARegisters *id);
5870     };
5871 
5872 #define K(op0, op1, crn, crm, op2) \
5873     ENCODE_AA64_CP_REG(CP_REG_ARM64_SYSREG_CP, crn, crm, op0, op1, op2)
5874 
5875     static const struct E2HAlias aliases[] = {
5876         { K(3, 0,  1, 0, 0), K(3, 4,  1, 0, 0), K(3, 5, 1, 0, 0),
5877           "SCTLR", "SCTLR_EL2", "SCTLR_EL12" },
5878         { K(3, 0,  1, 0, 2), K(3, 4,  1, 1, 2), K(3, 5, 1, 0, 2),
5879           "CPACR", "CPTR_EL2", "CPACR_EL12" },
5880         { K(3, 0,  2, 0, 0), K(3, 4,  2, 0, 0), K(3, 5, 2, 0, 0),
5881           "TTBR0_EL1", "TTBR0_EL2", "TTBR0_EL12" },
5882         { K(3, 0,  2, 0, 1), K(3, 4,  2, 0, 1), K(3, 5, 2, 0, 1),
5883           "TTBR1_EL1", "TTBR1_EL2", "TTBR1_EL12" },
5884         { K(3, 0,  2, 0, 2), K(3, 4,  2, 0, 2), K(3, 5, 2, 0, 2),
5885           "TCR_EL1", "TCR_EL2", "TCR_EL12" },
5886         { K(3, 0,  4, 0, 0), K(3, 4,  4, 0, 0), K(3, 5, 4, 0, 0),
5887           "SPSR_EL1", "SPSR_EL2", "SPSR_EL12" },
5888         { K(3, 0,  4, 0, 1), K(3, 4,  4, 0, 1), K(3, 5, 4, 0, 1),
5889           "ELR_EL1", "ELR_EL2", "ELR_EL12" },
5890         { K(3, 0,  5, 1, 0), K(3, 4,  5, 1, 0), K(3, 5, 5, 1, 0),
5891           "AFSR0_EL1", "AFSR0_EL2", "AFSR0_EL12" },
5892         { K(3, 0,  5, 1, 1), K(3, 4,  5, 1, 1), K(3, 5, 5, 1, 1),
5893           "AFSR1_EL1", "AFSR1_EL2", "AFSR1_EL12" },
5894         { K(3, 0,  5, 2, 0), K(3, 4,  5, 2, 0), K(3, 5, 5, 2, 0),
5895           "ESR_EL1", "ESR_EL2", "ESR_EL12" },
5896         { K(3, 0,  6, 0, 0), K(3, 4,  6, 0, 0), K(3, 5, 6, 0, 0),
5897           "FAR_EL1", "FAR_EL2", "FAR_EL12" },
5898         { K(3, 0, 10, 2, 0), K(3, 4, 10, 2, 0), K(3, 5, 10, 2, 0),
5899           "MAIR_EL1", "MAIR_EL2", "MAIR_EL12" },
5900         { K(3, 0, 10, 3, 0), K(3, 4, 10, 3, 0), K(3, 5, 10, 3, 0),
5901           "AMAIR0", "AMAIR_EL2", "AMAIR_EL12" },
5902         { K(3, 0, 12, 0, 0), K(3, 4, 12, 0, 0), K(3, 5, 12, 0, 0),
5903           "VBAR", "VBAR_EL2", "VBAR_EL12" },
5904         { K(3, 0, 13, 0, 1), K(3, 4, 13, 0, 1), K(3, 5, 13, 0, 1),
5905           "CONTEXTIDR_EL1", "CONTEXTIDR_EL2", "CONTEXTIDR_EL12" },
5906         { K(3, 0, 14, 1, 0), K(3, 4, 14, 1, 0), K(3, 5, 14, 1, 0),
5907           "CNTKCTL", "CNTHCTL_EL2", "CNTKCTL_EL12" },
5908 
5909         /*
5910          * Note that redirection of ZCR is mentioned in the description
5911          * of ZCR_EL2, and aliasing in the description of ZCR_EL1, but
5912          * not in the summary table.
5913          */
5914         { K(3, 0,  1, 2, 0), K(3, 4,  1, 2, 0), K(3, 5, 1, 2, 0),
5915           "ZCR_EL1", "ZCR_EL2", "ZCR_EL12", isar_feature_aa64_sve },
5916 
5917         { K(3, 0,  5, 6, 0), K(3, 4,  5, 6, 0), K(3, 5, 5, 6, 0),
5918           "TFSR_EL1", "TFSR_EL2", "TFSR_EL12", isar_feature_aa64_mte },
5919 
5920         /* TODO: ARMv8.2-SPE -- PMSCR_EL2 */
5921         /* TODO: ARMv8.4-Trace -- TRFCR_EL2 */
5922     };
5923 #undef K
5924 
5925     size_t i;
5926 
5927     for (i = 0; i < ARRAY_SIZE(aliases); i++) {
5928         const struct E2HAlias *a = &aliases[i];
5929         ARMCPRegInfo *src_reg, *dst_reg;
5930 
5931         if (a->feature && !a->feature(&cpu->isar)) {
5932             continue;
5933         }
5934 
5935         src_reg = g_hash_table_lookup(cpu->cp_regs, &a->src_key);
5936         dst_reg = g_hash_table_lookup(cpu->cp_regs, &a->dst_key);
5937         g_assert(src_reg != NULL);
5938         g_assert(dst_reg != NULL);
5939 
5940         /* Cross-compare names to detect typos in the keys.  */
5941         g_assert(strcmp(src_reg->name, a->src_name) == 0);
5942         g_assert(strcmp(dst_reg->name, a->dst_name) == 0);
5943 
5944         /* None of the core system registers use opaque; we will.  */
5945         g_assert(src_reg->opaque == NULL);
5946 
5947         /* Create alias before redirection so we dup the right data. */
5948         if (a->new_key) {
5949             ARMCPRegInfo *new_reg = g_memdup(src_reg, sizeof(ARMCPRegInfo));
5950             uint32_t *new_key = g_memdup(&a->new_key, sizeof(uint32_t));
5951             bool ok;
5952 
5953             new_reg->name = a->new_name;
5954             new_reg->type |= ARM_CP_ALIAS;
5955             /* Remove PL1/PL0 access, leaving PL2/PL3 R/W in place.  */
5956             new_reg->access &= PL2_RW | PL3_RW;
5957 
5958             ok = g_hash_table_insert(cpu->cp_regs, new_key, new_reg);
5959             g_assert(ok);
5960         }
5961 
5962         src_reg->opaque = dst_reg;
5963         src_reg->orig_readfn = src_reg->readfn ?: raw_read;
5964         src_reg->orig_writefn = src_reg->writefn ?: raw_write;
5965         if (!src_reg->raw_readfn) {
5966             src_reg->raw_readfn = raw_read;
5967         }
5968         if (!src_reg->raw_writefn) {
5969             src_reg->raw_writefn = raw_write;
5970         }
5971         src_reg->readfn = el2_e2h_read;
5972         src_reg->writefn = el2_e2h_write;
5973     }
5974 }
5975 #endif
5976 
5977 static CPAccessResult ctr_el0_access(CPUARMState *env, const ARMCPRegInfo *ri,
5978                                      bool isread)
5979 {
5980     int cur_el = arm_current_el(env);
5981 
5982     if (cur_el < 2) {
5983         uint64_t hcr = arm_hcr_el2_eff(env);
5984 
5985         if (cur_el == 0) {
5986             if ((hcr & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)) {
5987                 if (!(env->cp15.sctlr_el[2] & SCTLR_UCT)) {
5988                     return CP_ACCESS_TRAP_EL2;
5989                 }
5990             } else {
5991                 if (!(env->cp15.sctlr_el[1] & SCTLR_UCT)) {
5992                     return CP_ACCESS_TRAP;
5993                 }
5994                 if (hcr & HCR_TID2) {
5995                     return CP_ACCESS_TRAP_EL2;
5996                 }
5997             }
5998         } else if (hcr & HCR_TID2) {
5999             return CP_ACCESS_TRAP_EL2;
6000         }
6001     }
6002 
6003     if (arm_current_el(env) < 2 && arm_hcr_el2_eff(env) & HCR_TID2) {
6004         return CP_ACCESS_TRAP_EL2;
6005     }
6006 
6007     return CP_ACCESS_OK;
6008 }
6009 
6010 static void oslar_write(CPUARMState *env, const ARMCPRegInfo *ri,
6011                         uint64_t value)
6012 {
6013     /* Writes to OSLAR_EL1 may update the OS lock status, which can be
6014      * read via a bit in OSLSR_EL1.
6015      */
6016     int oslock;
6017 
6018     if (ri->state == ARM_CP_STATE_AA32) {
6019         oslock = (value == 0xC5ACCE55);
6020     } else {
6021         oslock = value & 1;
6022     }
6023 
6024     env->cp15.oslsr_el1 = deposit32(env->cp15.oslsr_el1, 1, 1, oslock);
6025 }
6026 
6027 static const ARMCPRegInfo debug_cp_reginfo[] = {
6028     /* DBGDRAR, DBGDSAR: always RAZ since we don't implement memory mapped
6029      * debug components. The AArch64 version of DBGDRAR is named MDRAR_EL1;
6030      * unlike DBGDRAR it is never accessible from EL0.
6031      * DBGDSAR is deprecated and must RAZ from v8 anyway, so it has no AArch64
6032      * accessor.
6033      */
6034     { .name = "DBGDRAR", .cp = 14, .crn = 1, .crm = 0, .opc1 = 0, .opc2 = 0,
6035       .access = PL0_R, .accessfn = access_tdra,
6036       .type = ARM_CP_CONST, .resetvalue = 0 },
6037     { .name = "MDRAR_EL1", .state = ARM_CP_STATE_AA64,
6038       .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 0,
6039       .access = PL1_R, .accessfn = access_tdra,
6040       .type = ARM_CP_CONST, .resetvalue = 0 },
6041     { .name = "DBGDSAR", .cp = 14, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 0,
6042       .access = PL0_R, .accessfn = access_tdra,
6043       .type = ARM_CP_CONST, .resetvalue = 0 },
6044     /* Monitor debug system control register; the 32-bit alias is DBGDSCRext. */
6045     { .name = "MDSCR_EL1", .state = ARM_CP_STATE_BOTH,
6046       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 2,
6047       .access = PL1_RW, .accessfn = access_tda,
6048       .fieldoffset = offsetof(CPUARMState, cp15.mdscr_el1),
6049       .resetvalue = 0 },
6050     /* MDCCSR_EL0, aka DBGDSCRint. This is a read-only mirror of MDSCR_EL1.
6051      * We don't implement the configurable EL0 access.
6052      */
6053     { .name = "MDCCSR_EL0", .state = ARM_CP_STATE_BOTH,
6054       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 0,
6055       .type = ARM_CP_ALIAS,
6056       .access = PL1_R, .accessfn = access_tda,
6057       .fieldoffset = offsetof(CPUARMState, cp15.mdscr_el1), },
6058     { .name = "OSLAR_EL1", .state = ARM_CP_STATE_BOTH,
6059       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 4,
6060       .access = PL1_W, .type = ARM_CP_NO_RAW,
6061       .accessfn = access_tdosa,
6062       .writefn = oslar_write },
6063     { .name = "OSLSR_EL1", .state = ARM_CP_STATE_BOTH,
6064       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 4,
6065       .access = PL1_R, .resetvalue = 10,
6066       .accessfn = access_tdosa,
6067       .fieldoffset = offsetof(CPUARMState, cp15.oslsr_el1) },
6068     /* Dummy OSDLR_EL1: 32-bit Linux will read this */
6069     { .name = "OSDLR_EL1", .state = ARM_CP_STATE_BOTH,
6070       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 3, .opc2 = 4,
6071       .access = PL1_RW, .accessfn = access_tdosa,
6072       .type = ARM_CP_NOP },
6073     /* Dummy DBGVCR: Linux wants to clear this on startup, but we don't
6074      * implement vector catch debug events yet.
6075      */
6076     { .name = "DBGVCR",
6077       .cp = 14, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 0,
6078       .access = PL1_RW, .accessfn = access_tda,
6079       .type = ARM_CP_NOP },
6080     /* Dummy DBGVCR32_EL2 (which is only for a 64-bit hypervisor
6081      * to save and restore a 32-bit guest's DBGVCR)
6082      */
6083     { .name = "DBGVCR32_EL2", .state = ARM_CP_STATE_AA64,
6084       .opc0 = 2, .opc1 = 4, .crn = 0, .crm = 7, .opc2 = 0,
6085       .access = PL2_RW, .accessfn = access_tda,
6086       .type = ARM_CP_NOP },
6087     /* Dummy MDCCINT_EL1, since we don't implement the Debug Communications
6088      * Channel but Linux may try to access this register. The 32-bit
6089      * alias is DBGDCCINT.
6090      */
6091     { .name = "MDCCINT_EL1", .state = ARM_CP_STATE_BOTH,
6092       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 0,
6093       .access = PL1_RW, .accessfn = access_tda,
6094       .type = ARM_CP_NOP },
6095     REGINFO_SENTINEL
6096 };
6097 
6098 static const ARMCPRegInfo debug_lpae_cp_reginfo[] = {
6099     /* 64 bit access versions of the (dummy) debug registers */
6100     { .name = "DBGDRAR", .cp = 14, .crm = 1, .opc1 = 0,
6101       .access = PL0_R, .type = ARM_CP_CONST|ARM_CP_64BIT, .resetvalue = 0 },
6102     { .name = "DBGDSAR", .cp = 14, .crm = 2, .opc1 = 0,
6103       .access = PL0_R, .type = ARM_CP_CONST|ARM_CP_64BIT, .resetvalue = 0 },
6104     REGINFO_SENTINEL
6105 };
6106 
6107 /* Return the exception level to which exceptions should be taken
6108  * via SVEAccessTrap.  If an exception should be routed through
6109  * AArch64.AdvSIMDFPAccessTrap, return 0; fp_exception_el should
6110  * take care of raising that exception.
6111  * C.f. the ARM pseudocode function CheckSVEEnabled.
6112  */
6113 int sve_exception_el(CPUARMState *env, int el)
6114 {
6115 #ifndef CONFIG_USER_ONLY
6116     uint64_t hcr_el2 = arm_hcr_el2_eff(env);
6117 
6118     if (el <= 1 && (hcr_el2 & (HCR_E2H | HCR_TGE)) != (HCR_E2H | HCR_TGE)) {
6119         bool disabled = false;
6120 
6121         /* The CPACR.ZEN controls traps to EL1:
6122          * 0, 2 : trap EL0 and EL1 accesses
6123          * 1    : trap only EL0 accesses
6124          * 3    : trap no accesses
6125          */
6126         if (!extract32(env->cp15.cpacr_el1, 16, 1)) {
6127             disabled = true;
6128         } else if (!extract32(env->cp15.cpacr_el1, 17, 1)) {
6129             disabled = el == 0;
6130         }
6131         if (disabled) {
6132             /* route_to_el2 */
6133             return hcr_el2 & HCR_TGE ? 2 : 1;
6134         }
6135 
6136         /* Check CPACR.FPEN.  */
6137         if (!extract32(env->cp15.cpacr_el1, 20, 1)) {
6138             disabled = true;
6139         } else if (!extract32(env->cp15.cpacr_el1, 21, 1)) {
6140             disabled = el == 0;
6141         }
6142         if (disabled) {
6143             return 0;
6144         }
6145     }
6146 
6147     /* CPTR_EL2.  Since TZ and TFP are positive,
6148      * they will be zero when EL2 is not present.
6149      */
6150     if (el <= 2 && !arm_is_secure_below_el3(env)) {
6151         if (env->cp15.cptr_el[2] & CPTR_TZ) {
6152             return 2;
6153         }
6154         if (env->cp15.cptr_el[2] & CPTR_TFP) {
6155             return 0;
6156         }
6157     }
6158 
6159     /* CPTR_EL3.  Since EZ is negative we must check for EL3.  */
6160     if (arm_feature(env, ARM_FEATURE_EL3)
6161         && !(env->cp15.cptr_el[3] & CPTR_EZ)) {
6162         return 3;
6163     }
6164 #endif
6165     return 0;
6166 }
6167 
6168 static uint32_t sve_zcr_get_valid_len(ARMCPU *cpu, uint32_t start_len)
6169 {
6170     uint32_t end_len;
6171 
6172     end_len = start_len &= 0xf;
6173     if (!test_bit(start_len, cpu->sve_vq_map)) {
6174         end_len = find_last_bit(cpu->sve_vq_map, start_len);
6175         assert(end_len < start_len);
6176     }
6177     return end_len;
6178 }
6179 
6180 /*
6181  * Given that SVE is enabled, return the vector length for EL.
6182  */
6183 uint32_t sve_zcr_len_for_el(CPUARMState *env, int el)
6184 {
6185     ARMCPU *cpu = env_archcpu(env);
6186     uint32_t zcr_len = cpu->sve_max_vq - 1;
6187 
6188     if (el <= 1) {
6189         zcr_len = MIN(zcr_len, 0xf & (uint32_t)env->vfp.zcr_el[1]);
6190     }
6191     if (el <= 2 && arm_feature(env, ARM_FEATURE_EL2)) {
6192         zcr_len = MIN(zcr_len, 0xf & (uint32_t)env->vfp.zcr_el[2]);
6193     }
6194     if (arm_feature(env, ARM_FEATURE_EL3)) {
6195         zcr_len = MIN(zcr_len, 0xf & (uint32_t)env->vfp.zcr_el[3]);
6196     }
6197 
6198     return sve_zcr_get_valid_len(cpu, zcr_len);
6199 }
6200 
6201 static void zcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
6202                       uint64_t value)
6203 {
6204     int cur_el = arm_current_el(env);
6205     int old_len = sve_zcr_len_for_el(env, cur_el);
6206     int new_len;
6207 
6208     /* Bits other than [3:0] are RAZ/WI.  */
6209     QEMU_BUILD_BUG_ON(ARM_MAX_VQ > 16);
6210     raw_write(env, ri, value & 0xf);
6211 
6212     /*
6213      * Because we arrived here, we know both FP and SVE are enabled;
6214      * otherwise we would have trapped access to the ZCR_ELn register.
6215      */
6216     new_len = sve_zcr_len_for_el(env, cur_el);
6217     if (new_len < old_len) {
6218         aarch64_sve_narrow_vq(env, new_len + 1);
6219     }
6220 }
6221 
6222 static const ARMCPRegInfo zcr_el1_reginfo = {
6223     .name = "ZCR_EL1", .state = ARM_CP_STATE_AA64,
6224     .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 2, .opc2 = 0,
6225     .access = PL1_RW, .type = ARM_CP_SVE,
6226     .fieldoffset = offsetof(CPUARMState, vfp.zcr_el[1]),
6227     .writefn = zcr_write, .raw_writefn = raw_write
6228 };
6229 
6230 static const ARMCPRegInfo zcr_el2_reginfo = {
6231     .name = "ZCR_EL2", .state = ARM_CP_STATE_AA64,
6232     .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 2, .opc2 = 0,
6233     .access = PL2_RW, .type = ARM_CP_SVE,
6234     .fieldoffset = offsetof(CPUARMState, vfp.zcr_el[2]),
6235     .writefn = zcr_write, .raw_writefn = raw_write
6236 };
6237 
6238 static const ARMCPRegInfo zcr_no_el2_reginfo = {
6239     .name = "ZCR_EL2", .state = ARM_CP_STATE_AA64,
6240     .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 2, .opc2 = 0,
6241     .access = PL2_RW, .type = ARM_CP_SVE,
6242     .readfn = arm_cp_read_zero, .writefn = arm_cp_write_ignore
6243 };
6244 
6245 static const ARMCPRegInfo zcr_el3_reginfo = {
6246     .name = "ZCR_EL3", .state = ARM_CP_STATE_AA64,
6247     .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 2, .opc2 = 0,
6248     .access = PL3_RW, .type = ARM_CP_SVE,
6249     .fieldoffset = offsetof(CPUARMState, vfp.zcr_el[3]),
6250     .writefn = zcr_write, .raw_writefn = raw_write
6251 };
6252 
6253 void hw_watchpoint_update(ARMCPU *cpu, int n)
6254 {
6255     CPUARMState *env = &cpu->env;
6256     vaddr len = 0;
6257     vaddr wvr = env->cp15.dbgwvr[n];
6258     uint64_t wcr = env->cp15.dbgwcr[n];
6259     int mask;
6260     int flags = BP_CPU | BP_STOP_BEFORE_ACCESS;
6261 
6262     if (env->cpu_watchpoint[n]) {
6263         cpu_watchpoint_remove_by_ref(CPU(cpu), env->cpu_watchpoint[n]);
6264         env->cpu_watchpoint[n] = NULL;
6265     }
6266 
6267     if (!extract64(wcr, 0, 1)) {
6268         /* E bit clear : watchpoint disabled */
6269         return;
6270     }
6271 
6272     switch (extract64(wcr, 3, 2)) {
6273     case 0:
6274         /* LSC 00 is reserved and must behave as if the wp is disabled */
6275         return;
6276     case 1:
6277         flags |= BP_MEM_READ;
6278         break;
6279     case 2:
6280         flags |= BP_MEM_WRITE;
6281         break;
6282     case 3:
6283         flags |= BP_MEM_ACCESS;
6284         break;
6285     }
6286 
6287     /* Attempts to use both MASK and BAS fields simultaneously are
6288      * CONSTRAINED UNPREDICTABLE; we opt to ignore BAS in this case,
6289      * thus generating a watchpoint for every byte in the masked region.
6290      */
6291     mask = extract64(wcr, 24, 4);
6292     if (mask == 1 || mask == 2) {
6293         /* Reserved values of MASK; we must act as if the mask value was
6294          * some non-reserved value, or as if the watchpoint were disabled.
6295          * We choose the latter.
6296          */
6297         return;
6298     } else if (mask) {
6299         /* Watchpoint covers an aligned area up to 2GB in size */
6300         len = 1ULL << mask;
6301         /* If masked bits in WVR are not zero it's CONSTRAINED UNPREDICTABLE
6302          * whether the watchpoint fires when the unmasked bits match; we opt
6303          * to generate the exceptions.
6304          */
6305         wvr &= ~(len - 1);
6306     } else {
6307         /* Watchpoint covers bytes defined by the byte address select bits */
6308         int bas = extract64(wcr, 5, 8);
6309         int basstart;
6310 
6311         if (extract64(wvr, 2, 1)) {
6312             /* Deprecated case of an only 4-aligned address. BAS[7:4] are
6313              * ignored, and BAS[3:0] define which bytes to watch.
6314              */
6315             bas &= 0xf;
6316         }
6317 
6318         if (bas == 0) {
6319             /* This must act as if the watchpoint is disabled */
6320             return;
6321         }
6322 
6323         /* The BAS bits are supposed to be programmed to indicate a contiguous
6324          * range of bytes. Otherwise it is CONSTRAINED UNPREDICTABLE whether
6325          * we fire for each byte in the word/doubleword addressed by the WVR.
6326          * We choose to ignore any non-zero bits after the first range of 1s.
6327          */
6328         basstart = ctz32(bas);
6329         len = cto32(bas >> basstart);
6330         wvr += basstart;
6331     }
6332 
6333     cpu_watchpoint_insert(CPU(cpu), wvr, len, flags,
6334                           &env->cpu_watchpoint[n]);
6335 }
6336 
6337 void hw_watchpoint_update_all(ARMCPU *cpu)
6338 {
6339     int i;
6340     CPUARMState *env = &cpu->env;
6341 
6342     /* Completely clear out existing QEMU watchpoints and our array, to
6343      * avoid possible stale entries following migration load.
6344      */
6345     cpu_watchpoint_remove_all(CPU(cpu), BP_CPU);
6346     memset(env->cpu_watchpoint, 0, sizeof(env->cpu_watchpoint));
6347 
6348     for (i = 0; i < ARRAY_SIZE(cpu->env.cpu_watchpoint); i++) {
6349         hw_watchpoint_update(cpu, i);
6350     }
6351 }
6352 
6353 static void dbgwvr_write(CPUARMState *env, const ARMCPRegInfo *ri,
6354                          uint64_t value)
6355 {
6356     ARMCPU *cpu = env_archcpu(env);
6357     int i = ri->crm;
6358 
6359     /* Bits [63:49] are hardwired to the value of bit [48]; that is, the
6360      * register reads and behaves as if values written are sign extended.
6361      * Bits [1:0] are RES0.
6362      */
6363     value = sextract64(value, 0, 49) & ~3ULL;
6364 
6365     raw_write(env, ri, value);
6366     hw_watchpoint_update(cpu, i);
6367 }
6368 
6369 static void dbgwcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
6370                          uint64_t value)
6371 {
6372     ARMCPU *cpu = env_archcpu(env);
6373     int i = ri->crm;
6374 
6375     raw_write(env, ri, value);
6376     hw_watchpoint_update(cpu, i);
6377 }
6378 
6379 void hw_breakpoint_update(ARMCPU *cpu, int n)
6380 {
6381     CPUARMState *env = &cpu->env;
6382     uint64_t bvr = env->cp15.dbgbvr[n];
6383     uint64_t bcr = env->cp15.dbgbcr[n];
6384     vaddr addr;
6385     int bt;
6386     int flags = BP_CPU;
6387 
6388     if (env->cpu_breakpoint[n]) {
6389         cpu_breakpoint_remove_by_ref(CPU(cpu), env->cpu_breakpoint[n]);
6390         env->cpu_breakpoint[n] = NULL;
6391     }
6392 
6393     if (!extract64(bcr, 0, 1)) {
6394         /* E bit clear : watchpoint disabled */
6395         return;
6396     }
6397 
6398     bt = extract64(bcr, 20, 4);
6399 
6400     switch (bt) {
6401     case 4: /* unlinked address mismatch (reserved if AArch64) */
6402     case 5: /* linked address mismatch (reserved if AArch64) */
6403         qemu_log_mask(LOG_UNIMP,
6404                       "arm: address mismatch breakpoint types not implemented\n");
6405         return;
6406     case 0: /* unlinked address match */
6407     case 1: /* linked address match */
6408     {
6409         /* Bits [63:49] are hardwired to the value of bit [48]; that is,
6410          * we behave as if the register was sign extended. Bits [1:0] are
6411          * RES0. The BAS field is used to allow setting breakpoints on 16
6412          * bit wide instructions; it is CONSTRAINED UNPREDICTABLE whether
6413          * a bp will fire if the addresses covered by the bp and the addresses
6414          * covered by the insn overlap but the insn doesn't start at the
6415          * start of the bp address range. We choose to require the insn and
6416          * the bp to have the same address. The constraints on writing to
6417          * BAS enforced in dbgbcr_write mean we have only four cases:
6418          *  0b0000  => no breakpoint
6419          *  0b0011  => breakpoint on addr
6420          *  0b1100  => breakpoint on addr + 2
6421          *  0b1111  => breakpoint on addr
6422          * See also figure D2-3 in the v8 ARM ARM (DDI0487A.c).
6423          */
6424         int bas = extract64(bcr, 5, 4);
6425         addr = sextract64(bvr, 0, 49) & ~3ULL;
6426         if (bas == 0) {
6427             return;
6428         }
6429         if (bas == 0xc) {
6430             addr += 2;
6431         }
6432         break;
6433     }
6434     case 2: /* unlinked context ID match */
6435     case 8: /* unlinked VMID match (reserved if no EL2) */
6436     case 10: /* unlinked context ID and VMID match (reserved if no EL2) */
6437         qemu_log_mask(LOG_UNIMP,
6438                       "arm: unlinked context breakpoint types not implemented\n");
6439         return;
6440     case 9: /* linked VMID match (reserved if no EL2) */
6441     case 11: /* linked context ID and VMID match (reserved if no EL2) */
6442     case 3: /* linked context ID match */
6443     default:
6444         /* We must generate no events for Linked context matches (unless
6445          * they are linked to by some other bp/wp, which is handled in
6446          * updates for the linking bp/wp). We choose to also generate no events
6447          * for reserved values.
6448          */
6449         return;
6450     }
6451 
6452     cpu_breakpoint_insert(CPU(cpu), addr, flags, &env->cpu_breakpoint[n]);
6453 }
6454 
6455 void hw_breakpoint_update_all(ARMCPU *cpu)
6456 {
6457     int i;
6458     CPUARMState *env = &cpu->env;
6459 
6460     /* Completely clear out existing QEMU breakpoints and our array, to
6461      * avoid possible stale entries following migration load.
6462      */
6463     cpu_breakpoint_remove_all(CPU(cpu), BP_CPU);
6464     memset(env->cpu_breakpoint, 0, sizeof(env->cpu_breakpoint));
6465 
6466     for (i = 0; i < ARRAY_SIZE(cpu->env.cpu_breakpoint); i++) {
6467         hw_breakpoint_update(cpu, i);
6468     }
6469 }
6470 
6471 static void dbgbvr_write(CPUARMState *env, const ARMCPRegInfo *ri,
6472                          uint64_t value)
6473 {
6474     ARMCPU *cpu = env_archcpu(env);
6475     int i = ri->crm;
6476 
6477     raw_write(env, ri, value);
6478     hw_breakpoint_update(cpu, i);
6479 }
6480 
6481 static void dbgbcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
6482                          uint64_t value)
6483 {
6484     ARMCPU *cpu = env_archcpu(env);
6485     int i = ri->crm;
6486 
6487     /* BAS[3] is a read-only copy of BAS[2], and BAS[1] a read-only
6488      * copy of BAS[0].
6489      */
6490     value = deposit64(value, 6, 1, extract64(value, 5, 1));
6491     value = deposit64(value, 8, 1, extract64(value, 7, 1));
6492 
6493     raw_write(env, ri, value);
6494     hw_breakpoint_update(cpu, i);
6495 }
6496 
6497 static void define_debug_regs(ARMCPU *cpu)
6498 {
6499     /* Define v7 and v8 architectural debug registers.
6500      * These are just dummy implementations for now.
6501      */
6502     int i;
6503     int wrps, brps, ctx_cmps;
6504     ARMCPRegInfo dbgdidr = {
6505         .name = "DBGDIDR", .cp = 14, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 0,
6506         .access = PL0_R, .accessfn = access_tda,
6507         .type = ARM_CP_CONST, .resetvalue = cpu->isar.dbgdidr,
6508     };
6509 
6510     /* Note that all these register fields hold "number of Xs minus 1". */
6511     brps = arm_num_brps(cpu);
6512     wrps = arm_num_wrps(cpu);
6513     ctx_cmps = arm_num_ctx_cmps(cpu);
6514 
6515     assert(ctx_cmps <= brps);
6516 
6517     define_one_arm_cp_reg(cpu, &dbgdidr);
6518     define_arm_cp_regs(cpu, debug_cp_reginfo);
6519 
6520     if (arm_feature(&cpu->env, ARM_FEATURE_LPAE)) {
6521         define_arm_cp_regs(cpu, debug_lpae_cp_reginfo);
6522     }
6523 
6524     for (i = 0; i < brps; i++) {
6525         ARMCPRegInfo dbgregs[] = {
6526             { .name = "DBGBVR", .state = ARM_CP_STATE_BOTH,
6527               .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 4,
6528               .access = PL1_RW, .accessfn = access_tda,
6529               .fieldoffset = offsetof(CPUARMState, cp15.dbgbvr[i]),
6530               .writefn = dbgbvr_write, .raw_writefn = raw_write
6531             },
6532             { .name = "DBGBCR", .state = ARM_CP_STATE_BOTH,
6533               .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 5,
6534               .access = PL1_RW, .accessfn = access_tda,
6535               .fieldoffset = offsetof(CPUARMState, cp15.dbgbcr[i]),
6536               .writefn = dbgbcr_write, .raw_writefn = raw_write
6537             },
6538             REGINFO_SENTINEL
6539         };
6540         define_arm_cp_regs(cpu, dbgregs);
6541     }
6542 
6543     for (i = 0; i < wrps; i++) {
6544         ARMCPRegInfo dbgregs[] = {
6545             { .name = "DBGWVR", .state = ARM_CP_STATE_BOTH,
6546               .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 6,
6547               .access = PL1_RW, .accessfn = access_tda,
6548               .fieldoffset = offsetof(CPUARMState, cp15.dbgwvr[i]),
6549               .writefn = dbgwvr_write, .raw_writefn = raw_write
6550             },
6551             { .name = "DBGWCR", .state = ARM_CP_STATE_BOTH,
6552               .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 7,
6553               .access = PL1_RW, .accessfn = access_tda,
6554               .fieldoffset = offsetof(CPUARMState, cp15.dbgwcr[i]),
6555               .writefn = dbgwcr_write, .raw_writefn = raw_write
6556             },
6557             REGINFO_SENTINEL
6558         };
6559         define_arm_cp_regs(cpu, dbgregs);
6560     }
6561 }
6562 
6563 static void define_pmu_regs(ARMCPU *cpu)
6564 {
6565     /*
6566      * v7 performance monitor control register: same implementor
6567      * field as main ID register, and we implement four counters in
6568      * addition to the cycle count register.
6569      */
6570     unsigned int i, pmcrn = 4;
6571     ARMCPRegInfo pmcr = {
6572         .name = "PMCR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 0,
6573         .access = PL0_RW,
6574         .type = ARM_CP_IO | ARM_CP_ALIAS,
6575         .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmcr),
6576         .accessfn = pmreg_access, .writefn = pmcr_write,
6577         .raw_writefn = raw_write,
6578     };
6579     ARMCPRegInfo pmcr64 = {
6580         .name = "PMCR_EL0", .state = ARM_CP_STATE_AA64,
6581         .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 0,
6582         .access = PL0_RW, .accessfn = pmreg_access,
6583         .type = ARM_CP_IO,
6584         .fieldoffset = offsetof(CPUARMState, cp15.c9_pmcr),
6585         .resetvalue = (cpu->midr & 0xff000000) | (pmcrn << PMCRN_SHIFT) |
6586                       PMCRLC,
6587         .writefn = pmcr_write, .raw_writefn = raw_write,
6588     };
6589     define_one_arm_cp_reg(cpu, &pmcr);
6590     define_one_arm_cp_reg(cpu, &pmcr64);
6591     for (i = 0; i < pmcrn; i++) {
6592         char *pmevcntr_name = g_strdup_printf("PMEVCNTR%d", i);
6593         char *pmevcntr_el0_name = g_strdup_printf("PMEVCNTR%d_EL0", i);
6594         char *pmevtyper_name = g_strdup_printf("PMEVTYPER%d", i);
6595         char *pmevtyper_el0_name = g_strdup_printf("PMEVTYPER%d_EL0", i);
6596         ARMCPRegInfo pmev_regs[] = {
6597             { .name = pmevcntr_name, .cp = 15, .crn = 14,
6598               .crm = 8 | (3 & (i >> 3)), .opc1 = 0, .opc2 = i & 7,
6599               .access = PL0_RW, .type = ARM_CP_IO | ARM_CP_ALIAS,
6600               .readfn = pmevcntr_readfn, .writefn = pmevcntr_writefn,
6601               .accessfn = pmreg_access },
6602             { .name = pmevcntr_el0_name, .state = ARM_CP_STATE_AA64,
6603               .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 8 | (3 & (i >> 3)),
6604               .opc2 = i & 7, .access = PL0_RW, .accessfn = pmreg_access,
6605               .type = ARM_CP_IO,
6606               .readfn = pmevcntr_readfn, .writefn = pmevcntr_writefn,
6607               .raw_readfn = pmevcntr_rawread,
6608               .raw_writefn = pmevcntr_rawwrite },
6609             { .name = pmevtyper_name, .cp = 15, .crn = 14,
6610               .crm = 12 | (3 & (i >> 3)), .opc1 = 0, .opc2 = i & 7,
6611               .access = PL0_RW, .type = ARM_CP_IO | ARM_CP_ALIAS,
6612               .readfn = pmevtyper_readfn, .writefn = pmevtyper_writefn,
6613               .accessfn = pmreg_access },
6614             { .name = pmevtyper_el0_name, .state = ARM_CP_STATE_AA64,
6615               .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 12 | (3 & (i >> 3)),
6616               .opc2 = i & 7, .access = PL0_RW, .accessfn = pmreg_access,
6617               .type = ARM_CP_IO,
6618               .readfn = pmevtyper_readfn, .writefn = pmevtyper_writefn,
6619               .raw_writefn = pmevtyper_rawwrite },
6620             REGINFO_SENTINEL
6621         };
6622         define_arm_cp_regs(cpu, pmev_regs);
6623         g_free(pmevcntr_name);
6624         g_free(pmevcntr_el0_name);
6625         g_free(pmevtyper_name);
6626         g_free(pmevtyper_el0_name);
6627     }
6628     if (cpu_isar_feature(aa32_pmu_8_1, cpu)) {
6629         ARMCPRegInfo v81_pmu_regs[] = {
6630             { .name = "PMCEID2", .state = ARM_CP_STATE_AA32,
6631               .cp = 15, .opc1 = 0, .crn = 9, .crm = 14, .opc2 = 4,
6632               .access = PL0_R, .accessfn = pmreg_access, .type = ARM_CP_CONST,
6633               .resetvalue = extract64(cpu->pmceid0, 32, 32) },
6634             { .name = "PMCEID3", .state = ARM_CP_STATE_AA32,
6635               .cp = 15, .opc1 = 0, .crn = 9, .crm = 14, .opc2 = 5,
6636               .access = PL0_R, .accessfn = pmreg_access, .type = ARM_CP_CONST,
6637               .resetvalue = extract64(cpu->pmceid1, 32, 32) },
6638             REGINFO_SENTINEL
6639         };
6640         define_arm_cp_regs(cpu, v81_pmu_regs);
6641     }
6642     if (cpu_isar_feature(any_pmu_8_4, cpu)) {
6643         static const ARMCPRegInfo v84_pmmir = {
6644             .name = "PMMIR_EL1", .state = ARM_CP_STATE_BOTH,
6645             .opc0 = 3, .opc1 = 0, .crn = 9, .crm = 14, .opc2 = 6,
6646             .access = PL1_R, .accessfn = pmreg_access, .type = ARM_CP_CONST,
6647             .resetvalue = 0
6648         };
6649         define_one_arm_cp_reg(cpu, &v84_pmmir);
6650     }
6651 }
6652 
6653 /* We don't know until after realize whether there's a GICv3
6654  * attached, and that is what registers the gicv3 sysregs.
6655  * So we have to fill in the GIC fields in ID_PFR/ID_PFR1_EL1/ID_AA64PFR0_EL1
6656  * at runtime.
6657  */
6658 static uint64_t id_pfr1_read(CPUARMState *env, const ARMCPRegInfo *ri)
6659 {
6660     ARMCPU *cpu = env_archcpu(env);
6661     uint64_t pfr1 = cpu->isar.id_pfr1;
6662 
6663     if (env->gicv3state) {
6664         pfr1 |= 1 << 28;
6665     }
6666     return pfr1;
6667 }
6668 
6669 #ifndef CONFIG_USER_ONLY
6670 static uint64_t id_aa64pfr0_read(CPUARMState *env, const ARMCPRegInfo *ri)
6671 {
6672     ARMCPU *cpu = env_archcpu(env);
6673     uint64_t pfr0 = cpu->isar.id_aa64pfr0;
6674 
6675     if (env->gicv3state) {
6676         pfr0 |= 1 << 24;
6677     }
6678     return pfr0;
6679 }
6680 #endif
6681 
6682 /* Shared logic between LORID and the rest of the LOR* registers.
6683  * Secure state has already been delt with.
6684  */
6685 static CPAccessResult access_lor_ns(CPUARMState *env)
6686 {
6687     int el = arm_current_el(env);
6688 
6689     if (el < 2 && (arm_hcr_el2_eff(env) & HCR_TLOR)) {
6690         return CP_ACCESS_TRAP_EL2;
6691     }
6692     if (el < 3 && (env->cp15.scr_el3 & SCR_TLOR)) {
6693         return CP_ACCESS_TRAP_EL3;
6694     }
6695     return CP_ACCESS_OK;
6696 }
6697 
6698 static CPAccessResult access_lorid(CPUARMState *env, const ARMCPRegInfo *ri,
6699                                    bool isread)
6700 {
6701     if (arm_is_secure_below_el3(env)) {
6702         /* Access ok in secure mode.  */
6703         return CP_ACCESS_OK;
6704     }
6705     return access_lor_ns(env);
6706 }
6707 
6708 static CPAccessResult access_lor_other(CPUARMState *env,
6709                                        const ARMCPRegInfo *ri, bool isread)
6710 {
6711     if (arm_is_secure_below_el3(env)) {
6712         /* Access denied in secure mode.  */
6713         return CP_ACCESS_TRAP;
6714     }
6715     return access_lor_ns(env);
6716 }
6717 
6718 /*
6719  * A trivial implementation of ARMv8.1-LOR leaves all of these
6720  * registers fixed at 0, which indicates that there are zero
6721  * supported Limited Ordering regions.
6722  */
6723 static const ARMCPRegInfo lor_reginfo[] = {
6724     { .name = "LORSA_EL1", .state = ARM_CP_STATE_AA64,
6725       .opc0 = 3, .opc1 = 0, .crn = 10, .crm = 4, .opc2 = 0,
6726       .access = PL1_RW, .accessfn = access_lor_other,
6727       .type = ARM_CP_CONST, .resetvalue = 0 },
6728     { .name = "LOREA_EL1", .state = ARM_CP_STATE_AA64,
6729       .opc0 = 3, .opc1 = 0, .crn = 10, .crm = 4, .opc2 = 1,
6730       .access = PL1_RW, .accessfn = access_lor_other,
6731       .type = ARM_CP_CONST, .resetvalue = 0 },
6732     { .name = "LORN_EL1", .state = ARM_CP_STATE_AA64,
6733       .opc0 = 3, .opc1 = 0, .crn = 10, .crm = 4, .opc2 = 2,
6734       .access = PL1_RW, .accessfn = access_lor_other,
6735       .type = ARM_CP_CONST, .resetvalue = 0 },
6736     { .name = "LORC_EL1", .state = ARM_CP_STATE_AA64,
6737       .opc0 = 3, .opc1 = 0, .crn = 10, .crm = 4, .opc2 = 3,
6738       .access = PL1_RW, .accessfn = access_lor_other,
6739       .type = ARM_CP_CONST, .resetvalue = 0 },
6740     { .name = "LORID_EL1", .state = ARM_CP_STATE_AA64,
6741       .opc0 = 3, .opc1 = 0, .crn = 10, .crm = 4, .opc2 = 7,
6742       .access = PL1_R, .accessfn = access_lorid,
6743       .type = ARM_CP_CONST, .resetvalue = 0 },
6744     REGINFO_SENTINEL
6745 };
6746 
6747 #ifdef TARGET_AARCH64
6748 static CPAccessResult access_pauth(CPUARMState *env, const ARMCPRegInfo *ri,
6749                                    bool isread)
6750 {
6751     int el = arm_current_el(env);
6752 
6753     if (el < 2 &&
6754         arm_feature(env, ARM_FEATURE_EL2) &&
6755         !(arm_hcr_el2_eff(env) & HCR_APK)) {
6756         return CP_ACCESS_TRAP_EL2;
6757     }
6758     if (el < 3 &&
6759         arm_feature(env, ARM_FEATURE_EL3) &&
6760         !(env->cp15.scr_el3 & SCR_APK)) {
6761         return CP_ACCESS_TRAP_EL3;
6762     }
6763     return CP_ACCESS_OK;
6764 }
6765 
6766 static const ARMCPRegInfo pauth_reginfo[] = {
6767     { .name = "APDAKEYLO_EL1", .state = ARM_CP_STATE_AA64,
6768       .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 2, .opc2 = 0,
6769       .access = PL1_RW, .accessfn = access_pauth,
6770       .fieldoffset = offsetof(CPUARMState, keys.apda.lo) },
6771     { .name = "APDAKEYHI_EL1", .state = ARM_CP_STATE_AA64,
6772       .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 2, .opc2 = 1,
6773       .access = PL1_RW, .accessfn = access_pauth,
6774       .fieldoffset = offsetof(CPUARMState, keys.apda.hi) },
6775     { .name = "APDBKEYLO_EL1", .state = ARM_CP_STATE_AA64,
6776       .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 2, .opc2 = 2,
6777       .access = PL1_RW, .accessfn = access_pauth,
6778       .fieldoffset = offsetof(CPUARMState, keys.apdb.lo) },
6779     { .name = "APDBKEYHI_EL1", .state = ARM_CP_STATE_AA64,
6780       .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 2, .opc2 = 3,
6781       .access = PL1_RW, .accessfn = access_pauth,
6782       .fieldoffset = offsetof(CPUARMState, keys.apdb.hi) },
6783     { .name = "APGAKEYLO_EL1", .state = ARM_CP_STATE_AA64,
6784       .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 3, .opc2 = 0,
6785       .access = PL1_RW, .accessfn = access_pauth,
6786       .fieldoffset = offsetof(CPUARMState, keys.apga.lo) },
6787     { .name = "APGAKEYHI_EL1", .state = ARM_CP_STATE_AA64,
6788       .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 3, .opc2 = 1,
6789       .access = PL1_RW, .accessfn = access_pauth,
6790       .fieldoffset = offsetof(CPUARMState, keys.apga.hi) },
6791     { .name = "APIAKEYLO_EL1", .state = ARM_CP_STATE_AA64,
6792       .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 1, .opc2 = 0,
6793       .access = PL1_RW, .accessfn = access_pauth,
6794       .fieldoffset = offsetof(CPUARMState, keys.apia.lo) },
6795     { .name = "APIAKEYHI_EL1", .state = ARM_CP_STATE_AA64,
6796       .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 1, .opc2 = 1,
6797       .access = PL1_RW, .accessfn = access_pauth,
6798       .fieldoffset = offsetof(CPUARMState, keys.apia.hi) },
6799     { .name = "APIBKEYLO_EL1", .state = ARM_CP_STATE_AA64,
6800       .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 1, .opc2 = 2,
6801       .access = PL1_RW, .accessfn = access_pauth,
6802       .fieldoffset = offsetof(CPUARMState, keys.apib.lo) },
6803     { .name = "APIBKEYHI_EL1", .state = ARM_CP_STATE_AA64,
6804       .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 1, .opc2 = 3,
6805       .access = PL1_RW, .accessfn = access_pauth,
6806       .fieldoffset = offsetof(CPUARMState, keys.apib.hi) },
6807     REGINFO_SENTINEL
6808 };
6809 
6810 static uint64_t rndr_readfn(CPUARMState *env, const ARMCPRegInfo *ri)
6811 {
6812     Error *err = NULL;
6813     uint64_t ret;
6814 
6815     /* Success sets NZCV = 0000.  */
6816     env->NF = env->CF = env->VF = 0, env->ZF = 1;
6817 
6818     if (qemu_guest_getrandom(&ret, sizeof(ret), &err) < 0) {
6819         /*
6820          * ??? Failed, for unknown reasons in the crypto subsystem.
6821          * The best we can do is log the reason and return the
6822          * timed-out indication to the guest.  There is no reason
6823          * we know to expect this failure to be transitory, so the
6824          * guest may well hang retrying the operation.
6825          */
6826         qemu_log_mask(LOG_UNIMP, "%s: Crypto failure: %s",
6827                       ri->name, error_get_pretty(err));
6828         error_free(err);
6829 
6830         env->ZF = 0; /* NZCF = 0100 */
6831         return 0;
6832     }
6833     return ret;
6834 }
6835 
6836 /* We do not support re-seeding, so the two registers operate the same.  */
6837 static const ARMCPRegInfo rndr_reginfo[] = {
6838     { .name = "RNDR", .state = ARM_CP_STATE_AA64,
6839       .type = ARM_CP_NO_RAW | ARM_CP_SUPPRESS_TB_END | ARM_CP_IO,
6840       .opc0 = 3, .opc1 = 3, .crn = 2, .crm = 4, .opc2 = 0,
6841       .access = PL0_R, .readfn = rndr_readfn },
6842     { .name = "RNDRRS", .state = ARM_CP_STATE_AA64,
6843       .type = ARM_CP_NO_RAW | ARM_CP_SUPPRESS_TB_END | ARM_CP_IO,
6844       .opc0 = 3, .opc1 = 3, .crn = 2, .crm = 4, .opc2 = 1,
6845       .access = PL0_R, .readfn = rndr_readfn },
6846     REGINFO_SENTINEL
6847 };
6848 
6849 #ifndef CONFIG_USER_ONLY
6850 static void dccvap_writefn(CPUARMState *env, const ARMCPRegInfo *opaque,
6851                           uint64_t value)
6852 {
6853     ARMCPU *cpu = env_archcpu(env);
6854     /* CTR_EL0 System register -> DminLine, bits [19:16] */
6855     uint64_t dline_size = 4 << ((cpu->ctr >> 16) & 0xF);
6856     uint64_t vaddr_in = (uint64_t) value;
6857     uint64_t vaddr = vaddr_in & ~(dline_size - 1);
6858     void *haddr;
6859     int mem_idx = cpu_mmu_index(env, false);
6860 
6861     /* This won't be crossing page boundaries */
6862     haddr = probe_read(env, vaddr, dline_size, mem_idx, GETPC());
6863     if (haddr) {
6864 
6865         ram_addr_t offset;
6866         MemoryRegion *mr;
6867 
6868         /* RCU lock is already being held */
6869         mr = memory_region_from_host(haddr, &offset);
6870 
6871         if (mr) {
6872             memory_region_writeback(mr, offset, dline_size);
6873         }
6874     }
6875 }
6876 
6877 static const ARMCPRegInfo dcpop_reg[] = {
6878     { .name = "DC_CVAP", .state = ARM_CP_STATE_AA64,
6879       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 12, .opc2 = 1,
6880       .access = PL0_W, .type = ARM_CP_NO_RAW | ARM_CP_SUPPRESS_TB_END,
6881       .accessfn = aa64_cacheop_poc_access, .writefn = dccvap_writefn },
6882     REGINFO_SENTINEL
6883 };
6884 
6885 static const ARMCPRegInfo dcpodp_reg[] = {
6886     { .name = "DC_CVADP", .state = ARM_CP_STATE_AA64,
6887       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 13, .opc2 = 1,
6888       .access = PL0_W, .type = ARM_CP_NO_RAW | ARM_CP_SUPPRESS_TB_END,
6889       .accessfn = aa64_cacheop_poc_access, .writefn = dccvap_writefn },
6890     REGINFO_SENTINEL
6891 };
6892 #endif /*CONFIG_USER_ONLY*/
6893 
6894 static CPAccessResult access_aa64_tid5(CPUARMState *env, const ARMCPRegInfo *ri,
6895                                        bool isread)
6896 {
6897     if ((arm_current_el(env) < 2) && (arm_hcr_el2_eff(env) & HCR_TID5)) {
6898         return CP_ACCESS_TRAP_EL2;
6899     }
6900 
6901     return CP_ACCESS_OK;
6902 }
6903 
6904 static CPAccessResult access_mte(CPUARMState *env, const ARMCPRegInfo *ri,
6905                                  bool isread)
6906 {
6907     int el = arm_current_el(env);
6908 
6909     if (el < 2 &&
6910         arm_feature(env, ARM_FEATURE_EL2) &&
6911         !(arm_hcr_el2_eff(env) & HCR_ATA)) {
6912         return CP_ACCESS_TRAP_EL2;
6913     }
6914     if (el < 3 &&
6915         arm_feature(env, ARM_FEATURE_EL3) &&
6916         !(env->cp15.scr_el3 & SCR_ATA)) {
6917         return CP_ACCESS_TRAP_EL3;
6918     }
6919     return CP_ACCESS_OK;
6920 }
6921 
6922 static uint64_t tco_read(CPUARMState *env, const ARMCPRegInfo *ri)
6923 {
6924     return env->pstate & PSTATE_TCO;
6925 }
6926 
6927 static void tco_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t val)
6928 {
6929     env->pstate = (env->pstate & ~PSTATE_TCO) | (val & PSTATE_TCO);
6930 }
6931 
6932 static const ARMCPRegInfo mte_reginfo[] = {
6933     { .name = "TFSRE0_EL1", .state = ARM_CP_STATE_AA64,
6934       .opc0 = 3, .opc1 = 0, .crn = 5, .crm = 6, .opc2 = 1,
6935       .access = PL1_RW, .accessfn = access_mte,
6936       .fieldoffset = offsetof(CPUARMState, cp15.tfsr_el[0]) },
6937     { .name = "TFSR_EL1", .state = ARM_CP_STATE_AA64,
6938       .opc0 = 3, .opc1 = 0, .crn = 5, .crm = 6, .opc2 = 0,
6939       .access = PL1_RW, .accessfn = access_mte,
6940       .fieldoffset = offsetof(CPUARMState, cp15.tfsr_el[1]) },
6941     { .name = "TFSR_EL2", .state = ARM_CP_STATE_AA64,
6942       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 6, .opc2 = 0,
6943       .access = PL2_RW, .accessfn = access_mte,
6944       .fieldoffset = offsetof(CPUARMState, cp15.tfsr_el[2]) },
6945     { .name = "TFSR_EL3", .state = ARM_CP_STATE_AA64,
6946       .opc0 = 3, .opc1 = 6, .crn = 5, .crm = 6, .opc2 = 0,
6947       .access = PL3_RW,
6948       .fieldoffset = offsetof(CPUARMState, cp15.tfsr_el[3]) },
6949     { .name = "RGSR_EL1", .state = ARM_CP_STATE_AA64,
6950       .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 5,
6951       .access = PL1_RW, .accessfn = access_mte,
6952       .fieldoffset = offsetof(CPUARMState, cp15.rgsr_el1) },
6953     { .name = "GCR_EL1", .state = ARM_CP_STATE_AA64,
6954       .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 6,
6955       .access = PL1_RW, .accessfn = access_mte,
6956       .fieldoffset = offsetof(CPUARMState, cp15.gcr_el1) },
6957     { .name = "GMID_EL1", .state = ARM_CP_STATE_AA64,
6958       .opc0 = 3, .opc1 = 1, .crn = 0, .crm = 0, .opc2 = 4,
6959       .access = PL1_R, .accessfn = access_aa64_tid5,
6960       .type = ARM_CP_CONST, .resetvalue = GMID_EL1_BS },
6961     { .name = "TCO", .state = ARM_CP_STATE_AA64,
6962       .opc0 = 3, .opc1 = 3, .crn = 4, .crm = 2, .opc2 = 7,
6963       .type = ARM_CP_NO_RAW,
6964       .access = PL0_RW, .readfn = tco_read, .writefn = tco_write },
6965     { .name = "DC_IGVAC", .state = ARM_CP_STATE_AA64,
6966       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 3,
6967       .type = ARM_CP_NOP, .access = PL1_W,
6968       .accessfn = aa64_cacheop_poc_access },
6969     { .name = "DC_IGSW", .state = ARM_CP_STATE_AA64,
6970       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 4,
6971       .type = ARM_CP_NOP, .access = PL1_W, .accessfn = access_tsw },
6972     { .name = "DC_IGDVAC", .state = ARM_CP_STATE_AA64,
6973       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 5,
6974       .type = ARM_CP_NOP, .access = PL1_W,
6975       .accessfn = aa64_cacheop_poc_access },
6976     { .name = "DC_IGDSW", .state = ARM_CP_STATE_AA64,
6977       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 6,
6978       .type = ARM_CP_NOP, .access = PL1_W, .accessfn = access_tsw },
6979     { .name = "DC_CGSW", .state = ARM_CP_STATE_AA64,
6980       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 10, .opc2 = 4,
6981       .type = ARM_CP_NOP, .access = PL1_W, .accessfn = access_tsw },
6982     { .name = "DC_CGDSW", .state = ARM_CP_STATE_AA64,
6983       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 10, .opc2 = 6,
6984       .type = ARM_CP_NOP, .access = PL1_W, .accessfn = access_tsw },
6985     { .name = "DC_CIGSW", .state = ARM_CP_STATE_AA64,
6986       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 14, .opc2 = 4,
6987       .type = ARM_CP_NOP, .access = PL1_W, .accessfn = access_tsw },
6988     { .name = "DC_CIGDSW", .state = ARM_CP_STATE_AA64,
6989       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 14, .opc2 = 6,
6990       .type = ARM_CP_NOP, .access = PL1_W, .accessfn = access_tsw },
6991     REGINFO_SENTINEL
6992 };
6993 
6994 static const ARMCPRegInfo mte_tco_ro_reginfo[] = {
6995     { .name = "TCO", .state = ARM_CP_STATE_AA64,
6996       .opc0 = 3, .opc1 = 3, .crn = 4, .crm = 2, .opc2 = 7,
6997       .type = ARM_CP_CONST, .access = PL0_RW, },
6998     REGINFO_SENTINEL
6999 };
7000 
7001 static const ARMCPRegInfo mte_el0_cacheop_reginfo[] = {
7002     { .name = "DC_CGVAC", .state = ARM_CP_STATE_AA64,
7003       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 10, .opc2 = 3,
7004       .type = ARM_CP_NOP, .access = PL0_W,
7005       .accessfn = aa64_cacheop_poc_access },
7006     { .name = "DC_CGDVAC", .state = ARM_CP_STATE_AA64,
7007       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 10, .opc2 = 5,
7008       .type = ARM_CP_NOP, .access = PL0_W,
7009       .accessfn = aa64_cacheop_poc_access },
7010     { .name = "DC_CGVAP", .state = ARM_CP_STATE_AA64,
7011       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 12, .opc2 = 3,
7012       .type = ARM_CP_NOP, .access = PL0_W,
7013       .accessfn = aa64_cacheop_poc_access },
7014     { .name = "DC_CGDVAP", .state = ARM_CP_STATE_AA64,
7015       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 12, .opc2 = 5,
7016       .type = ARM_CP_NOP, .access = PL0_W,
7017       .accessfn = aa64_cacheop_poc_access },
7018     { .name = "DC_CGVADP", .state = ARM_CP_STATE_AA64,
7019       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 13, .opc2 = 3,
7020       .type = ARM_CP_NOP, .access = PL0_W,
7021       .accessfn = aa64_cacheop_poc_access },
7022     { .name = "DC_CGDVADP", .state = ARM_CP_STATE_AA64,
7023       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 13, .opc2 = 5,
7024       .type = ARM_CP_NOP, .access = PL0_W,
7025       .accessfn = aa64_cacheop_poc_access },
7026     { .name = "DC_CIGVAC", .state = ARM_CP_STATE_AA64,
7027       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 14, .opc2 = 3,
7028       .type = ARM_CP_NOP, .access = PL0_W,
7029       .accessfn = aa64_cacheop_poc_access },
7030     { .name = "DC_CIGDVAC", .state = ARM_CP_STATE_AA64,
7031       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 14, .opc2 = 5,
7032       .type = ARM_CP_NOP, .access = PL0_W,
7033       .accessfn = aa64_cacheop_poc_access },
7034     { .name = "DC_GVA", .state = ARM_CP_STATE_AA64,
7035       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 4, .opc2 = 3,
7036       .access = PL0_W, .type = ARM_CP_DC_GVA,
7037 #ifndef CONFIG_USER_ONLY
7038       /* Avoid overhead of an access check that always passes in user-mode */
7039       .accessfn = aa64_zva_access,
7040 #endif
7041     },
7042     { .name = "DC_GZVA", .state = ARM_CP_STATE_AA64,
7043       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 4, .opc2 = 4,
7044       .access = PL0_W, .type = ARM_CP_DC_GZVA,
7045 #ifndef CONFIG_USER_ONLY
7046       /* Avoid overhead of an access check that always passes in user-mode */
7047       .accessfn = aa64_zva_access,
7048 #endif
7049     },
7050     REGINFO_SENTINEL
7051 };
7052 
7053 #endif
7054 
7055 static CPAccessResult access_predinv(CPUARMState *env, const ARMCPRegInfo *ri,
7056                                      bool isread)
7057 {
7058     int el = arm_current_el(env);
7059 
7060     if (el == 0) {
7061         uint64_t sctlr = arm_sctlr(env, el);
7062         if (!(sctlr & SCTLR_EnRCTX)) {
7063             return CP_ACCESS_TRAP;
7064         }
7065     } else if (el == 1) {
7066         uint64_t hcr = arm_hcr_el2_eff(env);
7067         if (hcr & HCR_NV) {
7068             return CP_ACCESS_TRAP_EL2;
7069         }
7070     }
7071     return CP_ACCESS_OK;
7072 }
7073 
7074 static const ARMCPRegInfo predinv_reginfo[] = {
7075     { .name = "CFP_RCTX", .state = ARM_CP_STATE_AA64,
7076       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 3, .opc2 = 4,
7077       .type = ARM_CP_NOP, .access = PL0_W, .accessfn = access_predinv },
7078     { .name = "DVP_RCTX", .state = ARM_CP_STATE_AA64,
7079       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 3, .opc2 = 5,
7080       .type = ARM_CP_NOP, .access = PL0_W, .accessfn = access_predinv },
7081     { .name = "CPP_RCTX", .state = ARM_CP_STATE_AA64,
7082       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 3, .opc2 = 7,
7083       .type = ARM_CP_NOP, .access = PL0_W, .accessfn = access_predinv },
7084     /*
7085      * Note the AArch32 opcodes have a different OPC1.
7086      */
7087     { .name = "CFPRCTX", .state = ARM_CP_STATE_AA32,
7088       .cp = 15, .opc1 = 0, .crn = 7, .crm = 3, .opc2 = 4,
7089       .type = ARM_CP_NOP, .access = PL0_W, .accessfn = access_predinv },
7090     { .name = "DVPRCTX", .state = ARM_CP_STATE_AA32,
7091       .cp = 15, .opc1 = 0, .crn = 7, .crm = 3, .opc2 = 5,
7092       .type = ARM_CP_NOP, .access = PL0_W, .accessfn = access_predinv },
7093     { .name = "CPPRCTX", .state = ARM_CP_STATE_AA32,
7094       .cp = 15, .opc1 = 0, .crn = 7, .crm = 3, .opc2 = 7,
7095       .type = ARM_CP_NOP, .access = PL0_W, .accessfn = access_predinv },
7096     REGINFO_SENTINEL
7097 };
7098 
7099 static uint64_t ccsidr2_read(CPUARMState *env, const ARMCPRegInfo *ri)
7100 {
7101     /* Read the high 32 bits of the current CCSIDR */
7102     return extract64(ccsidr_read(env, ri), 32, 32);
7103 }
7104 
7105 static const ARMCPRegInfo ccsidr2_reginfo[] = {
7106     { .name = "CCSIDR2", .state = ARM_CP_STATE_BOTH,
7107       .opc0 = 3, .opc1 = 1, .crn = 0, .crm = 0, .opc2 = 2,
7108       .access = PL1_R,
7109       .accessfn = access_aa64_tid2,
7110       .readfn = ccsidr2_read, .type = ARM_CP_NO_RAW },
7111     REGINFO_SENTINEL
7112 };
7113 
7114 static CPAccessResult access_aa64_tid3(CPUARMState *env, const ARMCPRegInfo *ri,
7115                                        bool isread)
7116 {
7117     if ((arm_current_el(env) < 2) && (arm_hcr_el2_eff(env) & HCR_TID3)) {
7118         return CP_ACCESS_TRAP_EL2;
7119     }
7120 
7121     return CP_ACCESS_OK;
7122 }
7123 
7124 static CPAccessResult access_aa32_tid3(CPUARMState *env, const ARMCPRegInfo *ri,
7125                                        bool isread)
7126 {
7127     if (arm_feature(env, ARM_FEATURE_V8)) {
7128         return access_aa64_tid3(env, ri, isread);
7129     }
7130 
7131     return CP_ACCESS_OK;
7132 }
7133 
7134 static CPAccessResult access_jazelle(CPUARMState *env, const ARMCPRegInfo *ri,
7135                                      bool isread)
7136 {
7137     if (arm_current_el(env) == 1 && (arm_hcr_el2_eff(env) & HCR_TID0)) {
7138         return CP_ACCESS_TRAP_EL2;
7139     }
7140 
7141     return CP_ACCESS_OK;
7142 }
7143 
7144 static const ARMCPRegInfo jazelle_regs[] = {
7145     { .name = "JIDR",
7146       .cp = 14, .crn = 0, .crm = 0, .opc1 = 7, .opc2 = 0,
7147       .access = PL1_R, .accessfn = access_jazelle,
7148       .type = ARM_CP_CONST, .resetvalue = 0 },
7149     { .name = "JOSCR",
7150       .cp = 14, .crn = 1, .crm = 0, .opc1 = 7, .opc2 = 0,
7151       .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
7152     { .name = "JMCR",
7153       .cp = 14, .crn = 2, .crm = 0, .opc1 = 7, .opc2 = 0,
7154       .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
7155     REGINFO_SENTINEL
7156 };
7157 
7158 static const ARMCPRegInfo vhe_reginfo[] = {
7159     { .name = "CONTEXTIDR_EL2", .state = ARM_CP_STATE_AA64,
7160       .opc0 = 3, .opc1 = 4, .crn = 13, .crm = 0, .opc2 = 1,
7161       .access = PL2_RW,
7162       .fieldoffset = offsetof(CPUARMState, cp15.contextidr_el[2]) },
7163     { .name = "TTBR1_EL2", .state = ARM_CP_STATE_AA64,
7164       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 1,
7165       .access = PL2_RW, .writefn = vmsa_tcr_ttbr_el2_write,
7166       .fieldoffset = offsetof(CPUARMState, cp15.ttbr1_el[2]) },
7167 #ifndef CONFIG_USER_ONLY
7168     { .name = "CNTHV_CVAL_EL2", .state = ARM_CP_STATE_AA64,
7169       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 3, .opc2 = 2,
7170       .fieldoffset =
7171         offsetof(CPUARMState, cp15.c14_timer[GTIMER_HYPVIRT].cval),
7172       .type = ARM_CP_IO, .access = PL2_RW,
7173       .writefn = gt_hv_cval_write, .raw_writefn = raw_write },
7174     { .name = "CNTHV_TVAL_EL2", .state = ARM_CP_STATE_BOTH,
7175       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 3, .opc2 = 0,
7176       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL2_RW,
7177       .resetfn = gt_hv_timer_reset,
7178       .readfn = gt_hv_tval_read, .writefn = gt_hv_tval_write },
7179     { .name = "CNTHV_CTL_EL2", .state = ARM_CP_STATE_BOTH,
7180       .type = ARM_CP_IO,
7181       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 3, .opc2 = 1,
7182       .access = PL2_RW,
7183       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_HYPVIRT].ctl),
7184       .writefn = gt_hv_ctl_write, .raw_writefn = raw_write },
7185     { .name = "CNTP_CTL_EL02", .state = ARM_CP_STATE_AA64,
7186       .opc0 = 3, .opc1 = 5, .crn = 14, .crm = 2, .opc2 = 1,
7187       .type = ARM_CP_IO | ARM_CP_ALIAS,
7188       .access = PL2_RW, .accessfn = e2h_access,
7189       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].ctl),
7190       .writefn = gt_phys_ctl_write, .raw_writefn = raw_write },
7191     { .name = "CNTV_CTL_EL02", .state = ARM_CP_STATE_AA64,
7192       .opc0 = 3, .opc1 = 5, .crn = 14, .crm = 3, .opc2 = 1,
7193       .type = ARM_CP_IO | ARM_CP_ALIAS,
7194       .access = PL2_RW, .accessfn = e2h_access,
7195       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].ctl),
7196       .writefn = gt_virt_ctl_write, .raw_writefn = raw_write },
7197     { .name = "CNTP_TVAL_EL02", .state = ARM_CP_STATE_AA64,
7198       .opc0 = 3, .opc1 = 5, .crn = 14, .crm = 2, .opc2 = 0,
7199       .type = ARM_CP_NO_RAW | ARM_CP_IO | ARM_CP_ALIAS,
7200       .access = PL2_RW, .accessfn = e2h_access,
7201       .readfn = gt_phys_tval_read, .writefn = gt_phys_tval_write },
7202     { .name = "CNTV_TVAL_EL02", .state = ARM_CP_STATE_AA64,
7203       .opc0 = 3, .opc1 = 5, .crn = 14, .crm = 3, .opc2 = 0,
7204       .type = ARM_CP_NO_RAW | ARM_CP_IO | ARM_CP_ALIAS,
7205       .access = PL2_RW, .accessfn = e2h_access,
7206       .readfn = gt_virt_tval_read, .writefn = gt_virt_tval_write },
7207     { .name = "CNTP_CVAL_EL02", .state = ARM_CP_STATE_AA64,
7208       .opc0 = 3, .opc1 = 5, .crn = 14, .crm = 2, .opc2 = 2,
7209       .type = ARM_CP_IO | ARM_CP_ALIAS,
7210       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].cval),
7211       .access = PL2_RW, .accessfn = e2h_access,
7212       .writefn = gt_phys_cval_write, .raw_writefn = raw_write },
7213     { .name = "CNTV_CVAL_EL02", .state = ARM_CP_STATE_AA64,
7214       .opc0 = 3, .opc1 = 5, .crn = 14, .crm = 3, .opc2 = 2,
7215       .type = ARM_CP_IO | ARM_CP_ALIAS,
7216       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].cval),
7217       .access = PL2_RW, .accessfn = e2h_access,
7218       .writefn = gt_virt_cval_write, .raw_writefn = raw_write },
7219 #endif
7220     REGINFO_SENTINEL
7221 };
7222 
7223 #ifndef CONFIG_USER_ONLY
7224 static const ARMCPRegInfo ats1e1_reginfo[] = {
7225     { .name = "AT_S1E1R", .state = ARM_CP_STATE_AA64,
7226       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 9, .opc2 = 0,
7227       .access = PL1_W, .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC,
7228       .writefn = ats_write64 },
7229     { .name = "AT_S1E1W", .state = ARM_CP_STATE_AA64,
7230       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 9, .opc2 = 1,
7231       .access = PL1_W, .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC,
7232       .writefn = ats_write64 },
7233     REGINFO_SENTINEL
7234 };
7235 
7236 static const ARMCPRegInfo ats1cp_reginfo[] = {
7237     { .name = "ATS1CPRP",
7238       .cp = 15, .opc1 = 0, .crn = 7, .crm = 9, .opc2 = 0,
7239       .access = PL1_W, .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC,
7240       .writefn = ats_write },
7241     { .name = "ATS1CPWP",
7242       .cp = 15, .opc1 = 0, .crn = 7, .crm = 9, .opc2 = 1,
7243       .access = PL1_W, .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC,
7244       .writefn = ats_write },
7245     REGINFO_SENTINEL
7246 };
7247 #endif
7248 
7249 /*
7250  * ACTLR2 and HACTLR2 map to ACTLR_EL1[63:32] and
7251  * ACTLR_EL2[63:32]. They exist only if the ID_MMFR4.AC2 field
7252  * is non-zero, which is never for ARMv7, optionally in ARMv8
7253  * and mandatorily for ARMv8.2 and up.
7254  * ACTLR2 is banked for S and NS if EL3 is AArch32. Since QEMU's
7255  * implementation is RAZ/WI we can ignore this detail, as we
7256  * do for ACTLR.
7257  */
7258 static const ARMCPRegInfo actlr2_hactlr2_reginfo[] = {
7259     { .name = "ACTLR2", .state = ARM_CP_STATE_AA32,
7260       .cp = 15, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 3,
7261       .access = PL1_RW, .accessfn = access_tacr,
7262       .type = ARM_CP_CONST, .resetvalue = 0 },
7263     { .name = "HACTLR2", .state = ARM_CP_STATE_AA32,
7264       .cp = 15, .opc1 = 4, .crn = 1, .crm = 0, .opc2 = 3,
7265       .access = PL2_RW, .type = ARM_CP_CONST,
7266       .resetvalue = 0 },
7267     REGINFO_SENTINEL
7268 };
7269 
7270 void register_cp_regs_for_features(ARMCPU *cpu)
7271 {
7272     /* Register all the coprocessor registers based on feature bits */
7273     CPUARMState *env = &cpu->env;
7274     if (arm_feature(env, ARM_FEATURE_M)) {
7275         /* M profile has no coprocessor registers */
7276         return;
7277     }
7278 
7279     define_arm_cp_regs(cpu, cp_reginfo);
7280     if (!arm_feature(env, ARM_FEATURE_V8)) {
7281         /* Must go early as it is full of wildcards that may be
7282          * overridden by later definitions.
7283          */
7284         define_arm_cp_regs(cpu, not_v8_cp_reginfo);
7285     }
7286 
7287     if (arm_feature(env, ARM_FEATURE_V6)) {
7288         /* The ID registers all have impdef reset values */
7289         ARMCPRegInfo v6_idregs[] = {
7290             { .name = "ID_PFR0", .state = ARM_CP_STATE_BOTH,
7291               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 0,
7292               .access = PL1_R, .type = ARM_CP_CONST,
7293               .accessfn = access_aa32_tid3,
7294               .resetvalue = cpu->isar.id_pfr0 },
7295             /* ID_PFR1 is not a plain ARM_CP_CONST because we don't know
7296              * the value of the GIC field until after we define these regs.
7297              */
7298             { .name = "ID_PFR1", .state = ARM_CP_STATE_BOTH,
7299               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 1,
7300               .access = PL1_R, .type = ARM_CP_NO_RAW,
7301               .accessfn = access_aa32_tid3,
7302               .readfn = id_pfr1_read,
7303               .writefn = arm_cp_write_ignore },
7304             { .name = "ID_DFR0", .state = ARM_CP_STATE_BOTH,
7305               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 2,
7306               .access = PL1_R, .type = ARM_CP_CONST,
7307               .accessfn = access_aa32_tid3,
7308               .resetvalue = cpu->isar.id_dfr0 },
7309             { .name = "ID_AFR0", .state = ARM_CP_STATE_BOTH,
7310               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 3,
7311               .access = PL1_R, .type = ARM_CP_CONST,
7312               .accessfn = access_aa32_tid3,
7313               .resetvalue = cpu->id_afr0 },
7314             { .name = "ID_MMFR0", .state = ARM_CP_STATE_BOTH,
7315               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 4,
7316               .access = PL1_R, .type = ARM_CP_CONST,
7317               .accessfn = access_aa32_tid3,
7318               .resetvalue = cpu->isar.id_mmfr0 },
7319             { .name = "ID_MMFR1", .state = ARM_CP_STATE_BOTH,
7320               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 5,
7321               .access = PL1_R, .type = ARM_CP_CONST,
7322               .accessfn = access_aa32_tid3,
7323               .resetvalue = cpu->isar.id_mmfr1 },
7324             { .name = "ID_MMFR2", .state = ARM_CP_STATE_BOTH,
7325               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 6,
7326               .access = PL1_R, .type = ARM_CP_CONST,
7327               .accessfn = access_aa32_tid3,
7328               .resetvalue = cpu->isar.id_mmfr2 },
7329             { .name = "ID_MMFR3", .state = ARM_CP_STATE_BOTH,
7330               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 7,
7331               .access = PL1_R, .type = ARM_CP_CONST,
7332               .accessfn = access_aa32_tid3,
7333               .resetvalue = cpu->isar.id_mmfr3 },
7334             { .name = "ID_ISAR0", .state = ARM_CP_STATE_BOTH,
7335               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 0,
7336               .access = PL1_R, .type = ARM_CP_CONST,
7337               .accessfn = access_aa32_tid3,
7338               .resetvalue = cpu->isar.id_isar0 },
7339             { .name = "ID_ISAR1", .state = ARM_CP_STATE_BOTH,
7340               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 1,
7341               .access = PL1_R, .type = ARM_CP_CONST,
7342               .accessfn = access_aa32_tid3,
7343               .resetvalue = cpu->isar.id_isar1 },
7344             { .name = "ID_ISAR2", .state = ARM_CP_STATE_BOTH,
7345               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 2,
7346               .access = PL1_R, .type = ARM_CP_CONST,
7347               .accessfn = access_aa32_tid3,
7348               .resetvalue = cpu->isar.id_isar2 },
7349             { .name = "ID_ISAR3", .state = ARM_CP_STATE_BOTH,
7350               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 3,
7351               .access = PL1_R, .type = ARM_CP_CONST,
7352               .accessfn = access_aa32_tid3,
7353               .resetvalue = cpu->isar.id_isar3 },
7354             { .name = "ID_ISAR4", .state = ARM_CP_STATE_BOTH,
7355               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 4,
7356               .access = PL1_R, .type = ARM_CP_CONST,
7357               .accessfn = access_aa32_tid3,
7358               .resetvalue = cpu->isar.id_isar4 },
7359             { .name = "ID_ISAR5", .state = ARM_CP_STATE_BOTH,
7360               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 5,
7361               .access = PL1_R, .type = ARM_CP_CONST,
7362               .accessfn = access_aa32_tid3,
7363               .resetvalue = cpu->isar.id_isar5 },
7364             { .name = "ID_MMFR4", .state = ARM_CP_STATE_BOTH,
7365               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 6,
7366               .access = PL1_R, .type = ARM_CP_CONST,
7367               .accessfn = access_aa32_tid3,
7368               .resetvalue = cpu->isar.id_mmfr4 },
7369             { .name = "ID_ISAR6", .state = ARM_CP_STATE_BOTH,
7370               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 7,
7371               .access = PL1_R, .type = ARM_CP_CONST,
7372               .accessfn = access_aa32_tid3,
7373               .resetvalue = cpu->isar.id_isar6 },
7374             REGINFO_SENTINEL
7375         };
7376         define_arm_cp_regs(cpu, v6_idregs);
7377         define_arm_cp_regs(cpu, v6_cp_reginfo);
7378     } else {
7379         define_arm_cp_regs(cpu, not_v6_cp_reginfo);
7380     }
7381     if (arm_feature(env, ARM_FEATURE_V6K)) {
7382         define_arm_cp_regs(cpu, v6k_cp_reginfo);
7383     }
7384     if (arm_feature(env, ARM_FEATURE_V7MP) &&
7385         !arm_feature(env, ARM_FEATURE_PMSA)) {
7386         define_arm_cp_regs(cpu, v7mp_cp_reginfo);
7387     }
7388     if (arm_feature(env, ARM_FEATURE_V7VE)) {
7389         define_arm_cp_regs(cpu, pmovsset_cp_reginfo);
7390     }
7391     if (arm_feature(env, ARM_FEATURE_V7)) {
7392         ARMCPRegInfo clidr = {
7393             .name = "CLIDR", .state = ARM_CP_STATE_BOTH,
7394             .opc0 = 3, .crn = 0, .crm = 0, .opc1 = 1, .opc2 = 1,
7395             .access = PL1_R, .type = ARM_CP_CONST,
7396             .accessfn = access_aa64_tid2,
7397             .resetvalue = cpu->clidr
7398         };
7399         define_one_arm_cp_reg(cpu, &clidr);
7400         define_arm_cp_regs(cpu, v7_cp_reginfo);
7401         define_debug_regs(cpu);
7402         define_pmu_regs(cpu);
7403     } else {
7404         define_arm_cp_regs(cpu, not_v7_cp_reginfo);
7405     }
7406     if (arm_feature(env, ARM_FEATURE_V8)) {
7407         /* AArch64 ID registers, which all have impdef reset values.
7408          * Note that within the ID register ranges the unused slots
7409          * must all RAZ, not UNDEF; future architecture versions may
7410          * define new registers here.
7411          */
7412         ARMCPRegInfo v8_idregs[] = {
7413             /*
7414              * ID_AA64PFR0_EL1 is not a plain ARM_CP_CONST in system
7415              * emulation because we don't know the right value for the
7416              * GIC field until after we define these regs.
7417              */
7418             { .name = "ID_AA64PFR0_EL1", .state = ARM_CP_STATE_AA64,
7419               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 0,
7420               .access = PL1_R,
7421 #ifdef CONFIG_USER_ONLY
7422               .type = ARM_CP_CONST,
7423               .resetvalue = cpu->isar.id_aa64pfr0
7424 #else
7425               .type = ARM_CP_NO_RAW,
7426               .accessfn = access_aa64_tid3,
7427               .readfn = id_aa64pfr0_read,
7428               .writefn = arm_cp_write_ignore
7429 #endif
7430             },
7431             { .name = "ID_AA64PFR1_EL1", .state = ARM_CP_STATE_AA64,
7432               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 1,
7433               .access = PL1_R, .type = ARM_CP_CONST,
7434               .accessfn = access_aa64_tid3,
7435               .resetvalue = cpu->isar.id_aa64pfr1},
7436             { .name = "ID_AA64PFR2_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
7437               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 2,
7438               .access = PL1_R, .type = ARM_CP_CONST,
7439               .accessfn = access_aa64_tid3,
7440               .resetvalue = 0 },
7441             { .name = "ID_AA64PFR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
7442               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 3,
7443               .access = PL1_R, .type = ARM_CP_CONST,
7444               .accessfn = access_aa64_tid3,
7445               .resetvalue = 0 },
7446             { .name = "ID_AA64ZFR0_EL1", .state = ARM_CP_STATE_AA64,
7447               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 4,
7448               .access = PL1_R, .type = ARM_CP_CONST,
7449               .accessfn = access_aa64_tid3,
7450               /* At present, only SVEver == 0 is defined anyway.  */
7451               .resetvalue = 0 },
7452             { .name = "ID_AA64PFR5_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
7453               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 5,
7454               .access = PL1_R, .type = ARM_CP_CONST,
7455               .accessfn = access_aa64_tid3,
7456               .resetvalue = 0 },
7457             { .name = "ID_AA64PFR6_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
7458               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 6,
7459               .access = PL1_R, .type = ARM_CP_CONST,
7460               .accessfn = access_aa64_tid3,
7461               .resetvalue = 0 },
7462             { .name = "ID_AA64PFR7_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
7463               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 7,
7464               .access = PL1_R, .type = ARM_CP_CONST,
7465               .accessfn = access_aa64_tid3,
7466               .resetvalue = 0 },
7467             { .name = "ID_AA64DFR0_EL1", .state = ARM_CP_STATE_AA64,
7468               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 0,
7469               .access = PL1_R, .type = ARM_CP_CONST,
7470               .accessfn = access_aa64_tid3,
7471               .resetvalue = cpu->isar.id_aa64dfr0 },
7472             { .name = "ID_AA64DFR1_EL1", .state = ARM_CP_STATE_AA64,
7473               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 1,
7474               .access = PL1_R, .type = ARM_CP_CONST,
7475               .accessfn = access_aa64_tid3,
7476               .resetvalue = cpu->isar.id_aa64dfr1 },
7477             { .name = "ID_AA64DFR2_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
7478               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 2,
7479               .access = PL1_R, .type = ARM_CP_CONST,
7480               .accessfn = access_aa64_tid3,
7481               .resetvalue = 0 },
7482             { .name = "ID_AA64DFR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
7483               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 3,
7484               .access = PL1_R, .type = ARM_CP_CONST,
7485               .accessfn = access_aa64_tid3,
7486               .resetvalue = 0 },
7487             { .name = "ID_AA64AFR0_EL1", .state = ARM_CP_STATE_AA64,
7488               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 4,
7489               .access = PL1_R, .type = ARM_CP_CONST,
7490               .accessfn = access_aa64_tid3,
7491               .resetvalue = cpu->id_aa64afr0 },
7492             { .name = "ID_AA64AFR1_EL1", .state = ARM_CP_STATE_AA64,
7493               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 5,
7494               .access = PL1_R, .type = ARM_CP_CONST,
7495               .accessfn = access_aa64_tid3,
7496               .resetvalue = cpu->id_aa64afr1 },
7497             { .name = "ID_AA64AFR2_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
7498               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 6,
7499               .access = PL1_R, .type = ARM_CP_CONST,
7500               .accessfn = access_aa64_tid3,
7501               .resetvalue = 0 },
7502             { .name = "ID_AA64AFR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
7503               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 7,
7504               .access = PL1_R, .type = ARM_CP_CONST,
7505               .accessfn = access_aa64_tid3,
7506               .resetvalue = 0 },
7507             { .name = "ID_AA64ISAR0_EL1", .state = ARM_CP_STATE_AA64,
7508               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 0,
7509               .access = PL1_R, .type = ARM_CP_CONST,
7510               .accessfn = access_aa64_tid3,
7511               .resetvalue = cpu->isar.id_aa64isar0 },
7512             { .name = "ID_AA64ISAR1_EL1", .state = ARM_CP_STATE_AA64,
7513               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 1,
7514               .access = PL1_R, .type = ARM_CP_CONST,
7515               .accessfn = access_aa64_tid3,
7516               .resetvalue = cpu->isar.id_aa64isar1 },
7517             { .name = "ID_AA64ISAR2_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
7518               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 2,
7519               .access = PL1_R, .type = ARM_CP_CONST,
7520               .accessfn = access_aa64_tid3,
7521               .resetvalue = 0 },
7522             { .name = "ID_AA64ISAR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
7523               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 3,
7524               .access = PL1_R, .type = ARM_CP_CONST,
7525               .accessfn = access_aa64_tid3,
7526               .resetvalue = 0 },
7527             { .name = "ID_AA64ISAR4_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
7528               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 4,
7529               .access = PL1_R, .type = ARM_CP_CONST,
7530               .accessfn = access_aa64_tid3,
7531               .resetvalue = 0 },
7532             { .name = "ID_AA64ISAR5_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
7533               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 5,
7534               .access = PL1_R, .type = ARM_CP_CONST,
7535               .accessfn = access_aa64_tid3,
7536               .resetvalue = 0 },
7537             { .name = "ID_AA64ISAR6_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
7538               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 6,
7539               .access = PL1_R, .type = ARM_CP_CONST,
7540               .accessfn = access_aa64_tid3,
7541               .resetvalue = 0 },
7542             { .name = "ID_AA64ISAR7_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
7543               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 7,
7544               .access = PL1_R, .type = ARM_CP_CONST,
7545               .accessfn = access_aa64_tid3,
7546               .resetvalue = 0 },
7547             { .name = "ID_AA64MMFR0_EL1", .state = ARM_CP_STATE_AA64,
7548               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 0,
7549               .access = PL1_R, .type = ARM_CP_CONST,
7550               .accessfn = access_aa64_tid3,
7551               .resetvalue = cpu->isar.id_aa64mmfr0 },
7552             { .name = "ID_AA64MMFR1_EL1", .state = ARM_CP_STATE_AA64,
7553               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 1,
7554               .access = PL1_R, .type = ARM_CP_CONST,
7555               .accessfn = access_aa64_tid3,
7556               .resetvalue = cpu->isar.id_aa64mmfr1 },
7557             { .name = "ID_AA64MMFR2_EL1", .state = ARM_CP_STATE_AA64,
7558               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 2,
7559               .access = PL1_R, .type = ARM_CP_CONST,
7560               .accessfn = access_aa64_tid3,
7561               .resetvalue = cpu->isar.id_aa64mmfr2 },
7562             { .name = "ID_AA64MMFR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
7563               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 3,
7564               .access = PL1_R, .type = ARM_CP_CONST,
7565               .accessfn = access_aa64_tid3,
7566               .resetvalue = 0 },
7567             { .name = "ID_AA64MMFR4_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
7568               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 4,
7569               .access = PL1_R, .type = ARM_CP_CONST,
7570               .accessfn = access_aa64_tid3,
7571               .resetvalue = 0 },
7572             { .name = "ID_AA64MMFR5_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
7573               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 5,
7574               .access = PL1_R, .type = ARM_CP_CONST,
7575               .accessfn = access_aa64_tid3,
7576               .resetvalue = 0 },
7577             { .name = "ID_AA64MMFR6_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
7578               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 6,
7579               .access = PL1_R, .type = ARM_CP_CONST,
7580               .accessfn = access_aa64_tid3,
7581               .resetvalue = 0 },
7582             { .name = "ID_AA64MMFR7_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
7583               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 7,
7584               .access = PL1_R, .type = ARM_CP_CONST,
7585               .accessfn = access_aa64_tid3,
7586               .resetvalue = 0 },
7587             { .name = "MVFR0_EL1", .state = ARM_CP_STATE_AA64,
7588               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 0,
7589               .access = PL1_R, .type = ARM_CP_CONST,
7590               .accessfn = access_aa64_tid3,
7591               .resetvalue = cpu->isar.mvfr0 },
7592             { .name = "MVFR1_EL1", .state = ARM_CP_STATE_AA64,
7593               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 1,
7594               .access = PL1_R, .type = ARM_CP_CONST,
7595               .accessfn = access_aa64_tid3,
7596               .resetvalue = cpu->isar.mvfr1 },
7597             { .name = "MVFR2_EL1", .state = ARM_CP_STATE_AA64,
7598               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 2,
7599               .access = PL1_R, .type = ARM_CP_CONST,
7600               .accessfn = access_aa64_tid3,
7601               .resetvalue = cpu->isar.mvfr2 },
7602             { .name = "MVFR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
7603               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 3,
7604               .access = PL1_R, .type = ARM_CP_CONST,
7605               .accessfn = access_aa64_tid3,
7606               .resetvalue = 0 },
7607             { .name = "MVFR4_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
7608               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 4,
7609               .access = PL1_R, .type = ARM_CP_CONST,
7610               .accessfn = access_aa64_tid3,
7611               .resetvalue = 0 },
7612             { .name = "MVFR5_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
7613               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 5,
7614               .access = PL1_R, .type = ARM_CP_CONST,
7615               .accessfn = access_aa64_tid3,
7616               .resetvalue = 0 },
7617             { .name = "MVFR6_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
7618               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 6,
7619               .access = PL1_R, .type = ARM_CP_CONST,
7620               .accessfn = access_aa64_tid3,
7621               .resetvalue = 0 },
7622             { .name = "MVFR7_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
7623               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 7,
7624               .access = PL1_R, .type = ARM_CP_CONST,
7625               .accessfn = access_aa64_tid3,
7626               .resetvalue = 0 },
7627             { .name = "PMCEID0", .state = ARM_CP_STATE_AA32,
7628               .cp = 15, .opc1 = 0, .crn = 9, .crm = 12, .opc2 = 6,
7629               .access = PL0_R, .accessfn = pmreg_access, .type = ARM_CP_CONST,
7630               .resetvalue = extract64(cpu->pmceid0, 0, 32) },
7631             { .name = "PMCEID0_EL0", .state = ARM_CP_STATE_AA64,
7632               .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 6,
7633               .access = PL0_R, .accessfn = pmreg_access, .type = ARM_CP_CONST,
7634               .resetvalue = cpu->pmceid0 },
7635             { .name = "PMCEID1", .state = ARM_CP_STATE_AA32,
7636               .cp = 15, .opc1 = 0, .crn = 9, .crm = 12, .opc2 = 7,
7637               .access = PL0_R, .accessfn = pmreg_access, .type = ARM_CP_CONST,
7638               .resetvalue = extract64(cpu->pmceid1, 0, 32) },
7639             { .name = "PMCEID1_EL0", .state = ARM_CP_STATE_AA64,
7640               .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 7,
7641               .access = PL0_R, .accessfn = pmreg_access, .type = ARM_CP_CONST,
7642               .resetvalue = cpu->pmceid1 },
7643             REGINFO_SENTINEL
7644         };
7645 #ifdef CONFIG_USER_ONLY
7646         ARMCPRegUserSpaceInfo v8_user_idregs[] = {
7647             { .name = "ID_AA64PFR0_EL1",
7648               .exported_bits = 0x000f000f00ff0000,
7649               .fixed_bits    = 0x0000000000000011 },
7650             { .name = "ID_AA64PFR1_EL1",
7651               .exported_bits = 0x00000000000000f0 },
7652             { .name = "ID_AA64PFR*_EL1_RESERVED",
7653               .is_glob = true                     },
7654             { .name = "ID_AA64ZFR0_EL1"           },
7655             { .name = "ID_AA64MMFR0_EL1",
7656               .fixed_bits    = 0x00000000ff000000 },
7657             { .name = "ID_AA64MMFR1_EL1"          },
7658             { .name = "ID_AA64MMFR*_EL1_RESERVED",
7659               .is_glob = true                     },
7660             { .name = "ID_AA64DFR0_EL1",
7661               .fixed_bits    = 0x0000000000000006 },
7662             { .name = "ID_AA64DFR1_EL1"           },
7663             { .name = "ID_AA64DFR*_EL1_RESERVED",
7664               .is_glob = true                     },
7665             { .name = "ID_AA64AFR*",
7666               .is_glob = true                     },
7667             { .name = "ID_AA64ISAR0_EL1",
7668               .exported_bits = 0x00fffffff0fffff0 },
7669             { .name = "ID_AA64ISAR1_EL1",
7670               .exported_bits = 0x000000f0ffffffff },
7671             { .name = "ID_AA64ISAR*_EL1_RESERVED",
7672               .is_glob = true                     },
7673             REGUSERINFO_SENTINEL
7674         };
7675         modify_arm_cp_regs(v8_idregs, v8_user_idregs);
7676 #endif
7677         /* RVBAR_EL1 is only implemented if EL1 is the highest EL */
7678         if (!arm_feature(env, ARM_FEATURE_EL3) &&
7679             !arm_feature(env, ARM_FEATURE_EL2)) {
7680             ARMCPRegInfo rvbar = {
7681                 .name = "RVBAR_EL1", .state = ARM_CP_STATE_AA64,
7682                 .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 0, .opc2 = 1,
7683                 .type = ARM_CP_CONST, .access = PL1_R, .resetvalue = cpu->rvbar
7684             };
7685             define_one_arm_cp_reg(cpu, &rvbar);
7686         }
7687         define_arm_cp_regs(cpu, v8_idregs);
7688         define_arm_cp_regs(cpu, v8_cp_reginfo);
7689     }
7690     if (arm_feature(env, ARM_FEATURE_EL2)) {
7691         uint64_t vmpidr_def = mpidr_read_val(env);
7692         ARMCPRegInfo vpidr_regs[] = {
7693             { .name = "VPIDR", .state = ARM_CP_STATE_AA32,
7694               .cp = 15, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 0,
7695               .access = PL2_RW, .accessfn = access_el3_aa32ns,
7696               .resetvalue = cpu->midr, .type = ARM_CP_ALIAS,
7697               .fieldoffset = offsetoflow32(CPUARMState, cp15.vpidr_el2) },
7698             { .name = "VPIDR_EL2", .state = ARM_CP_STATE_AA64,
7699               .opc0 = 3, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 0,
7700               .access = PL2_RW, .resetvalue = cpu->midr,
7701               .fieldoffset = offsetof(CPUARMState, cp15.vpidr_el2) },
7702             { .name = "VMPIDR", .state = ARM_CP_STATE_AA32,
7703               .cp = 15, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 5,
7704               .access = PL2_RW, .accessfn = access_el3_aa32ns,
7705               .resetvalue = vmpidr_def, .type = ARM_CP_ALIAS,
7706               .fieldoffset = offsetoflow32(CPUARMState, cp15.vmpidr_el2) },
7707             { .name = "VMPIDR_EL2", .state = ARM_CP_STATE_AA64,
7708               .opc0 = 3, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 5,
7709               .access = PL2_RW,
7710               .resetvalue = vmpidr_def,
7711               .fieldoffset = offsetof(CPUARMState, cp15.vmpidr_el2) },
7712             REGINFO_SENTINEL
7713         };
7714         define_arm_cp_regs(cpu, vpidr_regs);
7715         define_arm_cp_regs(cpu, el2_cp_reginfo);
7716         if (arm_feature(env, ARM_FEATURE_V8)) {
7717             define_arm_cp_regs(cpu, el2_v8_cp_reginfo);
7718         }
7719         /* RVBAR_EL2 is only implemented if EL2 is the highest EL */
7720         if (!arm_feature(env, ARM_FEATURE_EL3)) {
7721             ARMCPRegInfo rvbar = {
7722                 .name = "RVBAR_EL2", .state = ARM_CP_STATE_AA64,
7723                 .opc0 = 3, .opc1 = 4, .crn = 12, .crm = 0, .opc2 = 1,
7724                 .type = ARM_CP_CONST, .access = PL2_R, .resetvalue = cpu->rvbar
7725             };
7726             define_one_arm_cp_reg(cpu, &rvbar);
7727         }
7728     } else {
7729         /* If EL2 is missing but higher ELs are enabled, we need to
7730          * register the no_el2 reginfos.
7731          */
7732         if (arm_feature(env, ARM_FEATURE_EL3)) {
7733             /* When EL3 exists but not EL2, VPIDR and VMPIDR take the value
7734              * of MIDR_EL1 and MPIDR_EL1.
7735              */
7736             ARMCPRegInfo vpidr_regs[] = {
7737                 { .name = "VPIDR_EL2", .state = ARM_CP_STATE_BOTH,
7738                   .opc0 = 3, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 0,
7739                   .access = PL2_RW, .accessfn = access_el3_aa32ns,
7740                   .type = ARM_CP_CONST, .resetvalue = cpu->midr,
7741                   .fieldoffset = offsetof(CPUARMState, cp15.vpidr_el2) },
7742                 { .name = "VMPIDR_EL2", .state = ARM_CP_STATE_BOTH,
7743                   .opc0 = 3, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 5,
7744                   .access = PL2_RW, .accessfn = access_el3_aa32ns,
7745                   .type = ARM_CP_NO_RAW,
7746                   .writefn = arm_cp_write_ignore, .readfn = mpidr_read },
7747                 REGINFO_SENTINEL
7748             };
7749             define_arm_cp_regs(cpu, vpidr_regs);
7750             define_arm_cp_regs(cpu, el3_no_el2_cp_reginfo);
7751             if (arm_feature(env, ARM_FEATURE_V8)) {
7752                 define_arm_cp_regs(cpu, el3_no_el2_v8_cp_reginfo);
7753             }
7754         }
7755     }
7756     if (arm_feature(env, ARM_FEATURE_EL3)) {
7757         define_arm_cp_regs(cpu, el3_cp_reginfo);
7758         ARMCPRegInfo el3_regs[] = {
7759             { .name = "RVBAR_EL3", .state = ARM_CP_STATE_AA64,
7760               .opc0 = 3, .opc1 = 6, .crn = 12, .crm = 0, .opc2 = 1,
7761               .type = ARM_CP_CONST, .access = PL3_R, .resetvalue = cpu->rvbar },
7762             { .name = "SCTLR_EL3", .state = ARM_CP_STATE_AA64,
7763               .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 0, .opc2 = 0,
7764               .access = PL3_RW,
7765               .raw_writefn = raw_write, .writefn = sctlr_write,
7766               .fieldoffset = offsetof(CPUARMState, cp15.sctlr_el[3]),
7767               .resetvalue = cpu->reset_sctlr },
7768             REGINFO_SENTINEL
7769         };
7770 
7771         define_arm_cp_regs(cpu, el3_regs);
7772     }
7773     /* The behaviour of NSACR is sufficiently various that we don't
7774      * try to describe it in a single reginfo:
7775      *  if EL3 is 64 bit, then trap to EL3 from S EL1,
7776      *     reads as constant 0xc00 from NS EL1 and NS EL2
7777      *  if EL3 is 32 bit, then RW at EL3, RO at NS EL1 and NS EL2
7778      *  if v7 without EL3, register doesn't exist
7779      *  if v8 without EL3, reads as constant 0xc00 from NS EL1 and NS EL2
7780      */
7781     if (arm_feature(env, ARM_FEATURE_EL3)) {
7782         if (arm_feature(env, ARM_FEATURE_AARCH64)) {
7783             ARMCPRegInfo nsacr = {
7784                 .name = "NSACR", .type = ARM_CP_CONST,
7785                 .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 2,
7786                 .access = PL1_RW, .accessfn = nsacr_access,
7787                 .resetvalue = 0xc00
7788             };
7789             define_one_arm_cp_reg(cpu, &nsacr);
7790         } else {
7791             ARMCPRegInfo nsacr = {
7792                 .name = "NSACR",
7793                 .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 2,
7794                 .access = PL3_RW | PL1_R,
7795                 .resetvalue = 0,
7796                 .fieldoffset = offsetof(CPUARMState, cp15.nsacr)
7797             };
7798             define_one_arm_cp_reg(cpu, &nsacr);
7799         }
7800     } else {
7801         if (arm_feature(env, ARM_FEATURE_V8)) {
7802             ARMCPRegInfo nsacr = {
7803                 .name = "NSACR", .type = ARM_CP_CONST,
7804                 .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 2,
7805                 .access = PL1_R,
7806                 .resetvalue = 0xc00
7807             };
7808             define_one_arm_cp_reg(cpu, &nsacr);
7809         }
7810     }
7811 
7812     if (arm_feature(env, ARM_FEATURE_PMSA)) {
7813         if (arm_feature(env, ARM_FEATURE_V6)) {
7814             /* PMSAv6 not implemented */
7815             assert(arm_feature(env, ARM_FEATURE_V7));
7816             define_arm_cp_regs(cpu, vmsa_pmsa_cp_reginfo);
7817             define_arm_cp_regs(cpu, pmsav7_cp_reginfo);
7818         } else {
7819             define_arm_cp_regs(cpu, pmsav5_cp_reginfo);
7820         }
7821     } else {
7822         define_arm_cp_regs(cpu, vmsa_pmsa_cp_reginfo);
7823         define_arm_cp_regs(cpu, vmsa_cp_reginfo);
7824         /* TTCBR2 is introduced with ARMv8.2-AA32HPD.  */
7825         if (cpu_isar_feature(aa32_hpd, cpu)) {
7826             define_one_arm_cp_reg(cpu, &ttbcr2_reginfo);
7827         }
7828     }
7829     if (arm_feature(env, ARM_FEATURE_THUMB2EE)) {
7830         define_arm_cp_regs(cpu, t2ee_cp_reginfo);
7831     }
7832     if (arm_feature(env, ARM_FEATURE_GENERIC_TIMER)) {
7833         define_arm_cp_regs(cpu, generic_timer_cp_reginfo);
7834     }
7835     if (arm_feature(env, ARM_FEATURE_VAPA)) {
7836         define_arm_cp_regs(cpu, vapa_cp_reginfo);
7837     }
7838     if (arm_feature(env, ARM_FEATURE_CACHE_TEST_CLEAN)) {
7839         define_arm_cp_regs(cpu, cache_test_clean_cp_reginfo);
7840     }
7841     if (arm_feature(env, ARM_FEATURE_CACHE_DIRTY_REG)) {
7842         define_arm_cp_regs(cpu, cache_dirty_status_cp_reginfo);
7843     }
7844     if (arm_feature(env, ARM_FEATURE_CACHE_BLOCK_OPS)) {
7845         define_arm_cp_regs(cpu, cache_block_ops_cp_reginfo);
7846     }
7847     if (arm_feature(env, ARM_FEATURE_OMAPCP)) {
7848         define_arm_cp_regs(cpu, omap_cp_reginfo);
7849     }
7850     if (arm_feature(env, ARM_FEATURE_STRONGARM)) {
7851         define_arm_cp_regs(cpu, strongarm_cp_reginfo);
7852     }
7853     if (arm_feature(env, ARM_FEATURE_XSCALE)) {
7854         define_arm_cp_regs(cpu, xscale_cp_reginfo);
7855     }
7856     if (arm_feature(env, ARM_FEATURE_DUMMY_C15_REGS)) {
7857         define_arm_cp_regs(cpu, dummy_c15_cp_reginfo);
7858     }
7859     if (arm_feature(env, ARM_FEATURE_LPAE)) {
7860         define_arm_cp_regs(cpu, lpae_cp_reginfo);
7861     }
7862     if (cpu_isar_feature(aa32_jazelle, cpu)) {
7863         define_arm_cp_regs(cpu, jazelle_regs);
7864     }
7865     /* Slightly awkwardly, the OMAP and StrongARM cores need all of
7866      * cp15 crn=0 to be writes-ignored, whereas for other cores they should
7867      * be read-only (ie write causes UNDEF exception).
7868      */
7869     {
7870         ARMCPRegInfo id_pre_v8_midr_cp_reginfo[] = {
7871             /* Pre-v8 MIDR space.
7872              * Note that the MIDR isn't a simple constant register because
7873              * of the TI925 behaviour where writes to another register can
7874              * cause the MIDR value to change.
7875              *
7876              * Unimplemented registers in the c15 0 0 0 space default to
7877              * MIDR. Define MIDR first as this entire space, then CTR, TCMTR
7878              * and friends override accordingly.
7879              */
7880             { .name = "MIDR",
7881               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = CP_ANY,
7882               .access = PL1_R, .resetvalue = cpu->midr,
7883               .writefn = arm_cp_write_ignore, .raw_writefn = raw_write,
7884               .readfn = midr_read,
7885               .fieldoffset = offsetof(CPUARMState, cp15.c0_cpuid),
7886               .type = ARM_CP_OVERRIDE },
7887             /* crn = 0 op1 = 0 crm = 3..7 : currently unassigned; we RAZ. */
7888             { .name = "DUMMY",
7889               .cp = 15, .crn = 0, .crm = 3, .opc1 = 0, .opc2 = CP_ANY,
7890               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
7891             { .name = "DUMMY",
7892               .cp = 15, .crn = 0, .crm = 4, .opc1 = 0, .opc2 = CP_ANY,
7893               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
7894             { .name = "DUMMY",
7895               .cp = 15, .crn = 0, .crm = 5, .opc1 = 0, .opc2 = CP_ANY,
7896               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
7897             { .name = "DUMMY",
7898               .cp = 15, .crn = 0, .crm = 6, .opc1 = 0, .opc2 = CP_ANY,
7899               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
7900             { .name = "DUMMY",
7901               .cp = 15, .crn = 0, .crm = 7, .opc1 = 0, .opc2 = CP_ANY,
7902               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
7903             REGINFO_SENTINEL
7904         };
7905         ARMCPRegInfo id_v8_midr_cp_reginfo[] = {
7906             { .name = "MIDR_EL1", .state = ARM_CP_STATE_BOTH,
7907               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 0, .opc2 = 0,
7908               .access = PL1_R, .type = ARM_CP_NO_RAW, .resetvalue = cpu->midr,
7909               .fieldoffset = offsetof(CPUARMState, cp15.c0_cpuid),
7910               .readfn = midr_read },
7911             /* crn = 0 op1 = 0 crm = 0 op2 = 4,7 : AArch32 aliases of MIDR */
7912             { .name = "MIDR", .type = ARM_CP_ALIAS | ARM_CP_CONST,
7913               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 4,
7914               .access = PL1_R, .resetvalue = cpu->midr },
7915             { .name = "MIDR", .type = ARM_CP_ALIAS | ARM_CP_CONST,
7916               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 7,
7917               .access = PL1_R, .resetvalue = cpu->midr },
7918             { .name = "REVIDR_EL1", .state = ARM_CP_STATE_BOTH,
7919               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 0, .opc2 = 6,
7920               .access = PL1_R,
7921               .accessfn = access_aa64_tid1,
7922               .type = ARM_CP_CONST, .resetvalue = cpu->revidr },
7923             REGINFO_SENTINEL
7924         };
7925         ARMCPRegInfo id_cp_reginfo[] = {
7926             /* These are common to v8 and pre-v8 */
7927             { .name = "CTR",
7928               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 1,
7929               .access = PL1_R, .accessfn = ctr_el0_access,
7930               .type = ARM_CP_CONST, .resetvalue = cpu->ctr },
7931             { .name = "CTR_EL0", .state = ARM_CP_STATE_AA64,
7932               .opc0 = 3, .opc1 = 3, .opc2 = 1, .crn = 0, .crm = 0,
7933               .access = PL0_R, .accessfn = ctr_el0_access,
7934               .type = ARM_CP_CONST, .resetvalue = cpu->ctr },
7935             /* TCMTR and TLBTR exist in v8 but have no 64-bit versions */
7936             { .name = "TCMTR",
7937               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 2,
7938               .access = PL1_R,
7939               .accessfn = access_aa32_tid1,
7940               .type = ARM_CP_CONST, .resetvalue = 0 },
7941             REGINFO_SENTINEL
7942         };
7943         /* TLBTR is specific to VMSA */
7944         ARMCPRegInfo id_tlbtr_reginfo = {
7945               .name = "TLBTR",
7946               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 3,
7947               .access = PL1_R,
7948               .accessfn = access_aa32_tid1,
7949               .type = ARM_CP_CONST, .resetvalue = 0,
7950         };
7951         /* MPUIR is specific to PMSA V6+ */
7952         ARMCPRegInfo id_mpuir_reginfo = {
7953               .name = "MPUIR",
7954               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 4,
7955               .access = PL1_R, .type = ARM_CP_CONST,
7956               .resetvalue = cpu->pmsav7_dregion << 8
7957         };
7958         ARMCPRegInfo crn0_wi_reginfo = {
7959             .name = "CRN0_WI", .cp = 15, .crn = 0, .crm = CP_ANY,
7960             .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_W,
7961             .type = ARM_CP_NOP | ARM_CP_OVERRIDE
7962         };
7963 #ifdef CONFIG_USER_ONLY
7964         ARMCPRegUserSpaceInfo id_v8_user_midr_cp_reginfo[] = {
7965             { .name = "MIDR_EL1",
7966               .exported_bits = 0x00000000ffffffff },
7967             { .name = "REVIDR_EL1"                },
7968             REGUSERINFO_SENTINEL
7969         };
7970         modify_arm_cp_regs(id_v8_midr_cp_reginfo, id_v8_user_midr_cp_reginfo);
7971 #endif
7972         if (arm_feature(env, ARM_FEATURE_OMAPCP) ||
7973             arm_feature(env, ARM_FEATURE_STRONGARM)) {
7974             ARMCPRegInfo *r;
7975             /* Register the blanket "writes ignored" value first to cover the
7976              * whole space. Then update the specific ID registers to allow write
7977              * access, so that they ignore writes rather than causing them to
7978              * UNDEF.
7979              */
7980             define_one_arm_cp_reg(cpu, &crn0_wi_reginfo);
7981             for (r = id_pre_v8_midr_cp_reginfo;
7982                  r->type != ARM_CP_SENTINEL; r++) {
7983                 r->access = PL1_RW;
7984             }
7985             for (r = id_cp_reginfo; r->type != ARM_CP_SENTINEL; r++) {
7986                 r->access = PL1_RW;
7987             }
7988             id_mpuir_reginfo.access = PL1_RW;
7989             id_tlbtr_reginfo.access = PL1_RW;
7990         }
7991         if (arm_feature(env, ARM_FEATURE_V8)) {
7992             define_arm_cp_regs(cpu, id_v8_midr_cp_reginfo);
7993         } else {
7994             define_arm_cp_regs(cpu, id_pre_v8_midr_cp_reginfo);
7995         }
7996         define_arm_cp_regs(cpu, id_cp_reginfo);
7997         if (!arm_feature(env, ARM_FEATURE_PMSA)) {
7998             define_one_arm_cp_reg(cpu, &id_tlbtr_reginfo);
7999         } else if (arm_feature(env, ARM_FEATURE_V7)) {
8000             define_one_arm_cp_reg(cpu, &id_mpuir_reginfo);
8001         }
8002     }
8003 
8004     if (arm_feature(env, ARM_FEATURE_MPIDR)) {
8005         ARMCPRegInfo mpidr_cp_reginfo[] = {
8006             { .name = "MPIDR_EL1", .state = ARM_CP_STATE_BOTH,
8007               .opc0 = 3, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 5,
8008               .access = PL1_R, .readfn = mpidr_read, .type = ARM_CP_NO_RAW },
8009             REGINFO_SENTINEL
8010         };
8011 #ifdef CONFIG_USER_ONLY
8012         ARMCPRegUserSpaceInfo mpidr_user_cp_reginfo[] = {
8013             { .name = "MPIDR_EL1",
8014               .fixed_bits = 0x0000000080000000 },
8015             REGUSERINFO_SENTINEL
8016         };
8017         modify_arm_cp_regs(mpidr_cp_reginfo, mpidr_user_cp_reginfo);
8018 #endif
8019         define_arm_cp_regs(cpu, mpidr_cp_reginfo);
8020     }
8021 
8022     if (arm_feature(env, ARM_FEATURE_AUXCR)) {
8023         ARMCPRegInfo auxcr_reginfo[] = {
8024             { .name = "ACTLR_EL1", .state = ARM_CP_STATE_BOTH,
8025               .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 1,
8026               .access = PL1_RW, .accessfn = access_tacr,
8027               .type = ARM_CP_CONST, .resetvalue = cpu->reset_auxcr },
8028             { .name = "ACTLR_EL2", .state = ARM_CP_STATE_BOTH,
8029               .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 0, .opc2 = 1,
8030               .access = PL2_RW, .type = ARM_CP_CONST,
8031               .resetvalue = 0 },
8032             { .name = "ACTLR_EL3", .state = ARM_CP_STATE_AA64,
8033               .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 0, .opc2 = 1,
8034               .access = PL3_RW, .type = ARM_CP_CONST,
8035               .resetvalue = 0 },
8036             REGINFO_SENTINEL
8037         };
8038         define_arm_cp_regs(cpu, auxcr_reginfo);
8039         if (cpu_isar_feature(aa32_ac2, cpu)) {
8040             define_arm_cp_regs(cpu, actlr2_hactlr2_reginfo);
8041         }
8042     }
8043 
8044     if (arm_feature(env, ARM_FEATURE_CBAR)) {
8045         /*
8046          * CBAR is IMPDEF, but common on Arm Cortex-A implementations.
8047          * There are two flavours:
8048          *  (1) older 32-bit only cores have a simple 32-bit CBAR
8049          *  (2) 64-bit cores have a 64-bit CBAR visible to AArch64, plus a
8050          *      32-bit register visible to AArch32 at a different encoding
8051          *      to the "flavour 1" register and with the bits rearranged to
8052          *      be able to squash a 64-bit address into the 32-bit view.
8053          * We distinguish the two via the ARM_FEATURE_AARCH64 flag, but
8054          * in future if we support AArch32-only configs of some of the
8055          * AArch64 cores we might need to add a specific feature flag
8056          * to indicate cores with "flavour 2" CBAR.
8057          */
8058         if (arm_feature(env, ARM_FEATURE_AARCH64)) {
8059             /* 32 bit view is [31:18] 0...0 [43:32]. */
8060             uint32_t cbar32 = (extract64(cpu->reset_cbar, 18, 14) << 18)
8061                 | extract64(cpu->reset_cbar, 32, 12);
8062             ARMCPRegInfo cbar_reginfo[] = {
8063                 { .name = "CBAR",
8064                   .type = ARM_CP_CONST,
8065                   .cp = 15, .crn = 15, .crm = 3, .opc1 = 1, .opc2 = 0,
8066                   .access = PL1_R, .resetvalue = cbar32 },
8067                 { .name = "CBAR_EL1", .state = ARM_CP_STATE_AA64,
8068                   .type = ARM_CP_CONST,
8069                   .opc0 = 3, .opc1 = 1, .crn = 15, .crm = 3, .opc2 = 0,
8070                   .access = PL1_R, .resetvalue = cpu->reset_cbar },
8071                 REGINFO_SENTINEL
8072             };
8073             /* We don't implement a r/w 64 bit CBAR currently */
8074             assert(arm_feature(env, ARM_FEATURE_CBAR_RO));
8075             define_arm_cp_regs(cpu, cbar_reginfo);
8076         } else {
8077             ARMCPRegInfo cbar = {
8078                 .name = "CBAR",
8079                 .cp = 15, .crn = 15, .crm = 0, .opc1 = 4, .opc2 = 0,
8080                 .access = PL1_R|PL3_W, .resetvalue = cpu->reset_cbar,
8081                 .fieldoffset = offsetof(CPUARMState,
8082                                         cp15.c15_config_base_address)
8083             };
8084             if (arm_feature(env, ARM_FEATURE_CBAR_RO)) {
8085                 cbar.access = PL1_R;
8086                 cbar.fieldoffset = 0;
8087                 cbar.type = ARM_CP_CONST;
8088             }
8089             define_one_arm_cp_reg(cpu, &cbar);
8090         }
8091     }
8092 
8093     if (arm_feature(env, ARM_FEATURE_VBAR)) {
8094         ARMCPRegInfo vbar_cp_reginfo[] = {
8095             { .name = "VBAR", .state = ARM_CP_STATE_BOTH,
8096               .opc0 = 3, .crn = 12, .crm = 0, .opc1 = 0, .opc2 = 0,
8097               .access = PL1_RW, .writefn = vbar_write,
8098               .bank_fieldoffsets = { offsetof(CPUARMState, cp15.vbar_s),
8099                                      offsetof(CPUARMState, cp15.vbar_ns) },
8100               .resetvalue = 0 },
8101             REGINFO_SENTINEL
8102         };
8103         define_arm_cp_regs(cpu, vbar_cp_reginfo);
8104     }
8105 
8106     /* Generic registers whose values depend on the implementation */
8107     {
8108         ARMCPRegInfo sctlr = {
8109             .name = "SCTLR", .state = ARM_CP_STATE_BOTH,
8110             .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 0,
8111             .access = PL1_RW, .accessfn = access_tvm_trvm,
8112             .bank_fieldoffsets = { offsetof(CPUARMState, cp15.sctlr_s),
8113                                    offsetof(CPUARMState, cp15.sctlr_ns) },
8114             .writefn = sctlr_write, .resetvalue = cpu->reset_sctlr,
8115             .raw_writefn = raw_write,
8116         };
8117         if (arm_feature(env, ARM_FEATURE_XSCALE)) {
8118             /* Normally we would always end the TB on an SCTLR write, but Linux
8119              * arch/arm/mach-pxa/sleep.S expects two instructions following
8120              * an MMU enable to execute from cache.  Imitate this behaviour.
8121              */
8122             sctlr.type |= ARM_CP_SUPPRESS_TB_END;
8123         }
8124         define_one_arm_cp_reg(cpu, &sctlr);
8125     }
8126 
8127     if (cpu_isar_feature(aa64_lor, cpu)) {
8128         define_arm_cp_regs(cpu, lor_reginfo);
8129     }
8130     if (cpu_isar_feature(aa64_pan, cpu)) {
8131         define_one_arm_cp_reg(cpu, &pan_reginfo);
8132     }
8133 #ifndef CONFIG_USER_ONLY
8134     if (cpu_isar_feature(aa64_ats1e1, cpu)) {
8135         define_arm_cp_regs(cpu, ats1e1_reginfo);
8136     }
8137     if (cpu_isar_feature(aa32_ats1e1, cpu)) {
8138         define_arm_cp_regs(cpu, ats1cp_reginfo);
8139     }
8140 #endif
8141     if (cpu_isar_feature(aa64_uao, cpu)) {
8142         define_one_arm_cp_reg(cpu, &uao_reginfo);
8143     }
8144 
8145     if (arm_feature(env, ARM_FEATURE_EL2) && cpu_isar_feature(aa64_vh, cpu)) {
8146         define_arm_cp_regs(cpu, vhe_reginfo);
8147     }
8148 
8149     if (cpu_isar_feature(aa64_sve, cpu)) {
8150         define_one_arm_cp_reg(cpu, &zcr_el1_reginfo);
8151         if (arm_feature(env, ARM_FEATURE_EL2)) {
8152             define_one_arm_cp_reg(cpu, &zcr_el2_reginfo);
8153         } else {
8154             define_one_arm_cp_reg(cpu, &zcr_no_el2_reginfo);
8155         }
8156         if (arm_feature(env, ARM_FEATURE_EL3)) {
8157             define_one_arm_cp_reg(cpu, &zcr_el3_reginfo);
8158         }
8159     }
8160 
8161 #ifdef TARGET_AARCH64
8162     if (cpu_isar_feature(aa64_pauth, cpu)) {
8163         define_arm_cp_regs(cpu, pauth_reginfo);
8164     }
8165     if (cpu_isar_feature(aa64_rndr, cpu)) {
8166         define_arm_cp_regs(cpu, rndr_reginfo);
8167     }
8168 #ifndef CONFIG_USER_ONLY
8169     /* Data Cache clean instructions up to PoP */
8170     if (cpu_isar_feature(aa64_dcpop, cpu)) {
8171         define_one_arm_cp_reg(cpu, dcpop_reg);
8172 
8173         if (cpu_isar_feature(aa64_dcpodp, cpu)) {
8174             define_one_arm_cp_reg(cpu, dcpodp_reg);
8175         }
8176     }
8177 #endif /*CONFIG_USER_ONLY*/
8178 
8179     /*
8180      * If full MTE is enabled, add all of the system registers.
8181      * If only "instructions available at EL0" are enabled,
8182      * then define only a RAZ/WI version of PSTATE.TCO.
8183      */
8184     if (cpu_isar_feature(aa64_mte, cpu)) {
8185         define_arm_cp_regs(cpu, mte_reginfo);
8186         define_arm_cp_regs(cpu, mte_el0_cacheop_reginfo);
8187     } else if (cpu_isar_feature(aa64_mte_insn_reg, cpu)) {
8188         define_arm_cp_regs(cpu, mte_tco_ro_reginfo);
8189         define_arm_cp_regs(cpu, mte_el0_cacheop_reginfo);
8190     }
8191 #endif
8192 
8193     if (cpu_isar_feature(any_predinv, cpu)) {
8194         define_arm_cp_regs(cpu, predinv_reginfo);
8195     }
8196 
8197     if (cpu_isar_feature(any_ccidx, cpu)) {
8198         define_arm_cp_regs(cpu, ccsidr2_reginfo);
8199     }
8200 
8201 #ifndef CONFIG_USER_ONLY
8202     /*
8203      * Register redirections and aliases must be done last,
8204      * after the registers from the other extensions have been defined.
8205      */
8206     if (arm_feature(env, ARM_FEATURE_EL2) && cpu_isar_feature(aa64_vh, cpu)) {
8207         define_arm_vh_e2h_redirects_aliases(cpu);
8208     }
8209 #endif
8210 }
8211 
8212 void arm_cpu_register_gdb_regs_for_features(ARMCPU *cpu)
8213 {
8214     CPUState *cs = CPU(cpu);
8215     CPUARMState *env = &cpu->env;
8216 
8217     if (arm_feature(env, ARM_FEATURE_AARCH64)) {
8218         /*
8219          * The lower part of each SVE register aliases to the FPU
8220          * registers so we don't need to include both.
8221          */
8222 #ifdef TARGET_AARCH64
8223         if (isar_feature_aa64_sve(&cpu->isar)) {
8224             gdb_register_coprocessor(cs, arm_gdb_get_svereg, arm_gdb_set_svereg,
8225                                      arm_gen_dynamic_svereg_xml(cs, cs->gdb_num_regs),
8226                                      "sve-registers.xml", 0);
8227         } else
8228 #endif
8229         {
8230             gdb_register_coprocessor(cs, aarch64_fpu_gdb_get_reg,
8231                                      aarch64_fpu_gdb_set_reg,
8232                                      34, "aarch64-fpu.xml", 0);
8233         }
8234     } else if (arm_feature(env, ARM_FEATURE_NEON)) {
8235         gdb_register_coprocessor(cs, vfp_gdb_get_reg, vfp_gdb_set_reg,
8236                                  51, "arm-neon.xml", 0);
8237     } else if (cpu_isar_feature(aa32_simd_r32, cpu)) {
8238         gdb_register_coprocessor(cs, vfp_gdb_get_reg, vfp_gdb_set_reg,
8239                                  35, "arm-vfp3.xml", 0);
8240     } else if (cpu_isar_feature(aa32_vfp_simd, cpu)) {
8241         gdb_register_coprocessor(cs, vfp_gdb_get_reg, vfp_gdb_set_reg,
8242                                  19, "arm-vfp.xml", 0);
8243     }
8244     gdb_register_coprocessor(cs, arm_gdb_get_sysreg, arm_gdb_set_sysreg,
8245                              arm_gen_dynamic_sysreg_xml(cs, cs->gdb_num_regs),
8246                              "system-registers.xml", 0);
8247 
8248 }
8249 
8250 /* Sort alphabetically by type name, except for "any". */
8251 static gint arm_cpu_list_compare(gconstpointer a, gconstpointer b)
8252 {
8253     ObjectClass *class_a = (ObjectClass *)a;
8254     ObjectClass *class_b = (ObjectClass *)b;
8255     const char *name_a, *name_b;
8256 
8257     name_a = object_class_get_name(class_a);
8258     name_b = object_class_get_name(class_b);
8259     if (strcmp(name_a, "any-" TYPE_ARM_CPU) == 0) {
8260         return 1;
8261     } else if (strcmp(name_b, "any-" TYPE_ARM_CPU) == 0) {
8262         return -1;
8263     } else {
8264         return strcmp(name_a, name_b);
8265     }
8266 }
8267 
8268 static void arm_cpu_list_entry(gpointer data, gpointer user_data)
8269 {
8270     ObjectClass *oc = data;
8271     const char *typename;
8272     char *name;
8273 
8274     typename = object_class_get_name(oc);
8275     name = g_strndup(typename, strlen(typename) - strlen("-" TYPE_ARM_CPU));
8276     qemu_printf("  %s\n", name);
8277     g_free(name);
8278 }
8279 
8280 void arm_cpu_list(void)
8281 {
8282     GSList *list;
8283 
8284     list = object_class_get_list(TYPE_ARM_CPU, false);
8285     list = g_slist_sort(list, arm_cpu_list_compare);
8286     qemu_printf("Available CPUs:\n");
8287     g_slist_foreach(list, arm_cpu_list_entry, NULL);
8288     g_slist_free(list);
8289 }
8290 
8291 static void arm_cpu_add_definition(gpointer data, gpointer user_data)
8292 {
8293     ObjectClass *oc = data;
8294     CpuDefinitionInfoList **cpu_list = user_data;
8295     CpuDefinitionInfoList *entry;
8296     CpuDefinitionInfo *info;
8297     const char *typename;
8298 
8299     typename = object_class_get_name(oc);
8300     info = g_malloc0(sizeof(*info));
8301     info->name = g_strndup(typename,
8302                            strlen(typename) - strlen("-" TYPE_ARM_CPU));
8303     info->q_typename = g_strdup(typename);
8304 
8305     entry = g_malloc0(sizeof(*entry));
8306     entry->value = info;
8307     entry->next = *cpu_list;
8308     *cpu_list = entry;
8309 }
8310 
8311 CpuDefinitionInfoList *qmp_query_cpu_definitions(Error **errp)
8312 {
8313     CpuDefinitionInfoList *cpu_list = NULL;
8314     GSList *list;
8315 
8316     list = object_class_get_list(TYPE_ARM_CPU, false);
8317     g_slist_foreach(list, arm_cpu_add_definition, &cpu_list);
8318     g_slist_free(list);
8319 
8320     return cpu_list;
8321 }
8322 
8323 static void add_cpreg_to_hashtable(ARMCPU *cpu, const ARMCPRegInfo *r,
8324                                    void *opaque, int state, int secstate,
8325                                    int crm, int opc1, int opc2,
8326                                    const char *name)
8327 {
8328     /* Private utility function for define_one_arm_cp_reg_with_opaque():
8329      * add a single reginfo struct to the hash table.
8330      */
8331     uint32_t *key = g_new(uint32_t, 1);
8332     ARMCPRegInfo *r2 = g_memdup(r, sizeof(ARMCPRegInfo));
8333     int is64 = (r->type & ARM_CP_64BIT) ? 1 : 0;
8334     int ns = (secstate & ARM_CP_SECSTATE_NS) ? 1 : 0;
8335 
8336     r2->name = g_strdup(name);
8337     /* Reset the secure state to the specific incoming state.  This is
8338      * necessary as the register may have been defined with both states.
8339      */
8340     r2->secure = secstate;
8341 
8342     if (r->bank_fieldoffsets[0] && r->bank_fieldoffsets[1]) {
8343         /* Register is banked (using both entries in array).
8344          * Overwriting fieldoffset as the array is only used to define
8345          * banked registers but later only fieldoffset is used.
8346          */
8347         r2->fieldoffset = r->bank_fieldoffsets[ns];
8348     }
8349 
8350     if (state == ARM_CP_STATE_AA32) {
8351         if (r->bank_fieldoffsets[0] && r->bank_fieldoffsets[1]) {
8352             /* If the register is banked then we don't need to migrate or
8353              * reset the 32-bit instance in certain cases:
8354              *
8355              * 1) If the register has both 32-bit and 64-bit instances then we
8356              *    can count on the 64-bit instance taking care of the
8357              *    non-secure bank.
8358              * 2) If ARMv8 is enabled then we can count on a 64-bit version
8359              *    taking care of the secure bank.  This requires that separate
8360              *    32 and 64-bit definitions are provided.
8361              */
8362             if ((r->state == ARM_CP_STATE_BOTH && ns) ||
8363                 (arm_feature(&cpu->env, ARM_FEATURE_V8) && !ns)) {
8364                 r2->type |= ARM_CP_ALIAS;
8365             }
8366         } else if ((secstate != r->secure) && !ns) {
8367             /* The register is not banked so we only want to allow migration of
8368              * the non-secure instance.
8369              */
8370             r2->type |= ARM_CP_ALIAS;
8371         }
8372 
8373         if (r->state == ARM_CP_STATE_BOTH) {
8374             /* We assume it is a cp15 register if the .cp field is left unset.
8375              */
8376             if (r2->cp == 0) {
8377                 r2->cp = 15;
8378             }
8379 
8380 #ifdef HOST_WORDS_BIGENDIAN
8381             if (r2->fieldoffset) {
8382                 r2->fieldoffset += sizeof(uint32_t);
8383             }
8384 #endif
8385         }
8386     }
8387     if (state == ARM_CP_STATE_AA64) {
8388         /* To allow abbreviation of ARMCPRegInfo
8389          * definitions, we treat cp == 0 as equivalent to
8390          * the value for "standard guest-visible sysreg".
8391          * STATE_BOTH definitions are also always "standard
8392          * sysreg" in their AArch64 view (the .cp value may
8393          * be non-zero for the benefit of the AArch32 view).
8394          */
8395         if (r->cp == 0 || r->state == ARM_CP_STATE_BOTH) {
8396             r2->cp = CP_REG_ARM64_SYSREG_CP;
8397         }
8398         *key = ENCODE_AA64_CP_REG(r2->cp, r2->crn, crm,
8399                                   r2->opc0, opc1, opc2);
8400     } else {
8401         *key = ENCODE_CP_REG(r2->cp, is64, ns, r2->crn, crm, opc1, opc2);
8402     }
8403     if (opaque) {
8404         r2->opaque = opaque;
8405     }
8406     /* reginfo passed to helpers is correct for the actual access,
8407      * and is never ARM_CP_STATE_BOTH:
8408      */
8409     r2->state = state;
8410     /* Make sure reginfo passed to helpers for wildcarded regs
8411      * has the correct crm/opc1/opc2 for this reg, not CP_ANY:
8412      */
8413     r2->crm = crm;
8414     r2->opc1 = opc1;
8415     r2->opc2 = opc2;
8416     /* By convention, for wildcarded registers only the first
8417      * entry is used for migration; the others are marked as
8418      * ALIAS so we don't try to transfer the register
8419      * multiple times. Special registers (ie NOP/WFI) are
8420      * never migratable and not even raw-accessible.
8421      */
8422     if ((r->type & ARM_CP_SPECIAL)) {
8423         r2->type |= ARM_CP_NO_RAW;
8424     }
8425     if (((r->crm == CP_ANY) && crm != 0) ||
8426         ((r->opc1 == CP_ANY) && opc1 != 0) ||
8427         ((r->opc2 == CP_ANY) && opc2 != 0)) {
8428         r2->type |= ARM_CP_ALIAS | ARM_CP_NO_GDB;
8429     }
8430 
8431     /* Check that raw accesses are either forbidden or handled. Note that
8432      * we can't assert this earlier because the setup of fieldoffset for
8433      * banked registers has to be done first.
8434      */
8435     if (!(r2->type & ARM_CP_NO_RAW)) {
8436         assert(!raw_accessors_invalid(r2));
8437     }
8438 
8439     /* Overriding of an existing definition must be explicitly
8440      * requested.
8441      */
8442     if (!(r->type & ARM_CP_OVERRIDE)) {
8443         ARMCPRegInfo *oldreg;
8444         oldreg = g_hash_table_lookup(cpu->cp_regs, key);
8445         if (oldreg && !(oldreg->type & ARM_CP_OVERRIDE)) {
8446             fprintf(stderr, "Register redefined: cp=%d %d bit "
8447                     "crn=%d crm=%d opc1=%d opc2=%d, "
8448                     "was %s, now %s\n", r2->cp, 32 + 32 * is64,
8449                     r2->crn, r2->crm, r2->opc1, r2->opc2,
8450                     oldreg->name, r2->name);
8451             g_assert_not_reached();
8452         }
8453     }
8454     g_hash_table_insert(cpu->cp_regs, key, r2);
8455 }
8456 
8457 
8458 void define_one_arm_cp_reg_with_opaque(ARMCPU *cpu,
8459                                        const ARMCPRegInfo *r, void *opaque)
8460 {
8461     /* Define implementations of coprocessor registers.
8462      * We store these in a hashtable because typically
8463      * there are less than 150 registers in a space which
8464      * is 16*16*16*8*8 = 262144 in size.
8465      * Wildcarding is supported for the crm, opc1 and opc2 fields.
8466      * If a register is defined twice then the second definition is
8467      * used, so this can be used to define some generic registers and
8468      * then override them with implementation specific variations.
8469      * At least one of the original and the second definition should
8470      * include ARM_CP_OVERRIDE in its type bits -- this is just a guard
8471      * against accidental use.
8472      *
8473      * The state field defines whether the register is to be
8474      * visible in the AArch32 or AArch64 execution state. If the
8475      * state is set to ARM_CP_STATE_BOTH then we synthesise a
8476      * reginfo structure for the AArch32 view, which sees the lower
8477      * 32 bits of the 64 bit register.
8478      *
8479      * Only registers visible in AArch64 may set r->opc0; opc0 cannot
8480      * be wildcarded. AArch64 registers are always considered to be 64
8481      * bits; the ARM_CP_64BIT* flag applies only to the AArch32 view of
8482      * the register, if any.
8483      */
8484     int crm, opc1, opc2, state;
8485     int crmmin = (r->crm == CP_ANY) ? 0 : r->crm;
8486     int crmmax = (r->crm == CP_ANY) ? 15 : r->crm;
8487     int opc1min = (r->opc1 == CP_ANY) ? 0 : r->opc1;
8488     int opc1max = (r->opc1 == CP_ANY) ? 7 : r->opc1;
8489     int opc2min = (r->opc2 == CP_ANY) ? 0 : r->opc2;
8490     int opc2max = (r->opc2 == CP_ANY) ? 7 : r->opc2;
8491     /* 64 bit registers have only CRm and Opc1 fields */
8492     assert(!((r->type & ARM_CP_64BIT) && (r->opc2 || r->crn)));
8493     /* op0 only exists in the AArch64 encodings */
8494     assert((r->state != ARM_CP_STATE_AA32) || (r->opc0 == 0));
8495     /* AArch64 regs are all 64 bit so ARM_CP_64BIT is meaningless */
8496     assert((r->state != ARM_CP_STATE_AA64) || !(r->type & ARM_CP_64BIT));
8497     /*
8498      * This API is only for Arm's system coprocessors (14 and 15) or
8499      * (M-profile or v7A-and-earlier only) for implementation defined
8500      * coprocessors in the range 0..7.  Our decode assumes this, since
8501      * 8..13 can be used for other insns including VFP and Neon. See
8502      * valid_cp() in translate.c.  Assert here that we haven't tried
8503      * to use an invalid coprocessor number.
8504      */
8505     switch (r->state) {
8506     case ARM_CP_STATE_BOTH:
8507         /* 0 has a special meaning, but otherwise the same rules as AA32. */
8508         if (r->cp == 0) {
8509             break;
8510         }
8511         /* fall through */
8512     case ARM_CP_STATE_AA32:
8513         if (arm_feature(&cpu->env, ARM_FEATURE_V8) &&
8514             !arm_feature(&cpu->env, ARM_FEATURE_M)) {
8515             assert(r->cp >= 14 && r->cp <= 15);
8516         } else {
8517             assert(r->cp < 8 || (r->cp >= 14 && r->cp <= 15));
8518         }
8519         break;
8520     case ARM_CP_STATE_AA64:
8521         assert(r->cp == 0 || r->cp == CP_REG_ARM64_SYSREG_CP);
8522         break;
8523     default:
8524         g_assert_not_reached();
8525     }
8526     /* The AArch64 pseudocode CheckSystemAccess() specifies that op1
8527      * encodes a minimum access level for the register. We roll this
8528      * runtime check into our general permission check code, so check
8529      * here that the reginfo's specified permissions are strict enough
8530      * to encompass the generic architectural permission check.
8531      */
8532     if (r->state != ARM_CP_STATE_AA32) {
8533         int mask = 0;
8534         switch (r->opc1) {
8535         case 0:
8536             /* min_EL EL1, but some accessible to EL0 via kernel ABI */
8537             mask = PL0U_R | PL1_RW;
8538             break;
8539         case 1: case 2:
8540             /* min_EL EL1 */
8541             mask = PL1_RW;
8542             break;
8543         case 3:
8544             /* min_EL EL0 */
8545             mask = PL0_RW;
8546             break;
8547         case 4:
8548         case 5:
8549             /* min_EL EL2 */
8550             mask = PL2_RW;
8551             break;
8552         case 6:
8553             /* min_EL EL3 */
8554             mask = PL3_RW;
8555             break;
8556         case 7:
8557             /* min_EL EL1, secure mode only (we don't check the latter) */
8558             mask = PL1_RW;
8559             break;
8560         default:
8561             /* broken reginfo with out-of-range opc1 */
8562             assert(false);
8563             break;
8564         }
8565         /* assert our permissions are not too lax (stricter is fine) */
8566         assert((r->access & ~mask) == 0);
8567     }
8568 
8569     /* Check that the register definition has enough info to handle
8570      * reads and writes if they are permitted.
8571      */
8572     if (!(r->type & (ARM_CP_SPECIAL|ARM_CP_CONST))) {
8573         if (r->access & PL3_R) {
8574             assert((r->fieldoffset ||
8575                    (r->bank_fieldoffsets[0] && r->bank_fieldoffsets[1])) ||
8576                    r->readfn);
8577         }
8578         if (r->access & PL3_W) {
8579             assert((r->fieldoffset ||
8580                    (r->bank_fieldoffsets[0] && r->bank_fieldoffsets[1])) ||
8581                    r->writefn);
8582         }
8583     }
8584     /* Bad type field probably means missing sentinel at end of reg list */
8585     assert(cptype_valid(r->type));
8586     for (crm = crmmin; crm <= crmmax; crm++) {
8587         for (opc1 = opc1min; opc1 <= opc1max; opc1++) {
8588             for (opc2 = opc2min; opc2 <= opc2max; opc2++) {
8589                 for (state = ARM_CP_STATE_AA32;
8590                      state <= ARM_CP_STATE_AA64; state++) {
8591                     if (r->state != state && r->state != ARM_CP_STATE_BOTH) {
8592                         continue;
8593                     }
8594                     if (state == ARM_CP_STATE_AA32) {
8595                         /* Under AArch32 CP registers can be common
8596                          * (same for secure and non-secure world) or banked.
8597                          */
8598                         char *name;
8599 
8600                         switch (r->secure) {
8601                         case ARM_CP_SECSTATE_S:
8602                         case ARM_CP_SECSTATE_NS:
8603                             add_cpreg_to_hashtable(cpu, r, opaque, state,
8604                                                    r->secure, crm, opc1, opc2,
8605                                                    r->name);
8606                             break;
8607                         default:
8608                             name = g_strdup_printf("%s_S", r->name);
8609                             add_cpreg_to_hashtable(cpu, r, opaque, state,
8610                                                    ARM_CP_SECSTATE_S,
8611                                                    crm, opc1, opc2, name);
8612                             g_free(name);
8613                             add_cpreg_to_hashtable(cpu, r, opaque, state,
8614                                                    ARM_CP_SECSTATE_NS,
8615                                                    crm, opc1, opc2, r->name);
8616                             break;
8617                         }
8618                     } else {
8619                         /* AArch64 registers get mapped to non-secure instance
8620                          * of AArch32 */
8621                         add_cpreg_to_hashtable(cpu, r, opaque, state,
8622                                                ARM_CP_SECSTATE_NS,
8623                                                crm, opc1, opc2, r->name);
8624                     }
8625                 }
8626             }
8627         }
8628     }
8629 }
8630 
8631 void define_arm_cp_regs_with_opaque(ARMCPU *cpu,
8632                                     const ARMCPRegInfo *regs, void *opaque)
8633 {
8634     /* Define a whole list of registers */
8635     const ARMCPRegInfo *r;
8636     for (r = regs; r->type != ARM_CP_SENTINEL; r++) {
8637         define_one_arm_cp_reg_with_opaque(cpu, r, opaque);
8638     }
8639 }
8640 
8641 /*
8642  * Modify ARMCPRegInfo for access from userspace.
8643  *
8644  * This is a data driven modification directed by
8645  * ARMCPRegUserSpaceInfo. All registers become ARM_CP_CONST as
8646  * user-space cannot alter any values and dynamic values pertaining to
8647  * execution state are hidden from user space view anyway.
8648  */
8649 void modify_arm_cp_regs(ARMCPRegInfo *regs, const ARMCPRegUserSpaceInfo *mods)
8650 {
8651     const ARMCPRegUserSpaceInfo *m;
8652     ARMCPRegInfo *r;
8653 
8654     for (m = mods; m->name; m++) {
8655         GPatternSpec *pat = NULL;
8656         if (m->is_glob) {
8657             pat = g_pattern_spec_new(m->name);
8658         }
8659         for (r = regs; r->type != ARM_CP_SENTINEL; r++) {
8660             if (pat && g_pattern_match_string(pat, r->name)) {
8661                 r->type = ARM_CP_CONST;
8662                 r->access = PL0U_R;
8663                 r->resetvalue = 0;
8664                 /* continue */
8665             } else if (strcmp(r->name, m->name) == 0) {
8666                 r->type = ARM_CP_CONST;
8667                 r->access = PL0U_R;
8668                 r->resetvalue &= m->exported_bits;
8669                 r->resetvalue |= m->fixed_bits;
8670                 break;
8671             }
8672         }
8673         if (pat) {
8674             g_pattern_spec_free(pat);
8675         }
8676     }
8677 }
8678 
8679 const ARMCPRegInfo *get_arm_cp_reginfo(GHashTable *cpregs, uint32_t encoded_cp)
8680 {
8681     return g_hash_table_lookup(cpregs, &encoded_cp);
8682 }
8683 
8684 void arm_cp_write_ignore(CPUARMState *env, const ARMCPRegInfo *ri,
8685                          uint64_t value)
8686 {
8687     /* Helper coprocessor write function for write-ignore registers */
8688 }
8689 
8690 uint64_t arm_cp_read_zero(CPUARMState *env, const ARMCPRegInfo *ri)
8691 {
8692     /* Helper coprocessor write function for read-as-zero registers */
8693     return 0;
8694 }
8695 
8696 void arm_cp_reset_ignore(CPUARMState *env, const ARMCPRegInfo *opaque)
8697 {
8698     /* Helper coprocessor reset function for do-nothing-on-reset registers */
8699 }
8700 
8701 static int bad_mode_switch(CPUARMState *env, int mode, CPSRWriteType write_type)
8702 {
8703     /* Return true if it is not valid for us to switch to
8704      * this CPU mode (ie all the UNPREDICTABLE cases in
8705      * the ARM ARM CPSRWriteByInstr pseudocode).
8706      */
8707 
8708     /* Changes to or from Hyp via MSR and CPS are illegal. */
8709     if (write_type == CPSRWriteByInstr &&
8710         ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_HYP ||
8711          mode == ARM_CPU_MODE_HYP)) {
8712         return 1;
8713     }
8714 
8715     switch (mode) {
8716     case ARM_CPU_MODE_USR:
8717         return 0;
8718     case ARM_CPU_MODE_SYS:
8719     case ARM_CPU_MODE_SVC:
8720     case ARM_CPU_MODE_ABT:
8721     case ARM_CPU_MODE_UND:
8722     case ARM_CPU_MODE_IRQ:
8723     case ARM_CPU_MODE_FIQ:
8724         /* Note that we don't implement the IMPDEF NSACR.RFR which in v7
8725          * allows FIQ mode to be Secure-only. (In v8 this doesn't exist.)
8726          */
8727         /* If HCR.TGE is set then changes from Monitor to NS PL1 via MSR
8728          * and CPS are treated as illegal mode changes.
8729          */
8730         if (write_type == CPSRWriteByInstr &&
8731             (env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_MON &&
8732             (arm_hcr_el2_eff(env) & HCR_TGE)) {
8733             return 1;
8734         }
8735         return 0;
8736     case ARM_CPU_MODE_HYP:
8737         return !arm_feature(env, ARM_FEATURE_EL2)
8738             || arm_current_el(env) < 2 || arm_is_secure_below_el3(env);
8739     case ARM_CPU_MODE_MON:
8740         return arm_current_el(env) < 3;
8741     default:
8742         return 1;
8743     }
8744 }
8745 
8746 uint32_t cpsr_read(CPUARMState *env)
8747 {
8748     int ZF;
8749     ZF = (env->ZF == 0);
8750     return env->uncached_cpsr | (env->NF & 0x80000000) | (ZF << 30) |
8751         (env->CF << 29) | ((env->VF & 0x80000000) >> 3) | (env->QF << 27)
8752         | (env->thumb << 5) | ((env->condexec_bits & 3) << 25)
8753         | ((env->condexec_bits & 0xfc) << 8)
8754         | (env->GE << 16) | (env->daif & CPSR_AIF);
8755 }
8756 
8757 void cpsr_write(CPUARMState *env, uint32_t val, uint32_t mask,
8758                 CPSRWriteType write_type)
8759 {
8760     uint32_t changed_daif;
8761 
8762     if (mask & CPSR_NZCV) {
8763         env->ZF = (~val) & CPSR_Z;
8764         env->NF = val;
8765         env->CF = (val >> 29) & 1;
8766         env->VF = (val << 3) & 0x80000000;
8767     }
8768     if (mask & CPSR_Q)
8769         env->QF = ((val & CPSR_Q) != 0);
8770     if (mask & CPSR_T)
8771         env->thumb = ((val & CPSR_T) != 0);
8772     if (mask & CPSR_IT_0_1) {
8773         env->condexec_bits &= ~3;
8774         env->condexec_bits |= (val >> 25) & 3;
8775     }
8776     if (mask & CPSR_IT_2_7) {
8777         env->condexec_bits &= 3;
8778         env->condexec_bits |= (val >> 8) & 0xfc;
8779     }
8780     if (mask & CPSR_GE) {
8781         env->GE = (val >> 16) & 0xf;
8782     }
8783 
8784     /* In a V7 implementation that includes the security extensions but does
8785      * not include Virtualization Extensions the SCR.FW and SCR.AW bits control
8786      * whether non-secure software is allowed to change the CPSR_F and CPSR_A
8787      * bits respectively.
8788      *
8789      * In a V8 implementation, it is permitted for privileged software to
8790      * change the CPSR A/F bits regardless of the SCR.AW/FW bits.
8791      */
8792     if (write_type != CPSRWriteRaw && !arm_feature(env, ARM_FEATURE_V8) &&
8793         arm_feature(env, ARM_FEATURE_EL3) &&
8794         !arm_feature(env, ARM_FEATURE_EL2) &&
8795         !arm_is_secure(env)) {
8796 
8797         changed_daif = (env->daif ^ val) & mask;
8798 
8799         if (changed_daif & CPSR_A) {
8800             /* Check to see if we are allowed to change the masking of async
8801              * abort exceptions from a non-secure state.
8802              */
8803             if (!(env->cp15.scr_el3 & SCR_AW)) {
8804                 qemu_log_mask(LOG_GUEST_ERROR,
8805                               "Ignoring attempt to switch CPSR_A flag from "
8806                               "non-secure world with SCR.AW bit clear\n");
8807                 mask &= ~CPSR_A;
8808             }
8809         }
8810 
8811         if (changed_daif & CPSR_F) {
8812             /* Check to see if we are allowed to change the masking of FIQ
8813              * exceptions from a non-secure state.
8814              */
8815             if (!(env->cp15.scr_el3 & SCR_FW)) {
8816                 qemu_log_mask(LOG_GUEST_ERROR,
8817                               "Ignoring attempt to switch CPSR_F flag from "
8818                               "non-secure world with SCR.FW bit clear\n");
8819                 mask &= ~CPSR_F;
8820             }
8821 
8822             /* Check whether non-maskable FIQ (NMFI) support is enabled.
8823              * If this bit is set software is not allowed to mask
8824              * FIQs, but is allowed to set CPSR_F to 0.
8825              */
8826             if ((A32_BANKED_CURRENT_REG_GET(env, sctlr) & SCTLR_NMFI) &&
8827                 (val & CPSR_F)) {
8828                 qemu_log_mask(LOG_GUEST_ERROR,
8829                               "Ignoring attempt to enable CPSR_F flag "
8830                               "(non-maskable FIQ [NMFI] support enabled)\n");
8831                 mask &= ~CPSR_F;
8832             }
8833         }
8834     }
8835 
8836     env->daif &= ~(CPSR_AIF & mask);
8837     env->daif |= val & CPSR_AIF & mask;
8838 
8839     if (write_type != CPSRWriteRaw &&
8840         ((env->uncached_cpsr ^ val) & mask & CPSR_M)) {
8841         if ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_USR) {
8842             /* Note that we can only get here in USR mode if this is a
8843              * gdb stub write; for this case we follow the architectural
8844              * behaviour for guest writes in USR mode of ignoring an attempt
8845              * to switch mode. (Those are caught by translate.c for writes
8846              * triggered by guest instructions.)
8847              */
8848             mask &= ~CPSR_M;
8849         } else if (bad_mode_switch(env, val & CPSR_M, write_type)) {
8850             /* Attempt to switch to an invalid mode: this is UNPREDICTABLE in
8851              * v7, and has defined behaviour in v8:
8852              *  + leave CPSR.M untouched
8853              *  + allow changes to the other CPSR fields
8854              *  + set PSTATE.IL
8855              * For user changes via the GDB stub, we don't set PSTATE.IL,
8856              * as this would be unnecessarily harsh for a user error.
8857              */
8858             mask &= ~CPSR_M;
8859             if (write_type != CPSRWriteByGDBStub &&
8860                 arm_feature(env, ARM_FEATURE_V8)) {
8861                 mask |= CPSR_IL;
8862                 val |= CPSR_IL;
8863             }
8864             qemu_log_mask(LOG_GUEST_ERROR,
8865                           "Illegal AArch32 mode switch attempt from %s to %s\n",
8866                           aarch32_mode_name(env->uncached_cpsr),
8867                           aarch32_mode_name(val));
8868         } else {
8869             qemu_log_mask(CPU_LOG_INT, "%s %s to %s PC 0x%" PRIx32 "\n",
8870                           write_type == CPSRWriteExceptionReturn ?
8871                           "Exception return from AArch32" :
8872                           "AArch32 mode switch from",
8873                           aarch32_mode_name(env->uncached_cpsr),
8874                           aarch32_mode_name(val), env->regs[15]);
8875             switch_mode(env, val & CPSR_M);
8876         }
8877     }
8878     mask &= ~CACHED_CPSR_BITS;
8879     env->uncached_cpsr = (env->uncached_cpsr & ~mask) | (val & mask);
8880 }
8881 
8882 /* Sign/zero extend */
8883 uint32_t HELPER(sxtb16)(uint32_t x)
8884 {
8885     uint32_t res;
8886     res = (uint16_t)(int8_t)x;
8887     res |= (uint32_t)(int8_t)(x >> 16) << 16;
8888     return res;
8889 }
8890 
8891 uint32_t HELPER(uxtb16)(uint32_t x)
8892 {
8893     uint32_t res;
8894     res = (uint16_t)(uint8_t)x;
8895     res |= (uint32_t)(uint8_t)(x >> 16) << 16;
8896     return res;
8897 }
8898 
8899 int32_t HELPER(sdiv)(int32_t num, int32_t den)
8900 {
8901     if (den == 0)
8902       return 0;
8903     if (num == INT_MIN && den == -1)
8904       return INT_MIN;
8905     return num / den;
8906 }
8907 
8908 uint32_t HELPER(udiv)(uint32_t num, uint32_t den)
8909 {
8910     if (den == 0)
8911       return 0;
8912     return num / den;
8913 }
8914 
8915 uint32_t HELPER(rbit)(uint32_t x)
8916 {
8917     return revbit32(x);
8918 }
8919 
8920 #ifdef CONFIG_USER_ONLY
8921 
8922 static void switch_mode(CPUARMState *env, int mode)
8923 {
8924     ARMCPU *cpu = env_archcpu(env);
8925 
8926     if (mode != ARM_CPU_MODE_USR) {
8927         cpu_abort(CPU(cpu), "Tried to switch out of user mode\n");
8928     }
8929 }
8930 
8931 uint32_t arm_phys_excp_target_el(CPUState *cs, uint32_t excp_idx,
8932                                  uint32_t cur_el, bool secure)
8933 {
8934     return 1;
8935 }
8936 
8937 void aarch64_sync_64_to_32(CPUARMState *env)
8938 {
8939     g_assert_not_reached();
8940 }
8941 
8942 #else
8943 
8944 static void switch_mode(CPUARMState *env, int mode)
8945 {
8946     int old_mode;
8947     int i;
8948 
8949     old_mode = env->uncached_cpsr & CPSR_M;
8950     if (mode == old_mode)
8951         return;
8952 
8953     if (old_mode == ARM_CPU_MODE_FIQ) {
8954         memcpy (env->fiq_regs, env->regs + 8, 5 * sizeof(uint32_t));
8955         memcpy (env->regs + 8, env->usr_regs, 5 * sizeof(uint32_t));
8956     } else if (mode == ARM_CPU_MODE_FIQ) {
8957         memcpy (env->usr_regs, env->regs + 8, 5 * sizeof(uint32_t));
8958         memcpy (env->regs + 8, env->fiq_regs, 5 * sizeof(uint32_t));
8959     }
8960 
8961     i = bank_number(old_mode);
8962     env->banked_r13[i] = env->regs[13];
8963     env->banked_spsr[i] = env->spsr;
8964 
8965     i = bank_number(mode);
8966     env->regs[13] = env->banked_r13[i];
8967     env->spsr = env->banked_spsr[i];
8968 
8969     env->banked_r14[r14_bank_number(old_mode)] = env->regs[14];
8970     env->regs[14] = env->banked_r14[r14_bank_number(mode)];
8971 }
8972 
8973 /* Physical Interrupt Target EL Lookup Table
8974  *
8975  * [ From ARM ARM section G1.13.4 (Table G1-15) ]
8976  *
8977  * The below multi-dimensional table is used for looking up the target
8978  * exception level given numerous condition criteria.  Specifically, the
8979  * target EL is based on SCR and HCR routing controls as well as the
8980  * currently executing EL and secure state.
8981  *
8982  *    Dimensions:
8983  *    target_el_table[2][2][2][2][2][4]
8984  *                    |  |  |  |  |  +--- Current EL
8985  *                    |  |  |  |  +------ Non-secure(0)/Secure(1)
8986  *                    |  |  |  +--------- HCR mask override
8987  *                    |  |  +------------ SCR exec state control
8988  *                    |  +--------------- SCR mask override
8989  *                    +------------------ 32-bit(0)/64-bit(1) EL3
8990  *
8991  *    The table values are as such:
8992  *    0-3 = EL0-EL3
8993  *     -1 = Cannot occur
8994  *
8995  * The ARM ARM target EL table includes entries indicating that an "exception
8996  * is not taken".  The two cases where this is applicable are:
8997  *    1) An exception is taken from EL3 but the SCR does not have the exception
8998  *    routed to EL3.
8999  *    2) An exception is taken from EL2 but the HCR does not have the exception
9000  *    routed to EL2.
9001  * In these two cases, the below table contain a target of EL1.  This value is
9002  * returned as it is expected that the consumer of the table data will check
9003  * for "target EL >= current EL" to ensure the exception is not taken.
9004  *
9005  *            SCR     HCR
9006  *         64  EA     AMO                 From
9007  *        BIT IRQ     IMO      Non-secure         Secure
9008  *        EL3 FIQ  RW FMO   EL0 EL1 EL2 EL3   EL0 EL1 EL2 EL3
9009  */
9010 static const int8_t target_el_table[2][2][2][2][2][4] = {
9011     {{{{/* 0   0   0   0 */{ 1,  1,  2, -1 },{ 3, -1, -1,  3 },},
9012        {/* 0   0   0   1 */{ 2,  2,  2, -1 },{ 3, -1, -1,  3 },},},
9013       {{/* 0   0   1   0 */{ 1,  1,  2, -1 },{ 3, -1, -1,  3 },},
9014        {/* 0   0   1   1 */{ 2,  2,  2, -1 },{ 3, -1, -1,  3 },},},},
9015      {{{/* 0   1   0   0 */{ 3,  3,  3, -1 },{ 3, -1, -1,  3 },},
9016        {/* 0   1   0   1 */{ 3,  3,  3, -1 },{ 3, -1, -1,  3 },},},
9017       {{/* 0   1   1   0 */{ 3,  3,  3, -1 },{ 3, -1, -1,  3 },},
9018        {/* 0   1   1   1 */{ 3,  3,  3, -1 },{ 3, -1, -1,  3 },},},},},
9019     {{{{/* 1   0   0   0 */{ 1,  1,  2, -1 },{ 1,  1, -1,  1 },},
9020        {/* 1   0   0   1 */{ 2,  2,  2, -1 },{ 1,  1, -1,  1 },},},
9021       {{/* 1   0   1   0 */{ 1,  1,  1, -1 },{ 1,  1, -1,  1 },},
9022        {/* 1   0   1   1 */{ 2,  2,  2, -1 },{ 1,  1, -1,  1 },},},},
9023      {{{/* 1   1   0   0 */{ 3,  3,  3, -1 },{ 3,  3, -1,  3 },},
9024        {/* 1   1   0   1 */{ 3,  3,  3, -1 },{ 3,  3, -1,  3 },},},
9025       {{/* 1   1   1   0 */{ 3,  3,  3, -1 },{ 3,  3, -1,  3 },},
9026        {/* 1   1   1   1 */{ 3,  3,  3, -1 },{ 3,  3, -1,  3 },},},},},
9027 };
9028 
9029 /*
9030  * Determine the target EL for physical exceptions
9031  */
9032 uint32_t arm_phys_excp_target_el(CPUState *cs, uint32_t excp_idx,
9033                                  uint32_t cur_el, bool secure)
9034 {
9035     CPUARMState *env = cs->env_ptr;
9036     bool rw;
9037     bool scr;
9038     bool hcr;
9039     int target_el;
9040     /* Is the highest EL AArch64? */
9041     bool is64 = arm_feature(env, ARM_FEATURE_AARCH64);
9042     uint64_t hcr_el2;
9043 
9044     if (arm_feature(env, ARM_FEATURE_EL3)) {
9045         rw = ((env->cp15.scr_el3 & SCR_RW) == SCR_RW);
9046     } else {
9047         /* Either EL2 is the highest EL (and so the EL2 register width
9048          * is given by is64); or there is no EL2 or EL3, in which case
9049          * the value of 'rw' does not affect the table lookup anyway.
9050          */
9051         rw = is64;
9052     }
9053 
9054     hcr_el2 = arm_hcr_el2_eff(env);
9055     switch (excp_idx) {
9056     case EXCP_IRQ:
9057         scr = ((env->cp15.scr_el3 & SCR_IRQ) == SCR_IRQ);
9058         hcr = hcr_el2 & HCR_IMO;
9059         break;
9060     case EXCP_FIQ:
9061         scr = ((env->cp15.scr_el3 & SCR_FIQ) == SCR_FIQ);
9062         hcr = hcr_el2 & HCR_FMO;
9063         break;
9064     default:
9065         scr = ((env->cp15.scr_el3 & SCR_EA) == SCR_EA);
9066         hcr = hcr_el2 & HCR_AMO;
9067         break;
9068     };
9069 
9070     /*
9071      * For these purposes, TGE and AMO/IMO/FMO both force the
9072      * interrupt to EL2.  Fold TGE into the bit extracted above.
9073      */
9074     hcr |= (hcr_el2 & HCR_TGE) != 0;
9075 
9076     /* Perform a table-lookup for the target EL given the current state */
9077     target_el = target_el_table[is64][scr][rw][hcr][secure][cur_el];
9078 
9079     assert(target_el > 0);
9080 
9081     return target_el;
9082 }
9083 
9084 void arm_log_exception(int idx)
9085 {
9086     if (qemu_loglevel_mask(CPU_LOG_INT)) {
9087         const char *exc = NULL;
9088         static const char * const excnames[] = {
9089             [EXCP_UDEF] = "Undefined Instruction",
9090             [EXCP_SWI] = "SVC",
9091             [EXCP_PREFETCH_ABORT] = "Prefetch Abort",
9092             [EXCP_DATA_ABORT] = "Data Abort",
9093             [EXCP_IRQ] = "IRQ",
9094             [EXCP_FIQ] = "FIQ",
9095             [EXCP_BKPT] = "Breakpoint",
9096             [EXCP_EXCEPTION_EXIT] = "QEMU v7M exception exit",
9097             [EXCP_KERNEL_TRAP] = "QEMU intercept of kernel commpage",
9098             [EXCP_HVC] = "Hypervisor Call",
9099             [EXCP_HYP_TRAP] = "Hypervisor Trap",
9100             [EXCP_SMC] = "Secure Monitor Call",
9101             [EXCP_VIRQ] = "Virtual IRQ",
9102             [EXCP_VFIQ] = "Virtual FIQ",
9103             [EXCP_SEMIHOST] = "Semihosting call",
9104             [EXCP_NOCP] = "v7M NOCP UsageFault",
9105             [EXCP_INVSTATE] = "v7M INVSTATE UsageFault",
9106             [EXCP_STKOF] = "v8M STKOF UsageFault",
9107             [EXCP_LAZYFP] = "v7M exception during lazy FP stacking",
9108             [EXCP_LSERR] = "v8M LSERR UsageFault",
9109             [EXCP_UNALIGNED] = "v7M UNALIGNED UsageFault",
9110         };
9111 
9112         if (idx >= 0 && idx < ARRAY_SIZE(excnames)) {
9113             exc = excnames[idx];
9114         }
9115         if (!exc) {
9116             exc = "unknown";
9117         }
9118         qemu_log_mask(CPU_LOG_INT, "Taking exception %d [%s]\n", idx, exc);
9119     }
9120 }
9121 
9122 /*
9123  * Function used to synchronize QEMU's AArch64 register set with AArch32
9124  * register set.  This is necessary when switching between AArch32 and AArch64
9125  * execution state.
9126  */
9127 void aarch64_sync_32_to_64(CPUARMState *env)
9128 {
9129     int i;
9130     uint32_t mode = env->uncached_cpsr & CPSR_M;
9131 
9132     /* We can blanket copy R[0:7] to X[0:7] */
9133     for (i = 0; i < 8; i++) {
9134         env->xregs[i] = env->regs[i];
9135     }
9136 
9137     /*
9138      * Unless we are in FIQ mode, x8-x12 come from the user registers r8-r12.
9139      * Otherwise, they come from the banked user regs.
9140      */
9141     if (mode == ARM_CPU_MODE_FIQ) {
9142         for (i = 8; i < 13; i++) {
9143             env->xregs[i] = env->usr_regs[i - 8];
9144         }
9145     } else {
9146         for (i = 8; i < 13; i++) {
9147             env->xregs[i] = env->regs[i];
9148         }
9149     }
9150 
9151     /*
9152      * Registers x13-x23 are the various mode SP and FP registers. Registers
9153      * r13 and r14 are only copied if we are in that mode, otherwise we copy
9154      * from the mode banked register.
9155      */
9156     if (mode == ARM_CPU_MODE_USR || mode == ARM_CPU_MODE_SYS) {
9157         env->xregs[13] = env->regs[13];
9158         env->xregs[14] = env->regs[14];
9159     } else {
9160         env->xregs[13] = env->banked_r13[bank_number(ARM_CPU_MODE_USR)];
9161         /* HYP is an exception in that it is copied from r14 */
9162         if (mode == ARM_CPU_MODE_HYP) {
9163             env->xregs[14] = env->regs[14];
9164         } else {
9165             env->xregs[14] = env->banked_r14[r14_bank_number(ARM_CPU_MODE_USR)];
9166         }
9167     }
9168 
9169     if (mode == ARM_CPU_MODE_HYP) {
9170         env->xregs[15] = env->regs[13];
9171     } else {
9172         env->xregs[15] = env->banked_r13[bank_number(ARM_CPU_MODE_HYP)];
9173     }
9174 
9175     if (mode == ARM_CPU_MODE_IRQ) {
9176         env->xregs[16] = env->regs[14];
9177         env->xregs[17] = env->regs[13];
9178     } else {
9179         env->xregs[16] = env->banked_r14[r14_bank_number(ARM_CPU_MODE_IRQ)];
9180         env->xregs[17] = env->banked_r13[bank_number(ARM_CPU_MODE_IRQ)];
9181     }
9182 
9183     if (mode == ARM_CPU_MODE_SVC) {
9184         env->xregs[18] = env->regs[14];
9185         env->xregs[19] = env->regs[13];
9186     } else {
9187         env->xregs[18] = env->banked_r14[r14_bank_number(ARM_CPU_MODE_SVC)];
9188         env->xregs[19] = env->banked_r13[bank_number(ARM_CPU_MODE_SVC)];
9189     }
9190 
9191     if (mode == ARM_CPU_MODE_ABT) {
9192         env->xregs[20] = env->regs[14];
9193         env->xregs[21] = env->regs[13];
9194     } else {
9195         env->xregs[20] = env->banked_r14[r14_bank_number(ARM_CPU_MODE_ABT)];
9196         env->xregs[21] = env->banked_r13[bank_number(ARM_CPU_MODE_ABT)];
9197     }
9198 
9199     if (mode == ARM_CPU_MODE_UND) {
9200         env->xregs[22] = env->regs[14];
9201         env->xregs[23] = env->regs[13];
9202     } else {
9203         env->xregs[22] = env->banked_r14[r14_bank_number(ARM_CPU_MODE_UND)];
9204         env->xregs[23] = env->banked_r13[bank_number(ARM_CPU_MODE_UND)];
9205     }
9206 
9207     /*
9208      * Registers x24-x30 are mapped to r8-r14 in FIQ mode.  If we are in FIQ
9209      * mode, then we can copy from r8-r14.  Otherwise, we copy from the
9210      * FIQ bank for r8-r14.
9211      */
9212     if (mode == ARM_CPU_MODE_FIQ) {
9213         for (i = 24; i < 31; i++) {
9214             env->xregs[i] = env->regs[i - 16];   /* X[24:30] <- R[8:14] */
9215         }
9216     } else {
9217         for (i = 24; i < 29; i++) {
9218             env->xregs[i] = env->fiq_regs[i - 24];
9219         }
9220         env->xregs[29] = env->banked_r13[bank_number(ARM_CPU_MODE_FIQ)];
9221         env->xregs[30] = env->banked_r14[r14_bank_number(ARM_CPU_MODE_FIQ)];
9222     }
9223 
9224     env->pc = env->regs[15];
9225 }
9226 
9227 /*
9228  * Function used to synchronize QEMU's AArch32 register set with AArch64
9229  * register set.  This is necessary when switching between AArch32 and AArch64
9230  * execution state.
9231  */
9232 void aarch64_sync_64_to_32(CPUARMState *env)
9233 {
9234     int i;
9235     uint32_t mode = env->uncached_cpsr & CPSR_M;
9236 
9237     /* We can blanket copy X[0:7] to R[0:7] */
9238     for (i = 0; i < 8; i++) {
9239         env->regs[i] = env->xregs[i];
9240     }
9241 
9242     /*
9243      * Unless we are in FIQ mode, r8-r12 come from the user registers x8-x12.
9244      * Otherwise, we copy x8-x12 into the banked user regs.
9245      */
9246     if (mode == ARM_CPU_MODE_FIQ) {
9247         for (i = 8; i < 13; i++) {
9248             env->usr_regs[i - 8] = env->xregs[i];
9249         }
9250     } else {
9251         for (i = 8; i < 13; i++) {
9252             env->regs[i] = env->xregs[i];
9253         }
9254     }
9255 
9256     /*
9257      * Registers r13 & r14 depend on the current mode.
9258      * If we are in a given mode, we copy the corresponding x registers to r13
9259      * and r14.  Otherwise, we copy the x register to the banked r13 and r14
9260      * for the mode.
9261      */
9262     if (mode == ARM_CPU_MODE_USR || mode == ARM_CPU_MODE_SYS) {
9263         env->regs[13] = env->xregs[13];
9264         env->regs[14] = env->xregs[14];
9265     } else {
9266         env->banked_r13[bank_number(ARM_CPU_MODE_USR)] = env->xregs[13];
9267 
9268         /*
9269          * HYP is an exception in that it does not have its own banked r14 but
9270          * shares the USR r14
9271          */
9272         if (mode == ARM_CPU_MODE_HYP) {
9273             env->regs[14] = env->xregs[14];
9274         } else {
9275             env->banked_r14[r14_bank_number(ARM_CPU_MODE_USR)] = env->xregs[14];
9276         }
9277     }
9278 
9279     if (mode == ARM_CPU_MODE_HYP) {
9280         env->regs[13] = env->xregs[15];
9281     } else {
9282         env->banked_r13[bank_number(ARM_CPU_MODE_HYP)] = env->xregs[15];
9283     }
9284 
9285     if (mode == ARM_CPU_MODE_IRQ) {
9286         env->regs[14] = env->xregs[16];
9287         env->regs[13] = env->xregs[17];
9288     } else {
9289         env->banked_r14[r14_bank_number(ARM_CPU_MODE_IRQ)] = env->xregs[16];
9290         env->banked_r13[bank_number(ARM_CPU_MODE_IRQ)] = env->xregs[17];
9291     }
9292 
9293     if (mode == ARM_CPU_MODE_SVC) {
9294         env->regs[14] = env->xregs[18];
9295         env->regs[13] = env->xregs[19];
9296     } else {
9297         env->banked_r14[r14_bank_number(ARM_CPU_MODE_SVC)] = env->xregs[18];
9298         env->banked_r13[bank_number(ARM_CPU_MODE_SVC)] = env->xregs[19];
9299     }
9300 
9301     if (mode == ARM_CPU_MODE_ABT) {
9302         env->regs[14] = env->xregs[20];
9303         env->regs[13] = env->xregs[21];
9304     } else {
9305         env->banked_r14[r14_bank_number(ARM_CPU_MODE_ABT)] = env->xregs[20];
9306         env->banked_r13[bank_number(ARM_CPU_MODE_ABT)] = env->xregs[21];
9307     }
9308 
9309     if (mode == ARM_CPU_MODE_UND) {
9310         env->regs[14] = env->xregs[22];
9311         env->regs[13] = env->xregs[23];
9312     } else {
9313         env->banked_r14[r14_bank_number(ARM_CPU_MODE_UND)] = env->xregs[22];
9314         env->banked_r13[bank_number(ARM_CPU_MODE_UND)] = env->xregs[23];
9315     }
9316 
9317     /* Registers x24-x30 are mapped to r8-r14 in FIQ mode.  If we are in FIQ
9318      * mode, then we can copy to r8-r14.  Otherwise, we copy to the
9319      * FIQ bank for r8-r14.
9320      */
9321     if (mode == ARM_CPU_MODE_FIQ) {
9322         for (i = 24; i < 31; i++) {
9323             env->regs[i - 16] = env->xregs[i];   /* X[24:30] -> R[8:14] */
9324         }
9325     } else {
9326         for (i = 24; i < 29; i++) {
9327             env->fiq_regs[i - 24] = env->xregs[i];
9328         }
9329         env->banked_r13[bank_number(ARM_CPU_MODE_FIQ)] = env->xregs[29];
9330         env->banked_r14[r14_bank_number(ARM_CPU_MODE_FIQ)] = env->xregs[30];
9331     }
9332 
9333     env->regs[15] = env->pc;
9334 }
9335 
9336 static void take_aarch32_exception(CPUARMState *env, int new_mode,
9337                                    uint32_t mask, uint32_t offset,
9338                                    uint32_t newpc)
9339 {
9340     int new_el;
9341 
9342     /* Change the CPU state so as to actually take the exception. */
9343     switch_mode(env, new_mode);
9344 
9345     /*
9346      * For exceptions taken to AArch32 we must clear the SS bit in both
9347      * PSTATE and in the old-state value we save to SPSR_<mode>, so zero it now.
9348      */
9349     env->uncached_cpsr &= ~PSTATE_SS;
9350     env->spsr = cpsr_read(env);
9351     /* Clear IT bits.  */
9352     env->condexec_bits = 0;
9353     /* Switch to the new mode, and to the correct instruction set.  */
9354     env->uncached_cpsr = (env->uncached_cpsr & ~CPSR_M) | new_mode;
9355 
9356     /* This must be after mode switching. */
9357     new_el = arm_current_el(env);
9358 
9359     /* Set new mode endianness */
9360     env->uncached_cpsr &= ~CPSR_E;
9361     if (env->cp15.sctlr_el[new_el] & SCTLR_EE) {
9362         env->uncached_cpsr |= CPSR_E;
9363     }
9364     /* J and IL must always be cleared for exception entry */
9365     env->uncached_cpsr &= ~(CPSR_IL | CPSR_J);
9366     env->daif |= mask;
9367 
9368     if (new_mode == ARM_CPU_MODE_HYP) {
9369         env->thumb = (env->cp15.sctlr_el[2] & SCTLR_TE) != 0;
9370         env->elr_el[2] = env->regs[15];
9371     } else {
9372         /* CPSR.PAN is normally preserved preserved unless...  */
9373         if (cpu_isar_feature(aa32_pan, env_archcpu(env))) {
9374             switch (new_el) {
9375             case 3:
9376                 if (!arm_is_secure_below_el3(env)) {
9377                     /* ... the target is EL3, from non-secure state.  */
9378                     env->uncached_cpsr &= ~CPSR_PAN;
9379                     break;
9380                 }
9381                 /* ... the target is EL3, from secure state ... */
9382                 /* fall through */
9383             case 1:
9384                 /* ... the target is EL1 and SCTLR.SPAN is 0.  */
9385                 if (!(env->cp15.sctlr_el[new_el] & SCTLR_SPAN)) {
9386                     env->uncached_cpsr |= CPSR_PAN;
9387                 }
9388                 break;
9389             }
9390         }
9391         /*
9392          * this is a lie, as there was no c1_sys on V4T/V5, but who cares
9393          * and we should just guard the thumb mode on V4
9394          */
9395         if (arm_feature(env, ARM_FEATURE_V4T)) {
9396             env->thumb =
9397                 (A32_BANKED_CURRENT_REG_GET(env, sctlr) & SCTLR_TE) != 0;
9398         }
9399         env->regs[14] = env->regs[15] + offset;
9400     }
9401     env->regs[15] = newpc;
9402     arm_rebuild_hflags(env);
9403 }
9404 
9405 static void arm_cpu_do_interrupt_aarch32_hyp(CPUState *cs)
9406 {
9407     /*
9408      * Handle exception entry to Hyp mode; this is sufficiently
9409      * different to entry to other AArch32 modes that we handle it
9410      * separately here.
9411      *
9412      * The vector table entry used is always the 0x14 Hyp mode entry point,
9413      * unless this is an UNDEF/HVC/abort taken from Hyp to Hyp.
9414      * The offset applied to the preferred return address is always zero
9415      * (see DDI0487C.a section G1.12.3).
9416      * PSTATE A/I/F masks are set based only on the SCR.EA/IRQ/FIQ values.
9417      */
9418     uint32_t addr, mask;
9419     ARMCPU *cpu = ARM_CPU(cs);
9420     CPUARMState *env = &cpu->env;
9421 
9422     switch (cs->exception_index) {
9423     case EXCP_UDEF:
9424         addr = 0x04;
9425         break;
9426     case EXCP_SWI:
9427         addr = 0x14;
9428         break;
9429     case EXCP_BKPT:
9430         /* Fall through to prefetch abort.  */
9431     case EXCP_PREFETCH_ABORT:
9432         env->cp15.ifar_s = env->exception.vaddress;
9433         qemu_log_mask(CPU_LOG_INT, "...with HIFAR 0x%x\n",
9434                       (uint32_t)env->exception.vaddress);
9435         addr = 0x0c;
9436         break;
9437     case EXCP_DATA_ABORT:
9438         env->cp15.dfar_s = env->exception.vaddress;
9439         qemu_log_mask(CPU_LOG_INT, "...with HDFAR 0x%x\n",
9440                       (uint32_t)env->exception.vaddress);
9441         addr = 0x10;
9442         break;
9443     case EXCP_IRQ:
9444         addr = 0x18;
9445         break;
9446     case EXCP_FIQ:
9447         addr = 0x1c;
9448         break;
9449     case EXCP_HVC:
9450         addr = 0x08;
9451         break;
9452     case EXCP_HYP_TRAP:
9453         addr = 0x14;
9454         break;
9455     default:
9456         cpu_abort(cs, "Unhandled exception 0x%x\n", cs->exception_index);
9457     }
9458 
9459     if (cs->exception_index != EXCP_IRQ && cs->exception_index != EXCP_FIQ) {
9460         if (!arm_feature(env, ARM_FEATURE_V8)) {
9461             /*
9462              * QEMU syndrome values are v8-style. v7 has the IL bit
9463              * UNK/SBZP for "field not valid" cases, where v8 uses RES1.
9464              * If this is a v7 CPU, squash the IL bit in those cases.
9465              */
9466             if (cs->exception_index == EXCP_PREFETCH_ABORT ||
9467                 (cs->exception_index == EXCP_DATA_ABORT &&
9468                  !(env->exception.syndrome & ARM_EL_ISV)) ||
9469                 syn_get_ec(env->exception.syndrome) == EC_UNCATEGORIZED) {
9470                 env->exception.syndrome &= ~ARM_EL_IL;
9471             }
9472         }
9473         env->cp15.esr_el[2] = env->exception.syndrome;
9474     }
9475 
9476     if (arm_current_el(env) != 2 && addr < 0x14) {
9477         addr = 0x14;
9478     }
9479 
9480     mask = 0;
9481     if (!(env->cp15.scr_el3 & SCR_EA)) {
9482         mask |= CPSR_A;
9483     }
9484     if (!(env->cp15.scr_el3 & SCR_IRQ)) {
9485         mask |= CPSR_I;
9486     }
9487     if (!(env->cp15.scr_el3 & SCR_FIQ)) {
9488         mask |= CPSR_F;
9489     }
9490 
9491     addr += env->cp15.hvbar;
9492 
9493     take_aarch32_exception(env, ARM_CPU_MODE_HYP, mask, 0, addr);
9494 }
9495 
9496 static void arm_cpu_do_interrupt_aarch32(CPUState *cs)
9497 {
9498     ARMCPU *cpu = ARM_CPU(cs);
9499     CPUARMState *env = &cpu->env;
9500     uint32_t addr;
9501     uint32_t mask;
9502     int new_mode;
9503     uint32_t offset;
9504     uint32_t moe;
9505 
9506     /* If this is a debug exception we must update the DBGDSCR.MOE bits */
9507     switch (syn_get_ec(env->exception.syndrome)) {
9508     case EC_BREAKPOINT:
9509     case EC_BREAKPOINT_SAME_EL:
9510         moe = 1;
9511         break;
9512     case EC_WATCHPOINT:
9513     case EC_WATCHPOINT_SAME_EL:
9514         moe = 10;
9515         break;
9516     case EC_AA32_BKPT:
9517         moe = 3;
9518         break;
9519     case EC_VECTORCATCH:
9520         moe = 5;
9521         break;
9522     default:
9523         moe = 0;
9524         break;
9525     }
9526 
9527     if (moe) {
9528         env->cp15.mdscr_el1 = deposit64(env->cp15.mdscr_el1, 2, 4, moe);
9529     }
9530 
9531     if (env->exception.target_el == 2) {
9532         arm_cpu_do_interrupt_aarch32_hyp(cs);
9533         return;
9534     }
9535 
9536     switch (cs->exception_index) {
9537     case EXCP_UDEF:
9538         new_mode = ARM_CPU_MODE_UND;
9539         addr = 0x04;
9540         mask = CPSR_I;
9541         if (env->thumb)
9542             offset = 2;
9543         else
9544             offset = 4;
9545         break;
9546     case EXCP_SWI:
9547         new_mode = ARM_CPU_MODE_SVC;
9548         addr = 0x08;
9549         mask = CPSR_I;
9550         /* The PC already points to the next instruction.  */
9551         offset = 0;
9552         break;
9553     case EXCP_BKPT:
9554         /* Fall through to prefetch abort.  */
9555     case EXCP_PREFETCH_ABORT:
9556         A32_BANKED_CURRENT_REG_SET(env, ifsr, env->exception.fsr);
9557         A32_BANKED_CURRENT_REG_SET(env, ifar, env->exception.vaddress);
9558         qemu_log_mask(CPU_LOG_INT, "...with IFSR 0x%x IFAR 0x%x\n",
9559                       env->exception.fsr, (uint32_t)env->exception.vaddress);
9560         new_mode = ARM_CPU_MODE_ABT;
9561         addr = 0x0c;
9562         mask = CPSR_A | CPSR_I;
9563         offset = 4;
9564         break;
9565     case EXCP_DATA_ABORT:
9566         A32_BANKED_CURRENT_REG_SET(env, dfsr, env->exception.fsr);
9567         A32_BANKED_CURRENT_REG_SET(env, dfar, env->exception.vaddress);
9568         qemu_log_mask(CPU_LOG_INT, "...with DFSR 0x%x DFAR 0x%x\n",
9569                       env->exception.fsr,
9570                       (uint32_t)env->exception.vaddress);
9571         new_mode = ARM_CPU_MODE_ABT;
9572         addr = 0x10;
9573         mask = CPSR_A | CPSR_I;
9574         offset = 8;
9575         break;
9576     case EXCP_IRQ:
9577         new_mode = ARM_CPU_MODE_IRQ;
9578         addr = 0x18;
9579         /* Disable IRQ and imprecise data aborts.  */
9580         mask = CPSR_A | CPSR_I;
9581         offset = 4;
9582         if (env->cp15.scr_el3 & SCR_IRQ) {
9583             /* IRQ routed to monitor mode */
9584             new_mode = ARM_CPU_MODE_MON;
9585             mask |= CPSR_F;
9586         }
9587         break;
9588     case EXCP_FIQ:
9589         new_mode = ARM_CPU_MODE_FIQ;
9590         addr = 0x1c;
9591         /* Disable FIQ, IRQ and imprecise data aborts.  */
9592         mask = CPSR_A | CPSR_I | CPSR_F;
9593         if (env->cp15.scr_el3 & SCR_FIQ) {
9594             /* FIQ routed to monitor mode */
9595             new_mode = ARM_CPU_MODE_MON;
9596         }
9597         offset = 4;
9598         break;
9599     case EXCP_VIRQ:
9600         new_mode = ARM_CPU_MODE_IRQ;
9601         addr = 0x18;
9602         /* Disable IRQ and imprecise data aborts.  */
9603         mask = CPSR_A | CPSR_I;
9604         offset = 4;
9605         break;
9606     case EXCP_VFIQ:
9607         new_mode = ARM_CPU_MODE_FIQ;
9608         addr = 0x1c;
9609         /* Disable FIQ, IRQ and imprecise data aborts.  */
9610         mask = CPSR_A | CPSR_I | CPSR_F;
9611         offset = 4;
9612         break;
9613     case EXCP_SMC:
9614         new_mode = ARM_CPU_MODE_MON;
9615         addr = 0x08;
9616         mask = CPSR_A | CPSR_I | CPSR_F;
9617         offset = 0;
9618         break;
9619     default:
9620         cpu_abort(cs, "Unhandled exception 0x%x\n", cs->exception_index);
9621         return; /* Never happens.  Keep compiler happy.  */
9622     }
9623 
9624     if (new_mode == ARM_CPU_MODE_MON) {
9625         addr += env->cp15.mvbar;
9626     } else if (A32_BANKED_CURRENT_REG_GET(env, sctlr) & SCTLR_V) {
9627         /* High vectors. When enabled, base address cannot be remapped. */
9628         addr += 0xffff0000;
9629     } else {
9630         /* ARM v7 architectures provide a vector base address register to remap
9631          * the interrupt vector table.
9632          * This register is only followed in non-monitor mode, and is banked.
9633          * Note: only bits 31:5 are valid.
9634          */
9635         addr += A32_BANKED_CURRENT_REG_GET(env, vbar);
9636     }
9637 
9638     if ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_MON) {
9639         env->cp15.scr_el3 &= ~SCR_NS;
9640     }
9641 
9642     take_aarch32_exception(env, new_mode, mask, offset, addr);
9643 }
9644 
9645 static int aarch64_regnum(CPUARMState *env, int aarch32_reg)
9646 {
9647     /*
9648      * Return the register number of the AArch64 view of the AArch32
9649      * register @aarch32_reg. The CPUARMState CPSR is assumed to still
9650      * be that of the AArch32 mode the exception came from.
9651      */
9652     int mode = env->uncached_cpsr & CPSR_M;
9653 
9654     switch (aarch32_reg) {
9655     case 0 ... 7:
9656         return aarch32_reg;
9657     case 8 ... 12:
9658         return mode == ARM_CPU_MODE_FIQ ? aarch32_reg + 16 : aarch32_reg;
9659     case 13:
9660         switch (mode) {
9661         case ARM_CPU_MODE_USR:
9662         case ARM_CPU_MODE_SYS:
9663             return 13;
9664         case ARM_CPU_MODE_HYP:
9665             return 15;
9666         case ARM_CPU_MODE_IRQ:
9667             return 17;
9668         case ARM_CPU_MODE_SVC:
9669             return 19;
9670         case ARM_CPU_MODE_ABT:
9671             return 21;
9672         case ARM_CPU_MODE_UND:
9673             return 23;
9674         case ARM_CPU_MODE_FIQ:
9675             return 29;
9676         default:
9677             g_assert_not_reached();
9678         }
9679     case 14:
9680         switch (mode) {
9681         case ARM_CPU_MODE_USR:
9682         case ARM_CPU_MODE_SYS:
9683         case ARM_CPU_MODE_HYP:
9684             return 14;
9685         case ARM_CPU_MODE_IRQ:
9686             return 16;
9687         case ARM_CPU_MODE_SVC:
9688             return 18;
9689         case ARM_CPU_MODE_ABT:
9690             return 20;
9691         case ARM_CPU_MODE_UND:
9692             return 22;
9693         case ARM_CPU_MODE_FIQ:
9694             return 30;
9695         default:
9696             g_assert_not_reached();
9697         }
9698     case 15:
9699         return 31;
9700     default:
9701         g_assert_not_reached();
9702     }
9703 }
9704 
9705 /* Handle exception entry to a target EL which is using AArch64 */
9706 static void arm_cpu_do_interrupt_aarch64(CPUState *cs)
9707 {
9708     ARMCPU *cpu = ARM_CPU(cs);
9709     CPUARMState *env = &cpu->env;
9710     unsigned int new_el = env->exception.target_el;
9711     target_ulong addr = env->cp15.vbar_el[new_el];
9712     unsigned int new_mode = aarch64_pstate_mode(new_el, true);
9713     unsigned int old_mode;
9714     unsigned int cur_el = arm_current_el(env);
9715     int rt;
9716 
9717     /*
9718      * Note that new_el can never be 0.  If cur_el is 0, then
9719      * el0_a64 is is_a64(), else el0_a64 is ignored.
9720      */
9721     aarch64_sve_change_el(env, cur_el, new_el, is_a64(env));
9722 
9723     if (cur_el < new_el) {
9724         /* Entry vector offset depends on whether the implemented EL
9725          * immediately lower than the target level is using AArch32 or AArch64
9726          */
9727         bool is_aa64;
9728         uint64_t hcr;
9729 
9730         switch (new_el) {
9731         case 3:
9732             is_aa64 = (env->cp15.scr_el3 & SCR_RW) != 0;
9733             break;
9734         case 2:
9735             hcr = arm_hcr_el2_eff(env);
9736             if ((hcr & (HCR_E2H | HCR_TGE)) != (HCR_E2H | HCR_TGE)) {
9737                 is_aa64 = (hcr & HCR_RW) != 0;
9738                 break;
9739             }
9740             /* fall through */
9741         case 1:
9742             is_aa64 = is_a64(env);
9743             break;
9744         default:
9745             g_assert_not_reached();
9746         }
9747 
9748         if (is_aa64) {
9749             addr += 0x400;
9750         } else {
9751             addr += 0x600;
9752         }
9753     } else if (pstate_read(env) & PSTATE_SP) {
9754         addr += 0x200;
9755     }
9756 
9757     switch (cs->exception_index) {
9758     case EXCP_PREFETCH_ABORT:
9759     case EXCP_DATA_ABORT:
9760         env->cp15.far_el[new_el] = env->exception.vaddress;
9761         qemu_log_mask(CPU_LOG_INT, "...with FAR 0x%" PRIx64 "\n",
9762                       env->cp15.far_el[new_el]);
9763         /* fall through */
9764     case EXCP_BKPT:
9765     case EXCP_UDEF:
9766     case EXCP_SWI:
9767     case EXCP_HVC:
9768     case EXCP_HYP_TRAP:
9769     case EXCP_SMC:
9770         switch (syn_get_ec(env->exception.syndrome)) {
9771         case EC_ADVSIMDFPACCESSTRAP:
9772             /*
9773              * QEMU internal FP/SIMD syndromes from AArch32 include the
9774              * TA and coproc fields which are only exposed if the exception
9775              * is taken to AArch32 Hyp mode. Mask them out to get a valid
9776              * AArch64 format syndrome.
9777              */
9778             env->exception.syndrome &= ~MAKE_64BIT_MASK(0, 20);
9779             break;
9780         case EC_CP14RTTRAP:
9781         case EC_CP15RTTRAP:
9782         case EC_CP14DTTRAP:
9783             /*
9784              * For a trap on AArch32 MRC/MCR/LDC/STC the Rt field is currently
9785              * the raw register field from the insn; when taking this to
9786              * AArch64 we must convert it to the AArch64 view of the register
9787              * number. Notice that we read a 4-bit AArch32 register number and
9788              * write back a 5-bit AArch64 one.
9789              */
9790             rt = extract32(env->exception.syndrome, 5, 4);
9791             rt = aarch64_regnum(env, rt);
9792             env->exception.syndrome = deposit32(env->exception.syndrome,
9793                                                 5, 5, rt);
9794             break;
9795         case EC_CP15RRTTRAP:
9796         case EC_CP14RRTTRAP:
9797             /* Similarly for MRRC/MCRR traps for Rt and Rt2 fields */
9798             rt = extract32(env->exception.syndrome, 5, 4);
9799             rt = aarch64_regnum(env, rt);
9800             env->exception.syndrome = deposit32(env->exception.syndrome,
9801                                                 5, 5, rt);
9802             rt = extract32(env->exception.syndrome, 10, 4);
9803             rt = aarch64_regnum(env, rt);
9804             env->exception.syndrome = deposit32(env->exception.syndrome,
9805                                                 10, 5, rt);
9806             break;
9807         }
9808         env->cp15.esr_el[new_el] = env->exception.syndrome;
9809         break;
9810     case EXCP_IRQ:
9811     case EXCP_VIRQ:
9812         addr += 0x80;
9813         break;
9814     case EXCP_FIQ:
9815     case EXCP_VFIQ:
9816         addr += 0x100;
9817         break;
9818     default:
9819         cpu_abort(cs, "Unhandled exception 0x%x\n", cs->exception_index);
9820     }
9821 
9822     if (is_a64(env)) {
9823         old_mode = pstate_read(env);
9824         aarch64_save_sp(env, arm_current_el(env));
9825         env->elr_el[new_el] = env->pc;
9826     } else {
9827         old_mode = cpsr_read(env);
9828         env->elr_el[new_el] = env->regs[15];
9829 
9830         aarch64_sync_32_to_64(env);
9831 
9832         env->condexec_bits = 0;
9833     }
9834     env->banked_spsr[aarch64_banked_spsr_index(new_el)] = old_mode;
9835 
9836     qemu_log_mask(CPU_LOG_INT, "...with ELR 0x%" PRIx64 "\n",
9837                   env->elr_el[new_el]);
9838 
9839     if (cpu_isar_feature(aa64_pan, cpu)) {
9840         /* The value of PSTATE.PAN is normally preserved, except when ... */
9841         new_mode |= old_mode & PSTATE_PAN;
9842         switch (new_el) {
9843         case 2:
9844             /* ... the target is EL2 with HCR_EL2.{E2H,TGE} == '11' ...  */
9845             if ((arm_hcr_el2_eff(env) & (HCR_E2H | HCR_TGE))
9846                 != (HCR_E2H | HCR_TGE)) {
9847                 break;
9848             }
9849             /* fall through */
9850         case 1:
9851             /* ... the target is EL1 ... */
9852             /* ... and SCTLR_ELx.SPAN == 0, then set to 1.  */
9853             if ((env->cp15.sctlr_el[new_el] & SCTLR_SPAN) == 0) {
9854                 new_mode |= PSTATE_PAN;
9855             }
9856             break;
9857         }
9858     }
9859     if (cpu_isar_feature(aa64_mte, cpu)) {
9860         new_mode |= PSTATE_TCO;
9861     }
9862 
9863     pstate_write(env, PSTATE_DAIF | new_mode);
9864     env->aarch64 = 1;
9865     aarch64_restore_sp(env, new_el);
9866     helper_rebuild_hflags_a64(env, new_el);
9867 
9868     env->pc = addr;
9869 
9870     qemu_log_mask(CPU_LOG_INT, "...to EL%d PC 0x%" PRIx64 " PSTATE 0x%x\n",
9871                   new_el, env->pc, pstate_read(env));
9872 }
9873 
9874 /*
9875  * Do semihosting call and set the appropriate return value. All the
9876  * permission and validity checks have been done at translate time.
9877  *
9878  * We only see semihosting exceptions in TCG only as they are not
9879  * trapped to the hypervisor in KVM.
9880  */
9881 #ifdef CONFIG_TCG
9882 static void handle_semihosting(CPUState *cs)
9883 {
9884     ARMCPU *cpu = ARM_CPU(cs);
9885     CPUARMState *env = &cpu->env;
9886 
9887     if (is_a64(env)) {
9888         qemu_log_mask(CPU_LOG_INT,
9889                       "...handling as semihosting call 0x%" PRIx64 "\n",
9890                       env->xregs[0]);
9891         env->xregs[0] = do_arm_semihosting(env);
9892         env->pc += 4;
9893     } else {
9894         qemu_log_mask(CPU_LOG_INT,
9895                       "...handling as semihosting call 0x%x\n",
9896                       env->regs[0]);
9897         env->regs[0] = do_arm_semihosting(env);
9898         env->regs[15] += env->thumb ? 2 : 4;
9899     }
9900 }
9901 #endif
9902 
9903 /* Handle a CPU exception for A and R profile CPUs.
9904  * Do any appropriate logging, handle PSCI calls, and then hand off
9905  * to the AArch64-entry or AArch32-entry function depending on the
9906  * target exception level's register width.
9907  */
9908 void arm_cpu_do_interrupt(CPUState *cs)
9909 {
9910     ARMCPU *cpu = ARM_CPU(cs);
9911     CPUARMState *env = &cpu->env;
9912     unsigned int new_el = env->exception.target_el;
9913 
9914     assert(!arm_feature(env, ARM_FEATURE_M));
9915 
9916     arm_log_exception(cs->exception_index);
9917     qemu_log_mask(CPU_LOG_INT, "...from EL%d to EL%d\n", arm_current_el(env),
9918                   new_el);
9919     if (qemu_loglevel_mask(CPU_LOG_INT)
9920         && !excp_is_internal(cs->exception_index)) {
9921         qemu_log_mask(CPU_LOG_INT, "...with ESR 0x%x/0x%" PRIx32 "\n",
9922                       syn_get_ec(env->exception.syndrome),
9923                       env->exception.syndrome);
9924     }
9925 
9926     if (arm_is_psci_call(cpu, cs->exception_index)) {
9927         arm_handle_psci_call(cpu);
9928         qemu_log_mask(CPU_LOG_INT, "...handled as PSCI call\n");
9929         return;
9930     }
9931 
9932     /*
9933      * Semihosting semantics depend on the register width of the code
9934      * that caused the exception, not the target exception level, so
9935      * must be handled here.
9936      */
9937 #ifdef CONFIG_TCG
9938     if (cs->exception_index == EXCP_SEMIHOST) {
9939         handle_semihosting(cs);
9940         return;
9941     }
9942 #endif
9943 
9944     /* Hooks may change global state so BQL should be held, also the
9945      * BQL needs to be held for any modification of
9946      * cs->interrupt_request.
9947      */
9948     g_assert(qemu_mutex_iothread_locked());
9949 
9950     arm_call_pre_el_change_hook(cpu);
9951 
9952     assert(!excp_is_internal(cs->exception_index));
9953     if (arm_el_is_aa64(env, new_el)) {
9954         arm_cpu_do_interrupt_aarch64(cs);
9955     } else {
9956         arm_cpu_do_interrupt_aarch32(cs);
9957     }
9958 
9959     arm_call_el_change_hook(cpu);
9960 
9961     if (!kvm_enabled()) {
9962         cs->interrupt_request |= CPU_INTERRUPT_EXITTB;
9963     }
9964 }
9965 #endif /* !CONFIG_USER_ONLY */
9966 
9967 uint64_t arm_sctlr(CPUARMState *env, int el)
9968 {
9969     /* Only EL0 needs to be adjusted for EL1&0 or EL2&0. */
9970     if (el == 0) {
9971         ARMMMUIdx mmu_idx = arm_mmu_idx_el(env, 0);
9972         el = (mmu_idx == ARMMMUIdx_E20_0 ? 2 : 1);
9973     }
9974     return env->cp15.sctlr_el[el];
9975 }
9976 
9977 /* Return the SCTLR value which controls this address translation regime */
9978 static inline uint64_t regime_sctlr(CPUARMState *env, ARMMMUIdx mmu_idx)
9979 {
9980     return env->cp15.sctlr_el[regime_el(env, mmu_idx)];
9981 }
9982 
9983 #ifndef CONFIG_USER_ONLY
9984 
9985 /* Return true if the specified stage of address translation is disabled */
9986 static inline bool regime_translation_disabled(CPUARMState *env,
9987                                                ARMMMUIdx mmu_idx)
9988 {
9989     if (arm_feature(env, ARM_FEATURE_M)) {
9990         switch (env->v7m.mpu_ctrl[regime_is_secure(env, mmu_idx)] &
9991                 (R_V7M_MPU_CTRL_ENABLE_MASK | R_V7M_MPU_CTRL_HFNMIENA_MASK)) {
9992         case R_V7M_MPU_CTRL_ENABLE_MASK:
9993             /* Enabled, but not for HardFault and NMI */
9994             return mmu_idx & ARM_MMU_IDX_M_NEGPRI;
9995         case R_V7M_MPU_CTRL_ENABLE_MASK | R_V7M_MPU_CTRL_HFNMIENA_MASK:
9996             /* Enabled for all cases */
9997             return false;
9998         case 0:
9999         default:
10000             /* HFNMIENA set and ENABLE clear is UNPREDICTABLE, but
10001              * we warned about that in armv7m_nvic.c when the guest set it.
10002              */
10003             return true;
10004         }
10005     }
10006 
10007     if (mmu_idx == ARMMMUIdx_Stage2) {
10008         /* HCR.DC means HCR.VM behaves as 1 */
10009         return (env->cp15.hcr_el2 & (HCR_DC | HCR_VM)) == 0;
10010     }
10011 
10012     if (env->cp15.hcr_el2 & HCR_TGE) {
10013         /* TGE means that NS EL0/1 act as if SCTLR_EL1.M is zero */
10014         if (!regime_is_secure(env, mmu_idx) && regime_el(env, mmu_idx) == 1) {
10015             return true;
10016         }
10017     }
10018 
10019     if ((env->cp15.hcr_el2 & HCR_DC) && arm_mmu_idx_is_stage1_of_2(mmu_idx)) {
10020         /* HCR.DC means SCTLR_EL1.M behaves as 0 */
10021         return true;
10022     }
10023 
10024     return (regime_sctlr(env, mmu_idx) & SCTLR_M) == 0;
10025 }
10026 
10027 static inline bool regime_translation_big_endian(CPUARMState *env,
10028                                                  ARMMMUIdx mmu_idx)
10029 {
10030     return (regime_sctlr(env, mmu_idx) & SCTLR_EE) != 0;
10031 }
10032 
10033 /* Return the TTBR associated with this translation regime */
10034 static inline uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx,
10035                                    int ttbrn)
10036 {
10037     if (mmu_idx == ARMMMUIdx_Stage2) {
10038         return env->cp15.vttbr_el2;
10039     }
10040     if (ttbrn == 0) {
10041         return env->cp15.ttbr0_el[regime_el(env, mmu_idx)];
10042     } else {
10043         return env->cp15.ttbr1_el[regime_el(env, mmu_idx)];
10044     }
10045 }
10046 
10047 #endif /* !CONFIG_USER_ONLY */
10048 
10049 /* Convert a possible stage1+2 MMU index into the appropriate
10050  * stage 1 MMU index
10051  */
10052 static inline ARMMMUIdx stage_1_mmu_idx(ARMMMUIdx mmu_idx)
10053 {
10054     switch (mmu_idx) {
10055     case ARMMMUIdx_E10_0:
10056         return ARMMMUIdx_Stage1_E0;
10057     case ARMMMUIdx_E10_1:
10058         return ARMMMUIdx_Stage1_E1;
10059     case ARMMMUIdx_E10_1_PAN:
10060         return ARMMMUIdx_Stage1_E1_PAN;
10061     default:
10062         return mmu_idx;
10063     }
10064 }
10065 
10066 /* Return true if the translation regime is using LPAE format page tables */
10067 static inline bool regime_using_lpae_format(CPUARMState *env,
10068                                             ARMMMUIdx mmu_idx)
10069 {
10070     int el = regime_el(env, mmu_idx);
10071     if (el == 2 || arm_el_is_aa64(env, el)) {
10072         return true;
10073     }
10074     if (arm_feature(env, ARM_FEATURE_LPAE)
10075         && (regime_tcr(env, mmu_idx)->raw_tcr & TTBCR_EAE)) {
10076         return true;
10077     }
10078     return false;
10079 }
10080 
10081 /* Returns true if the stage 1 translation regime is using LPAE format page
10082  * tables. Used when raising alignment exceptions, whose FSR changes depending
10083  * on whether the long or short descriptor format is in use. */
10084 bool arm_s1_regime_using_lpae_format(CPUARMState *env, ARMMMUIdx mmu_idx)
10085 {
10086     mmu_idx = stage_1_mmu_idx(mmu_idx);
10087 
10088     return regime_using_lpae_format(env, mmu_idx);
10089 }
10090 
10091 #ifndef CONFIG_USER_ONLY
10092 static inline bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx)
10093 {
10094     switch (mmu_idx) {
10095     case ARMMMUIdx_SE10_0:
10096     case ARMMMUIdx_E20_0:
10097     case ARMMMUIdx_Stage1_E0:
10098     case ARMMMUIdx_MUser:
10099     case ARMMMUIdx_MSUser:
10100     case ARMMMUIdx_MUserNegPri:
10101     case ARMMMUIdx_MSUserNegPri:
10102         return true;
10103     default:
10104         return false;
10105     case ARMMMUIdx_E10_0:
10106     case ARMMMUIdx_E10_1:
10107     case ARMMMUIdx_E10_1_PAN:
10108         g_assert_not_reached();
10109     }
10110 }
10111 
10112 /* Translate section/page access permissions to page
10113  * R/W protection flags
10114  *
10115  * @env:         CPUARMState
10116  * @mmu_idx:     MMU index indicating required translation regime
10117  * @ap:          The 3-bit access permissions (AP[2:0])
10118  * @domain_prot: The 2-bit domain access permissions
10119  */
10120 static inline int ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx,
10121                                 int ap, int domain_prot)
10122 {
10123     bool is_user = regime_is_user(env, mmu_idx);
10124 
10125     if (domain_prot == 3) {
10126         return PAGE_READ | PAGE_WRITE;
10127     }
10128 
10129     switch (ap) {
10130     case 0:
10131         if (arm_feature(env, ARM_FEATURE_V7)) {
10132             return 0;
10133         }
10134         switch (regime_sctlr(env, mmu_idx) & (SCTLR_S | SCTLR_R)) {
10135         case SCTLR_S:
10136             return is_user ? 0 : PAGE_READ;
10137         case SCTLR_R:
10138             return PAGE_READ;
10139         default:
10140             return 0;
10141         }
10142     case 1:
10143         return is_user ? 0 : PAGE_READ | PAGE_WRITE;
10144     case 2:
10145         if (is_user) {
10146             return PAGE_READ;
10147         } else {
10148             return PAGE_READ | PAGE_WRITE;
10149         }
10150     case 3:
10151         return PAGE_READ | PAGE_WRITE;
10152     case 4: /* Reserved.  */
10153         return 0;
10154     case 5:
10155         return is_user ? 0 : PAGE_READ;
10156     case 6:
10157         return PAGE_READ;
10158     case 7:
10159         if (!arm_feature(env, ARM_FEATURE_V6K)) {
10160             return 0;
10161         }
10162         return PAGE_READ;
10163     default:
10164         g_assert_not_reached();
10165     }
10166 }
10167 
10168 /* Translate section/page access permissions to page
10169  * R/W protection flags.
10170  *
10171  * @ap:      The 2-bit simple AP (AP[2:1])
10172  * @is_user: TRUE if accessing from PL0
10173  */
10174 static inline int simple_ap_to_rw_prot_is_user(int ap, bool is_user)
10175 {
10176     switch (ap) {
10177     case 0:
10178         return is_user ? 0 : PAGE_READ | PAGE_WRITE;
10179     case 1:
10180         return PAGE_READ | PAGE_WRITE;
10181     case 2:
10182         return is_user ? 0 : PAGE_READ;
10183     case 3:
10184         return PAGE_READ;
10185     default:
10186         g_assert_not_reached();
10187     }
10188 }
10189 
10190 static inline int
10191 simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap)
10192 {
10193     return simple_ap_to_rw_prot_is_user(ap, regime_is_user(env, mmu_idx));
10194 }
10195 
10196 /* Translate S2 section/page access permissions to protection flags
10197  *
10198  * @env:     CPUARMState
10199  * @s2ap:    The 2-bit stage2 access permissions (S2AP)
10200  * @xn:      XN (execute-never) bits
10201  * @s1_is_el0: true if this is S2 of an S1+2 walk for EL0
10202  */
10203 static int get_S2prot(CPUARMState *env, int s2ap, int xn, bool s1_is_el0)
10204 {
10205     int prot = 0;
10206 
10207     if (s2ap & 1) {
10208         prot |= PAGE_READ;
10209     }
10210     if (s2ap & 2) {
10211         prot |= PAGE_WRITE;
10212     }
10213 
10214     if (cpu_isar_feature(any_tts2uxn, env_archcpu(env))) {
10215         switch (xn) {
10216         case 0:
10217             prot |= PAGE_EXEC;
10218             break;
10219         case 1:
10220             if (s1_is_el0) {
10221                 prot |= PAGE_EXEC;
10222             }
10223             break;
10224         case 2:
10225             break;
10226         case 3:
10227             if (!s1_is_el0) {
10228                 prot |= PAGE_EXEC;
10229             }
10230             break;
10231         default:
10232             g_assert_not_reached();
10233         }
10234     } else {
10235         if (!extract32(xn, 1, 1)) {
10236             if (arm_el_is_aa64(env, 2) || prot & PAGE_READ) {
10237                 prot |= PAGE_EXEC;
10238             }
10239         }
10240     }
10241     return prot;
10242 }
10243 
10244 /* Translate section/page access permissions to protection flags
10245  *
10246  * @env:     CPUARMState
10247  * @mmu_idx: MMU index indicating required translation regime
10248  * @is_aa64: TRUE if AArch64
10249  * @ap:      The 2-bit simple AP (AP[2:1])
10250  * @ns:      NS (non-secure) bit
10251  * @xn:      XN (execute-never) bit
10252  * @pxn:     PXN (privileged execute-never) bit
10253  */
10254 static int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64,
10255                       int ap, int ns, int xn, int pxn)
10256 {
10257     bool is_user = regime_is_user(env, mmu_idx);
10258     int prot_rw, user_rw;
10259     bool have_wxn;
10260     int wxn = 0;
10261 
10262     assert(mmu_idx != ARMMMUIdx_Stage2);
10263 
10264     user_rw = simple_ap_to_rw_prot_is_user(ap, true);
10265     if (is_user) {
10266         prot_rw = user_rw;
10267     } else {
10268         if (user_rw && regime_is_pan(env, mmu_idx)) {
10269             /* PAN forbids data accesses but doesn't affect insn fetch */
10270             prot_rw = 0;
10271         } else {
10272             prot_rw = simple_ap_to_rw_prot_is_user(ap, false);
10273         }
10274     }
10275 
10276     if (ns && arm_is_secure(env) && (env->cp15.scr_el3 & SCR_SIF)) {
10277         return prot_rw;
10278     }
10279 
10280     /* TODO have_wxn should be replaced with
10281      *   ARM_FEATURE_V8 || (ARM_FEATURE_V7 && ARM_FEATURE_EL2)
10282      * when ARM_FEATURE_EL2 starts getting set. For now we assume all LPAE
10283      * compatible processors have EL2, which is required for [U]WXN.
10284      */
10285     have_wxn = arm_feature(env, ARM_FEATURE_LPAE);
10286 
10287     if (have_wxn) {
10288         wxn = regime_sctlr(env, mmu_idx) & SCTLR_WXN;
10289     }
10290 
10291     if (is_aa64) {
10292         if (regime_has_2_ranges(mmu_idx) && !is_user) {
10293             xn = pxn || (user_rw & PAGE_WRITE);
10294         }
10295     } else if (arm_feature(env, ARM_FEATURE_V7)) {
10296         switch (regime_el(env, mmu_idx)) {
10297         case 1:
10298         case 3:
10299             if (is_user) {
10300                 xn = xn || !(user_rw & PAGE_READ);
10301             } else {
10302                 int uwxn = 0;
10303                 if (have_wxn) {
10304                     uwxn = regime_sctlr(env, mmu_idx) & SCTLR_UWXN;
10305                 }
10306                 xn = xn || !(prot_rw & PAGE_READ) || pxn ||
10307                      (uwxn && (user_rw & PAGE_WRITE));
10308             }
10309             break;
10310         case 2:
10311             break;
10312         }
10313     } else {
10314         xn = wxn = 0;
10315     }
10316 
10317     if (xn || (wxn && (prot_rw & PAGE_WRITE))) {
10318         return prot_rw;
10319     }
10320     return prot_rw | PAGE_EXEC;
10321 }
10322 
10323 static bool get_level1_table_address(CPUARMState *env, ARMMMUIdx mmu_idx,
10324                                      uint32_t *table, uint32_t address)
10325 {
10326     /* Note that we can only get here for an AArch32 PL0/PL1 lookup */
10327     TCR *tcr = regime_tcr(env, mmu_idx);
10328 
10329     if (address & tcr->mask) {
10330         if (tcr->raw_tcr & TTBCR_PD1) {
10331             /* Translation table walk disabled for TTBR1 */
10332             return false;
10333         }
10334         *table = regime_ttbr(env, mmu_idx, 1) & 0xffffc000;
10335     } else {
10336         if (tcr->raw_tcr & TTBCR_PD0) {
10337             /* Translation table walk disabled for TTBR0 */
10338             return false;
10339         }
10340         *table = regime_ttbr(env, mmu_idx, 0) & tcr->base_mask;
10341     }
10342     *table |= (address >> 18) & 0x3ffc;
10343     return true;
10344 }
10345 
10346 /* Translate a S1 pagetable walk through S2 if needed.  */
10347 static hwaddr S1_ptw_translate(CPUARMState *env, ARMMMUIdx mmu_idx,
10348                                hwaddr addr, MemTxAttrs txattrs,
10349                                ARMMMUFaultInfo *fi)
10350 {
10351     if (arm_mmu_idx_is_stage1_of_2(mmu_idx) &&
10352         !regime_translation_disabled(env, ARMMMUIdx_Stage2)) {
10353         target_ulong s2size;
10354         hwaddr s2pa;
10355         int s2prot;
10356         int ret;
10357         ARMCacheAttrs cacheattrs = {};
10358 
10359         ret = get_phys_addr_lpae(env, addr, MMU_DATA_LOAD, ARMMMUIdx_Stage2,
10360                                  false,
10361                                  &s2pa, &txattrs, &s2prot, &s2size, fi,
10362                                  &cacheattrs);
10363         if (ret) {
10364             assert(fi->type != ARMFault_None);
10365             fi->s2addr = addr;
10366             fi->stage2 = true;
10367             fi->s1ptw = true;
10368             return ~0;
10369         }
10370         if ((env->cp15.hcr_el2 & HCR_PTW) && (cacheattrs.attrs & 0xf0) == 0) {
10371             /*
10372              * PTW set and S1 walk touched S2 Device memory:
10373              * generate Permission fault.
10374              */
10375             fi->type = ARMFault_Permission;
10376             fi->s2addr = addr;
10377             fi->stage2 = true;
10378             fi->s1ptw = true;
10379             return ~0;
10380         }
10381         addr = s2pa;
10382     }
10383     return addr;
10384 }
10385 
10386 /* All loads done in the course of a page table walk go through here. */
10387 static uint32_t arm_ldl_ptw(CPUState *cs, hwaddr addr, bool is_secure,
10388                             ARMMMUIdx mmu_idx, ARMMMUFaultInfo *fi)
10389 {
10390     ARMCPU *cpu = ARM_CPU(cs);
10391     CPUARMState *env = &cpu->env;
10392     MemTxAttrs attrs = {};
10393     MemTxResult result = MEMTX_OK;
10394     AddressSpace *as;
10395     uint32_t data;
10396 
10397     attrs.secure = is_secure;
10398     as = arm_addressspace(cs, attrs);
10399     addr = S1_ptw_translate(env, mmu_idx, addr, attrs, fi);
10400     if (fi->s1ptw) {
10401         return 0;
10402     }
10403     if (regime_translation_big_endian(env, mmu_idx)) {
10404         data = address_space_ldl_be(as, addr, attrs, &result);
10405     } else {
10406         data = address_space_ldl_le(as, addr, attrs, &result);
10407     }
10408     if (result == MEMTX_OK) {
10409         return data;
10410     }
10411     fi->type = ARMFault_SyncExternalOnWalk;
10412     fi->ea = arm_extabort_type(result);
10413     return 0;
10414 }
10415 
10416 static uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure,
10417                             ARMMMUIdx mmu_idx, ARMMMUFaultInfo *fi)
10418 {
10419     ARMCPU *cpu = ARM_CPU(cs);
10420     CPUARMState *env = &cpu->env;
10421     MemTxAttrs attrs = {};
10422     MemTxResult result = MEMTX_OK;
10423     AddressSpace *as;
10424     uint64_t data;
10425 
10426     attrs.secure = is_secure;
10427     as = arm_addressspace(cs, attrs);
10428     addr = S1_ptw_translate(env, mmu_idx, addr, attrs, fi);
10429     if (fi->s1ptw) {
10430         return 0;
10431     }
10432     if (regime_translation_big_endian(env, mmu_idx)) {
10433         data = address_space_ldq_be(as, addr, attrs, &result);
10434     } else {
10435         data = address_space_ldq_le(as, addr, attrs, &result);
10436     }
10437     if (result == MEMTX_OK) {
10438         return data;
10439     }
10440     fi->type = ARMFault_SyncExternalOnWalk;
10441     fi->ea = arm_extabort_type(result);
10442     return 0;
10443 }
10444 
10445 static bool get_phys_addr_v5(CPUARMState *env, uint32_t address,
10446                              MMUAccessType access_type, ARMMMUIdx mmu_idx,
10447                              hwaddr *phys_ptr, int *prot,
10448                              target_ulong *page_size,
10449                              ARMMMUFaultInfo *fi)
10450 {
10451     CPUState *cs = env_cpu(env);
10452     int level = 1;
10453     uint32_t table;
10454     uint32_t desc;
10455     int type;
10456     int ap;
10457     int domain = 0;
10458     int domain_prot;
10459     hwaddr phys_addr;
10460     uint32_t dacr;
10461 
10462     /* Pagetable walk.  */
10463     /* Lookup l1 descriptor.  */
10464     if (!get_level1_table_address(env, mmu_idx, &table, address)) {
10465         /* Section translation fault if page walk is disabled by PD0 or PD1 */
10466         fi->type = ARMFault_Translation;
10467         goto do_fault;
10468     }
10469     desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx),
10470                        mmu_idx, fi);
10471     if (fi->type != ARMFault_None) {
10472         goto do_fault;
10473     }
10474     type = (desc & 3);
10475     domain = (desc >> 5) & 0x0f;
10476     if (regime_el(env, mmu_idx) == 1) {
10477         dacr = env->cp15.dacr_ns;
10478     } else {
10479         dacr = env->cp15.dacr_s;
10480     }
10481     domain_prot = (dacr >> (domain * 2)) & 3;
10482     if (type == 0) {
10483         /* Section translation fault.  */
10484         fi->type = ARMFault_Translation;
10485         goto do_fault;
10486     }
10487     if (type != 2) {
10488         level = 2;
10489     }
10490     if (domain_prot == 0 || domain_prot == 2) {
10491         fi->type = ARMFault_Domain;
10492         goto do_fault;
10493     }
10494     if (type == 2) {
10495         /* 1Mb section.  */
10496         phys_addr = (desc & 0xfff00000) | (address & 0x000fffff);
10497         ap = (desc >> 10) & 3;
10498         *page_size = 1024 * 1024;
10499     } else {
10500         /* Lookup l2 entry.  */
10501         if (type == 1) {
10502             /* Coarse pagetable.  */
10503             table = (desc & 0xfffffc00) | ((address >> 10) & 0x3fc);
10504         } else {
10505             /* Fine pagetable.  */
10506             table = (desc & 0xfffff000) | ((address >> 8) & 0xffc);
10507         }
10508         desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx),
10509                            mmu_idx, fi);
10510         if (fi->type != ARMFault_None) {
10511             goto do_fault;
10512         }
10513         switch (desc & 3) {
10514         case 0: /* Page translation fault.  */
10515             fi->type = ARMFault_Translation;
10516             goto do_fault;
10517         case 1: /* 64k page.  */
10518             phys_addr = (desc & 0xffff0000) | (address & 0xffff);
10519             ap = (desc >> (4 + ((address >> 13) & 6))) & 3;
10520             *page_size = 0x10000;
10521             break;
10522         case 2: /* 4k page.  */
10523             phys_addr = (desc & 0xfffff000) | (address & 0xfff);
10524             ap = (desc >> (4 + ((address >> 9) & 6))) & 3;
10525             *page_size = 0x1000;
10526             break;
10527         case 3: /* 1k page, or ARMv6/XScale "extended small (4k) page" */
10528             if (type == 1) {
10529                 /* ARMv6/XScale extended small page format */
10530                 if (arm_feature(env, ARM_FEATURE_XSCALE)
10531                     || arm_feature(env, ARM_FEATURE_V6)) {
10532                     phys_addr = (desc & 0xfffff000) | (address & 0xfff);
10533                     *page_size = 0x1000;
10534                 } else {
10535                     /* UNPREDICTABLE in ARMv5; we choose to take a
10536                      * page translation fault.
10537                      */
10538                     fi->type = ARMFault_Translation;
10539                     goto do_fault;
10540                 }
10541             } else {
10542                 phys_addr = (desc & 0xfffffc00) | (address & 0x3ff);
10543                 *page_size = 0x400;
10544             }
10545             ap = (desc >> 4) & 3;
10546             break;
10547         default:
10548             /* Never happens, but compiler isn't smart enough to tell.  */
10549             abort();
10550         }
10551     }
10552     *prot = ap_to_rw_prot(env, mmu_idx, ap, domain_prot);
10553     *prot |= *prot ? PAGE_EXEC : 0;
10554     if (!(*prot & (1 << access_type))) {
10555         /* Access permission fault.  */
10556         fi->type = ARMFault_Permission;
10557         goto do_fault;
10558     }
10559     *phys_ptr = phys_addr;
10560     return false;
10561 do_fault:
10562     fi->domain = domain;
10563     fi->level = level;
10564     return true;
10565 }
10566 
10567 static bool get_phys_addr_v6(CPUARMState *env, uint32_t address,
10568                              MMUAccessType access_type, ARMMMUIdx mmu_idx,
10569                              hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot,
10570                              target_ulong *page_size, ARMMMUFaultInfo *fi)
10571 {
10572     CPUState *cs = env_cpu(env);
10573     ARMCPU *cpu = env_archcpu(env);
10574     int level = 1;
10575     uint32_t table;
10576     uint32_t desc;
10577     uint32_t xn;
10578     uint32_t pxn = 0;
10579     int type;
10580     int ap;
10581     int domain = 0;
10582     int domain_prot;
10583     hwaddr phys_addr;
10584     uint32_t dacr;
10585     bool ns;
10586 
10587     /* Pagetable walk.  */
10588     /* Lookup l1 descriptor.  */
10589     if (!get_level1_table_address(env, mmu_idx, &table, address)) {
10590         /* Section translation fault if page walk is disabled by PD0 or PD1 */
10591         fi->type = ARMFault_Translation;
10592         goto do_fault;
10593     }
10594     desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx),
10595                        mmu_idx, fi);
10596     if (fi->type != ARMFault_None) {
10597         goto do_fault;
10598     }
10599     type = (desc & 3);
10600     if (type == 0 || (type == 3 && !cpu_isar_feature(aa32_pxn, cpu))) {
10601         /* Section translation fault, or attempt to use the encoding
10602          * which is Reserved on implementations without PXN.
10603          */
10604         fi->type = ARMFault_Translation;
10605         goto do_fault;
10606     }
10607     if ((type == 1) || !(desc & (1 << 18))) {
10608         /* Page or Section.  */
10609         domain = (desc >> 5) & 0x0f;
10610     }
10611     if (regime_el(env, mmu_idx) == 1) {
10612         dacr = env->cp15.dacr_ns;
10613     } else {
10614         dacr = env->cp15.dacr_s;
10615     }
10616     if (type == 1) {
10617         level = 2;
10618     }
10619     domain_prot = (dacr >> (domain * 2)) & 3;
10620     if (domain_prot == 0 || domain_prot == 2) {
10621         /* Section or Page domain fault */
10622         fi->type = ARMFault_Domain;
10623         goto do_fault;
10624     }
10625     if (type != 1) {
10626         if (desc & (1 << 18)) {
10627             /* Supersection.  */
10628             phys_addr = (desc & 0xff000000) | (address & 0x00ffffff);
10629             phys_addr |= (uint64_t)extract32(desc, 20, 4) << 32;
10630             phys_addr |= (uint64_t)extract32(desc, 5, 4) << 36;
10631             *page_size = 0x1000000;
10632         } else {
10633             /* Section.  */
10634             phys_addr = (desc & 0xfff00000) | (address & 0x000fffff);
10635             *page_size = 0x100000;
10636         }
10637         ap = ((desc >> 10) & 3) | ((desc >> 13) & 4);
10638         xn = desc & (1 << 4);
10639         pxn = desc & 1;
10640         ns = extract32(desc, 19, 1);
10641     } else {
10642         if (cpu_isar_feature(aa32_pxn, cpu)) {
10643             pxn = (desc >> 2) & 1;
10644         }
10645         ns = extract32(desc, 3, 1);
10646         /* Lookup l2 entry.  */
10647         table = (desc & 0xfffffc00) | ((address >> 10) & 0x3fc);
10648         desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx),
10649                            mmu_idx, fi);
10650         if (fi->type != ARMFault_None) {
10651             goto do_fault;
10652         }
10653         ap = ((desc >> 4) & 3) | ((desc >> 7) & 4);
10654         switch (desc & 3) {
10655         case 0: /* Page translation fault.  */
10656             fi->type = ARMFault_Translation;
10657             goto do_fault;
10658         case 1: /* 64k page.  */
10659             phys_addr = (desc & 0xffff0000) | (address & 0xffff);
10660             xn = desc & (1 << 15);
10661             *page_size = 0x10000;
10662             break;
10663         case 2: case 3: /* 4k page.  */
10664             phys_addr = (desc & 0xfffff000) | (address & 0xfff);
10665             xn = desc & 1;
10666             *page_size = 0x1000;
10667             break;
10668         default:
10669             /* Never happens, but compiler isn't smart enough to tell.  */
10670             abort();
10671         }
10672     }
10673     if (domain_prot == 3) {
10674         *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
10675     } else {
10676         if (pxn && !regime_is_user(env, mmu_idx)) {
10677             xn = 1;
10678         }
10679         if (xn && access_type == MMU_INST_FETCH) {
10680             fi->type = ARMFault_Permission;
10681             goto do_fault;
10682         }
10683 
10684         if (arm_feature(env, ARM_FEATURE_V6K) &&
10685                 (regime_sctlr(env, mmu_idx) & SCTLR_AFE)) {
10686             /* The simplified model uses AP[0] as an access control bit.  */
10687             if ((ap & 1) == 0) {
10688                 /* Access flag fault.  */
10689                 fi->type = ARMFault_AccessFlag;
10690                 goto do_fault;
10691             }
10692             *prot = simple_ap_to_rw_prot(env, mmu_idx, ap >> 1);
10693         } else {
10694             *prot = ap_to_rw_prot(env, mmu_idx, ap, domain_prot);
10695         }
10696         if (*prot && !xn) {
10697             *prot |= PAGE_EXEC;
10698         }
10699         if (!(*prot & (1 << access_type))) {
10700             /* Access permission fault.  */
10701             fi->type = ARMFault_Permission;
10702             goto do_fault;
10703         }
10704     }
10705     if (ns) {
10706         /* The NS bit will (as required by the architecture) have no effect if
10707          * the CPU doesn't support TZ or this is a non-secure translation
10708          * regime, because the attribute will already be non-secure.
10709          */
10710         attrs->secure = false;
10711     }
10712     *phys_ptr = phys_addr;
10713     return false;
10714 do_fault:
10715     fi->domain = domain;
10716     fi->level = level;
10717     return true;
10718 }
10719 
10720 /*
10721  * check_s2_mmu_setup
10722  * @cpu:        ARMCPU
10723  * @is_aa64:    True if the translation regime is in AArch64 state
10724  * @startlevel: Suggested starting level
10725  * @inputsize:  Bitsize of IPAs
10726  * @stride:     Page-table stride (See the ARM ARM)
10727  *
10728  * Returns true if the suggested S2 translation parameters are OK and
10729  * false otherwise.
10730  */
10731 static bool check_s2_mmu_setup(ARMCPU *cpu, bool is_aa64, int level,
10732                                int inputsize, int stride)
10733 {
10734     const int grainsize = stride + 3;
10735     int startsizecheck;
10736 
10737     /* Negative levels are never allowed.  */
10738     if (level < 0) {
10739         return false;
10740     }
10741 
10742     startsizecheck = inputsize - ((3 - level) * stride + grainsize);
10743     if (startsizecheck < 1 || startsizecheck > stride + 4) {
10744         return false;
10745     }
10746 
10747     if (is_aa64) {
10748         CPUARMState *env = &cpu->env;
10749         unsigned int pamax = arm_pamax(cpu);
10750 
10751         switch (stride) {
10752         case 13: /* 64KB Pages.  */
10753             if (level == 0 || (level == 1 && pamax <= 42)) {
10754                 return false;
10755             }
10756             break;
10757         case 11: /* 16KB Pages.  */
10758             if (level == 0 || (level == 1 && pamax <= 40)) {
10759                 return false;
10760             }
10761             break;
10762         case 9: /* 4KB Pages.  */
10763             if (level == 0 && pamax <= 42) {
10764                 return false;
10765             }
10766             break;
10767         default:
10768             g_assert_not_reached();
10769         }
10770 
10771         /* Inputsize checks.  */
10772         if (inputsize > pamax &&
10773             (arm_el_is_aa64(env, 1) || inputsize > 40)) {
10774             /* This is CONSTRAINED UNPREDICTABLE and we choose to fault.  */
10775             return false;
10776         }
10777     } else {
10778         /* AArch32 only supports 4KB pages. Assert on that.  */
10779         assert(stride == 9);
10780 
10781         if (level == 0) {
10782             return false;
10783         }
10784     }
10785     return true;
10786 }
10787 
10788 /* Translate from the 4-bit stage 2 representation of
10789  * memory attributes (without cache-allocation hints) to
10790  * the 8-bit representation of the stage 1 MAIR registers
10791  * (which includes allocation hints).
10792  *
10793  * ref: shared/translation/attrs/S2AttrDecode()
10794  *      .../S2ConvertAttrsHints()
10795  */
10796 static uint8_t convert_stage2_attrs(CPUARMState *env, uint8_t s2attrs)
10797 {
10798     uint8_t hiattr = extract32(s2attrs, 2, 2);
10799     uint8_t loattr = extract32(s2attrs, 0, 2);
10800     uint8_t hihint = 0, lohint = 0;
10801 
10802     if (hiattr != 0) { /* normal memory */
10803         if ((env->cp15.hcr_el2 & HCR_CD) != 0) { /* cache disabled */
10804             hiattr = loattr = 1; /* non-cacheable */
10805         } else {
10806             if (hiattr != 1) { /* Write-through or write-back */
10807                 hihint = 3; /* RW allocate */
10808             }
10809             if (loattr != 1) { /* Write-through or write-back */
10810                 lohint = 3; /* RW allocate */
10811             }
10812         }
10813     }
10814 
10815     return (hiattr << 6) | (hihint << 4) | (loattr << 2) | lohint;
10816 }
10817 #endif /* !CONFIG_USER_ONLY */
10818 
10819 static int aa64_va_parameter_tbi(uint64_t tcr, ARMMMUIdx mmu_idx)
10820 {
10821     if (regime_has_2_ranges(mmu_idx)) {
10822         return extract64(tcr, 37, 2);
10823     } else if (mmu_idx == ARMMMUIdx_Stage2) {
10824         return 0; /* VTCR_EL2 */
10825     } else {
10826         /* Replicate the single TBI bit so we always have 2 bits.  */
10827         return extract32(tcr, 20, 1) * 3;
10828     }
10829 }
10830 
10831 static int aa64_va_parameter_tbid(uint64_t tcr, ARMMMUIdx mmu_idx)
10832 {
10833     if (regime_has_2_ranges(mmu_idx)) {
10834         return extract64(tcr, 51, 2);
10835     } else if (mmu_idx == ARMMMUIdx_Stage2) {
10836         return 0; /* VTCR_EL2 */
10837     } else {
10838         /* Replicate the single TBID bit so we always have 2 bits.  */
10839         return extract32(tcr, 29, 1) * 3;
10840     }
10841 }
10842 
10843 static int aa64_va_parameter_tcma(uint64_t tcr, ARMMMUIdx mmu_idx)
10844 {
10845     if (regime_has_2_ranges(mmu_idx)) {
10846         return extract64(tcr, 57, 2);
10847     } else {
10848         /* Replicate the single TCMA bit so we always have 2 bits.  */
10849         return extract32(tcr, 30, 1) * 3;
10850     }
10851 }
10852 
10853 ARMVAParameters aa64_va_parameters(CPUARMState *env, uint64_t va,
10854                                    ARMMMUIdx mmu_idx, bool data)
10855 {
10856     uint64_t tcr = regime_tcr(env, mmu_idx)->raw_tcr;
10857     bool epd, hpd, using16k, using64k;
10858     int select, tsz, tbi;
10859 
10860     if (!regime_has_2_ranges(mmu_idx)) {
10861         select = 0;
10862         tsz = extract32(tcr, 0, 6);
10863         using64k = extract32(tcr, 14, 1);
10864         using16k = extract32(tcr, 15, 1);
10865         if (mmu_idx == ARMMMUIdx_Stage2) {
10866             /* VTCR_EL2 */
10867             hpd = false;
10868         } else {
10869             hpd = extract32(tcr, 24, 1);
10870         }
10871         epd = false;
10872     } else {
10873         /*
10874          * Bit 55 is always between the two regions, and is canonical for
10875          * determining if address tagging is enabled.
10876          */
10877         select = extract64(va, 55, 1);
10878         if (!select) {
10879             tsz = extract32(tcr, 0, 6);
10880             epd = extract32(tcr, 7, 1);
10881             using64k = extract32(tcr, 14, 1);
10882             using16k = extract32(tcr, 15, 1);
10883             hpd = extract64(tcr, 41, 1);
10884         } else {
10885             int tg = extract32(tcr, 30, 2);
10886             using16k = tg == 1;
10887             using64k = tg == 3;
10888             tsz = extract32(tcr, 16, 6);
10889             epd = extract32(tcr, 23, 1);
10890             hpd = extract64(tcr, 42, 1);
10891         }
10892     }
10893     tsz = MIN(tsz, 39);  /* TODO: ARMv8.4-TTST */
10894     tsz = MAX(tsz, 16);  /* TODO: ARMv8.2-LVA  */
10895 
10896     /* Present TBI as a composite with TBID.  */
10897     tbi = aa64_va_parameter_tbi(tcr, mmu_idx);
10898     if (!data) {
10899         tbi &= ~aa64_va_parameter_tbid(tcr, mmu_idx);
10900     }
10901     tbi = (tbi >> select) & 1;
10902 
10903     return (ARMVAParameters) {
10904         .tsz = tsz,
10905         .select = select,
10906         .tbi = tbi,
10907         .epd = epd,
10908         .hpd = hpd,
10909         .using16k = using16k,
10910         .using64k = using64k,
10911     };
10912 }
10913 
10914 #ifndef CONFIG_USER_ONLY
10915 static ARMVAParameters aa32_va_parameters(CPUARMState *env, uint32_t va,
10916                                           ARMMMUIdx mmu_idx)
10917 {
10918     uint64_t tcr = regime_tcr(env, mmu_idx)->raw_tcr;
10919     uint32_t el = regime_el(env, mmu_idx);
10920     int select, tsz;
10921     bool epd, hpd;
10922 
10923     if (mmu_idx == ARMMMUIdx_Stage2) {
10924         /* VTCR */
10925         bool sext = extract32(tcr, 4, 1);
10926         bool sign = extract32(tcr, 3, 1);
10927 
10928         /*
10929          * If the sign-extend bit is not the same as t0sz[3], the result
10930          * is unpredictable. Flag this as a guest error.
10931          */
10932         if (sign != sext) {
10933             qemu_log_mask(LOG_GUEST_ERROR,
10934                           "AArch32: VTCR.S / VTCR.T0SZ[3] mismatch\n");
10935         }
10936         tsz = sextract32(tcr, 0, 4) + 8;
10937         select = 0;
10938         hpd = false;
10939         epd = false;
10940     } else if (el == 2) {
10941         /* HTCR */
10942         tsz = extract32(tcr, 0, 3);
10943         select = 0;
10944         hpd = extract64(tcr, 24, 1);
10945         epd = false;
10946     } else {
10947         int t0sz = extract32(tcr, 0, 3);
10948         int t1sz = extract32(tcr, 16, 3);
10949 
10950         if (t1sz == 0) {
10951             select = va > (0xffffffffu >> t0sz);
10952         } else {
10953             /* Note that we will detect errors later.  */
10954             select = va >= ~(0xffffffffu >> t1sz);
10955         }
10956         if (!select) {
10957             tsz = t0sz;
10958             epd = extract32(tcr, 7, 1);
10959             hpd = extract64(tcr, 41, 1);
10960         } else {
10961             tsz = t1sz;
10962             epd = extract32(tcr, 23, 1);
10963             hpd = extract64(tcr, 42, 1);
10964         }
10965         /* For aarch32, hpd0 is not enabled without t2e as well.  */
10966         hpd &= extract32(tcr, 6, 1);
10967     }
10968 
10969     return (ARMVAParameters) {
10970         .tsz = tsz,
10971         .select = select,
10972         .epd = epd,
10973         .hpd = hpd,
10974     };
10975 }
10976 
10977 /**
10978  * get_phys_addr_lpae: perform one stage of page table walk, LPAE format
10979  *
10980  * Returns false if the translation was successful. Otherwise, phys_ptr, attrs,
10981  * prot and page_size may not be filled in, and the populated fsr value provides
10982  * information on why the translation aborted, in the format of a long-format
10983  * DFSR/IFSR fault register, with the following caveats:
10984  *  * the WnR bit is never set (the caller must do this).
10985  *
10986  * @env: CPUARMState
10987  * @address: virtual address to get physical address for
10988  * @access_type: MMU_DATA_LOAD, MMU_DATA_STORE or MMU_INST_FETCH
10989  * @mmu_idx: MMU index indicating required translation regime
10990  * @s1_is_el0: if @mmu_idx is ARMMMUIdx_Stage2 (so this is a stage 2 page table
10991  *             walk), must be true if this is stage 2 of a stage 1+2 walk for an
10992  *             EL0 access). If @mmu_idx is anything else, @s1_is_el0 is ignored.
10993  * @phys_ptr: set to the physical address corresponding to the virtual address
10994  * @attrs: set to the memory transaction attributes to use
10995  * @prot: set to the permissions for the page containing phys_ptr
10996  * @page_size_ptr: set to the size of the page containing phys_ptr
10997  * @fi: set to fault info if the translation fails
10998  * @cacheattrs: (if non-NULL) set to the cacheability/shareability attributes
10999  */
11000 static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address,
11001                                MMUAccessType access_type, ARMMMUIdx mmu_idx,
11002                                bool s1_is_el0,
11003                                hwaddr *phys_ptr, MemTxAttrs *txattrs, int *prot,
11004                                target_ulong *page_size_ptr,
11005                                ARMMMUFaultInfo *fi, ARMCacheAttrs *cacheattrs)
11006 {
11007     ARMCPU *cpu = env_archcpu(env);
11008     CPUState *cs = CPU(cpu);
11009     /* Read an LPAE long-descriptor translation table. */
11010     ARMFaultType fault_type = ARMFault_Translation;
11011     uint32_t level;
11012     ARMVAParameters param;
11013     uint64_t ttbr;
11014     hwaddr descaddr, indexmask, indexmask_grainsize;
11015     uint32_t tableattrs;
11016     target_ulong page_size;
11017     uint32_t attrs;
11018     int32_t stride;
11019     int addrsize, inputsize;
11020     TCR *tcr = regime_tcr(env, mmu_idx);
11021     int ap, ns, xn, pxn;
11022     uint32_t el = regime_el(env, mmu_idx);
11023     uint64_t descaddrmask;
11024     bool aarch64 = arm_el_is_aa64(env, el);
11025     bool guarded = false;
11026 
11027     /* TODO: This code does not support shareability levels. */
11028     if (aarch64) {
11029         param = aa64_va_parameters(env, address, mmu_idx,
11030                                    access_type != MMU_INST_FETCH);
11031         level = 0;
11032         addrsize = 64 - 8 * param.tbi;
11033         inputsize = 64 - param.tsz;
11034     } else {
11035         param = aa32_va_parameters(env, address, mmu_idx);
11036         level = 1;
11037         addrsize = (mmu_idx == ARMMMUIdx_Stage2 ? 40 : 32);
11038         inputsize = addrsize - param.tsz;
11039     }
11040 
11041     /*
11042      * We determined the region when collecting the parameters, but we
11043      * have not yet validated that the address is valid for the region.
11044      * Extract the top bits and verify that they all match select.
11045      *
11046      * For aa32, if inputsize == addrsize, then we have selected the
11047      * region by exclusion in aa32_va_parameters and there is no more
11048      * validation to do here.
11049      */
11050     if (inputsize < addrsize) {
11051         target_ulong top_bits = sextract64(address, inputsize,
11052                                            addrsize - inputsize);
11053         if (-top_bits != param.select) {
11054             /* The gap between the two regions is a Translation fault */
11055             fault_type = ARMFault_Translation;
11056             goto do_fault;
11057         }
11058     }
11059 
11060     if (param.using64k) {
11061         stride = 13;
11062     } else if (param.using16k) {
11063         stride = 11;
11064     } else {
11065         stride = 9;
11066     }
11067 
11068     /* Note that QEMU ignores shareability and cacheability attributes,
11069      * so we don't need to do anything with the SH, ORGN, IRGN fields
11070      * in the TTBCR.  Similarly, TTBCR:A1 selects whether we get the
11071      * ASID from TTBR0 or TTBR1, but QEMU's TLB doesn't currently
11072      * implement any ASID-like capability so we can ignore it (instead
11073      * we will always flush the TLB any time the ASID is changed).
11074      */
11075     ttbr = regime_ttbr(env, mmu_idx, param.select);
11076 
11077     /* Here we should have set up all the parameters for the translation:
11078      * inputsize, ttbr, epd, stride, tbi
11079      */
11080 
11081     if (param.epd) {
11082         /* Translation table walk disabled => Translation fault on TLB miss
11083          * Note: This is always 0 on 64-bit EL2 and EL3.
11084          */
11085         goto do_fault;
11086     }
11087 
11088     if (mmu_idx != ARMMMUIdx_Stage2) {
11089         /* The starting level depends on the virtual address size (which can
11090          * be up to 48 bits) and the translation granule size. It indicates
11091          * the number of strides (stride bits at a time) needed to
11092          * consume the bits of the input address. In the pseudocode this is:
11093          *  level = 4 - RoundUp((inputsize - grainsize) / stride)
11094          * where their 'inputsize' is our 'inputsize', 'grainsize' is
11095          * our 'stride + 3' and 'stride' is our 'stride'.
11096          * Applying the usual "rounded up m/n is (m+n-1)/n" and simplifying:
11097          * = 4 - (inputsize - stride - 3 + stride - 1) / stride
11098          * = 4 - (inputsize - 4) / stride;
11099          */
11100         level = 4 - (inputsize - 4) / stride;
11101     } else {
11102         /* For stage 2 translations the starting level is specified by the
11103          * VTCR_EL2.SL0 field (whose interpretation depends on the page size)
11104          */
11105         uint32_t sl0 = extract32(tcr->raw_tcr, 6, 2);
11106         uint32_t startlevel;
11107         bool ok;
11108 
11109         if (!aarch64 || stride == 9) {
11110             /* AArch32 or 4KB pages */
11111             startlevel = 2 - sl0;
11112         } else {
11113             /* 16KB or 64KB pages */
11114             startlevel = 3 - sl0;
11115         }
11116 
11117         /* Check that the starting level is valid. */
11118         ok = check_s2_mmu_setup(cpu, aarch64, startlevel,
11119                                 inputsize, stride);
11120         if (!ok) {
11121             fault_type = ARMFault_Translation;
11122             goto do_fault;
11123         }
11124         level = startlevel;
11125     }
11126 
11127     indexmask_grainsize = (1ULL << (stride + 3)) - 1;
11128     indexmask = (1ULL << (inputsize - (stride * (4 - level)))) - 1;
11129 
11130     /* Now we can extract the actual base address from the TTBR */
11131     descaddr = extract64(ttbr, 0, 48);
11132     /*
11133      * We rely on this masking to clear the RES0 bits at the bottom of the TTBR
11134      * and also to mask out CnP (bit 0) which could validly be non-zero.
11135      */
11136     descaddr &= ~indexmask;
11137 
11138     /* The address field in the descriptor goes up to bit 39 for ARMv7
11139      * but up to bit 47 for ARMv8, but we use the descaddrmask
11140      * up to bit 39 for AArch32, because we don't need other bits in that case
11141      * to construct next descriptor address (anyway they should be all zeroes).
11142      */
11143     descaddrmask = ((1ull << (aarch64 ? 48 : 40)) - 1) &
11144                    ~indexmask_grainsize;
11145 
11146     /* Secure accesses start with the page table in secure memory and
11147      * can be downgraded to non-secure at any step. Non-secure accesses
11148      * remain non-secure. We implement this by just ORing in the NSTable/NS
11149      * bits at each step.
11150      */
11151     tableattrs = regime_is_secure(env, mmu_idx) ? 0 : (1 << 4);
11152     for (;;) {
11153         uint64_t descriptor;
11154         bool nstable;
11155 
11156         descaddr |= (address >> (stride * (4 - level))) & indexmask;
11157         descaddr &= ~7ULL;
11158         nstable = extract32(tableattrs, 4, 1);
11159         descriptor = arm_ldq_ptw(cs, descaddr, !nstable, mmu_idx, fi);
11160         if (fi->type != ARMFault_None) {
11161             goto do_fault;
11162         }
11163 
11164         if (!(descriptor & 1) ||
11165             (!(descriptor & 2) && (level == 3))) {
11166             /* Invalid, or the Reserved level 3 encoding */
11167             goto do_fault;
11168         }
11169         descaddr = descriptor & descaddrmask;
11170 
11171         if ((descriptor & 2) && (level < 3)) {
11172             /* Table entry. The top five bits are attributes which may
11173              * propagate down through lower levels of the table (and
11174              * which are all arranged so that 0 means "no effect", so
11175              * we can gather them up by ORing in the bits at each level).
11176              */
11177             tableattrs |= extract64(descriptor, 59, 5);
11178             level++;
11179             indexmask = indexmask_grainsize;
11180             continue;
11181         }
11182         /* Block entry at level 1 or 2, or page entry at level 3.
11183          * These are basically the same thing, although the number
11184          * of bits we pull in from the vaddr varies.
11185          */
11186         page_size = (1ULL << ((stride * (4 - level)) + 3));
11187         descaddr |= (address & (page_size - 1));
11188         /* Extract attributes from the descriptor */
11189         attrs = extract64(descriptor, 2, 10)
11190             | (extract64(descriptor, 52, 12) << 10);
11191 
11192         if (mmu_idx == ARMMMUIdx_Stage2) {
11193             /* Stage 2 table descriptors do not include any attribute fields */
11194             break;
11195         }
11196         /* Merge in attributes from table descriptors */
11197         attrs |= nstable << 3; /* NS */
11198         guarded = extract64(descriptor, 50, 1);  /* GP */
11199         if (param.hpd) {
11200             /* HPD disables all the table attributes except NSTable.  */
11201             break;
11202         }
11203         attrs |= extract32(tableattrs, 0, 2) << 11;     /* XN, PXN */
11204         /* The sense of AP[1] vs APTable[0] is reversed, as APTable[0] == 1
11205          * means "force PL1 access only", which means forcing AP[1] to 0.
11206          */
11207         attrs &= ~(extract32(tableattrs, 2, 1) << 4);   /* !APT[0] => AP[1] */
11208         attrs |= extract32(tableattrs, 3, 1) << 5;      /* APT[1] => AP[2] */
11209         break;
11210     }
11211     /* Here descaddr is the final physical address, and attributes
11212      * are all in attrs.
11213      */
11214     fault_type = ARMFault_AccessFlag;
11215     if ((attrs & (1 << 8)) == 0) {
11216         /* Access flag */
11217         goto do_fault;
11218     }
11219 
11220     ap = extract32(attrs, 4, 2);
11221 
11222     if (mmu_idx == ARMMMUIdx_Stage2) {
11223         ns = true;
11224         xn = extract32(attrs, 11, 2);
11225         *prot = get_S2prot(env, ap, xn, s1_is_el0);
11226     } else {
11227         ns = extract32(attrs, 3, 1);
11228         xn = extract32(attrs, 12, 1);
11229         pxn = extract32(attrs, 11, 1);
11230         *prot = get_S1prot(env, mmu_idx, aarch64, ap, ns, xn, pxn);
11231     }
11232 
11233     fault_type = ARMFault_Permission;
11234     if (!(*prot & (1 << access_type))) {
11235         goto do_fault;
11236     }
11237 
11238     if (ns) {
11239         /* The NS bit will (as required by the architecture) have no effect if
11240          * the CPU doesn't support TZ or this is a non-secure translation
11241          * regime, because the attribute will already be non-secure.
11242          */
11243         txattrs->secure = false;
11244     }
11245     /* When in aarch64 mode, and BTI is enabled, remember GP in the IOTLB.  */
11246     if (aarch64 && guarded && cpu_isar_feature(aa64_bti, cpu)) {
11247         arm_tlb_bti_gp(txattrs) = true;
11248     }
11249 
11250     if (mmu_idx == ARMMMUIdx_Stage2) {
11251         cacheattrs->attrs = convert_stage2_attrs(env, extract32(attrs, 0, 4));
11252     } else {
11253         /* Index into MAIR registers for cache attributes */
11254         uint8_t attrindx = extract32(attrs, 0, 3);
11255         uint64_t mair = env->cp15.mair_el[regime_el(env, mmu_idx)];
11256         assert(attrindx <= 7);
11257         cacheattrs->attrs = extract64(mair, attrindx * 8, 8);
11258     }
11259     cacheattrs->shareability = extract32(attrs, 6, 2);
11260 
11261     *phys_ptr = descaddr;
11262     *page_size_ptr = page_size;
11263     return false;
11264 
11265 do_fault:
11266     fi->type = fault_type;
11267     fi->level = level;
11268     /* Tag the error as S2 for failed S1 PTW at S2 or ordinary S2.  */
11269     fi->stage2 = fi->s1ptw || (mmu_idx == ARMMMUIdx_Stage2);
11270     return true;
11271 }
11272 
11273 static inline void get_phys_addr_pmsav7_default(CPUARMState *env,
11274                                                 ARMMMUIdx mmu_idx,
11275                                                 int32_t address, int *prot)
11276 {
11277     if (!arm_feature(env, ARM_FEATURE_M)) {
11278         *prot = PAGE_READ | PAGE_WRITE;
11279         switch (address) {
11280         case 0xF0000000 ... 0xFFFFFFFF:
11281             if (regime_sctlr(env, mmu_idx) & SCTLR_V) {
11282                 /* hivecs execing is ok */
11283                 *prot |= PAGE_EXEC;
11284             }
11285             break;
11286         case 0x00000000 ... 0x7FFFFFFF:
11287             *prot |= PAGE_EXEC;
11288             break;
11289         }
11290     } else {
11291         /* Default system address map for M profile cores.
11292          * The architecture specifies which regions are execute-never;
11293          * at the MPU level no other checks are defined.
11294          */
11295         switch (address) {
11296         case 0x00000000 ... 0x1fffffff: /* ROM */
11297         case 0x20000000 ... 0x3fffffff: /* SRAM */
11298         case 0x60000000 ... 0x7fffffff: /* RAM */
11299         case 0x80000000 ... 0x9fffffff: /* RAM */
11300             *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
11301             break;
11302         case 0x40000000 ... 0x5fffffff: /* Peripheral */
11303         case 0xa0000000 ... 0xbfffffff: /* Device */
11304         case 0xc0000000 ... 0xdfffffff: /* Device */
11305         case 0xe0000000 ... 0xffffffff: /* System */
11306             *prot = PAGE_READ | PAGE_WRITE;
11307             break;
11308         default:
11309             g_assert_not_reached();
11310         }
11311     }
11312 }
11313 
11314 static bool pmsav7_use_background_region(ARMCPU *cpu,
11315                                          ARMMMUIdx mmu_idx, bool is_user)
11316 {
11317     /* Return true if we should use the default memory map as a
11318      * "background" region if there are no hits against any MPU regions.
11319      */
11320     CPUARMState *env = &cpu->env;
11321 
11322     if (is_user) {
11323         return false;
11324     }
11325 
11326     if (arm_feature(env, ARM_FEATURE_M)) {
11327         return env->v7m.mpu_ctrl[regime_is_secure(env, mmu_idx)]
11328             & R_V7M_MPU_CTRL_PRIVDEFENA_MASK;
11329     } else {
11330         return regime_sctlr(env, mmu_idx) & SCTLR_BR;
11331     }
11332 }
11333 
11334 static inline bool m_is_ppb_region(CPUARMState *env, uint32_t address)
11335 {
11336     /* True if address is in the M profile PPB region 0xe0000000 - 0xe00fffff */
11337     return arm_feature(env, ARM_FEATURE_M) &&
11338         extract32(address, 20, 12) == 0xe00;
11339 }
11340 
11341 static inline bool m_is_system_region(CPUARMState *env, uint32_t address)
11342 {
11343     /* True if address is in the M profile system region
11344      * 0xe0000000 - 0xffffffff
11345      */
11346     return arm_feature(env, ARM_FEATURE_M) && extract32(address, 29, 3) == 0x7;
11347 }
11348 
11349 static bool get_phys_addr_pmsav7(CPUARMState *env, uint32_t address,
11350                                  MMUAccessType access_type, ARMMMUIdx mmu_idx,
11351                                  hwaddr *phys_ptr, int *prot,
11352                                  target_ulong *page_size,
11353                                  ARMMMUFaultInfo *fi)
11354 {
11355     ARMCPU *cpu = env_archcpu(env);
11356     int n;
11357     bool is_user = regime_is_user(env, mmu_idx);
11358 
11359     *phys_ptr = address;
11360     *page_size = TARGET_PAGE_SIZE;
11361     *prot = 0;
11362 
11363     if (regime_translation_disabled(env, mmu_idx) ||
11364         m_is_ppb_region(env, address)) {
11365         /* MPU disabled or M profile PPB access: use default memory map.
11366          * The other case which uses the default memory map in the
11367          * v7M ARM ARM pseudocode is exception vector reads from the vector
11368          * table. In QEMU those accesses are done in arm_v7m_load_vector(),
11369          * which always does a direct read using address_space_ldl(), rather
11370          * than going via this function, so we don't need to check that here.
11371          */
11372         get_phys_addr_pmsav7_default(env, mmu_idx, address, prot);
11373     } else { /* MPU enabled */
11374         for (n = (int)cpu->pmsav7_dregion - 1; n >= 0; n--) {
11375             /* region search */
11376             uint32_t base = env->pmsav7.drbar[n];
11377             uint32_t rsize = extract32(env->pmsav7.drsr[n], 1, 5);
11378             uint32_t rmask;
11379             bool srdis = false;
11380 
11381             if (!(env->pmsav7.drsr[n] & 0x1)) {
11382                 continue;
11383             }
11384 
11385             if (!rsize) {
11386                 qemu_log_mask(LOG_GUEST_ERROR,
11387                               "DRSR[%d]: Rsize field cannot be 0\n", n);
11388                 continue;
11389             }
11390             rsize++;
11391             rmask = (1ull << rsize) - 1;
11392 
11393             if (base & rmask) {
11394                 qemu_log_mask(LOG_GUEST_ERROR,
11395                               "DRBAR[%d]: 0x%" PRIx32 " misaligned "
11396                               "to DRSR region size, mask = 0x%" PRIx32 "\n",
11397                               n, base, rmask);
11398                 continue;
11399             }
11400 
11401             if (address < base || address > base + rmask) {
11402                 /*
11403                  * Address not in this region. We must check whether the
11404                  * region covers addresses in the same page as our address.
11405                  * In that case we must not report a size that covers the
11406                  * whole page for a subsequent hit against a different MPU
11407                  * region or the background region, because it would result in
11408                  * incorrect TLB hits for subsequent accesses to addresses that
11409                  * are in this MPU region.
11410                  */
11411                 if (ranges_overlap(base, rmask,
11412                                    address & TARGET_PAGE_MASK,
11413                                    TARGET_PAGE_SIZE)) {
11414                     *page_size = 1;
11415                 }
11416                 continue;
11417             }
11418 
11419             /* Region matched */
11420 
11421             if (rsize >= 8) { /* no subregions for regions < 256 bytes */
11422                 int i, snd;
11423                 uint32_t srdis_mask;
11424 
11425                 rsize -= 3; /* sub region size (power of 2) */
11426                 snd = ((address - base) >> rsize) & 0x7;
11427                 srdis = extract32(env->pmsav7.drsr[n], snd + 8, 1);
11428 
11429                 srdis_mask = srdis ? 0x3 : 0x0;
11430                 for (i = 2; i <= 8 && rsize < TARGET_PAGE_BITS; i *= 2) {
11431                     /* This will check in groups of 2, 4 and then 8, whether
11432                      * the subregion bits are consistent. rsize is incremented
11433                      * back up to give the region size, considering consistent
11434                      * adjacent subregions as one region. Stop testing if rsize
11435                      * is already big enough for an entire QEMU page.
11436                      */
11437                     int snd_rounded = snd & ~(i - 1);
11438                     uint32_t srdis_multi = extract32(env->pmsav7.drsr[n],
11439                                                      snd_rounded + 8, i);
11440                     if (srdis_mask ^ srdis_multi) {
11441                         break;
11442                     }
11443                     srdis_mask = (srdis_mask << i) | srdis_mask;
11444                     rsize++;
11445                 }
11446             }
11447             if (srdis) {
11448                 continue;
11449             }
11450             if (rsize < TARGET_PAGE_BITS) {
11451                 *page_size = 1 << rsize;
11452             }
11453             break;
11454         }
11455 
11456         if (n == -1) { /* no hits */
11457             if (!pmsav7_use_background_region(cpu, mmu_idx, is_user)) {
11458                 /* background fault */
11459                 fi->type = ARMFault_Background;
11460                 return true;
11461             }
11462             get_phys_addr_pmsav7_default(env, mmu_idx, address, prot);
11463         } else { /* a MPU hit! */
11464             uint32_t ap = extract32(env->pmsav7.dracr[n], 8, 3);
11465             uint32_t xn = extract32(env->pmsav7.dracr[n], 12, 1);
11466 
11467             if (m_is_system_region(env, address)) {
11468                 /* System space is always execute never */
11469                 xn = 1;
11470             }
11471 
11472             if (is_user) { /* User mode AP bit decoding */
11473                 switch (ap) {
11474                 case 0:
11475                 case 1:
11476                 case 5:
11477                     break; /* no access */
11478                 case 3:
11479                     *prot |= PAGE_WRITE;
11480                     /* fall through */
11481                 case 2:
11482                 case 6:
11483                     *prot |= PAGE_READ | PAGE_EXEC;
11484                     break;
11485                 case 7:
11486                     /* for v7M, same as 6; for R profile a reserved value */
11487                     if (arm_feature(env, ARM_FEATURE_M)) {
11488                         *prot |= PAGE_READ | PAGE_EXEC;
11489                         break;
11490                     }
11491                     /* fall through */
11492                 default:
11493                     qemu_log_mask(LOG_GUEST_ERROR,
11494                                   "DRACR[%d]: Bad value for AP bits: 0x%"
11495                                   PRIx32 "\n", n, ap);
11496                 }
11497             } else { /* Priv. mode AP bits decoding */
11498                 switch (ap) {
11499                 case 0:
11500                     break; /* no access */
11501                 case 1:
11502                 case 2:
11503                 case 3:
11504                     *prot |= PAGE_WRITE;
11505                     /* fall through */
11506                 case 5:
11507                 case 6:
11508                     *prot |= PAGE_READ | PAGE_EXEC;
11509                     break;
11510                 case 7:
11511                     /* for v7M, same as 6; for R profile a reserved value */
11512                     if (arm_feature(env, ARM_FEATURE_M)) {
11513                         *prot |= PAGE_READ | PAGE_EXEC;
11514                         break;
11515                     }
11516                     /* fall through */
11517                 default:
11518                     qemu_log_mask(LOG_GUEST_ERROR,
11519                                   "DRACR[%d]: Bad value for AP bits: 0x%"
11520                                   PRIx32 "\n", n, ap);
11521                 }
11522             }
11523 
11524             /* execute never */
11525             if (xn) {
11526                 *prot &= ~PAGE_EXEC;
11527             }
11528         }
11529     }
11530 
11531     fi->type = ARMFault_Permission;
11532     fi->level = 1;
11533     return !(*prot & (1 << access_type));
11534 }
11535 
11536 static bool v8m_is_sau_exempt(CPUARMState *env,
11537                               uint32_t address, MMUAccessType access_type)
11538 {
11539     /* The architecture specifies that certain address ranges are
11540      * exempt from v8M SAU/IDAU checks.
11541      */
11542     return
11543         (access_type == MMU_INST_FETCH && m_is_system_region(env, address)) ||
11544         (address >= 0xe0000000 && address <= 0xe0002fff) ||
11545         (address >= 0xe000e000 && address <= 0xe000efff) ||
11546         (address >= 0xe002e000 && address <= 0xe002efff) ||
11547         (address >= 0xe0040000 && address <= 0xe0041fff) ||
11548         (address >= 0xe00ff000 && address <= 0xe00fffff);
11549 }
11550 
11551 void v8m_security_lookup(CPUARMState *env, uint32_t address,
11552                                 MMUAccessType access_type, ARMMMUIdx mmu_idx,
11553                                 V8M_SAttributes *sattrs)
11554 {
11555     /* Look up the security attributes for this address. Compare the
11556      * pseudocode SecurityCheck() function.
11557      * We assume the caller has zero-initialized *sattrs.
11558      */
11559     ARMCPU *cpu = env_archcpu(env);
11560     int r;
11561     bool idau_exempt = false, idau_ns = true, idau_nsc = true;
11562     int idau_region = IREGION_NOTVALID;
11563     uint32_t addr_page_base = address & TARGET_PAGE_MASK;
11564     uint32_t addr_page_limit = addr_page_base + (TARGET_PAGE_SIZE - 1);
11565 
11566     if (cpu->idau) {
11567         IDAUInterfaceClass *iic = IDAU_INTERFACE_GET_CLASS(cpu->idau);
11568         IDAUInterface *ii = IDAU_INTERFACE(cpu->idau);
11569 
11570         iic->check(ii, address, &idau_region, &idau_exempt, &idau_ns,
11571                    &idau_nsc);
11572     }
11573 
11574     if (access_type == MMU_INST_FETCH && extract32(address, 28, 4) == 0xf) {
11575         /* 0xf0000000..0xffffffff is always S for insn fetches */
11576         return;
11577     }
11578 
11579     if (idau_exempt || v8m_is_sau_exempt(env, address, access_type)) {
11580         sattrs->ns = !regime_is_secure(env, mmu_idx);
11581         return;
11582     }
11583 
11584     if (idau_region != IREGION_NOTVALID) {
11585         sattrs->irvalid = true;
11586         sattrs->iregion = idau_region;
11587     }
11588 
11589     switch (env->sau.ctrl & 3) {
11590     case 0: /* SAU.ENABLE == 0, SAU.ALLNS == 0 */
11591         break;
11592     case 2: /* SAU.ENABLE == 0, SAU.ALLNS == 1 */
11593         sattrs->ns = true;
11594         break;
11595     default: /* SAU.ENABLE == 1 */
11596         for (r = 0; r < cpu->sau_sregion; r++) {
11597             if (env->sau.rlar[r] & 1) {
11598                 uint32_t base = env->sau.rbar[r] & ~0x1f;
11599                 uint32_t limit = env->sau.rlar[r] | 0x1f;
11600 
11601                 if (base <= address && limit >= address) {
11602                     if (base > addr_page_base || limit < addr_page_limit) {
11603                         sattrs->subpage = true;
11604                     }
11605                     if (sattrs->srvalid) {
11606                         /* If we hit in more than one region then we must report
11607                          * as Secure, not NS-Callable, with no valid region
11608                          * number info.
11609                          */
11610                         sattrs->ns = false;
11611                         sattrs->nsc = false;
11612                         sattrs->sregion = 0;
11613                         sattrs->srvalid = false;
11614                         break;
11615                     } else {
11616                         if (env->sau.rlar[r] & 2) {
11617                             sattrs->nsc = true;
11618                         } else {
11619                             sattrs->ns = true;
11620                         }
11621                         sattrs->srvalid = true;
11622                         sattrs->sregion = r;
11623                     }
11624                 } else {
11625                     /*
11626                      * Address not in this region. We must check whether the
11627                      * region covers addresses in the same page as our address.
11628                      * In that case we must not report a size that covers the
11629                      * whole page for a subsequent hit against a different MPU
11630                      * region or the background region, because it would result
11631                      * in incorrect TLB hits for subsequent accesses to
11632                      * addresses that are in this MPU region.
11633                      */
11634                     if (limit >= base &&
11635                         ranges_overlap(base, limit - base + 1,
11636                                        addr_page_base,
11637                                        TARGET_PAGE_SIZE)) {
11638                         sattrs->subpage = true;
11639                     }
11640                 }
11641             }
11642         }
11643         break;
11644     }
11645 
11646     /*
11647      * The IDAU will override the SAU lookup results if it specifies
11648      * higher security than the SAU does.
11649      */
11650     if (!idau_ns) {
11651         if (sattrs->ns || (!idau_nsc && sattrs->nsc)) {
11652             sattrs->ns = false;
11653             sattrs->nsc = idau_nsc;
11654         }
11655     }
11656 }
11657 
11658 bool pmsav8_mpu_lookup(CPUARMState *env, uint32_t address,
11659                               MMUAccessType access_type, ARMMMUIdx mmu_idx,
11660                               hwaddr *phys_ptr, MemTxAttrs *txattrs,
11661                               int *prot, bool *is_subpage,
11662                               ARMMMUFaultInfo *fi, uint32_t *mregion)
11663 {
11664     /* Perform a PMSAv8 MPU lookup (without also doing the SAU check
11665      * that a full phys-to-virt translation does).
11666      * mregion is (if not NULL) set to the region number which matched,
11667      * or -1 if no region number is returned (MPU off, address did not
11668      * hit a region, address hit in multiple regions).
11669      * We set is_subpage to true if the region hit doesn't cover the
11670      * entire TARGET_PAGE the address is within.
11671      */
11672     ARMCPU *cpu = env_archcpu(env);
11673     bool is_user = regime_is_user(env, mmu_idx);
11674     uint32_t secure = regime_is_secure(env, mmu_idx);
11675     int n;
11676     int matchregion = -1;
11677     bool hit = false;
11678     uint32_t addr_page_base = address & TARGET_PAGE_MASK;
11679     uint32_t addr_page_limit = addr_page_base + (TARGET_PAGE_SIZE - 1);
11680 
11681     *is_subpage = false;
11682     *phys_ptr = address;
11683     *prot = 0;
11684     if (mregion) {
11685         *mregion = -1;
11686     }
11687 
11688     /* Unlike the ARM ARM pseudocode, we don't need to check whether this
11689      * was an exception vector read from the vector table (which is always
11690      * done using the default system address map), because those accesses
11691      * are done in arm_v7m_load_vector(), which always does a direct
11692      * read using address_space_ldl(), rather than going via this function.
11693      */
11694     if (regime_translation_disabled(env, mmu_idx)) { /* MPU disabled */
11695         hit = true;
11696     } else if (m_is_ppb_region(env, address)) {
11697         hit = true;
11698     } else {
11699         if (pmsav7_use_background_region(cpu, mmu_idx, is_user)) {
11700             hit = true;
11701         }
11702 
11703         for (n = (int)cpu->pmsav7_dregion - 1; n >= 0; n--) {
11704             /* region search */
11705             /* Note that the base address is bits [31:5] from the register
11706              * with bits [4:0] all zeroes, but the limit address is bits
11707              * [31:5] from the register with bits [4:0] all ones.
11708              */
11709             uint32_t base = env->pmsav8.rbar[secure][n] & ~0x1f;
11710             uint32_t limit = env->pmsav8.rlar[secure][n] | 0x1f;
11711 
11712             if (!(env->pmsav8.rlar[secure][n] & 0x1)) {
11713                 /* Region disabled */
11714                 continue;
11715             }
11716 
11717             if (address < base || address > limit) {
11718                 /*
11719                  * Address not in this region. We must check whether the
11720                  * region covers addresses in the same page as our address.
11721                  * In that case we must not report a size that covers the
11722                  * whole page for a subsequent hit against a different MPU
11723                  * region or the background region, because it would result in
11724                  * incorrect TLB hits for subsequent accesses to addresses that
11725                  * are in this MPU region.
11726                  */
11727                 if (limit >= base &&
11728                     ranges_overlap(base, limit - base + 1,
11729                                    addr_page_base,
11730                                    TARGET_PAGE_SIZE)) {
11731                     *is_subpage = true;
11732                 }
11733                 continue;
11734             }
11735 
11736             if (base > addr_page_base || limit < addr_page_limit) {
11737                 *is_subpage = true;
11738             }
11739 
11740             if (matchregion != -1) {
11741                 /* Multiple regions match -- always a failure (unlike
11742                  * PMSAv7 where highest-numbered-region wins)
11743                  */
11744                 fi->type = ARMFault_Permission;
11745                 fi->level = 1;
11746                 return true;
11747             }
11748 
11749             matchregion = n;
11750             hit = true;
11751         }
11752     }
11753 
11754     if (!hit) {
11755         /* background fault */
11756         fi->type = ARMFault_Background;
11757         return true;
11758     }
11759 
11760     if (matchregion == -1) {
11761         /* hit using the background region */
11762         get_phys_addr_pmsav7_default(env, mmu_idx, address, prot);
11763     } else {
11764         uint32_t ap = extract32(env->pmsav8.rbar[secure][matchregion], 1, 2);
11765         uint32_t xn = extract32(env->pmsav8.rbar[secure][matchregion], 0, 1);
11766 
11767         if (m_is_system_region(env, address)) {
11768             /* System space is always execute never */
11769             xn = 1;
11770         }
11771 
11772         *prot = simple_ap_to_rw_prot(env, mmu_idx, ap);
11773         if (*prot && !xn) {
11774             *prot |= PAGE_EXEC;
11775         }
11776         /* We don't need to look the attribute up in the MAIR0/MAIR1
11777          * registers because that only tells us about cacheability.
11778          */
11779         if (mregion) {
11780             *mregion = matchregion;
11781         }
11782     }
11783 
11784     fi->type = ARMFault_Permission;
11785     fi->level = 1;
11786     return !(*prot & (1 << access_type));
11787 }
11788 
11789 
11790 static bool get_phys_addr_pmsav8(CPUARMState *env, uint32_t address,
11791                                  MMUAccessType access_type, ARMMMUIdx mmu_idx,
11792                                  hwaddr *phys_ptr, MemTxAttrs *txattrs,
11793                                  int *prot, target_ulong *page_size,
11794                                  ARMMMUFaultInfo *fi)
11795 {
11796     uint32_t secure = regime_is_secure(env, mmu_idx);
11797     V8M_SAttributes sattrs = {};
11798     bool ret;
11799     bool mpu_is_subpage;
11800 
11801     if (arm_feature(env, ARM_FEATURE_M_SECURITY)) {
11802         v8m_security_lookup(env, address, access_type, mmu_idx, &sattrs);
11803         if (access_type == MMU_INST_FETCH) {
11804             /* Instruction fetches always use the MMU bank and the
11805              * transaction attribute determined by the fetch address,
11806              * regardless of CPU state. This is painful for QEMU
11807              * to handle, because it would mean we need to encode
11808              * into the mmu_idx not just the (user, negpri) information
11809              * for the current security state but also that for the
11810              * other security state, which would balloon the number
11811              * of mmu_idx values needed alarmingly.
11812              * Fortunately we can avoid this because it's not actually
11813              * possible to arbitrarily execute code from memory with
11814              * the wrong security attribute: it will always generate
11815              * an exception of some kind or another, apart from the
11816              * special case of an NS CPU executing an SG instruction
11817              * in S&NSC memory. So we always just fail the translation
11818              * here and sort things out in the exception handler
11819              * (including possibly emulating an SG instruction).
11820              */
11821             if (sattrs.ns != !secure) {
11822                 if (sattrs.nsc) {
11823                     fi->type = ARMFault_QEMU_NSCExec;
11824                 } else {
11825                     fi->type = ARMFault_QEMU_SFault;
11826                 }
11827                 *page_size = sattrs.subpage ? 1 : TARGET_PAGE_SIZE;
11828                 *phys_ptr = address;
11829                 *prot = 0;
11830                 return true;
11831             }
11832         } else {
11833             /* For data accesses we always use the MMU bank indicated
11834              * by the current CPU state, but the security attributes
11835              * might downgrade a secure access to nonsecure.
11836              */
11837             if (sattrs.ns) {
11838                 txattrs->secure = false;
11839             } else if (!secure) {
11840                 /* NS access to S memory must fault.
11841                  * Architecturally we should first check whether the
11842                  * MPU information for this address indicates that we
11843                  * are doing an unaligned access to Device memory, which
11844                  * should generate a UsageFault instead. QEMU does not
11845                  * currently check for that kind of unaligned access though.
11846                  * If we added it we would need to do so as a special case
11847                  * for M_FAKE_FSR_SFAULT in arm_v7m_cpu_do_interrupt().
11848                  */
11849                 fi->type = ARMFault_QEMU_SFault;
11850                 *page_size = sattrs.subpage ? 1 : TARGET_PAGE_SIZE;
11851                 *phys_ptr = address;
11852                 *prot = 0;
11853                 return true;
11854             }
11855         }
11856     }
11857 
11858     ret = pmsav8_mpu_lookup(env, address, access_type, mmu_idx, phys_ptr,
11859                             txattrs, prot, &mpu_is_subpage, fi, NULL);
11860     *page_size = sattrs.subpage || mpu_is_subpage ? 1 : TARGET_PAGE_SIZE;
11861     return ret;
11862 }
11863 
11864 static bool get_phys_addr_pmsav5(CPUARMState *env, uint32_t address,
11865                                  MMUAccessType access_type, ARMMMUIdx mmu_idx,
11866                                  hwaddr *phys_ptr, int *prot,
11867                                  ARMMMUFaultInfo *fi)
11868 {
11869     int n;
11870     uint32_t mask;
11871     uint32_t base;
11872     bool is_user = regime_is_user(env, mmu_idx);
11873 
11874     if (regime_translation_disabled(env, mmu_idx)) {
11875         /* MPU disabled.  */
11876         *phys_ptr = address;
11877         *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
11878         return false;
11879     }
11880 
11881     *phys_ptr = address;
11882     for (n = 7; n >= 0; n--) {
11883         base = env->cp15.c6_region[n];
11884         if ((base & 1) == 0) {
11885             continue;
11886         }
11887         mask = 1 << ((base >> 1) & 0x1f);
11888         /* Keep this shift separate from the above to avoid an
11889            (undefined) << 32.  */
11890         mask = (mask << 1) - 1;
11891         if (((base ^ address) & ~mask) == 0) {
11892             break;
11893         }
11894     }
11895     if (n < 0) {
11896         fi->type = ARMFault_Background;
11897         return true;
11898     }
11899 
11900     if (access_type == MMU_INST_FETCH) {
11901         mask = env->cp15.pmsav5_insn_ap;
11902     } else {
11903         mask = env->cp15.pmsav5_data_ap;
11904     }
11905     mask = (mask >> (n * 4)) & 0xf;
11906     switch (mask) {
11907     case 0:
11908         fi->type = ARMFault_Permission;
11909         fi->level = 1;
11910         return true;
11911     case 1:
11912         if (is_user) {
11913             fi->type = ARMFault_Permission;
11914             fi->level = 1;
11915             return true;
11916         }
11917         *prot = PAGE_READ | PAGE_WRITE;
11918         break;
11919     case 2:
11920         *prot = PAGE_READ;
11921         if (!is_user) {
11922             *prot |= PAGE_WRITE;
11923         }
11924         break;
11925     case 3:
11926         *prot = PAGE_READ | PAGE_WRITE;
11927         break;
11928     case 5:
11929         if (is_user) {
11930             fi->type = ARMFault_Permission;
11931             fi->level = 1;
11932             return true;
11933         }
11934         *prot = PAGE_READ;
11935         break;
11936     case 6:
11937         *prot = PAGE_READ;
11938         break;
11939     default:
11940         /* Bad permission.  */
11941         fi->type = ARMFault_Permission;
11942         fi->level = 1;
11943         return true;
11944     }
11945     *prot |= PAGE_EXEC;
11946     return false;
11947 }
11948 
11949 /* Combine either inner or outer cacheability attributes for normal
11950  * memory, according to table D4-42 and pseudocode procedure
11951  * CombineS1S2AttrHints() of ARM DDI 0487B.b (the ARMv8 ARM).
11952  *
11953  * NB: only stage 1 includes allocation hints (RW bits), leading to
11954  * some asymmetry.
11955  */
11956 static uint8_t combine_cacheattr_nibble(uint8_t s1, uint8_t s2)
11957 {
11958     if (s1 == 4 || s2 == 4) {
11959         /* non-cacheable has precedence */
11960         return 4;
11961     } else if (extract32(s1, 2, 2) == 0 || extract32(s1, 2, 2) == 2) {
11962         /* stage 1 write-through takes precedence */
11963         return s1;
11964     } else if (extract32(s2, 2, 2) == 2) {
11965         /* stage 2 write-through takes precedence, but the allocation hint
11966          * is still taken from stage 1
11967          */
11968         return (2 << 2) | extract32(s1, 0, 2);
11969     } else { /* write-back */
11970         return s1;
11971     }
11972 }
11973 
11974 /* Combine S1 and S2 cacheability/shareability attributes, per D4.5.4
11975  * and CombineS1S2Desc()
11976  *
11977  * @s1:      Attributes from stage 1 walk
11978  * @s2:      Attributes from stage 2 walk
11979  */
11980 static ARMCacheAttrs combine_cacheattrs(ARMCacheAttrs s1, ARMCacheAttrs s2)
11981 {
11982     uint8_t s1lo, s2lo, s1hi, s2hi;
11983     ARMCacheAttrs ret;
11984     bool tagged = false;
11985 
11986     if (s1.attrs == 0xf0) {
11987         tagged = true;
11988         s1.attrs = 0xff;
11989     }
11990 
11991     s1lo = extract32(s1.attrs, 0, 4);
11992     s2lo = extract32(s2.attrs, 0, 4);
11993     s1hi = extract32(s1.attrs, 4, 4);
11994     s2hi = extract32(s2.attrs, 4, 4);
11995 
11996     /* Combine shareability attributes (table D4-43) */
11997     if (s1.shareability == 2 || s2.shareability == 2) {
11998         /* if either are outer-shareable, the result is outer-shareable */
11999         ret.shareability = 2;
12000     } else if (s1.shareability == 3 || s2.shareability == 3) {
12001         /* if either are inner-shareable, the result is inner-shareable */
12002         ret.shareability = 3;
12003     } else {
12004         /* both non-shareable */
12005         ret.shareability = 0;
12006     }
12007 
12008     /* Combine memory type and cacheability attributes */
12009     if (s1hi == 0 || s2hi == 0) {
12010         /* Device has precedence over normal */
12011         if (s1lo == 0 || s2lo == 0) {
12012             /* nGnRnE has precedence over anything */
12013             ret.attrs = 0;
12014         } else if (s1lo == 4 || s2lo == 4) {
12015             /* non-Reordering has precedence over Reordering */
12016             ret.attrs = 4;  /* nGnRE */
12017         } else if (s1lo == 8 || s2lo == 8) {
12018             /* non-Gathering has precedence over Gathering */
12019             ret.attrs = 8;  /* nGRE */
12020         } else {
12021             ret.attrs = 0xc; /* GRE */
12022         }
12023 
12024         /* Any location for which the resultant memory type is any
12025          * type of Device memory is always treated as Outer Shareable.
12026          */
12027         ret.shareability = 2;
12028     } else { /* Normal memory */
12029         /* Outer/inner cacheability combine independently */
12030         ret.attrs = combine_cacheattr_nibble(s1hi, s2hi) << 4
12031                   | combine_cacheattr_nibble(s1lo, s2lo);
12032 
12033         if (ret.attrs == 0x44) {
12034             /* Any location for which the resultant memory type is Normal
12035              * Inner Non-cacheable, Outer Non-cacheable is always treated
12036              * as Outer Shareable.
12037              */
12038             ret.shareability = 2;
12039         }
12040     }
12041 
12042     /* TODO: CombineS1S2Desc does not consider transient, only WB, RWA. */
12043     if (tagged && ret.attrs == 0xff) {
12044         ret.attrs = 0xf0;
12045     }
12046 
12047     return ret;
12048 }
12049 
12050 
12051 /* get_phys_addr - get the physical address for this virtual address
12052  *
12053  * Find the physical address corresponding to the given virtual address,
12054  * by doing a translation table walk on MMU based systems or using the
12055  * MPU state on MPU based systems.
12056  *
12057  * Returns false if the translation was successful. Otherwise, phys_ptr, attrs,
12058  * prot and page_size may not be filled in, and the populated fsr value provides
12059  * information on why the translation aborted, in the format of a
12060  * DFSR/IFSR fault register, with the following caveats:
12061  *  * we honour the short vs long DFSR format differences.
12062  *  * the WnR bit is never set (the caller must do this).
12063  *  * for PSMAv5 based systems we don't bother to return a full FSR format
12064  *    value.
12065  *
12066  * @env: CPUARMState
12067  * @address: virtual address to get physical address for
12068  * @access_type: 0 for read, 1 for write, 2 for execute
12069  * @mmu_idx: MMU index indicating required translation regime
12070  * @phys_ptr: set to the physical address corresponding to the virtual address
12071  * @attrs: set to the memory transaction attributes to use
12072  * @prot: set to the permissions for the page containing phys_ptr
12073  * @page_size: set to the size of the page containing phys_ptr
12074  * @fi: set to fault info if the translation fails
12075  * @cacheattrs: (if non-NULL) set to the cacheability/shareability attributes
12076  */
12077 bool get_phys_addr(CPUARMState *env, target_ulong address,
12078                    MMUAccessType access_type, ARMMMUIdx mmu_idx,
12079                    hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot,
12080                    target_ulong *page_size,
12081                    ARMMMUFaultInfo *fi, ARMCacheAttrs *cacheattrs)
12082 {
12083     if (mmu_idx == ARMMMUIdx_E10_0 ||
12084         mmu_idx == ARMMMUIdx_E10_1 ||
12085         mmu_idx == ARMMMUIdx_E10_1_PAN) {
12086         /* Call ourselves recursively to do the stage 1 and then stage 2
12087          * translations.
12088          */
12089         if (arm_feature(env, ARM_FEATURE_EL2)) {
12090             hwaddr ipa;
12091             int s2_prot;
12092             int ret;
12093             ARMCacheAttrs cacheattrs2 = {};
12094 
12095             ret = get_phys_addr(env, address, access_type,
12096                                 stage_1_mmu_idx(mmu_idx), &ipa, attrs,
12097                                 prot, page_size, fi, cacheattrs);
12098 
12099             /* If S1 fails or S2 is disabled, return early.  */
12100             if (ret || regime_translation_disabled(env, ARMMMUIdx_Stage2)) {
12101                 *phys_ptr = ipa;
12102                 return ret;
12103             }
12104 
12105             /* S1 is done. Now do S2 translation.  */
12106             ret = get_phys_addr_lpae(env, ipa, access_type, ARMMMUIdx_Stage2,
12107                                      mmu_idx == ARMMMUIdx_E10_0,
12108                                      phys_ptr, attrs, &s2_prot,
12109                                      page_size, fi, &cacheattrs2);
12110             fi->s2addr = ipa;
12111             /* Combine the S1 and S2 perms.  */
12112             *prot &= s2_prot;
12113 
12114             /* If S2 fails, return early.  */
12115             if (ret) {
12116                 return ret;
12117             }
12118 
12119             /* Combine the S1 and S2 cache attributes. */
12120             if (env->cp15.hcr_el2 & HCR_DC) {
12121                 /*
12122                  * HCR.DC forces the first stage attributes to
12123                  *  Normal Non-Shareable,
12124                  *  Inner Write-Back Read-Allocate Write-Allocate,
12125                  *  Outer Write-Back Read-Allocate Write-Allocate.
12126                  * Do not overwrite Tagged within attrs.
12127                  */
12128                 if (cacheattrs->attrs != 0xf0) {
12129                     cacheattrs->attrs = 0xff;
12130                 }
12131                 cacheattrs->shareability = 0;
12132             }
12133             *cacheattrs = combine_cacheattrs(*cacheattrs, cacheattrs2);
12134             return 0;
12135         } else {
12136             /*
12137              * For non-EL2 CPUs a stage1+stage2 translation is just stage 1.
12138              */
12139             mmu_idx = stage_1_mmu_idx(mmu_idx);
12140         }
12141     }
12142 
12143     /* The page table entries may downgrade secure to non-secure, but
12144      * cannot upgrade an non-secure translation regime's attributes
12145      * to secure.
12146      */
12147     attrs->secure = regime_is_secure(env, mmu_idx);
12148     attrs->user = regime_is_user(env, mmu_idx);
12149 
12150     /* Fast Context Switch Extension. This doesn't exist at all in v8.
12151      * In v7 and earlier it affects all stage 1 translations.
12152      */
12153     if (address < 0x02000000 && mmu_idx != ARMMMUIdx_Stage2
12154         && !arm_feature(env, ARM_FEATURE_V8)) {
12155         if (regime_el(env, mmu_idx) == 3) {
12156             address += env->cp15.fcseidr_s;
12157         } else {
12158             address += env->cp15.fcseidr_ns;
12159         }
12160     }
12161 
12162     if (arm_feature(env, ARM_FEATURE_PMSA)) {
12163         bool ret;
12164         *page_size = TARGET_PAGE_SIZE;
12165 
12166         if (arm_feature(env, ARM_FEATURE_V8)) {
12167             /* PMSAv8 */
12168             ret = get_phys_addr_pmsav8(env, address, access_type, mmu_idx,
12169                                        phys_ptr, attrs, prot, page_size, fi);
12170         } else if (arm_feature(env, ARM_FEATURE_V7)) {
12171             /* PMSAv7 */
12172             ret = get_phys_addr_pmsav7(env, address, access_type, mmu_idx,
12173                                        phys_ptr, prot, page_size, fi);
12174         } else {
12175             /* Pre-v7 MPU */
12176             ret = get_phys_addr_pmsav5(env, address, access_type, mmu_idx,
12177                                        phys_ptr, prot, fi);
12178         }
12179         qemu_log_mask(CPU_LOG_MMU, "PMSA MPU lookup for %s at 0x%08" PRIx32
12180                       " mmu_idx %u -> %s (prot %c%c%c)\n",
12181                       access_type == MMU_DATA_LOAD ? "reading" :
12182                       (access_type == MMU_DATA_STORE ? "writing" : "execute"),
12183                       (uint32_t)address, mmu_idx,
12184                       ret ? "Miss" : "Hit",
12185                       *prot & PAGE_READ ? 'r' : '-',
12186                       *prot & PAGE_WRITE ? 'w' : '-',
12187                       *prot & PAGE_EXEC ? 'x' : '-');
12188 
12189         return ret;
12190     }
12191 
12192     /* Definitely a real MMU, not an MPU */
12193 
12194     if (regime_translation_disabled(env, mmu_idx)) {
12195         uint64_t hcr;
12196         uint8_t memattr;
12197 
12198         /*
12199          * MMU disabled.  S1 addresses within aa64 translation regimes are
12200          * still checked for bounds -- see AArch64.TranslateAddressS1Off.
12201          */
12202         if (mmu_idx != ARMMMUIdx_Stage2) {
12203             int r_el = regime_el(env, mmu_idx);
12204             if (arm_el_is_aa64(env, r_el)) {
12205                 int pamax = arm_pamax(env_archcpu(env));
12206                 uint64_t tcr = env->cp15.tcr_el[r_el].raw_tcr;
12207                 int addrtop, tbi;
12208 
12209                 tbi = aa64_va_parameter_tbi(tcr, mmu_idx);
12210                 if (access_type == MMU_INST_FETCH) {
12211                     tbi &= ~aa64_va_parameter_tbid(tcr, mmu_idx);
12212                 }
12213                 tbi = (tbi >> extract64(address, 55, 1)) & 1;
12214                 addrtop = (tbi ? 55 : 63);
12215 
12216                 if (extract64(address, pamax, addrtop - pamax + 1) != 0) {
12217                     fi->type = ARMFault_AddressSize;
12218                     fi->level = 0;
12219                     fi->stage2 = false;
12220                     return 1;
12221                 }
12222 
12223                 /*
12224                  * When TBI is disabled, we've just validated that all of the
12225                  * bits above PAMax are zero, so logically we only need to
12226                  * clear the top byte for TBI.  But it's clearer to follow
12227                  * the pseudocode set of addrdesc.paddress.
12228                  */
12229                 address = extract64(address, 0, 52);
12230             }
12231         }
12232         *phys_ptr = address;
12233         *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
12234         *page_size = TARGET_PAGE_SIZE;
12235 
12236         /* Fill in cacheattr a-la AArch64.TranslateAddressS1Off. */
12237         hcr = arm_hcr_el2_eff(env);
12238         cacheattrs->shareability = 0;
12239         if (hcr & HCR_DC) {
12240             if (hcr & HCR_DCT) {
12241                 memattr = 0xf0;  /* Tagged, Normal, WB, RWA */
12242             } else {
12243                 memattr = 0xff;  /* Normal, WB, RWA */
12244             }
12245         } else if (access_type == MMU_INST_FETCH) {
12246             if (regime_sctlr(env, mmu_idx) & SCTLR_I) {
12247                 memattr = 0xee;  /* Normal, WT, RA, NT */
12248             } else {
12249                 memattr = 0x44;  /* Normal, NC, No */
12250             }
12251             cacheattrs->shareability = 2; /* outer sharable */
12252         } else {
12253             memattr = 0x00;      /* Device, nGnRnE */
12254         }
12255         cacheattrs->attrs = memattr;
12256         return 0;
12257     }
12258 
12259     if (regime_using_lpae_format(env, mmu_idx)) {
12260         return get_phys_addr_lpae(env, address, access_type, mmu_idx, false,
12261                                   phys_ptr, attrs, prot, page_size,
12262                                   fi, cacheattrs);
12263     } else if (regime_sctlr(env, mmu_idx) & SCTLR_XP) {
12264         return get_phys_addr_v6(env, address, access_type, mmu_idx,
12265                                 phys_ptr, attrs, prot, page_size, fi);
12266     } else {
12267         return get_phys_addr_v5(env, address, access_type, mmu_idx,
12268                                     phys_ptr, prot, page_size, fi);
12269     }
12270 }
12271 
12272 hwaddr arm_cpu_get_phys_page_attrs_debug(CPUState *cs, vaddr addr,
12273                                          MemTxAttrs *attrs)
12274 {
12275     ARMCPU *cpu = ARM_CPU(cs);
12276     CPUARMState *env = &cpu->env;
12277     hwaddr phys_addr;
12278     target_ulong page_size;
12279     int prot;
12280     bool ret;
12281     ARMMMUFaultInfo fi = {};
12282     ARMMMUIdx mmu_idx = arm_mmu_idx(env);
12283     ARMCacheAttrs cacheattrs = {};
12284 
12285     *attrs = (MemTxAttrs) {};
12286 
12287     ret = get_phys_addr(env, addr, 0, mmu_idx, &phys_addr,
12288                         attrs, &prot, &page_size, &fi, &cacheattrs);
12289 
12290     if (ret) {
12291         return -1;
12292     }
12293     return phys_addr;
12294 }
12295 
12296 #endif
12297 
12298 /* Note that signed overflow is undefined in C.  The following routines are
12299    careful to use unsigned types where modulo arithmetic is required.
12300    Failure to do so _will_ break on newer gcc.  */
12301 
12302 /* Signed saturating arithmetic.  */
12303 
12304 /* Perform 16-bit signed saturating addition.  */
12305 static inline uint16_t add16_sat(uint16_t a, uint16_t b)
12306 {
12307     uint16_t res;
12308 
12309     res = a + b;
12310     if (((res ^ a) & 0x8000) && !((a ^ b) & 0x8000)) {
12311         if (a & 0x8000)
12312             res = 0x8000;
12313         else
12314             res = 0x7fff;
12315     }
12316     return res;
12317 }
12318 
12319 /* Perform 8-bit signed saturating addition.  */
12320 static inline uint8_t add8_sat(uint8_t a, uint8_t b)
12321 {
12322     uint8_t res;
12323 
12324     res = a + b;
12325     if (((res ^ a) & 0x80) && !((a ^ b) & 0x80)) {
12326         if (a & 0x80)
12327             res = 0x80;
12328         else
12329             res = 0x7f;
12330     }
12331     return res;
12332 }
12333 
12334 /* Perform 16-bit signed saturating subtraction.  */
12335 static inline uint16_t sub16_sat(uint16_t a, uint16_t b)
12336 {
12337     uint16_t res;
12338 
12339     res = a - b;
12340     if (((res ^ a) & 0x8000) && ((a ^ b) & 0x8000)) {
12341         if (a & 0x8000)
12342             res = 0x8000;
12343         else
12344             res = 0x7fff;
12345     }
12346     return res;
12347 }
12348 
12349 /* Perform 8-bit signed saturating subtraction.  */
12350 static inline uint8_t sub8_sat(uint8_t a, uint8_t b)
12351 {
12352     uint8_t res;
12353 
12354     res = a - b;
12355     if (((res ^ a) & 0x80) && ((a ^ b) & 0x80)) {
12356         if (a & 0x80)
12357             res = 0x80;
12358         else
12359             res = 0x7f;
12360     }
12361     return res;
12362 }
12363 
12364 #define ADD16(a, b, n) RESULT(add16_sat(a, b), n, 16);
12365 #define SUB16(a, b, n) RESULT(sub16_sat(a, b), n, 16);
12366 #define ADD8(a, b, n)  RESULT(add8_sat(a, b), n, 8);
12367 #define SUB8(a, b, n)  RESULT(sub8_sat(a, b), n, 8);
12368 #define PFX q
12369 
12370 #include "op_addsub.h"
12371 
12372 /* Unsigned saturating arithmetic.  */
12373 static inline uint16_t add16_usat(uint16_t a, uint16_t b)
12374 {
12375     uint16_t res;
12376     res = a + b;
12377     if (res < a)
12378         res = 0xffff;
12379     return res;
12380 }
12381 
12382 static inline uint16_t sub16_usat(uint16_t a, uint16_t b)
12383 {
12384     if (a > b)
12385         return a - b;
12386     else
12387         return 0;
12388 }
12389 
12390 static inline uint8_t add8_usat(uint8_t a, uint8_t b)
12391 {
12392     uint8_t res;
12393     res = a + b;
12394     if (res < a)
12395         res = 0xff;
12396     return res;
12397 }
12398 
12399 static inline uint8_t sub8_usat(uint8_t a, uint8_t b)
12400 {
12401     if (a > b)
12402         return a - b;
12403     else
12404         return 0;
12405 }
12406 
12407 #define ADD16(a, b, n) RESULT(add16_usat(a, b), n, 16);
12408 #define SUB16(a, b, n) RESULT(sub16_usat(a, b), n, 16);
12409 #define ADD8(a, b, n)  RESULT(add8_usat(a, b), n, 8);
12410 #define SUB8(a, b, n)  RESULT(sub8_usat(a, b), n, 8);
12411 #define PFX uq
12412 
12413 #include "op_addsub.h"
12414 
12415 /* Signed modulo arithmetic.  */
12416 #define SARITH16(a, b, n, op) do { \
12417     int32_t sum; \
12418     sum = (int32_t)(int16_t)(a) op (int32_t)(int16_t)(b); \
12419     RESULT(sum, n, 16); \
12420     if (sum >= 0) \
12421         ge |= 3 << (n * 2); \
12422     } while(0)
12423 
12424 #define SARITH8(a, b, n, op) do { \
12425     int32_t sum; \
12426     sum = (int32_t)(int8_t)(a) op (int32_t)(int8_t)(b); \
12427     RESULT(sum, n, 8); \
12428     if (sum >= 0) \
12429         ge |= 1 << n; \
12430     } while(0)
12431 
12432 
12433 #define ADD16(a, b, n) SARITH16(a, b, n, +)
12434 #define SUB16(a, b, n) SARITH16(a, b, n, -)
12435 #define ADD8(a, b, n)  SARITH8(a, b, n, +)
12436 #define SUB8(a, b, n)  SARITH8(a, b, n, -)
12437 #define PFX s
12438 #define ARITH_GE
12439 
12440 #include "op_addsub.h"
12441 
12442 /* Unsigned modulo arithmetic.  */
12443 #define ADD16(a, b, n) do { \
12444     uint32_t sum; \
12445     sum = (uint32_t)(uint16_t)(a) + (uint32_t)(uint16_t)(b); \
12446     RESULT(sum, n, 16); \
12447     if ((sum >> 16) == 1) \
12448         ge |= 3 << (n * 2); \
12449     } while(0)
12450 
12451 #define ADD8(a, b, n) do { \
12452     uint32_t sum; \
12453     sum = (uint32_t)(uint8_t)(a) + (uint32_t)(uint8_t)(b); \
12454     RESULT(sum, n, 8); \
12455     if ((sum >> 8) == 1) \
12456         ge |= 1 << n; \
12457     } while(0)
12458 
12459 #define SUB16(a, b, n) do { \
12460     uint32_t sum; \
12461     sum = (uint32_t)(uint16_t)(a) - (uint32_t)(uint16_t)(b); \
12462     RESULT(sum, n, 16); \
12463     if ((sum >> 16) == 0) \
12464         ge |= 3 << (n * 2); \
12465     } while(0)
12466 
12467 #define SUB8(a, b, n) do { \
12468     uint32_t sum; \
12469     sum = (uint32_t)(uint8_t)(a) - (uint32_t)(uint8_t)(b); \
12470     RESULT(sum, n, 8); \
12471     if ((sum >> 8) == 0) \
12472         ge |= 1 << n; \
12473     } while(0)
12474 
12475 #define PFX u
12476 #define ARITH_GE
12477 
12478 #include "op_addsub.h"
12479 
12480 /* Halved signed arithmetic.  */
12481 #define ADD16(a, b, n) \
12482   RESULT(((int32_t)(int16_t)(a) + (int32_t)(int16_t)(b)) >> 1, n, 16)
12483 #define SUB16(a, b, n) \
12484   RESULT(((int32_t)(int16_t)(a) - (int32_t)(int16_t)(b)) >> 1, n, 16)
12485 #define ADD8(a, b, n) \
12486   RESULT(((int32_t)(int8_t)(a) + (int32_t)(int8_t)(b)) >> 1, n, 8)
12487 #define SUB8(a, b, n) \
12488   RESULT(((int32_t)(int8_t)(a) - (int32_t)(int8_t)(b)) >> 1, n, 8)
12489 #define PFX sh
12490 
12491 #include "op_addsub.h"
12492 
12493 /* Halved unsigned arithmetic.  */
12494 #define ADD16(a, b, n) \
12495   RESULT(((uint32_t)(uint16_t)(a) + (uint32_t)(uint16_t)(b)) >> 1, n, 16)
12496 #define SUB16(a, b, n) \
12497   RESULT(((uint32_t)(uint16_t)(a) - (uint32_t)(uint16_t)(b)) >> 1, n, 16)
12498 #define ADD8(a, b, n) \
12499   RESULT(((uint32_t)(uint8_t)(a) + (uint32_t)(uint8_t)(b)) >> 1, n, 8)
12500 #define SUB8(a, b, n) \
12501   RESULT(((uint32_t)(uint8_t)(a) - (uint32_t)(uint8_t)(b)) >> 1, n, 8)
12502 #define PFX uh
12503 
12504 #include "op_addsub.h"
12505 
12506 static inline uint8_t do_usad(uint8_t a, uint8_t b)
12507 {
12508     if (a > b)
12509         return a - b;
12510     else
12511         return b - a;
12512 }
12513 
12514 /* Unsigned sum of absolute byte differences.  */
12515 uint32_t HELPER(usad8)(uint32_t a, uint32_t b)
12516 {
12517     uint32_t sum;
12518     sum = do_usad(a, b);
12519     sum += do_usad(a >> 8, b >> 8);
12520     sum += do_usad(a >> 16, b >>16);
12521     sum += do_usad(a >> 24, b >> 24);
12522     return sum;
12523 }
12524 
12525 /* For ARMv6 SEL instruction.  */
12526 uint32_t HELPER(sel_flags)(uint32_t flags, uint32_t a, uint32_t b)
12527 {
12528     uint32_t mask;
12529 
12530     mask = 0;
12531     if (flags & 1)
12532         mask |= 0xff;
12533     if (flags & 2)
12534         mask |= 0xff00;
12535     if (flags & 4)
12536         mask |= 0xff0000;
12537     if (flags & 8)
12538         mask |= 0xff000000;
12539     return (a & mask) | (b & ~mask);
12540 }
12541 
12542 /* CRC helpers.
12543  * The upper bytes of val (above the number specified by 'bytes') must have
12544  * been zeroed out by the caller.
12545  */
12546 uint32_t HELPER(crc32)(uint32_t acc, uint32_t val, uint32_t bytes)
12547 {
12548     uint8_t buf[4];
12549 
12550     stl_le_p(buf, val);
12551 
12552     /* zlib crc32 converts the accumulator and output to one's complement.  */
12553     return crc32(acc ^ 0xffffffff, buf, bytes) ^ 0xffffffff;
12554 }
12555 
12556 uint32_t HELPER(crc32c)(uint32_t acc, uint32_t val, uint32_t bytes)
12557 {
12558     uint8_t buf[4];
12559 
12560     stl_le_p(buf, val);
12561 
12562     /* Linux crc32c converts the output to one's complement.  */
12563     return crc32c(acc, buf, bytes) ^ 0xffffffff;
12564 }
12565 
12566 /* Return the exception level to which FP-disabled exceptions should
12567  * be taken, or 0 if FP is enabled.
12568  */
12569 int fp_exception_el(CPUARMState *env, int cur_el)
12570 {
12571 #ifndef CONFIG_USER_ONLY
12572     /* CPACR and the CPTR registers don't exist before v6, so FP is
12573      * always accessible
12574      */
12575     if (!arm_feature(env, ARM_FEATURE_V6)) {
12576         return 0;
12577     }
12578 
12579     if (arm_feature(env, ARM_FEATURE_M)) {
12580         /* CPACR can cause a NOCP UsageFault taken to current security state */
12581         if (!v7m_cpacr_pass(env, env->v7m.secure, cur_el != 0)) {
12582             return 1;
12583         }
12584 
12585         if (arm_feature(env, ARM_FEATURE_M_SECURITY) && !env->v7m.secure) {
12586             if (!extract32(env->v7m.nsacr, 10, 1)) {
12587                 /* FP insns cause a NOCP UsageFault taken to Secure */
12588                 return 3;
12589             }
12590         }
12591 
12592         return 0;
12593     }
12594 
12595     /* The CPACR controls traps to EL1, or PL1 if we're 32 bit:
12596      * 0, 2 : trap EL0 and EL1/PL1 accesses
12597      * 1    : trap only EL0 accesses
12598      * 3    : trap no accesses
12599      * This register is ignored if E2H+TGE are both set.
12600      */
12601     if ((arm_hcr_el2_eff(env) & (HCR_E2H | HCR_TGE)) != (HCR_E2H | HCR_TGE)) {
12602         int fpen = extract32(env->cp15.cpacr_el1, 20, 2);
12603 
12604         switch (fpen) {
12605         case 0:
12606         case 2:
12607             if (cur_el == 0 || cur_el == 1) {
12608                 /* Trap to PL1, which might be EL1 or EL3 */
12609                 if (arm_is_secure(env) && !arm_el_is_aa64(env, 3)) {
12610                     return 3;
12611                 }
12612                 return 1;
12613             }
12614             if (cur_el == 3 && !is_a64(env)) {
12615                 /* Secure PL1 running at EL3 */
12616                 return 3;
12617             }
12618             break;
12619         case 1:
12620             if (cur_el == 0) {
12621                 return 1;
12622             }
12623             break;
12624         case 3:
12625             break;
12626         }
12627     }
12628 
12629     /*
12630      * The NSACR allows A-profile AArch32 EL3 and M-profile secure mode
12631      * to control non-secure access to the FPU. It doesn't have any
12632      * effect if EL3 is AArch64 or if EL3 doesn't exist at all.
12633      */
12634     if ((arm_feature(env, ARM_FEATURE_EL3) && !arm_el_is_aa64(env, 3) &&
12635          cur_el <= 2 && !arm_is_secure_below_el3(env))) {
12636         if (!extract32(env->cp15.nsacr, 10, 1)) {
12637             /* FP insns act as UNDEF */
12638             return cur_el == 2 ? 2 : 1;
12639         }
12640     }
12641 
12642     /* For the CPTR registers we don't need to guard with an ARM_FEATURE
12643      * check because zero bits in the registers mean "don't trap".
12644      */
12645 
12646     /* CPTR_EL2 : present in v7VE or v8 */
12647     if (cur_el <= 2 && extract32(env->cp15.cptr_el[2], 10, 1)
12648         && !arm_is_secure_below_el3(env)) {
12649         /* Trap FP ops at EL2, NS-EL1 or NS-EL0 to EL2 */
12650         return 2;
12651     }
12652 
12653     /* CPTR_EL3 : present in v8 */
12654     if (extract32(env->cp15.cptr_el[3], 10, 1)) {
12655         /* Trap all FP ops to EL3 */
12656         return 3;
12657     }
12658 #endif
12659     return 0;
12660 }
12661 
12662 /* Return the exception level we're running at if this is our mmu_idx */
12663 int arm_mmu_idx_to_el(ARMMMUIdx mmu_idx)
12664 {
12665     if (mmu_idx & ARM_MMU_IDX_M) {
12666         return mmu_idx & ARM_MMU_IDX_M_PRIV;
12667     }
12668 
12669     switch (mmu_idx) {
12670     case ARMMMUIdx_E10_0:
12671     case ARMMMUIdx_E20_0:
12672     case ARMMMUIdx_SE10_0:
12673         return 0;
12674     case ARMMMUIdx_E10_1:
12675     case ARMMMUIdx_E10_1_PAN:
12676     case ARMMMUIdx_SE10_1:
12677     case ARMMMUIdx_SE10_1_PAN:
12678         return 1;
12679     case ARMMMUIdx_E2:
12680     case ARMMMUIdx_E20_2:
12681     case ARMMMUIdx_E20_2_PAN:
12682         return 2;
12683     case ARMMMUIdx_SE3:
12684         return 3;
12685     default:
12686         g_assert_not_reached();
12687     }
12688 }
12689 
12690 #ifndef CONFIG_TCG
12691 ARMMMUIdx arm_v7m_mmu_idx_for_secstate(CPUARMState *env, bool secstate)
12692 {
12693     g_assert_not_reached();
12694 }
12695 #endif
12696 
12697 ARMMMUIdx arm_mmu_idx_el(CPUARMState *env, int el)
12698 {
12699     if (arm_feature(env, ARM_FEATURE_M)) {
12700         return arm_v7m_mmu_idx_for_secstate(env, env->v7m.secure);
12701     }
12702 
12703     /* See ARM pseudo-function ELIsInHost.  */
12704     switch (el) {
12705     case 0:
12706         if (arm_is_secure_below_el3(env)) {
12707             return ARMMMUIdx_SE10_0;
12708         }
12709         if ((env->cp15.hcr_el2 & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)
12710             && arm_el_is_aa64(env, 2)) {
12711             return ARMMMUIdx_E20_0;
12712         }
12713         return ARMMMUIdx_E10_0;
12714     case 1:
12715         if (arm_is_secure_below_el3(env)) {
12716             if (env->pstate & PSTATE_PAN) {
12717                 return ARMMMUIdx_SE10_1_PAN;
12718             }
12719             return ARMMMUIdx_SE10_1;
12720         }
12721         if (env->pstate & PSTATE_PAN) {
12722             return ARMMMUIdx_E10_1_PAN;
12723         }
12724         return ARMMMUIdx_E10_1;
12725     case 2:
12726         /* TODO: ARMv8.4-SecEL2 */
12727         /* Note that TGE does not apply at EL2.  */
12728         if ((env->cp15.hcr_el2 & HCR_E2H) && arm_el_is_aa64(env, 2)) {
12729             if (env->pstate & PSTATE_PAN) {
12730                 return ARMMMUIdx_E20_2_PAN;
12731             }
12732             return ARMMMUIdx_E20_2;
12733         }
12734         return ARMMMUIdx_E2;
12735     case 3:
12736         return ARMMMUIdx_SE3;
12737     default:
12738         g_assert_not_reached();
12739     }
12740 }
12741 
12742 ARMMMUIdx arm_mmu_idx(CPUARMState *env)
12743 {
12744     return arm_mmu_idx_el(env, arm_current_el(env));
12745 }
12746 
12747 #ifndef CONFIG_USER_ONLY
12748 ARMMMUIdx arm_stage1_mmu_idx(CPUARMState *env)
12749 {
12750     return stage_1_mmu_idx(arm_mmu_idx(env));
12751 }
12752 #endif
12753 
12754 static uint32_t rebuild_hflags_common(CPUARMState *env, int fp_el,
12755                                       ARMMMUIdx mmu_idx, uint32_t flags)
12756 {
12757     flags = FIELD_DP32(flags, TBFLAG_ANY, FPEXC_EL, fp_el);
12758     flags = FIELD_DP32(flags, TBFLAG_ANY, MMUIDX,
12759                        arm_to_core_mmu_idx(mmu_idx));
12760 
12761     if (arm_singlestep_active(env)) {
12762         flags = FIELD_DP32(flags, TBFLAG_ANY, SS_ACTIVE, 1);
12763     }
12764     return flags;
12765 }
12766 
12767 static uint32_t rebuild_hflags_common_32(CPUARMState *env, int fp_el,
12768                                          ARMMMUIdx mmu_idx, uint32_t flags)
12769 {
12770     bool sctlr_b = arm_sctlr_b(env);
12771 
12772     if (sctlr_b) {
12773         flags = FIELD_DP32(flags, TBFLAG_A32, SCTLR_B, 1);
12774     }
12775     if (arm_cpu_data_is_big_endian_a32(env, sctlr_b)) {
12776         flags = FIELD_DP32(flags, TBFLAG_ANY, BE_DATA, 1);
12777     }
12778     flags = FIELD_DP32(flags, TBFLAG_A32, NS, !access_secure_reg(env));
12779 
12780     return rebuild_hflags_common(env, fp_el, mmu_idx, flags);
12781 }
12782 
12783 static uint32_t rebuild_hflags_m32(CPUARMState *env, int fp_el,
12784                                    ARMMMUIdx mmu_idx)
12785 {
12786     uint32_t flags = 0;
12787 
12788     if (arm_v7m_is_handler_mode(env)) {
12789         flags = FIELD_DP32(flags, TBFLAG_M32, HANDLER, 1);
12790     }
12791 
12792     /*
12793      * v8M always applies stack limit checks unless CCR.STKOFHFNMIGN
12794      * is suppressing them because the requested execution priority
12795      * is less than 0.
12796      */
12797     if (arm_feature(env, ARM_FEATURE_V8) &&
12798         !((mmu_idx & ARM_MMU_IDX_M_NEGPRI) &&
12799           (env->v7m.ccr[env->v7m.secure] & R_V7M_CCR_STKOFHFNMIGN_MASK))) {
12800         flags = FIELD_DP32(flags, TBFLAG_M32, STACKCHECK, 1);
12801     }
12802 
12803     return rebuild_hflags_common_32(env, fp_el, mmu_idx, flags);
12804 }
12805 
12806 static uint32_t rebuild_hflags_aprofile(CPUARMState *env)
12807 {
12808     int flags = 0;
12809 
12810     flags = FIELD_DP32(flags, TBFLAG_ANY, DEBUG_TARGET_EL,
12811                        arm_debug_target_el(env));
12812     return flags;
12813 }
12814 
12815 static uint32_t rebuild_hflags_a32(CPUARMState *env, int fp_el,
12816                                    ARMMMUIdx mmu_idx)
12817 {
12818     uint32_t flags = rebuild_hflags_aprofile(env);
12819 
12820     if (arm_el_is_aa64(env, 1)) {
12821         flags = FIELD_DP32(flags, TBFLAG_A32, VFPEN, 1);
12822     }
12823 
12824     if (arm_current_el(env) < 2 && env->cp15.hstr_el2 &&
12825         (arm_hcr_el2_eff(env) & (HCR_E2H | HCR_TGE)) != (HCR_E2H | HCR_TGE)) {
12826         flags = FIELD_DP32(flags, TBFLAG_A32, HSTR_ACTIVE, 1);
12827     }
12828 
12829     return rebuild_hflags_common_32(env, fp_el, mmu_idx, flags);
12830 }
12831 
12832 static uint32_t rebuild_hflags_a64(CPUARMState *env, int el, int fp_el,
12833                                    ARMMMUIdx mmu_idx)
12834 {
12835     uint32_t flags = rebuild_hflags_aprofile(env);
12836     ARMMMUIdx stage1 = stage_1_mmu_idx(mmu_idx);
12837     uint64_t tcr = regime_tcr(env, mmu_idx)->raw_tcr;
12838     uint64_t sctlr;
12839     int tbii, tbid;
12840 
12841     flags = FIELD_DP32(flags, TBFLAG_ANY, AARCH64_STATE, 1);
12842 
12843     /* Get control bits for tagged addresses.  */
12844     tbid = aa64_va_parameter_tbi(tcr, mmu_idx);
12845     tbii = tbid & ~aa64_va_parameter_tbid(tcr, mmu_idx);
12846 
12847     flags = FIELD_DP32(flags, TBFLAG_A64, TBII, tbii);
12848     flags = FIELD_DP32(flags, TBFLAG_A64, TBID, tbid);
12849 
12850     if (cpu_isar_feature(aa64_sve, env_archcpu(env))) {
12851         int sve_el = sve_exception_el(env, el);
12852         uint32_t zcr_len;
12853 
12854         /*
12855          * If SVE is disabled, but FP is enabled,
12856          * then the effective len is 0.
12857          */
12858         if (sve_el != 0 && fp_el == 0) {
12859             zcr_len = 0;
12860         } else {
12861             zcr_len = sve_zcr_len_for_el(env, el);
12862         }
12863         flags = FIELD_DP32(flags, TBFLAG_A64, SVEEXC_EL, sve_el);
12864         flags = FIELD_DP32(flags, TBFLAG_A64, ZCR_LEN, zcr_len);
12865     }
12866 
12867     sctlr = regime_sctlr(env, stage1);
12868 
12869     if (arm_cpu_data_is_big_endian_a64(el, sctlr)) {
12870         flags = FIELD_DP32(flags, TBFLAG_ANY, BE_DATA, 1);
12871     }
12872 
12873     if (cpu_isar_feature(aa64_pauth, env_archcpu(env))) {
12874         /*
12875          * In order to save space in flags, we record only whether
12876          * pauth is "inactive", meaning all insns are implemented as
12877          * a nop, or "active" when some action must be performed.
12878          * The decision of which action to take is left to a helper.
12879          */
12880         if (sctlr & (SCTLR_EnIA | SCTLR_EnIB | SCTLR_EnDA | SCTLR_EnDB)) {
12881             flags = FIELD_DP32(flags, TBFLAG_A64, PAUTH_ACTIVE, 1);
12882         }
12883     }
12884 
12885     if (cpu_isar_feature(aa64_bti, env_archcpu(env))) {
12886         /* Note that SCTLR_EL[23].BT == SCTLR_BT1.  */
12887         if (sctlr & (el == 0 ? SCTLR_BT0 : SCTLR_BT1)) {
12888             flags = FIELD_DP32(flags, TBFLAG_A64, BT, 1);
12889         }
12890     }
12891 
12892     /* Compute the condition for using AccType_UNPRIV for LDTR et al. */
12893     if (!(env->pstate & PSTATE_UAO)) {
12894         switch (mmu_idx) {
12895         case ARMMMUIdx_E10_1:
12896         case ARMMMUIdx_E10_1_PAN:
12897         case ARMMMUIdx_SE10_1:
12898         case ARMMMUIdx_SE10_1_PAN:
12899             /* TODO: ARMv8.3-NV */
12900             flags = FIELD_DP32(flags, TBFLAG_A64, UNPRIV, 1);
12901             break;
12902         case ARMMMUIdx_E20_2:
12903         case ARMMMUIdx_E20_2_PAN:
12904             /* TODO: ARMv8.4-SecEL2 */
12905             /*
12906              * Note that EL20_2 is gated by HCR_EL2.E2H == 1, but EL20_0 is
12907              * gated by HCR_EL2.<E2H,TGE> == '11', and so is LDTR.
12908              */
12909             if (env->cp15.hcr_el2 & HCR_TGE) {
12910                 flags = FIELD_DP32(flags, TBFLAG_A64, UNPRIV, 1);
12911             }
12912             break;
12913         default:
12914             break;
12915         }
12916     }
12917 
12918     if (cpu_isar_feature(aa64_mte, env_archcpu(env))) {
12919         /*
12920          * Set MTE_ACTIVE if any access may be Checked, and leave clear
12921          * if all accesses must be Unchecked:
12922          * 1) If no TBI, then there are no tags in the address to check,
12923          * 2) If Tag Check Override, then all accesses are Unchecked,
12924          * 3) If Tag Check Fail == 0, then Checked access have no effect,
12925          * 4) If no Allocation Tag Access, then all accesses are Unchecked.
12926          */
12927         if (allocation_tag_access_enabled(env, el, sctlr)) {
12928             flags = FIELD_DP32(flags, TBFLAG_A64, ATA, 1);
12929             if (tbid
12930                 && !(env->pstate & PSTATE_TCO)
12931                 && (sctlr & (el == 0 ? SCTLR_TCF0 : SCTLR_TCF))) {
12932                 flags = FIELD_DP32(flags, TBFLAG_A64, MTE_ACTIVE, 1);
12933             }
12934         }
12935         /* And again for unprivileged accesses, if required.  */
12936         if (FIELD_EX32(flags, TBFLAG_A64, UNPRIV)
12937             && tbid
12938             && !(env->pstate & PSTATE_TCO)
12939             && (sctlr & SCTLR_TCF0)
12940             && allocation_tag_access_enabled(env, 0, sctlr)) {
12941             flags = FIELD_DP32(flags, TBFLAG_A64, MTE0_ACTIVE, 1);
12942         }
12943         /* Cache TCMA as well as TBI. */
12944         flags = FIELD_DP32(flags, TBFLAG_A64, TCMA,
12945                            aa64_va_parameter_tcma(tcr, mmu_idx));
12946     }
12947 
12948     return rebuild_hflags_common(env, fp_el, mmu_idx, flags);
12949 }
12950 
12951 static uint32_t rebuild_hflags_internal(CPUARMState *env)
12952 {
12953     int el = arm_current_el(env);
12954     int fp_el = fp_exception_el(env, el);
12955     ARMMMUIdx mmu_idx = arm_mmu_idx_el(env, el);
12956 
12957     if (is_a64(env)) {
12958         return rebuild_hflags_a64(env, el, fp_el, mmu_idx);
12959     } else if (arm_feature(env, ARM_FEATURE_M)) {
12960         return rebuild_hflags_m32(env, fp_el, mmu_idx);
12961     } else {
12962         return rebuild_hflags_a32(env, fp_el, mmu_idx);
12963     }
12964 }
12965 
12966 void arm_rebuild_hflags(CPUARMState *env)
12967 {
12968     env->hflags = rebuild_hflags_internal(env);
12969 }
12970 
12971 /*
12972  * If we have triggered a EL state change we can't rely on the
12973  * translator having passed it to us, we need to recompute.
12974  */
12975 void HELPER(rebuild_hflags_m32_newel)(CPUARMState *env)
12976 {
12977     int el = arm_current_el(env);
12978     int fp_el = fp_exception_el(env, el);
12979     ARMMMUIdx mmu_idx = arm_mmu_idx_el(env, el);
12980     env->hflags = rebuild_hflags_m32(env, fp_el, mmu_idx);
12981 }
12982 
12983 void HELPER(rebuild_hflags_m32)(CPUARMState *env, int el)
12984 {
12985     int fp_el = fp_exception_el(env, el);
12986     ARMMMUIdx mmu_idx = arm_mmu_idx_el(env, el);
12987 
12988     env->hflags = rebuild_hflags_m32(env, fp_el, mmu_idx);
12989 }
12990 
12991 /*
12992  * If we have triggered a EL state change we can't rely on the
12993  * translator having passed it to us, we need to recompute.
12994  */
12995 void HELPER(rebuild_hflags_a32_newel)(CPUARMState *env)
12996 {
12997     int el = arm_current_el(env);
12998     int fp_el = fp_exception_el(env, el);
12999     ARMMMUIdx mmu_idx = arm_mmu_idx_el(env, el);
13000     env->hflags = rebuild_hflags_a32(env, fp_el, mmu_idx);
13001 }
13002 
13003 void HELPER(rebuild_hflags_a32)(CPUARMState *env, int el)
13004 {
13005     int fp_el = fp_exception_el(env, el);
13006     ARMMMUIdx mmu_idx = arm_mmu_idx_el(env, el);
13007 
13008     env->hflags = rebuild_hflags_a32(env, fp_el, mmu_idx);
13009 }
13010 
13011 void HELPER(rebuild_hflags_a64)(CPUARMState *env, int el)
13012 {
13013     int fp_el = fp_exception_el(env, el);
13014     ARMMMUIdx mmu_idx = arm_mmu_idx_el(env, el);
13015 
13016     env->hflags = rebuild_hflags_a64(env, el, fp_el, mmu_idx);
13017 }
13018 
13019 static inline void assert_hflags_rebuild_correctly(CPUARMState *env)
13020 {
13021 #ifdef CONFIG_DEBUG_TCG
13022     uint32_t env_flags_current = env->hflags;
13023     uint32_t env_flags_rebuilt = rebuild_hflags_internal(env);
13024 
13025     if (unlikely(env_flags_current != env_flags_rebuilt)) {
13026         fprintf(stderr, "TCG hflags mismatch (current:0x%08x rebuilt:0x%08x)\n",
13027                 env_flags_current, env_flags_rebuilt);
13028         abort();
13029     }
13030 #endif
13031 }
13032 
13033 void cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc,
13034                           target_ulong *cs_base, uint32_t *pflags)
13035 {
13036     uint32_t flags = env->hflags;
13037     uint32_t pstate_for_ss;
13038 
13039     *cs_base = 0;
13040     assert_hflags_rebuild_correctly(env);
13041 
13042     if (FIELD_EX32(flags, TBFLAG_ANY, AARCH64_STATE)) {
13043         *pc = env->pc;
13044         if (cpu_isar_feature(aa64_bti, env_archcpu(env))) {
13045             flags = FIELD_DP32(flags, TBFLAG_A64, BTYPE, env->btype);
13046         }
13047         pstate_for_ss = env->pstate;
13048     } else {
13049         *pc = env->regs[15];
13050 
13051         if (arm_feature(env, ARM_FEATURE_M)) {
13052             if (arm_feature(env, ARM_FEATURE_M_SECURITY) &&
13053                 FIELD_EX32(env->v7m.fpccr[M_REG_S], V7M_FPCCR, S)
13054                 != env->v7m.secure) {
13055                 flags = FIELD_DP32(flags, TBFLAG_M32, FPCCR_S_WRONG, 1);
13056             }
13057 
13058             if ((env->v7m.fpccr[env->v7m.secure] & R_V7M_FPCCR_ASPEN_MASK) &&
13059                 (!(env->v7m.control[M_REG_S] & R_V7M_CONTROL_FPCA_MASK) ||
13060                  (env->v7m.secure &&
13061                   !(env->v7m.control[M_REG_S] & R_V7M_CONTROL_SFPA_MASK)))) {
13062                 /*
13063                  * ASPEN is set, but FPCA/SFPA indicate that there is no
13064                  * active FP context; we must create a new FP context before
13065                  * executing any FP insn.
13066                  */
13067                 flags = FIELD_DP32(flags, TBFLAG_M32, NEW_FP_CTXT_NEEDED, 1);
13068             }
13069 
13070             bool is_secure = env->v7m.fpccr[M_REG_S] & R_V7M_FPCCR_S_MASK;
13071             if (env->v7m.fpccr[is_secure] & R_V7M_FPCCR_LSPACT_MASK) {
13072                 flags = FIELD_DP32(flags, TBFLAG_M32, LSPACT, 1);
13073             }
13074         } else {
13075             /*
13076              * Note that XSCALE_CPAR shares bits with VECSTRIDE.
13077              * Note that VECLEN+VECSTRIDE are RES0 for M-profile.
13078              */
13079             if (arm_feature(env, ARM_FEATURE_XSCALE)) {
13080                 flags = FIELD_DP32(flags, TBFLAG_A32,
13081                                    XSCALE_CPAR, env->cp15.c15_cpar);
13082             } else {
13083                 flags = FIELD_DP32(flags, TBFLAG_A32, VECLEN,
13084                                    env->vfp.vec_len);
13085                 flags = FIELD_DP32(flags, TBFLAG_A32, VECSTRIDE,
13086                                    env->vfp.vec_stride);
13087             }
13088             if (env->vfp.xregs[ARM_VFP_FPEXC] & (1 << 30)) {
13089                 flags = FIELD_DP32(flags, TBFLAG_A32, VFPEN, 1);
13090             }
13091         }
13092 
13093         flags = FIELD_DP32(flags, TBFLAG_AM32, THUMB, env->thumb);
13094         flags = FIELD_DP32(flags, TBFLAG_AM32, CONDEXEC, env->condexec_bits);
13095         pstate_for_ss = env->uncached_cpsr;
13096     }
13097 
13098     /*
13099      * The SS_ACTIVE and PSTATE_SS bits correspond to the state machine
13100      * states defined in the ARM ARM for software singlestep:
13101      *  SS_ACTIVE   PSTATE.SS   State
13102      *     0            x       Inactive (the TB flag for SS is always 0)
13103      *     1            0       Active-pending
13104      *     1            1       Active-not-pending
13105      * SS_ACTIVE is set in hflags; PSTATE_SS is computed every TB.
13106      */
13107     if (FIELD_EX32(flags, TBFLAG_ANY, SS_ACTIVE) &&
13108         (pstate_for_ss & PSTATE_SS)) {
13109         flags = FIELD_DP32(flags, TBFLAG_ANY, PSTATE_SS, 1);
13110     }
13111 
13112     *pflags = flags;
13113 }
13114 
13115 #ifdef TARGET_AARCH64
13116 /*
13117  * The manual says that when SVE is enabled and VQ is widened the
13118  * implementation is allowed to zero the previously inaccessible
13119  * portion of the registers.  The corollary to that is that when
13120  * SVE is enabled and VQ is narrowed we are also allowed to zero
13121  * the now inaccessible portion of the registers.
13122  *
13123  * The intent of this is that no predicate bit beyond VQ is ever set.
13124  * Which means that some operations on predicate registers themselves
13125  * may operate on full uint64_t or even unrolled across the maximum
13126  * uint64_t[4].  Performing 4 bits of host arithmetic unconditionally
13127  * may well be cheaper than conditionals to restrict the operation
13128  * to the relevant portion of a uint16_t[16].
13129  */
13130 void aarch64_sve_narrow_vq(CPUARMState *env, unsigned vq)
13131 {
13132     int i, j;
13133     uint64_t pmask;
13134 
13135     assert(vq >= 1 && vq <= ARM_MAX_VQ);
13136     assert(vq <= env_archcpu(env)->sve_max_vq);
13137 
13138     /* Zap the high bits of the zregs.  */
13139     for (i = 0; i < 32; i++) {
13140         memset(&env->vfp.zregs[i].d[2 * vq], 0, 16 * (ARM_MAX_VQ - vq));
13141     }
13142 
13143     /* Zap the high bits of the pregs and ffr.  */
13144     pmask = 0;
13145     if (vq & 3) {
13146         pmask = ~(-1ULL << (16 * (vq & 3)));
13147     }
13148     for (j = vq / 4; j < ARM_MAX_VQ / 4; j++) {
13149         for (i = 0; i < 17; ++i) {
13150             env->vfp.pregs[i].p[j] &= pmask;
13151         }
13152         pmask = 0;
13153     }
13154 }
13155 
13156 /*
13157  * Notice a change in SVE vector size when changing EL.
13158  */
13159 void aarch64_sve_change_el(CPUARMState *env, int old_el,
13160                            int new_el, bool el0_a64)
13161 {
13162     ARMCPU *cpu = env_archcpu(env);
13163     int old_len, new_len;
13164     bool old_a64, new_a64;
13165 
13166     /* Nothing to do if no SVE.  */
13167     if (!cpu_isar_feature(aa64_sve, cpu)) {
13168         return;
13169     }
13170 
13171     /* Nothing to do if FP is disabled in either EL.  */
13172     if (fp_exception_el(env, old_el) || fp_exception_el(env, new_el)) {
13173         return;
13174     }
13175 
13176     /*
13177      * DDI0584A.d sec 3.2: "If SVE instructions are disabled or trapped
13178      * at ELx, or not available because the EL is in AArch32 state, then
13179      * for all purposes other than a direct read, the ZCR_ELx.LEN field
13180      * has an effective value of 0".
13181      *
13182      * Consider EL2 (aa64, vq=4) -> EL0 (aa32) -> EL1 (aa64, vq=0).
13183      * If we ignore aa32 state, we would fail to see the vq4->vq0 transition
13184      * from EL2->EL1.  Thus we go ahead and narrow when entering aa32 so that
13185      * we already have the correct register contents when encountering the
13186      * vq0->vq0 transition between EL0->EL1.
13187      */
13188     old_a64 = old_el ? arm_el_is_aa64(env, old_el) : el0_a64;
13189     old_len = (old_a64 && !sve_exception_el(env, old_el)
13190                ? sve_zcr_len_for_el(env, old_el) : 0);
13191     new_a64 = new_el ? arm_el_is_aa64(env, new_el) : el0_a64;
13192     new_len = (new_a64 && !sve_exception_el(env, new_el)
13193                ? sve_zcr_len_for_el(env, new_el) : 0);
13194 
13195     /* When changing vector length, clear inaccessible state.  */
13196     if (new_len < old_len) {
13197         aarch64_sve_narrow_vq(env, new_len + 1);
13198     }
13199 }
13200 #endif
13201