xref: /openbmc/qemu/target/arm/helper.c (revision f4f318b4)
1 /*
2  * ARM generic helpers.
3  *
4  * This code is licensed under the GNU GPL v2 or later.
5  *
6  * SPDX-License-Identifier: GPL-2.0-or-later
7  */
8 
9 #include "qemu/osdep.h"
10 #include "qemu/log.h"
11 #include "trace.h"
12 #include "cpu.h"
13 #include "internals.h"
14 #include "cpu-features.h"
15 #include "exec/helper-proto.h"
16 #include "qemu/main-loop.h"
17 #include "qemu/timer.h"
18 #include "qemu/bitops.h"
19 #include "qemu/crc32c.h"
20 #include "qemu/qemu-print.h"
21 #include "exec/exec-all.h"
22 #include <zlib.h> /* For crc32 */
23 #include "hw/irq.h"
24 #include "sysemu/cpu-timers.h"
25 #include "sysemu/kvm.h"
26 #include "sysemu/tcg.h"
27 #include "qapi/error.h"
28 #include "qemu/guest-random.h"
29 #ifdef CONFIG_TCG
30 #include "semihosting/common-semi.h"
31 #endif
32 #include "cpregs.h"
33 #include "target/arm/gtimer.h"
34 
35 #define ARM_CPU_FREQ 1000000000 /* FIXME: 1 GHz, should be configurable */
36 
37 static void switch_mode(CPUARMState *env, int mode);
38 
39 static uint64_t raw_read(CPUARMState *env, const ARMCPRegInfo *ri)
40 {
41     assert(ri->fieldoffset);
42     if (cpreg_field_is_64bit(ri)) {
43         return CPREG_FIELD64(env, ri);
44     } else {
45         return CPREG_FIELD32(env, ri);
46     }
47 }
48 
49 void raw_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
50 {
51     assert(ri->fieldoffset);
52     if (cpreg_field_is_64bit(ri)) {
53         CPREG_FIELD64(env, ri) = value;
54     } else {
55         CPREG_FIELD32(env, ri) = value;
56     }
57 }
58 
59 static void *raw_ptr(CPUARMState *env, const ARMCPRegInfo *ri)
60 {
61     return (char *)env + ri->fieldoffset;
62 }
63 
64 uint64_t read_raw_cp_reg(CPUARMState *env, const ARMCPRegInfo *ri)
65 {
66     /* Raw read of a coprocessor register (as needed for migration, etc). */
67     if (ri->type & ARM_CP_CONST) {
68         return ri->resetvalue;
69     } else if (ri->raw_readfn) {
70         return ri->raw_readfn(env, ri);
71     } else if (ri->readfn) {
72         return ri->readfn(env, ri);
73     } else {
74         return raw_read(env, ri);
75     }
76 }
77 
78 static void write_raw_cp_reg(CPUARMState *env, const ARMCPRegInfo *ri,
79                              uint64_t v)
80 {
81     /*
82      * Raw write of a coprocessor register (as needed for migration, etc).
83      * Note that constant registers are treated as write-ignored; the
84      * caller should check for success by whether a readback gives the
85      * value written.
86      */
87     if (ri->type & ARM_CP_CONST) {
88         return;
89     } else if (ri->raw_writefn) {
90         ri->raw_writefn(env, ri, v);
91     } else if (ri->writefn) {
92         ri->writefn(env, ri, v);
93     } else {
94         raw_write(env, ri, v);
95     }
96 }
97 
98 static bool raw_accessors_invalid(const ARMCPRegInfo *ri)
99 {
100    /*
101     * Return true if the regdef would cause an assertion if you called
102     * read_raw_cp_reg() or write_raw_cp_reg() on it (ie if it is a
103     * program bug for it not to have the NO_RAW flag).
104     * NB that returning false here doesn't necessarily mean that calling
105     * read/write_raw_cp_reg() is safe, because we can't distinguish "has
106     * read/write access functions which are safe for raw use" from "has
107     * read/write access functions which have side effects but has forgotten
108     * to provide raw access functions".
109     * The tests here line up with the conditions in read/write_raw_cp_reg()
110     * and assertions in raw_read()/raw_write().
111     */
112     if ((ri->type & ARM_CP_CONST) ||
113         ri->fieldoffset ||
114         ((ri->raw_writefn || ri->writefn) && (ri->raw_readfn || ri->readfn))) {
115         return false;
116     }
117     return true;
118 }
119 
120 bool write_cpustate_to_list(ARMCPU *cpu, bool kvm_sync)
121 {
122     /* Write the coprocessor state from cpu->env to the (index,value) list. */
123     int i;
124     bool ok = true;
125 
126     for (i = 0; i < cpu->cpreg_array_len; i++) {
127         uint32_t regidx = kvm_to_cpreg_id(cpu->cpreg_indexes[i]);
128         const ARMCPRegInfo *ri;
129         uint64_t newval;
130 
131         ri = get_arm_cp_reginfo(cpu->cp_regs, regidx);
132         if (!ri) {
133             ok = false;
134             continue;
135         }
136         if (ri->type & ARM_CP_NO_RAW) {
137             continue;
138         }
139 
140         newval = read_raw_cp_reg(&cpu->env, ri);
141         if (kvm_sync) {
142             /*
143              * Only sync if the previous list->cpustate sync succeeded.
144              * Rather than tracking the success/failure state for every
145              * item in the list, we just recheck "does the raw write we must
146              * have made in write_list_to_cpustate() read back OK" here.
147              */
148             uint64_t oldval = cpu->cpreg_values[i];
149 
150             if (oldval == newval) {
151                 continue;
152             }
153 
154             write_raw_cp_reg(&cpu->env, ri, oldval);
155             if (read_raw_cp_reg(&cpu->env, ri) != oldval) {
156                 continue;
157             }
158 
159             write_raw_cp_reg(&cpu->env, ri, newval);
160         }
161         cpu->cpreg_values[i] = newval;
162     }
163     return ok;
164 }
165 
166 bool write_list_to_cpustate(ARMCPU *cpu)
167 {
168     int i;
169     bool ok = true;
170 
171     for (i = 0; i < cpu->cpreg_array_len; i++) {
172         uint32_t regidx = kvm_to_cpreg_id(cpu->cpreg_indexes[i]);
173         uint64_t v = cpu->cpreg_values[i];
174         const ARMCPRegInfo *ri;
175 
176         ri = get_arm_cp_reginfo(cpu->cp_regs, regidx);
177         if (!ri) {
178             ok = false;
179             continue;
180         }
181         if (ri->type & ARM_CP_NO_RAW) {
182             continue;
183         }
184         /*
185          * Write value and confirm it reads back as written
186          * (to catch read-only registers and partially read-only
187          * registers where the incoming migration value doesn't match)
188          */
189         write_raw_cp_reg(&cpu->env, ri, v);
190         if (read_raw_cp_reg(&cpu->env, ri) != v) {
191             ok = false;
192         }
193     }
194     return ok;
195 }
196 
197 static void add_cpreg_to_list(gpointer key, gpointer opaque)
198 {
199     ARMCPU *cpu = opaque;
200     uint32_t regidx = (uintptr_t)key;
201     const ARMCPRegInfo *ri = get_arm_cp_reginfo(cpu->cp_regs, regidx);
202 
203     if (!(ri->type & (ARM_CP_NO_RAW | ARM_CP_ALIAS))) {
204         cpu->cpreg_indexes[cpu->cpreg_array_len] = cpreg_to_kvm_id(regidx);
205         /* The value array need not be initialized at this point */
206         cpu->cpreg_array_len++;
207     }
208 }
209 
210 static void count_cpreg(gpointer key, gpointer opaque)
211 {
212     ARMCPU *cpu = opaque;
213     const ARMCPRegInfo *ri;
214 
215     ri = g_hash_table_lookup(cpu->cp_regs, key);
216 
217     if (!(ri->type & (ARM_CP_NO_RAW | ARM_CP_ALIAS))) {
218         cpu->cpreg_array_len++;
219     }
220 }
221 
222 static gint cpreg_key_compare(gconstpointer a, gconstpointer b)
223 {
224     uint64_t aidx = cpreg_to_kvm_id((uintptr_t)a);
225     uint64_t bidx = cpreg_to_kvm_id((uintptr_t)b);
226 
227     if (aidx > bidx) {
228         return 1;
229     }
230     if (aidx < bidx) {
231         return -1;
232     }
233     return 0;
234 }
235 
236 void init_cpreg_list(ARMCPU *cpu)
237 {
238     /*
239      * Initialise the cpreg_tuples[] array based on the cp_regs hash.
240      * Note that we require cpreg_tuples[] to be sorted by key ID.
241      */
242     GList *keys;
243     int arraylen;
244 
245     keys = g_hash_table_get_keys(cpu->cp_regs);
246     keys = g_list_sort(keys, cpreg_key_compare);
247 
248     cpu->cpreg_array_len = 0;
249 
250     g_list_foreach(keys, count_cpreg, cpu);
251 
252     arraylen = cpu->cpreg_array_len;
253     cpu->cpreg_indexes = g_new(uint64_t, arraylen);
254     cpu->cpreg_values = g_new(uint64_t, arraylen);
255     cpu->cpreg_vmstate_indexes = g_new(uint64_t, arraylen);
256     cpu->cpreg_vmstate_values = g_new(uint64_t, arraylen);
257     cpu->cpreg_vmstate_array_len = cpu->cpreg_array_len;
258     cpu->cpreg_array_len = 0;
259 
260     g_list_foreach(keys, add_cpreg_to_list, cpu);
261 
262     assert(cpu->cpreg_array_len == arraylen);
263 
264     g_list_free(keys);
265 }
266 
267 static bool arm_pan_enabled(CPUARMState *env)
268 {
269     if (is_a64(env)) {
270         if ((arm_hcr_el2_eff(env) & (HCR_NV | HCR_NV1)) == (HCR_NV | HCR_NV1)) {
271             return false;
272         }
273         return env->pstate & PSTATE_PAN;
274     } else {
275         return env->uncached_cpsr & CPSR_PAN;
276     }
277 }
278 
279 /*
280  * Some registers are not accessible from AArch32 EL3 if SCR.NS == 0.
281  */
282 static CPAccessResult access_el3_aa32ns(CPUARMState *env,
283                                         const ARMCPRegInfo *ri,
284                                         bool isread)
285 {
286     if (!is_a64(env) && arm_current_el(env) == 3 &&
287         arm_is_secure_below_el3(env)) {
288         return CP_ACCESS_TRAP_UNCATEGORIZED;
289     }
290     return CP_ACCESS_OK;
291 }
292 
293 /*
294  * Some secure-only AArch32 registers trap to EL3 if used from
295  * Secure EL1 (but are just ordinary UNDEF in other non-EL3 contexts).
296  * Note that an access from Secure EL1 can only happen if EL3 is AArch64.
297  * We assume that the .access field is set to PL1_RW.
298  */
299 static CPAccessResult access_trap_aa32s_el1(CPUARMState *env,
300                                             const ARMCPRegInfo *ri,
301                                             bool isread)
302 {
303     if (arm_current_el(env) == 3) {
304         return CP_ACCESS_OK;
305     }
306     if (arm_is_secure_below_el3(env)) {
307         if (env->cp15.scr_el3 & SCR_EEL2) {
308             return CP_ACCESS_TRAP_EL2;
309         }
310         return CP_ACCESS_TRAP_EL3;
311     }
312     /* This will be EL1 NS and EL2 NS, which just UNDEF */
313     return CP_ACCESS_TRAP_UNCATEGORIZED;
314 }
315 
316 /*
317  * Check for traps to performance monitor registers, which are controlled
318  * by MDCR_EL2.TPM for EL2 and MDCR_EL3.TPM for EL3.
319  */
320 static CPAccessResult access_tpm(CPUARMState *env, const ARMCPRegInfo *ri,
321                                  bool isread)
322 {
323     int el = arm_current_el(env);
324     uint64_t mdcr_el2 = arm_mdcr_el2_eff(env);
325 
326     if (el < 2 && (mdcr_el2 & MDCR_TPM)) {
327         return CP_ACCESS_TRAP_EL2;
328     }
329     if (el < 3 && (env->cp15.mdcr_el3 & MDCR_TPM)) {
330         return CP_ACCESS_TRAP_EL3;
331     }
332     return CP_ACCESS_OK;
333 }
334 
335 /* Check for traps from EL1 due to HCR_EL2.TVM and HCR_EL2.TRVM.  */
336 CPAccessResult access_tvm_trvm(CPUARMState *env, const ARMCPRegInfo *ri,
337                                bool isread)
338 {
339     if (arm_current_el(env) == 1) {
340         uint64_t trap = isread ? HCR_TRVM : HCR_TVM;
341         if (arm_hcr_el2_eff(env) & trap) {
342             return CP_ACCESS_TRAP_EL2;
343         }
344     }
345     return CP_ACCESS_OK;
346 }
347 
348 /* Check for traps from EL1 due to HCR_EL2.TSW.  */
349 static CPAccessResult access_tsw(CPUARMState *env, const ARMCPRegInfo *ri,
350                                  bool isread)
351 {
352     if (arm_current_el(env) == 1 && (arm_hcr_el2_eff(env) & HCR_TSW)) {
353         return CP_ACCESS_TRAP_EL2;
354     }
355     return CP_ACCESS_OK;
356 }
357 
358 /* Check for traps from EL1 due to HCR_EL2.TACR.  */
359 static CPAccessResult access_tacr(CPUARMState *env, const ARMCPRegInfo *ri,
360                                   bool isread)
361 {
362     if (arm_current_el(env) == 1 && (arm_hcr_el2_eff(env) & HCR_TACR)) {
363         return CP_ACCESS_TRAP_EL2;
364     }
365     return CP_ACCESS_OK;
366 }
367 
368 /* Check for traps from EL1 due to HCR_EL2.TTLB. */
369 static CPAccessResult access_ttlb(CPUARMState *env, const ARMCPRegInfo *ri,
370                                   bool isread)
371 {
372     if (arm_current_el(env) == 1 && (arm_hcr_el2_eff(env) & HCR_TTLB)) {
373         return CP_ACCESS_TRAP_EL2;
374     }
375     return CP_ACCESS_OK;
376 }
377 
378 /* Check for traps from EL1 due to HCR_EL2.TTLB or TTLBIS. */
379 static CPAccessResult access_ttlbis(CPUARMState *env, const ARMCPRegInfo *ri,
380                                     bool isread)
381 {
382     if (arm_current_el(env) == 1 &&
383         (arm_hcr_el2_eff(env) & (HCR_TTLB | HCR_TTLBIS))) {
384         return CP_ACCESS_TRAP_EL2;
385     }
386     return CP_ACCESS_OK;
387 }
388 
389 #ifdef TARGET_AARCH64
390 /* Check for traps from EL1 due to HCR_EL2.TTLB or TTLBOS. */
391 static CPAccessResult access_ttlbos(CPUARMState *env, const ARMCPRegInfo *ri,
392                                     bool isread)
393 {
394     if (arm_current_el(env) == 1 &&
395         (arm_hcr_el2_eff(env) & (HCR_TTLB | HCR_TTLBOS))) {
396         return CP_ACCESS_TRAP_EL2;
397     }
398     return CP_ACCESS_OK;
399 }
400 #endif
401 
402 static void dacr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
403 {
404     ARMCPU *cpu = env_archcpu(env);
405 
406     raw_write(env, ri, value);
407     tlb_flush(CPU(cpu)); /* Flush TLB as domain not tracked in TLB */
408 }
409 
410 static void fcse_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
411 {
412     ARMCPU *cpu = env_archcpu(env);
413 
414     if (raw_read(env, ri) != value) {
415         /*
416          * Unlike real hardware the qemu TLB uses virtual addresses,
417          * not modified virtual addresses, so this causes a TLB flush.
418          */
419         tlb_flush(CPU(cpu));
420         raw_write(env, ri, value);
421     }
422 }
423 
424 static void contextidr_write(CPUARMState *env, const ARMCPRegInfo *ri,
425                              uint64_t value)
426 {
427     ARMCPU *cpu = env_archcpu(env);
428 
429     if (raw_read(env, ri) != value && !arm_feature(env, ARM_FEATURE_PMSA)
430         && !extended_addresses_enabled(env)) {
431         /*
432          * For VMSA (when not using the LPAE long descriptor page table
433          * format) this register includes the ASID, so do a TLB flush.
434          * For PMSA it is purely a process ID and no action is needed.
435          */
436         tlb_flush(CPU(cpu));
437     }
438     raw_write(env, ri, value);
439 }
440 
441 static int alle1_tlbmask(CPUARMState *env)
442 {
443     /*
444      * Note that the 'ALL' scope must invalidate both stage 1 and
445      * stage 2 translations, whereas most other scopes only invalidate
446      * stage 1 translations.
447      */
448     return (ARMMMUIdxBit_E10_1 |
449             ARMMMUIdxBit_E10_1_PAN |
450             ARMMMUIdxBit_E10_0 |
451             ARMMMUIdxBit_Stage2 |
452             ARMMMUIdxBit_Stage2_S);
453 }
454 
455 
456 /* IS variants of TLB operations must affect all cores */
457 static void tlbiall_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
458                              uint64_t value)
459 {
460     CPUState *cs = env_cpu(env);
461 
462     tlb_flush_all_cpus_synced(cs);
463 }
464 
465 static void tlbiasid_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
466                              uint64_t value)
467 {
468     CPUState *cs = env_cpu(env);
469 
470     tlb_flush_all_cpus_synced(cs);
471 }
472 
473 static void tlbimva_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
474                              uint64_t value)
475 {
476     CPUState *cs = env_cpu(env);
477 
478     tlb_flush_page_all_cpus_synced(cs, value & TARGET_PAGE_MASK);
479 }
480 
481 static void tlbimvaa_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
482                              uint64_t value)
483 {
484     CPUState *cs = env_cpu(env);
485 
486     tlb_flush_page_all_cpus_synced(cs, value & TARGET_PAGE_MASK);
487 }
488 
489 /*
490  * Non-IS variants of TLB operations are upgraded to
491  * IS versions if we are at EL1 and HCR_EL2.FB is effectively set to
492  * force broadcast of these operations.
493  */
494 static bool tlb_force_broadcast(CPUARMState *env)
495 {
496     return arm_current_el(env) == 1 && (arm_hcr_el2_eff(env) & HCR_FB);
497 }
498 
499 static void tlbiall_write(CPUARMState *env, const ARMCPRegInfo *ri,
500                           uint64_t value)
501 {
502     /* Invalidate all (TLBIALL) */
503     CPUState *cs = env_cpu(env);
504 
505     if (tlb_force_broadcast(env)) {
506         tlb_flush_all_cpus_synced(cs);
507     } else {
508         tlb_flush(cs);
509     }
510 }
511 
512 static void tlbimva_write(CPUARMState *env, const ARMCPRegInfo *ri,
513                           uint64_t value)
514 {
515     /* Invalidate single TLB entry by MVA and ASID (TLBIMVA) */
516     CPUState *cs = env_cpu(env);
517 
518     value &= TARGET_PAGE_MASK;
519     if (tlb_force_broadcast(env)) {
520         tlb_flush_page_all_cpus_synced(cs, value);
521     } else {
522         tlb_flush_page(cs, value);
523     }
524 }
525 
526 static void tlbiasid_write(CPUARMState *env, const ARMCPRegInfo *ri,
527                            uint64_t value)
528 {
529     /* Invalidate by ASID (TLBIASID) */
530     CPUState *cs = env_cpu(env);
531 
532     if (tlb_force_broadcast(env)) {
533         tlb_flush_all_cpus_synced(cs);
534     } else {
535         tlb_flush(cs);
536     }
537 }
538 
539 static void tlbimvaa_write(CPUARMState *env, const ARMCPRegInfo *ri,
540                            uint64_t value)
541 {
542     /* Invalidate single entry by MVA, all ASIDs (TLBIMVAA) */
543     CPUState *cs = env_cpu(env);
544 
545     value &= TARGET_PAGE_MASK;
546     if (tlb_force_broadcast(env)) {
547         tlb_flush_page_all_cpus_synced(cs, value);
548     } else {
549         tlb_flush_page(cs, value);
550     }
551 }
552 
553 static void tlbiall_nsnh_write(CPUARMState *env, const ARMCPRegInfo *ri,
554                                uint64_t value)
555 {
556     CPUState *cs = env_cpu(env);
557 
558     tlb_flush_by_mmuidx(cs, alle1_tlbmask(env));
559 }
560 
561 static void tlbiall_nsnh_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
562                                   uint64_t value)
563 {
564     CPUState *cs = env_cpu(env);
565 
566     tlb_flush_by_mmuidx_all_cpus_synced(cs, alle1_tlbmask(env));
567 }
568 
569 
570 static void tlbiall_hyp_write(CPUARMState *env, const ARMCPRegInfo *ri,
571                               uint64_t value)
572 {
573     CPUState *cs = env_cpu(env);
574 
575     tlb_flush_by_mmuidx(cs, ARMMMUIdxBit_E2);
576 }
577 
578 static void tlbiall_hyp_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
579                                  uint64_t value)
580 {
581     CPUState *cs = env_cpu(env);
582 
583     tlb_flush_by_mmuidx_all_cpus_synced(cs, ARMMMUIdxBit_E2);
584 }
585 
586 static void tlbimva_hyp_write(CPUARMState *env, const ARMCPRegInfo *ri,
587                               uint64_t value)
588 {
589     CPUState *cs = env_cpu(env);
590     uint64_t pageaddr = value & ~MAKE_64BIT_MASK(0, 12);
591 
592     tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdxBit_E2);
593 }
594 
595 static void tlbimva_hyp_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
596                                  uint64_t value)
597 {
598     CPUState *cs = env_cpu(env);
599     uint64_t pageaddr = value & ~MAKE_64BIT_MASK(0, 12);
600 
601     tlb_flush_page_by_mmuidx_all_cpus_synced(cs, pageaddr,
602                                              ARMMMUIdxBit_E2);
603 }
604 
605 static void tlbiipas2_hyp_write(CPUARMState *env, const ARMCPRegInfo *ri,
606                                 uint64_t value)
607 {
608     CPUState *cs = env_cpu(env);
609     uint64_t pageaddr = (value & MAKE_64BIT_MASK(0, 28)) << 12;
610 
611     tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdxBit_Stage2);
612 }
613 
614 static void tlbiipas2is_hyp_write(CPUARMState *env, const ARMCPRegInfo *ri,
615                                 uint64_t value)
616 {
617     CPUState *cs = env_cpu(env);
618     uint64_t pageaddr = (value & MAKE_64BIT_MASK(0, 28)) << 12;
619 
620     tlb_flush_page_by_mmuidx_all_cpus_synced(cs, pageaddr, ARMMMUIdxBit_Stage2);
621 }
622 
623 static const ARMCPRegInfo cp_reginfo[] = {
624     /*
625      * Define the secure and non-secure FCSE identifier CP registers
626      * separately because there is no secure bank in V8 (no _EL3).  This allows
627      * the secure register to be properly reset and migrated. There is also no
628      * v8 EL1 version of the register so the non-secure instance stands alone.
629      */
630     { .name = "FCSEIDR",
631       .cp = 15, .opc1 = 0, .crn = 13, .crm = 0, .opc2 = 0,
632       .access = PL1_RW, .secure = ARM_CP_SECSTATE_NS,
633       .fieldoffset = offsetof(CPUARMState, cp15.fcseidr_ns),
634       .resetvalue = 0, .writefn = fcse_write, .raw_writefn = raw_write, },
635     { .name = "FCSEIDR_S",
636       .cp = 15, .opc1 = 0, .crn = 13, .crm = 0, .opc2 = 0,
637       .access = PL1_RW, .secure = ARM_CP_SECSTATE_S,
638       .fieldoffset = offsetof(CPUARMState, cp15.fcseidr_s),
639       .resetvalue = 0, .writefn = fcse_write, .raw_writefn = raw_write, },
640     /*
641      * Define the secure and non-secure context identifier CP registers
642      * separately because there is no secure bank in V8 (no _EL3).  This allows
643      * the secure register to be properly reset and migrated.  In the
644      * non-secure case, the 32-bit register will have reset and migration
645      * disabled during registration as it is handled by the 64-bit instance.
646      */
647     { .name = "CONTEXTIDR_EL1", .state = ARM_CP_STATE_BOTH,
648       .opc0 = 3, .opc1 = 0, .crn = 13, .crm = 0, .opc2 = 1,
649       .access = PL1_RW, .accessfn = access_tvm_trvm,
650       .fgt = FGT_CONTEXTIDR_EL1,
651       .nv2_redirect_offset = 0x108 | NV2_REDIR_NV1,
652       .secure = ARM_CP_SECSTATE_NS,
653       .fieldoffset = offsetof(CPUARMState, cp15.contextidr_el[1]),
654       .resetvalue = 0, .writefn = contextidr_write, .raw_writefn = raw_write, },
655     { .name = "CONTEXTIDR_S", .state = ARM_CP_STATE_AA32,
656       .cp = 15, .opc1 = 0, .crn = 13, .crm = 0, .opc2 = 1,
657       .access = PL1_RW, .accessfn = access_tvm_trvm,
658       .secure = ARM_CP_SECSTATE_S,
659       .fieldoffset = offsetof(CPUARMState, cp15.contextidr_s),
660       .resetvalue = 0, .writefn = contextidr_write, .raw_writefn = raw_write, },
661 };
662 
663 static const ARMCPRegInfo not_v8_cp_reginfo[] = {
664     /*
665      * NB: Some of these registers exist in v8 but with more precise
666      * definitions that don't use CP_ANY wildcards (mostly in v8_cp_reginfo[]).
667      */
668     /* MMU Domain access control / MPU write buffer control */
669     { .name = "DACR",
670       .cp = 15, .opc1 = CP_ANY, .crn = 3, .crm = CP_ANY, .opc2 = CP_ANY,
671       .access = PL1_RW, .accessfn = access_tvm_trvm, .resetvalue = 0,
672       .writefn = dacr_write, .raw_writefn = raw_write,
673       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.dacr_s),
674                              offsetoflow32(CPUARMState, cp15.dacr_ns) } },
675     /*
676      * ARMv7 allocates a range of implementation defined TLB LOCKDOWN regs.
677      * For v6 and v5, these mappings are overly broad.
678      */
679     { .name = "TLB_LOCKDOWN", .cp = 15, .crn = 10, .crm = 0,
680       .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_NOP },
681     { .name = "TLB_LOCKDOWN", .cp = 15, .crn = 10, .crm = 1,
682       .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_NOP },
683     { .name = "TLB_LOCKDOWN", .cp = 15, .crn = 10, .crm = 4,
684       .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_NOP },
685     { .name = "TLB_LOCKDOWN", .cp = 15, .crn = 10, .crm = 8,
686       .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_NOP },
687     /* Cache maintenance ops; some of this space may be overridden later. */
688     { .name = "CACHEMAINT", .cp = 15, .crn = 7, .crm = CP_ANY,
689       .opc1 = 0, .opc2 = CP_ANY, .access = PL1_W,
690       .type = ARM_CP_NOP | ARM_CP_OVERRIDE },
691 };
692 
693 static const ARMCPRegInfo not_v6_cp_reginfo[] = {
694     /*
695      * Not all pre-v6 cores implemented this WFI, so this is slightly
696      * over-broad.
697      */
698     { .name = "WFI_v5", .cp = 15, .crn = 7, .crm = 8, .opc1 = 0, .opc2 = 2,
699       .access = PL1_W, .type = ARM_CP_WFI },
700 };
701 
702 static const ARMCPRegInfo not_v7_cp_reginfo[] = {
703     /*
704      * Standard v6 WFI (also used in some pre-v6 cores); not in v7 (which
705      * is UNPREDICTABLE; we choose to NOP as most implementations do).
706      */
707     { .name = "WFI_v6", .cp = 15, .crn = 7, .crm = 0, .opc1 = 0, .opc2 = 4,
708       .access = PL1_W, .type = ARM_CP_WFI },
709     /*
710      * L1 cache lockdown. Not architectural in v6 and earlier but in practice
711      * implemented in 926, 946, 1026, 1136, 1176 and 11MPCore. StrongARM and
712      * OMAPCP will override this space.
713      */
714     { .name = "DLOCKDOWN", .cp = 15, .crn = 9, .crm = 0, .opc1 = 0, .opc2 = 0,
715       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c9_data),
716       .resetvalue = 0 },
717     { .name = "ILOCKDOWN", .cp = 15, .crn = 9, .crm = 0, .opc1 = 0, .opc2 = 1,
718       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c9_insn),
719       .resetvalue = 0 },
720     /* v6 doesn't have the cache ID registers but Linux reads them anyway */
721     { .name = "DUMMY", .cp = 15, .crn = 0, .crm = 0, .opc1 = 1, .opc2 = CP_ANY,
722       .access = PL1_R, .type = ARM_CP_CONST | ARM_CP_NO_RAW,
723       .resetvalue = 0 },
724     /*
725      * We don't implement pre-v7 debug but most CPUs had at least a DBGDIDR;
726      * implementing it as RAZ means the "debug architecture version" bits
727      * will read as a reserved value, which should cause Linux to not try
728      * to use the debug hardware.
729      */
730     { .name = "DBGDIDR", .cp = 14, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 0,
731       .access = PL0_R, .type = ARM_CP_CONST, .resetvalue = 0 },
732     /*
733      * MMU TLB control. Note that the wildcarding means we cover not just
734      * the unified TLB ops but also the dside/iside/inner-shareable variants.
735      */
736     { .name = "TLBIALL", .cp = 15, .crn = 8, .crm = CP_ANY,
737       .opc1 = CP_ANY, .opc2 = 0, .access = PL1_W, .writefn = tlbiall_write,
738       .type = ARM_CP_NO_RAW },
739     { .name = "TLBIMVA", .cp = 15, .crn = 8, .crm = CP_ANY,
740       .opc1 = CP_ANY, .opc2 = 1, .access = PL1_W, .writefn = tlbimva_write,
741       .type = ARM_CP_NO_RAW },
742     { .name = "TLBIASID", .cp = 15, .crn = 8, .crm = CP_ANY,
743       .opc1 = CP_ANY, .opc2 = 2, .access = PL1_W, .writefn = tlbiasid_write,
744       .type = ARM_CP_NO_RAW },
745     { .name = "TLBIMVAA", .cp = 15, .crn = 8, .crm = CP_ANY,
746       .opc1 = CP_ANY, .opc2 = 3, .access = PL1_W, .writefn = tlbimvaa_write,
747       .type = ARM_CP_NO_RAW },
748     { .name = "PRRR", .cp = 15, .crn = 10, .crm = 2,
749       .opc1 = 0, .opc2 = 0, .access = PL1_RW, .type = ARM_CP_NOP },
750     { .name = "NMRR", .cp = 15, .crn = 10, .crm = 2,
751       .opc1 = 0, .opc2 = 1, .access = PL1_RW, .type = ARM_CP_NOP },
752 };
753 
754 static void cpacr_write(CPUARMState *env, const ARMCPRegInfo *ri,
755                         uint64_t value)
756 {
757     uint32_t mask = 0;
758 
759     /* In ARMv8 most bits of CPACR_EL1 are RES0. */
760     if (!arm_feature(env, ARM_FEATURE_V8)) {
761         /*
762          * ARMv7 defines bits for unimplemented coprocessors as RAZ/WI.
763          * ASEDIS [31] and D32DIS [30] are both UNK/SBZP without VFP.
764          * TRCDIS [28] is RAZ/WI since we do not implement a trace macrocell.
765          */
766         if (cpu_isar_feature(aa32_vfp_simd, env_archcpu(env))) {
767             /* VFP coprocessor: cp10 & cp11 [23:20] */
768             mask |= R_CPACR_ASEDIS_MASK |
769                     R_CPACR_D32DIS_MASK |
770                     R_CPACR_CP11_MASK |
771                     R_CPACR_CP10_MASK;
772 
773             if (!arm_feature(env, ARM_FEATURE_NEON)) {
774                 /* ASEDIS [31] bit is RAO/WI */
775                 value |= R_CPACR_ASEDIS_MASK;
776             }
777 
778             /*
779              * VFPv3 and upwards with NEON implement 32 double precision
780              * registers (D0-D31).
781              */
782             if (!cpu_isar_feature(aa32_simd_r32, env_archcpu(env))) {
783                 /* D32DIS [30] is RAO/WI if D16-31 are not implemented. */
784                 value |= R_CPACR_D32DIS_MASK;
785             }
786         }
787         value &= mask;
788     }
789 
790     /*
791      * For A-profile AArch32 EL3 (but not M-profile secure mode), if NSACR.CP10
792      * is 0 then CPACR.{CP11,CP10} ignore writes and read as 0b00.
793      */
794     if (arm_feature(env, ARM_FEATURE_EL3) && !arm_el_is_aa64(env, 3) &&
795         !arm_is_secure(env) && !extract32(env->cp15.nsacr, 10, 1)) {
796         mask = R_CPACR_CP11_MASK | R_CPACR_CP10_MASK;
797         value = (value & ~mask) | (env->cp15.cpacr_el1 & mask);
798     }
799 
800     env->cp15.cpacr_el1 = value;
801 }
802 
803 static uint64_t cpacr_read(CPUARMState *env, const ARMCPRegInfo *ri)
804 {
805     /*
806      * For A-profile AArch32 EL3 (but not M-profile secure mode), if NSACR.CP10
807      * is 0 then CPACR.{CP11,CP10} ignore writes and read as 0b00.
808      */
809     uint64_t value = env->cp15.cpacr_el1;
810 
811     if (arm_feature(env, ARM_FEATURE_EL3) && !arm_el_is_aa64(env, 3) &&
812         !arm_is_secure(env) && !extract32(env->cp15.nsacr, 10, 1)) {
813         value = ~(R_CPACR_CP11_MASK | R_CPACR_CP10_MASK);
814     }
815     return value;
816 }
817 
818 
819 static void cpacr_reset(CPUARMState *env, const ARMCPRegInfo *ri)
820 {
821     /*
822      * Call cpacr_write() so that we reset with the correct RAO bits set
823      * for our CPU features.
824      */
825     cpacr_write(env, ri, 0);
826 }
827 
828 static CPAccessResult cpacr_access(CPUARMState *env, const ARMCPRegInfo *ri,
829                                    bool isread)
830 {
831     if (arm_feature(env, ARM_FEATURE_V8)) {
832         /* Check if CPACR accesses are to be trapped to EL2 */
833         if (arm_current_el(env) == 1 && arm_is_el2_enabled(env) &&
834             FIELD_EX64(env->cp15.cptr_el[2], CPTR_EL2, TCPAC)) {
835             return CP_ACCESS_TRAP_EL2;
836         /* Check if CPACR accesses are to be trapped to EL3 */
837         } else if (arm_current_el(env) < 3 &&
838                    FIELD_EX64(env->cp15.cptr_el[3], CPTR_EL3, TCPAC)) {
839             return CP_ACCESS_TRAP_EL3;
840         }
841     }
842 
843     return CP_ACCESS_OK;
844 }
845 
846 static CPAccessResult cptr_access(CPUARMState *env, const ARMCPRegInfo *ri,
847                                   bool isread)
848 {
849     /* Check if CPTR accesses are set to trap to EL3 */
850     if (arm_current_el(env) == 2 &&
851         FIELD_EX64(env->cp15.cptr_el[3], CPTR_EL3, TCPAC)) {
852         return CP_ACCESS_TRAP_EL3;
853     }
854 
855     return CP_ACCESS_OK;
856 }
857 
858 static const ARMCPRegInfo v6_cp_reginfo[] = {
859     /* prefetch by MVA in v6, NOP in v7 */
860     { .name = "MVA_prefetch",
861       .cp = 15, .crn = 7, .crm = 13, .opc1 = 0, .opc2 = 1,
862       .access = PL1_W, .type = ARM_CP_NOP },
863     /*
864      * We need to break the TB after ISB to execute self-modifying code
865      * correctly and also to take any pending interrupts immediately.
866      * So use arm_cp_write_ignore() function instead of ARM_CP_NOP flag.
867      */
868     { .name = "ISB", .cp = 15, .crn = 7, .crm = 5, .opc1 = 0, .opc2 = 4,
869       .access = PL0_W, .type = ARM_CP_NO_RAW, .writefn = arm_cp_write_ignore },
870     { .name = "DSB", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 4,
871       .access = PL0_W, .type = ARM_CP_NOP },
872     { .name = "DMB", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 5,
873       .access = PL0_W, .type = ARM_CP_NOP },
874     { .name = "IFAR", .cp = 15, .crn = 6, .crm = 0, .opc1 = 0, .opc2 = 2,
875       .access = PL1_RW, .accessfn = access_tvm_trvm,
876       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ifar_s),
877                              offsetof(CPUARMState, cp15.ifar_ns) },
878       .resetvalue = 0, },
879     /*
880      * Watchpoint Fault Address Register : should actually only be present
881      * for 1136, 1176, 11MPCore.
882      */
883     { .name = "WFAR", .cp = 15, .crn = 6, .crm = 0, .opc1 = 0, .opc2 = 1,
884       .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0, },
885     { .name = "CPACR", .state = ARM_CP_STATE_BOTH, .opc0 = 3,
886       .crn = 1, .crm = 0, .opc1 = 0, .opc2 = 2, .accessfn = cpacr_access,
887       .fgt = FGT_CPACR_EL1,
888       .nv2_redirect_offset = 0x100 | NV2_REDIR_NV1,
889       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.cpacr_el1),
890       .resetfn = cpacr_reset, .writefn = cpacr_write, .readfn = cpacr_read },
891 };
892 
893 typedef struct pm_event {
894     uint16_t number; /* PMEVTYPER.evtCount is 16 bits wide */
895     /* If the event is supported on this CPU (used to generate PMCEID[01]) */
896     bool (*supported)(CPUARMState *);
897     /*
898      * Retrieve the current count of the underlying event. The programmed
899      * counters hold a difference from the return value from this function
900      */
901     uint64_t (*get_count)(CPUARMState *);
902     /*
903      * Return how many nanoseconds it will take (at a minimum) for count events
904      * to occur. A negative value indicates the counter will never overflow, or
905      * that the counter has otherwise arranged for the overflow bit to be set
906      * and the PMU interrupt to be raised on overflow.
907      */
908     int64_t (*ns_per_count)(uint64_t);
909 } pm_event;
910 
911 static bool event_always_supported(CPUARMState *env)
912 {
913     return true;
914 }
915 
916 static uint64_t swinc_get_count(CPUARMState *env)
917 {
918     /*
919      * SW_INCR events are written directly to the pmevcntr's by writes to
920      * PMSWINC, so there is no underlying count maintained by the PMU itself
921      */
922     return 0;
923 }
924 
925 static int64_t swinc_ns_per(uint64_t ignored)
926 {
927     return -1;
928 }
929 
930 /*
931  * Return the underlying cycle count for the PMU cycle counters. If we're in
932  * usermode, simply return 0.
933  */
934 static uint64_t cycles_get_count(CPUARMState *env)
935 {
936 #ifndef CONFIG_USER_ONLY
937     return muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL),
938                    ARM_CPU_FREQ, NANOSECONDS_PER_SECOND);
939 #else
940     return cpu_get_host_ticks();
941 #endif
942 }
943 
944 #ifndef CONFIG_USER_ONLY
945 static int64_t cycles_ns_per(uint64_t cycles)
946 {
947     return (ARM_CPU_FREQ / NANOSECONDS_PER_SECOND) * cycles;
948 }
949 
950 static bool instructions_supported(CPUARMState *env)
951 {
952     /* Precise instruction counting */
953     return icount_enabled() == ICOUNT_PRECISE;
954 }
955 
956 static uint64_t instructions_get_count(CPUARMState *env)
957 {
958     assert(icount_enabled() == ICOUNT_PRECISE);
959     return (uint64_t)icount_get_raw();
960 }
961 
962 static int64_t instructions_ns_per(uint64_t icount)
963 {
964     assert(icount_enabled() == ICOUNT_PRECISE);
965     return icount_to_ns((int64_t)icount);
966 }
967 #endif
968 
969 static bool pmuv3p1_events_supported(CPUARMState *env)
970 {
971     /* For events which are supported in any v8.1 PMU */
972     return cpu_isar_feature(any_pmuv3p1, env_archcpu(env));
973 }
974 
975 static bool pmuv3p4_events_supported(CPUARMState *env)
976 {
977     /* For events which are supported in any v8.1 PMU */
978     return cpu_isar_feature(any_pmuv3p4, env_archcpu(env));
979 }
980 
981 static uint64_t zero_event_get_count(CPUARMState *env)
982 {
983     /* For events which on QEMU never fire, so their count is always zero */
984     return 0;
985 }
986 
987 static int64_t zero_event_ns_per(uint64_t cycles)
988 {
989     /* An event which never fires can never overflow */
990     return -1;
991 }
992 
993 static const pm_event pm_events[] = {
994     { .number = 0x000, /* SW_INCR */
995       .supported = event_always_supported,
996       .get_count = swinc_get_count,
997       .ns_per_count = swinc_ns_per,
998     },
999 #ifndef CONFIG_USER_ONLY
1000     { .number = 0x008, /* INST_RETIRED, Instruction architecturally executed */
1001       .supported = instructions_supported,
1002       .get_count = instructions_get_count,
1003       .ns_per_count = instructions_ns_per,
1004     },
1005     { .number = 0x011, /* CPU_CYCLES, Cycle */
1006       .supported = event_always_supported,
1007       .get_count = cycles_get_count,
1008       .ns_per_count = cycles_ns_per,
1009     },
1010 #endif
1011     { .number = 0x023, /* STALL_FRONTEND */
1012       .supported = pmuv3p1_events_supported,
1013       .get_count = zero_event_get_count,
1014       .ns_per_count = zero_event_ns_per,
1015     },
1016     { .number = 0x024, /* STALL_BACKEND */
1017       .supported = pmuv3p1_events_supported,
1018       .get_count = zero_event_get_count,
1019       .ns_per_count = zero_event_ns_per,
1020     },
1021     { .number = 0x03c, /* STALL */
1022       .supported = pmuv3p4_events_supported,
1023       .get_count = zero_event_get_count,
1024       .ns_per_count = zero_event_ns_per,
1025     },
1026 };
1027 
1028 /*
1029  * Note: Before increasing MAX_EVENT_ID beyond 0x3f into the 0x40xx range of
1030  * events (i.e. the statistical profiling extension), this implementation
1031  * should first be updated to something sparse instead of the current
1032  * supported_event_map[] array.
1033  */
1034 #define MAX_EVENT_ID 0x3c
1035 #define UNSUPPORTED_EVENT UINT16_MAX
1036 static uint16_t supported_event_map[MAX_EVENT_ID + 1];
1037 
1038 /*
1039  * Called upon CPU initialization to initialize PMCEID[01]_EL0 and build a map
1040  * of ARM event numbers to indices in our pm_events array.
1041  *
1042  * Note: Events in the 0x40XX range are not currently supported.
1043  */
1044 void pmu_init(ARMCPU *cpu)
1045 {
1046     unsigned int i;
1047 
1048     /*
1049      * Empty supported_event_map and cpu->pmceid[01] before adding supported
1050      * events to them
1051      */
1052     for (i = 0; i < ARRAY_SIZE(supported_event_map); i++) {
1053         supported_event_map[i] = UNSUPPORTED_EVENT;
1054     }
1055     cpu->pmceid0 = 0;
1056     cpu->pmceid1 = 0;
1057 
1058     for (i = 0; i < ARRAY_SIZE(pm_events); i++) {
1059         const pm_event *cnt = &pm_events[i];
1060         assert(cnt->number <= MAX_EVENT_ID);
1061         /* We do not currently support events in the 0x40xx range */
1062         assert(cnt->number <= 0x3f);
1063 
1064         if (cnt->supported(&cpu->env)) {
1065             supported_event_map[cnt->number] = i;
1066             uint64_t event_mask = 1ULL << (cnt->number & 0x1f);
1067             if (cnt->number & 0x20) {
1068                 cpu->pmceid1 |= event_mask;
1069             } else {
1070                 cpu->pmceid0 |= event_mask;
1071             }
1072         }
1073     }
1074 }
1075 
1076 /*
1077  * Check at runtime whether a PMU event is supported for the current machine
1078  */
1079 static bool event_supported(uint16_t number)
1080 {
1081     if (number > MAX_EVENT_ID) {
1082         return false;
1083     }
1084     return supported_event_map[number] != UNSUPPORTED_EVENT;
1085 }
1086 
1087 static CPAccessResult pmreg_access(CPUARMState *env, const ARMCPRegInfo *ri,
1088                                    bool isread)
1089 {
1090     /*
1091      * Performance monitor registers user accessibility is controlled
1092      * by PMUSERENR. MDCR_EL2.TPM and MDCR_EL3.TPM allow configurable
1093      * trapping to EL2 or EL3 for other accesses.
1094      */
1095     int el = arm_current_el(env);
1096     uint64_t mdcr_el2 = arm_mdcr_el2_eff(env);
1097 
1098     if (el == 0 && !(env->cp15.c9_pmuserenr & 1)) {
1099         return CP_ACCESS_TRAP;
1100     }
1101     if (el < 2 && (mdcr_el2 & MDCR_TPM)) {
1102         return CP_ACCESS_TRAP_EL2;
1103     }
1104     if (el < 3 && (env->cp15.mdcr_el3 & MDCR_TPM)) {
1105         return CP_ACCESS_TRAP_EL3;
1106     }
1107 
1108     return CP_ACCESS_OK;
1109 }
1110 
1111 static CPAccessResult pmreg_access_xevcntr(CPUARMState *env,
1112                                            const ARMCPRegInfo *ri,
1113                                            bool isread)
1114 {
1115     /* ER: event counter read trap control */
1116     if (arm_feature(env, ARM_FEATURE_V8)
1117         && arm_current_el(env) == 0
1118         && (env->cp15.c9_pmuserenr & (1 << 3)) != 0
1119         && isread) {
1120         return CP_ACCESS_OK;
1121     }
1122 
1123     return pmreg_access(env, ri, isread);
1124 }
1125 
1126 static CPAccessResult pmreg_access_swinc(CPUARMState *env,
1127                                          const ARMCPRegInfo *ri,
1128                                          bool isread)
1129 {
1130     /* SW: software increment write trap control */
1131     if (arm_feature(env, ARM_FEATURE_V8)
1132         && arm_current_el(env) == 0
1133         && (env->cp15.c9_pmuserenr & (1 << 1)) != 0
1134         && !isread) {
1135         return CP_ACCESS_OK;
1136     }
1137 
1138     return pmreg_access(env, ri, isread);
1139 }
1140 
1141 static CPAccessResult pmreg_access_selr(CPUARMState *env,
1142                                         const ARMCPRegInfo *ri,
1143                                         bool isread)
1144 {
1145     /* ER: event counter read trap control */
1146     if (arm_feature(env, ARM_FEATURE_V8)
1147         && arm_current_el(env) == 0
1148         && (env->cp15.c9_pmuserenr & (1 << 3)) != 0) {
1149         return CP_ACCESS_OK;
1150     }
1151 
1152     return pmreg_access(env, ri, isread);
1153 }
1154 
1155 static CPAccessResult pmreg_access_ccntr(CPUARMState *env,
1156                                          const ARMCPRegInfo *ri,
1157                                          bool isread)
1158 {
1159     /* CR: cycle counter read trap control */
1160     if (arm_feature(env, ARM_FEATURE_V8)
1161         && arm_current_el(env) == 0
1162         && (env->cp15.c9_pmuserenr & (1 << 2)) != 0
1163         && isread) {
1164         return CP_ACCESS_OK;
1165     }
1166 
1167     return pmreg_access(env, ri, isread);
1168 }
1169 
1170 /*
1171  * Bits in MDCR_EL2 and MDCR_EL3 which pmu_counter_enabled() looks at.
1172  * We use these to decide whether we need to wrap a write to MDCR_EL2
1173  * or MDCR_EL3 in pmu_op_start()/pmu_op_finish() calls.
1174  */
1175 #define MDCR_EL2_PMU_ENABLE_BITS \
1176     (MDCR_HPME | MDCR_HPMD | MDCR_HPMN | MDCR_HCCD | MDCR_HLP)
1177 #define MDCR_EL3_PMU_ENABLE_BITS (MDCR_SPME | MDCR_SCCD)
1178 
1179 /*
1180  * Returns true if the counter (pass 31 for PMCCNTR) should count events using
1181  * the current EL, security state, and register configuration.
1182  */
1183 static bool pmu_counter_enabled(CPUARMState *env, uint8_t counter)
1184 {
1185     uint64_t filter;
1186     bool e, p, u, nsk, nsu, nsh, m;
1187     bool enabled, prohibited = false, filtered;
1188     bool secure = arm_is_secure(env);
1189     int el = arm_current_el(env);
1190     uint64_t mdcr_el2 = arm_mdcr_el2_eff(env);
1191     uint8_t hpmn = mdcr_el2 & MDCR_HPMN;
1192 
1193     if (!arm_feature(env, ARM_FEATURE_PMU)) {
1194         return false;
1195     }
1196 
1197     if (!arm_feature(env, ARM_FEATURE_EL2) ||
1198             (counter < hpmn || counter == 31)) {
1199         e = env->cp15.c9_pmcr & PMCRE;
1200     } else {
1201         e = mdcr_el2 & MDCR_HPME;
1202     }
1203     enabled = e && (env->cp15.c9_pmcnten & (1 << counter));
1204 
1205     /* Is event counting prohibited? */
1206     if (el == 2 && (counter < hpmn || counter == 31)) {
1207         prohibited = mdcr_el2 & MDCR_HPMD;
1208     }
1209     if (secure) {
1210         prohibited = prohibited || !(env->cp15.mdcr_el3 & MDCR_SPME);
1211     }
1212 
1213     if (counter == 31) {
1214         /*
1215          * The cycle counter defaults to running. PMCR.DP says "disable
1216          * the cycle counter when event counting is prohibited".
1217          * Some MDCR bits disable the cycle counter specifically.
1218          */
1219         prohibited = prohibited && env->cp15.c9_pmcr & PMCRDP;
1220         if (cpu_isar_feature(any_pmuv3p5, env_archcpu(env))) {
1221             if (secure) {
1222                 prohibited = prohibited || (env->cp15.mdcr_el3 & MDCR_SCCD);
1223             }
1224             if (el == 2) {
1225                 prohibited = prohibited || (mdcr_el2 & MDCR_HCCD);
1226             }
1227         }
1228     }
1229 
1230     if (counter == 31) {
1231         filter = env->cp15.pmccfiltr_el0;
1232     } else {
1233         filter = env->cp15.c14_pmevtyper[counter];
1234     }
1235 
1236     p   = filter & PMXEVTYPER_P;
1237     u   = filter & PMXEVTYPER_U;
1238     nsk = arm_feature(env, ARM_FEATURE_EL3) && (filter & PMXEVTYPER_NSK);
1239     nsu = arm_feature(env, ARM_FEATURE_EL3) && (filter & PMXEVTYPER_NSU);
1240     nsh = arm_feature(env, ARM_FEATURE_EL2) && (filter & PMXEVTYPER_NSH);
1241     m   = arm_el_is_aa64(env, 1) &&
1242               arm_feature(env, ARM_FEATURE_EL3) && (filter & PMXEVTYPER_M);
1243 
1244     if (el == 0) {
1245         filtered = secure ? u : u != nsu;
1246     } else if (el == 1) {
1247         filtered = secure ? p : p != nsk;
1248     } else if (el == 2) {
1249         filtered = !nsh;
1250     } else { /* EL3 */
1251         filtered = m != p;
1252     }
1253 
1254     if (counter != 31) {
1255         /*
1256          * If not checking PMCCNTR, ensure the counter is setup to an event we
1257          * support
1258          */
1259         uint16_t event = filter & PMXEVTYPER_EVTCOUNT;
1260         if (!event_supported(event)) {
1261             return false;
1262         }
1263     }
1264 
1265     return enabled && !prohibited && !filtered;
1266 }
1267 
1268 static void pmu_update_irq(CPUARMState *env)
1269 {
1270     ARMCPU *cpu = env_archcpu(env);
1271     qemu_set_irq(cpu->pmu_interrupt, (env->cp15.c9_pmcr & PMCRE) &&
1272             (env->cp15.c9_pminten & env->cp15.c9_pmovsr));
1273 }
1274 
1275 static bool pmccntr_clockdiv_enabled(CPUARMState *env)
1276 {
1277     /*
1278      * Return true if the clock divider is enabled and the cycle counter
1279      * is supposed to tick only once every 64 clock cycles. This is
1280      * controlled by PMCR.D, but if PMCR.LC is set to enable the long
1281      * (64-bit) cycle counter PMCR.D has no effect.
1282      */
1283     return (env->cp15.c9_pmcr & (PMCRD | PMCRLC)) == PMCRD;
1284 }
1285 
1286 static bool pmevcntr_is_64_bit(CPUARMState *env, int counter)
1287 {
1288     /* Return true if the specified event counter is configured to be 64 bit */
1289 
1290     /* This isn't intended to be used with the cycle counter */
1291     assert(counter < 31);
1292 
1293     if (!cpu_isar_feature(any_pmuv3p5, env_archcpu(env))) {
1294         return false;
1295     }
1296 
1297     if (arm_feature(env, ARM_FEATURE_EL2)) {
1298         /*
1299          * MDCR_EL2.HLP still applies even when EL2 is disabled in the
1300          * current security state, so we don't use arm_mdcr_el2_eff() here.
1301          */
1302         bool hlp = env->cp15.mdcr_el2 & MDCR_HLP;
1303         int hpmn = env->cp15.mdcr_el2 & MDCR_HPMN;
1304 
1305         if (counter >= hpmn) {
1306             return hlp;
1307         }
1308     }
1309     return env->cp15.c9_pmcr & PMCRLP;
1310 }
1311 
1312 /*
1313  * Ensure c15_ccnt is the guest-visible count so that operations such as
1314  * enabling/disabling the counter or filtering, modifying the count itself,
1315  * etc. can be done logically. This is essentially a no-op if the counter is
1316  * not enabled at the time of the call.
1317  */
1318 static void pmccntr_op_start(CPUARMState *env)
1319 {
1320     uint64_t cycles = cycles_get_count(env);
1321 
1322     if (pmu_counter_enabled(env, 31)) {
1323         uint64_t eff_cycles = cycles;
1324         if (pmccntr_clockdiv_enabled(env)) {
1325             eff_cycles /= 64;
1326         }
1327 
1328         uint64_t new_pmccntr = eff_cycles - env->cp15.c15_ccnt_delta;
1329 
1330         uint64_t overflow_mask = env->cp15.c9_pmcr & PMCRLC ? \
1331                                  1ull << 63 : 1ull << 31;
1332         if (env->cp15.c15_ccnt & ~new_pmccntr & overflow_mask) {
1333             env->cp15.c9_pmovsr |= (1ULL << 31);
1334             pmu_update_irq(env);
1335         }
1336 
1337         env->cp15.c15_ccnt = new_pmccntr;
1338     }
1339     env->cp15.c15_ccnt_delta = cycles;
1340 }
1341 
1342 /*
1343  * If PMCCNTR is enabled, recalculate the delta between the clock and the
1344  * guest-visible count. A call to pmccntr_op_finish should follow every call to
1345  * pmccntr_op_start.
1346  */
1347 static void pmccntr_op_finish(CPUARMState *env)
1348 {
1349     if (pmu_counter_enabled(env, 31)) {
1350 #ifndef CONFIG_USER_ONLY
1351         /* Calculate when the counter will next overflow */
1352         uint64_t remaining_cycles = -env->cp15.c15_ccnt;
1353         if (!(env->cp15.c9_pmcr & PMCRLC)) {
1354             remaining_cycles = (uint32_t)remaining_cycles;
1355         }
1356         int64_t overflow_in = cycles_ns_per(remaining_cycles);
1357 
1358         if (overflow_in > 0) {
1359             int64_t overflow_at;
1360 
1361             if (!sadd64_overflow(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL),
1362                                  overflow_in, &overflow_at)) {
1363                 ARMCPU *cpu = env_archcpu(env);
1364                 timer_mod_anticipate_ns(cpu->pmu_timer, overflow_at);
1365             }
1366         }
1367 #endif
1368 
1369         uint64_t prev_cycles = env->cp15.c15_ccnt_delta;
1370         if (pmccntr_clockdiv_enabled(env)) {
1371             prev_cycles /= 64;
1372         }
1373         env->cp15.c15_ccnt_delta = prev_cycles - env->cp15.c15_ccnt;
1374     }
1375 }
1376 
1377 static void pmevcntr_op_start(CPUARMState *env, uint8_t counter)
1378 {
1379 
1380     uint16_t event = env->cp15.c14_pmevtyper[counter] & PMXEVTYPER_EVTCOUNT;
1381     uint64_t count = 0;
1382     if (event_supported(event)) {
1383         uint16_t event_idx = supported_event_map[event];
1384         count = pm_events[event_idx].get_count(env);
1385     }
1386 
1387     if (pmu_counter_enabled(env, counter)) {
1388         uint64_t new_pmevcntr = count - env->cp15.c14_pmevcntr_delta[counter];
1389         uint64_t overflow_mask = pmevcntr_is_64_bit(env, counter) ?
1390             1ULL << 63 : 1ULL << 31;
1391 
1392         if (env->cp15.c14_pmevcntr[counter] & ~new_pmevcntr & overflow_mask) {
1393             env->cp15.c9_pmovsr |= (1 << counter);
1394             pmu_update_irq(env);
1395         }
1396         env->cp15.c14_pmevcntr[counter] = new_pmevcntr;
1397     }
1398     env->cp15.c14_pmevcntr_delta[counter] = count;
1399 }
1400 
1401 static void pmevcntr_op_finish(CPUARMState *env, uint8_t counter)
1402 {
1403     if (pmu_counter_enabled(env, counter)) {
1404 #ifndef CONFIG_USER_ONLY
1405         uint16_t event = env->cp15.c14_pmevtyper[counter] & PMXEVTYPER_EVTCOUNT;
1406         uint16_t event_idx = supported_event_map[event];
1407         uint64_t delta = -(env->cp15.c14_pmevcntr[counter] + 1);
1408         int64_t overflow_in;
1409 
1410         if (!pmevcntr_is_64_bit(env, counter)) {
1411             delta = (uint32_t)delta;
1412         }
1413         overflow_in = pm_events[event_idx].ns_per_count(delta);
1414 
1415         if (overflow_in > 0) {
1416             int64_t overflow_at;
1417 
1418             if (!sadd64_overflow(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL),
1419                                  overflow_in, &overflow_at)) {
1420                 ARMCPU *cpu = env_archcpu(env);
1421                 timer_mod_anticipate_ns(cpu->pmu_timer, overflow_at);
1422             }
1423         }
1424 #endif
1425 
1426         env->cp15.c14_pmevcntr_delta[counter] -=
1427             env->cp15.c14_pmevcntr[counter];
1428     }
1429 }
1430 
1431 void pmu_op_start(CPUARMState *env)
1432 {
1433     unsigned int i;
1434     pmccntr_op_start(env);
1435     for (i = 0; i < pmu_num_counters(env); i++) {
1436         pmevcntr_op_start(env, i);
1437     }
1438 }
1439 
1440 void pmu_op_finish(CPUARMState *env)
1441 {
1442     unsigned int i;
1443     pmccntr_op_finish(env);
1444     for (i = 0; i < pmu_num_counters(env); i++) {
1445         pmevcntr_op_finish(env, i);
1446     }
1447 }
1448 
1449 void pmu_pre_el_change(ARMCPU *cpu, void *ignored)
1450 {
1451     pmu_op_start(&cpu->env);
1452 }
1453 
1454 void pmu_post_el_change(ARMCPU *cpu, void *ignored)
1455 {
1456     pmu_op_finish(&cpu->env);
1457 }
1458 
1459 void arm_pmu_timer_cb(void *opaque)
1460 {
1461     ARMCPU *cpu = opaque;
1462 
1463     /*
1464      * Update all the counter values based on the current underlying counts,
1465      * triggering interrupts to be raised, if necessary. pmu_op_finish() also
1466      * has the effect of setting the cpu->pmu_timer to the next earliest time a
1467      * counter may expire.
1468      */
1469     pmu_op_start(&cpu->env);
1470     pmu_op_finish(&cpu->env);
1471 }
1472 
1473 static void pmcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1474                        uint64_t value)
1475 {
1476     pmu_op_start(env);
1477 
1478     if (value & PMCRC) {
1479         /* The counter has been reset */
1480         env->cp15.c15_ccnt = 0;
1481     }
1482 
1483     if (value & PMCRP) {
1484         unsigned int i;
1485         for (i = 0; i < pmu_num_counters(env); i++) {
1486             env->cp15.c14_pmevcntr[i] = 0;
1487         }
1488     }
1489 
1490     env->cp15.c9_pmcr &= ~PMCR_WRITABLE_MASK;
1491     env->cp15.c9_pmcr |= (value & PMCR_WRITABLE_MASK);
1492 
1493     pmu_op_finish(env);
1494 }
1495 
1496 static uint64_t pmcr_read(CPUARMState *env, const ARMCPRegInfo *ri)
1497 {
1498     uint64_t pmcr = env->cp15.c9_pmcr;
1499 
1500     /*
1501      * If EL2 is implemented and enabled for the current security state, reads
1502      * of PMCR.N from EL1 or EL0 return the value of MDCR_EL2.HPMN or HDCR.HPMN.
1503      */
1504     if (arm_current_el(env) <= 1 && arm_is_el2_enabled(env)) {
1505         pmcr &= ~PMCRN_MASK;
1506         pmcr |= (env->cp15.mdcr_el2 & MDCR_HPMN) << PMCRN_SHIFT;
1507     }
1508 
1509     return pmcr;
1510 }
1511 
1512 static void pmswinc_write(CPUARMState *env, const ARMCPRegInfo *ri,
1513                           uint64_t value)
1514 {
1515     unsigned int i;
1516     uint64_t overflow_mask, new_pmswinc;
1517 
1518     for (i = 0; i < pmu_num_counters(env); i++) {
1519         /* Increment a counter's count iff: */
1520         if ((value & (1 << i)) && /* counter's bit is set */
1521                 /* counter is enabled and not filtered */
1522                 pmu_counter_enabled(env, i) &&
1523                 /* counter is SW_INCR */
1524                 (env->cp15.c14_pmevtyper[i] & PMXEVTYPER_EVTCOUNT) == 0x0) {
1525             pmevcntr_op_start(env, i);
1526 
1527             /*
1528              * Detect if this write causes an overflow since we can't predict
1529              * PMSWINC overflows like we can for other events
1530              */
1531             new_pmswinc = env->cp15.c14_pmevcntr[i] + 1;
1532 
1533             overflow_mask = pmevcntr_is_64_bit(env, i) ?
1534                 1ULL << 63 : 1ULL << 31;
1535 
1536             if (env->cp15.c14_pmevcntr[i] & ~new_pmswinc & overflow_mask) {
1537                 env->cp15.c9_pmovsr |= (1 << i);
1538                 pmu_update_irq(env);
1539             }
1540 
1541             env->cp15.c14_pmevcntr[i] = new_pmswinc;
1542 
1543             pmevcntr_op_finish(env, i);
1544         }
1545     }
1546 }
1547 
1548 static uint64_t pmccntr_read(CPUARMState *env, const ARMCPRegInfo *ri)
1549 {
1550     uint64_t ret;
1551     pmccntr_op_start(env);
1552     ret = env->cp15.c15_ccnt;
1553     pmccntr_op_finish(env);
1554     return ret;
1555 }
1556 
1557 static void pmselr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1558                          uint64_t value)
1559 {
1560     /*
1561      * The value of PMSELR.SEL affects the behavior of PMXEVTYPER and
1562      * PMXEVCNTR. We allow [0..31] to be written to PMSELR here; in the
1563      * meanwhile, we check PMSELR.SEL when PMXEVTYPER and PMXEVCNTR are
1564      * accessed.
1565      */
1566     env->cp15.c9_pmselr = value & 0x1f;
1567 }
1568 
1569 static void pmccntr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1570                         uint64_t value)
1571 {
1572     pmccntr_op_start(env);
1573     env->cp15.c15_ccnt = value;
1574     pmccntr_op_finish(env);
1575 }
1576 
1577 static void pmccntr_write32(CPUARMState *env, const ARMCPRegInfo *ri,
1578                             uint64_t value)
1579 {
1580     uint64_t cur_val = pmccntr_read(env, NULL);
1581 
1582     pmccntr_write(env, ri, deposit64(cur_val, 0, 32, value));
1583 }
1584 
1585 static void pmccfiltr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1586                             uint64_t value)
1587 {
1588     pmccntr_op_start(env);
1589     env->cp15.pmccfiltr_el0 = value & PMCCFILTR_EL0;
1590     pmccntr_op_finish(env);
1591 }
1592 
1593 static void pmccfiltr_write_a32(CPUARMState *env, const ARMCPRegInfo *ri,
1594                             uint64_t value)
1595 {
1596     pmccntr_op_start(env);
1597     /* M is not accessible from AArch32 */
1598     env->cp15.pmccfiltr_el0 = (env->cp15.pmccfiltr_el0 & PMCCFILTR_M) |
1599         (value & PMCCFILTR);
1600     pmccntr_op_finish(env);
1601 }
1602 
1603 static uint64_t pmccfiltr_read_a32(CPUARMState *env, const ARMCPRegInfo *ri)
1604 {
1605     /* M is not visible in AArch32 */
1606     return env->cp15.pmccfiltr_el0 & PMCCFILTR;
1607 }
1608 
1609 static void pmcntenset_write(CPUARMState *env, const ARMCPRegInfo *ri,
1610                             uint64_t value)
1611 {
1612     pmu_op_start(env);
1613     value &= pmu_counter_mask(env);
1614     env->cp15.c9_pmcnten |= value;
1615     pmu_op_finish(env);
1616 }
1617 
1618 static void pmcntenclr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1619                              uint64_t value)
1620 {
1621     pmu_op_start(env);
1622     value &= pmu_counter_mask(env);
1623     env->cp15.c9_pmcnten &= ~value;
1624     pmu_op_finish(env);
1625 }
1626 
1627 static void pmovsr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1628                          uint64_t value)
1629 {
1630     value &= pmu_counter_mask(env);
1631     env->cp15.c9_pmovsr &= ~value;
1632     pmu_update_irq(env);
1633 }
1634 
1635 static void pmovsset_write(CPUARMState *env, const ARMCPRegInfo *ri,
1636                          uint64_t value)
1637 {
1638     value &= pmu_counter_mask(env);
1639     env->cp15.c9_pmovsr |= value;
1640     pmu_update_irq(env);
1641 }
1642 
1643 static void pmevtyper_write(CPUARMState *env, const ARMCPRegInfo *ri,
1644                              uint64_t value, const uint8_t counter)
1645 {
1646     if (counter == 31) {
1647         pmccfiltr_write(env, ri, value);
1648     } else if (counter < pmu_num_counters(env)) {
1649         pmevcntr_op_start(env, counter);
1650 
1651         /*
1652          * If this counter's event type is changing, store the current
1653          * underlying count for the new type in c14_pmevcntr_delta[counter] so
1654          * pmevcntr_op_finish has the correct baseline when it converts back to
1655          * a delta.
1656          */
1657         uint16_t old_event = env->cp15.c14_pmevtyper[counter] &
1658             PMXEVTYPER_EVTCOUNT;
1659         uint16_t new_event = value & PMXEVTYPER_EVTCOUNT;
1660         if (old_event != new_event) {
1661             uint64_t count = 0;
1662             if (event_supported(new_event)) {
1663                 uint16_t event_idx = supported_event_map[new_event];
1664                 count = pm_events[event_idx].get_count(env);
1665             }
1666             env->cp15.c14_pmevcntr_delta[counter] = count;
1667         }
1668 
1669         env->cp15.c14_pmevtyper[counter] = value & PMXEVTYPER_MASK;
1670         pmevcntr_op_finish(env, counter);
1671     }
1672     /*
1673      * Attempts to access PMXEVTYPER are CONSTRAINED UNPREDICTABLE when
1674      * PMSELR value is equal to or greater than the number of implemented
1675      * counters, but not equal to 0x1f. We opt to behave as a RAZ/WI.
1676      */
1677 }
1678 
1679 static uint64_t pmevtyper_read(CPUARMState *env, const ARMCPRegInfo *ri,
1680                                const uint8_t counter)
1681 {
1682     if (counter == 31) {
1683         return env->cp15.pmccfiltr_el0;
1684     } else if (counter < pmu_num_counters(env)) {
1685         return env->cp15.c14_pmevtyper[counter];
1686     } else {
1687       /*
1688        * We opt to behave as a RAZ/WI when attempts to access PMXEVTYPER
1689        * are CONSTRAINED UNPREDICTABLE. See comments in pmevtyper_write().
1690        */
1691         return 0;
1692     }
1693 }
1694 
1695 static void pmevtyper_writefn(CPUARMState *env, const ARMCPRegInfo *ri,
1696                               uint64_t value)
1697 {
1698     uint8_t counter = ((ri->crm & 3) << 3) | (ri->opc2 & 7);
1699     pmevtyper_write(env, ri, value, counter);
1700 }
1701 
1702 static void pmevtyper_rawwrite(CPUARMState *env, const ARMCPRegInfo *ri,
1703                                uint64_t value)
1704 {
1705     uint8_t counter = ((ri->crm & 3) << 3) | (ri->opc2 & 7);
1706     env->cp15.c14_pmevtyper[counter] = value;
1707 
1708     /*
1709      * pmevtyper_rawwrite is called between a pair of pmu_op_start and
1710      * pmu_op_finish calls when loading saved state for a migration. Because
1711      * we're potentially updating the type of event here, the value written to
1712      * c14_pmevcntr_delta by the preceding pmu_op_start call may be for a
1713      * different counter type. Therefore, we need to set this value to the
1714      * current count for the counter type we're writing so that pmu_op_finish
1715      * has the correct count for its calculation.
1716      */
1717     uint16_t event = value & PMXEVTYPER_EVTCOUNT;
1718     if (event_supported(event)) {
1719         uint16_t event_idx = supported_event_map[event];
1720         env->cp15.c14_pmevcntr_delta[counter] =
1721             pm_events[event_idx].get_count(env);
1722     }
1723 }
1724 
1725 static uint64_t pmevtyper_readfn(CPUARMState *env, const ARMCPRegInfo *ri)
1726 {
1727     uint8_t counter = ((ri->crm & 3) << 3) | (ri->opc2 & 7);
1728     return pmevtyper_read(env, ri, counter);
1729 }
1730 
1731 static void pmxevtyper_write(CPUARMState *env, const ARMCPRegInfo *ri,
1732                              uint64_t value)
1733 {
1734     pmevtyper_write(env, ri, value, env->cp15.c9_pmselr & 31);
1735 }
1736 
1737 static uint64_t pmxevtyper_read(CPUARMState *env, const ARMCPRegInfo *ri)
1738 {
1739     return pmevtyper_read(env, ri, env->cp15.c9_pmselr & 31);
1740 }
1741 
1742 static void pmevcntr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1743                              uint64_t value, uint8_t counter)
1744 {
1745     if (!cpu_isar_feature(any_pmuv3p5, env_archcpu(env))) {
1746         /* Before FEAT_PMUv3p5, top 32 bits of event counters are RES0 */
1747         value &= MAKE_64BIT_MASK(0, 32);
1748     }
1749     if (counter < pmu_num_counters(env)) {
1750         pmevcntr_op_start(env, counter);
1751         env->cp15.c14_pmevcntr[counter] = value;
1752         pmevcntr_op_finish(env, counter);
1753     }
1754     /*
1755      * We opt to behave as a RAZ/WI when attempts to access PM[X]EVCNTR
1756      * are CONSTRAINED UNPREDICTABLE.
1757      */
1758 }
1759 
1760 static uint64_t pmevcntr_read(CPUARMState *env, const ARMCPRegInfo *ri,
1761                               uint8_t counter)
1762 {
1763     if (counter < pmu_num_counters(env)) {
1764         uint64_t ret;
1765         pmevcntr_op_start(env, counter);
1766         ret = env->cp15.c14_pmevcntr[counter];
1767         pmevcntr_op_finish(env, counter);
1768         if (!cpu_isar_feature(any_pmuv3p5, env_archcpu(env))) {
1769             /* Before FEAT_PMUv3p5, top 32 bits of event counters are RES0 */
1770             ret &= MAKE_64BIT_MASK(0, 32);
1771         }
1772         return ret;
1773     } else {
1774       /*
1775        * We opt to behave as a RAZ/WI when attempts to access PM[X]EVCNTR
1776        * are CONSTRAINED UNPREDICTABLE.
1777        */
1778         return 0;
1779     }
1780 }
1781 
1782 static void pmevcntr_writefn(CPUARMState *env, const ARMCPRegInfo *ri,
1783                              uint64_t value)
1784 {
1785     uint8_t counter = ((ri->crm & 3) << 3) | (ri->opc2 & 7);
1786     pmevcntr_write(env, ri, value, counter);
1787 }
1788 
1789 static uint64_t pmevcntr_readfn(CPUARMState *env, const ARMCPRegInfo *ri)
1790 {
1791     uint8_t counter = ((ri->crm & 3) << 3) | (ri->opc2 & 7);
1792     return pmevcntr_read(env, ri, counter);
1793 }
1794 
1795 static void pmevcntr_rawwrite(CPUARMState *env, const ARMCPRegInfo *ri,
1796                              uint64_t value)
1797 {
1798     uint8_t counter = ((ri->crm & 3) << 3) | (ri->opc2 & 7);
1799     assert(counter < pmu_num_counters(env));
1800     env->cp15.c14_pmevcntr[counter] = value;
1801     pmevcntr_write(env, ri, value, counter);
1802 }
1803 
1804 static uint64_t pmevcntr_rawread(CPUARMState *env, const ARMCPRegInfo *ri)
1805 {
1806     uint8_t counter = ((ri->crm & 3) << 3) | (ri->opc2 & 7);
1807     assert(counter < pmu_num_counters(env));
1808     return env->cp15.c14_pmevcntr[counter];
1809 }
1810 
1811 static void pmxevcntr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1812                              uint64_t value)
1813 {
1814     pmevcntr_write(env, ri, value, env->cp15.c9_pmselr & 31);
1815 }
1816 
1817 static uint64_t pmxevcntr_read(CPUARMState *env, const ARMCPRegInfo *ri)
1818 {
1819     return pmevcntr_read(env, ri, env->cp15.c9_pmselr & 31);
1820 }
1821 
1822 static void pmuserenr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1823                             uint64_t value)
1824 {
1825     if (arm_feature(env, ARM_FEATURE_V8)) {
1826         env->cp15.c9_pmuserenr = value & 0xf;
1827     } else {
1828         env->cp15.c9_pmuserenr = value & 1;
1829     }
1830 }
1831 
1832 static void pmintenset_write(CPUARMState *env, const ARMCPRegInfo *ri,
1833                              uint64_t value)
1834 {
1835     /* We have no event counters so only the C bit can be changed */
1836     value &= pmu_counter_mask(env);
1837     env->cp15.c9_pminten |= value;
1838     pmu_update_irq(env);
1839 }
1840 
1841 static void pmintenclr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1842                              uint64_t value)
1843 {
1844     value &= pmu_counter_mask(env);
1845     env->cp15.c9_pminten &= ~value;
1846     pmu_update_irq(env);
1847 }
1848 
1849 static void vbar_write(CPUARMState *env, const ARMCPRegInfo *ri,
1850                        uint64_t value)
1851 {
1852     /*
1853      * Note that even though the AArch64 view of this register has bits
1854      * [10:0] all RES0 we can only mask the bottom 5, to comply with the
1855      * architectural requirements for bits which are RES0 only in some
1856      * contexts. (ARMv8 would permit us to do no masking at all, but ARMv7
1857      * requires the bottom five bits to be RAZ/WI because they're UNK/SBZP.)
1858      */
1859     raw_write(env, ri, value & ~0x1FULL);
1860 }
1861 
1862 static void scr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
1863 {
1864     /* Begin with base v8.0 state.  */
1865     uint64_t valid_mask = 0x3fff;
1866     ARMCPU *cpu = env_archcpu(env);
1867     uint64_t changed;
1868 
1869     /*
1870      * Because SCR_EL3 is the "real" cpreg and SCR is the alias, reset always
1871      * passes the reginfo for SCR_EL3, which has type ARM_CP_STATE_AA64.
1872      * Instead, choose the format based on the mode of EL3.
1873      */
1874     if (arm_el_is_aa64(env, 3)) {
1875         value |= SCR_FW | SCR_AW;      /* RES1 */
1876         valid_mask &= ~SCR_NET;        /* RES0 */
1877 
1878         if (!cpu_isar_feature(aa64_aa32_el1, cpu) &&
1879             !cpu_isar_feature(aa64_aa32_el2, cpu)) {
1880             value |= SCR_RW;           /* RAO/WI */
1881         }
1882         if (cpu_isar_feature(aa64_ras, cpu)) {
1883             valid_mask |= SCR_TERR;
1884         }
1885         if (cpu_isar_feature(aa64_lor, cpu)) {
1886             valid_mask |= SCR_TLOR;
1887         }
1888         if (cpu_isar_feature(aa64_pauth, cpu)) {
1889             valid_mask |= SCR_API | SCR_APK;
1890         }
1891         if (cpu_isar_feature(aa64_sel2, cpu)) {
1892             valid_mask |= SCR_EEL2;
1893         } else if (cpu_isar_feature(aa64_rme, cpu)) {
1894             /* With RME and without SEL2, NS is RES1 (R_GSWWH, I_DJJQJ). */
1895             value |= SCR_NS;
1896         }
1897         if (cpu_isar_feature(aa64_mte, cpu)) {
1898             valid_mask |= SCR_ATA;
1899         }
1900         if (cpu_isar_feature(aa64_scxtnum, cpu)) {
1901             valid_mask |= SCR_ENSCXT;
1902         }
1903         if (cpu_isar_feature(aa64_doublefault, cpu)) {
1904             valid_mask |= SCR_EASE | SCR_NMEA;
1905         }
1906         if (cpu_isar_feature(aa64_sme, cpu)) {
1907             valid_mask |= SCR_ENTP2;
1908         }
1909         if (cpu_isar_feature(aa64_hcx, cpu)) {
1910             valid_mask |= SCR_HXEN;
1911         }
1912         if (cpu_isar_feature(aa64_fgt, cpu)) {
1913             valid_mask |= SCR_FGTEN;
1914         }
1915         if (cpu_isar_feature(aa64_rme, cpu)) {
1916             valid_mask |= SCR_NSE | SCR_GPF;
1917         }
1918     } else {
1919         valid_mask &= ~(SCR_RW | SCR_ST);
1920         if (cpu_isar_feature(aa32_ras, cpu)) {
1921             valid_mask |= SCR_TERR;
1922         }
1923     }
1924 
1925     if (!arm_feature(env, ARM_FEATURE_EL2)) {
1926         valid_mask &= ~SCR_HCE;
1927 
1928         /*
1929          * On ARMv7, SMD (or SCD as it is called in v7) is only
1930          * supported if EL2 exists. The bit is UNK/SBZP when
1931          * EL2 is unavailable. In QEMU ARMv7, we force it to always zero
1932          * when EL2 is unavailable.
1933          * On ARMv8, this bit is always available.
1934          */
1935         if (arm_feature(env, ARM_FEATURE_V7) &&
1936             !arm_feature(env, ARM_FEATURE_V8)) {
1937             valid_mask &= ~SCR_SMD;
1938         }
1939     }
1940 
1941     /* Clear all-context RES0 bits.  */
1942     value &= valid_mask;
1943     changed = env->cp15.scr_el3 ^ value;
1944     env->cp15.scr_el3 = value;
1945 
1946     /*
1947      * If SCR_EL3.{NS,NSE} changes, i.e. change of security state,
1948      * we must invalidate all TLBs below EL3.
1949      */
1950     if (changed & (SCR_NS | SCR_NSE)) {
1951         tlb_flush_by_mmuidx(env_cpu(env), (ARMMMUIdxBit_E10_0 |
1952                                            ARMMMUIdxBit_E20_0 |
1953                                            ARMMMUIdxBit_E10_1 |
1954                                            ARMMMUIdxBit_E20_2 |
1955                                            ARMMMUIdxBit_E10_1_PAN |
1956                                            ARMMMUIdxBit_E20_2_PAN |
1957                                            ARMMMUIdxBit_E2));
1958     }
1959 }
1960 
1961 static void scr_reset(CPUARMState *env, const ARMCPRegInfo *ri)
1962 {
1963     /*
1964      * scr_write will set the RES1 bits on an AArch64-only CPU.
1965      * The reset value will be 0x30 on an AArch64-only CPU and 0 otherwise.
1966      */
1967     scr_write(env, ri, 0);
1968 }
1969 
1970 static CPAccessResult access_tid4(CPUARMState *env,
1971                                   const ARMCPRegInfo *ri,
1972                                   bool isread)
1973 {
1974     if (arm_current_el(env) == 1 &&
1975         (arm_hcr_el2_eff(env) & (HCR_TID2 | HCR_TID4))) {
1976         return CP_ACCESS_TRAP_EL2;
1977     }
1978 
1979     return CP_ACCESS_OK;
1980 }
1981 
1982 static uint64_t ccsidr_read(CPUARMState *env, const ARMCPRegInfo *ri)
1983 {
1984     ARMCPU *cpu = env_archcpu(env);
1985 
1986     /*
1987      * Acquire the CSSELR index from the bank corresponding to the CCSIDR
1988      * bank
1989      */
1990     uint32_t index = A32_BANKED_REG_GET(env, csselr,
1991                                         ri->secure & ARM_CP_SECSTATE_S);
1992 
1993     return cpu->ccsidr[index];
1994 }
1995 
1996 static void csselr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1997                          uint64_t value)
1998 {
1999     raw_write(env, ri, value & 0xf);
2000 }
2001 
2002 static uint64_t isr_read(CPUARMState *env, const ARMCPRegInfo *ri)
2003 {
2004     CPUState *cs = env_cpu(env);
2005     bool el1 = arm_current_el(env) == 1;
2006     uint64_t hcr_el2 = el1 ? arm_hcr_el2_eff(env) : 0;
2007     uint64_t ret = 0;
2008 
2009     if (hcr_el2 & HCR_IMO) {
2010         if (cs->interrupt_request & CPU_INTERRUPT_VIRQ) {
2011             ret |= CPSR_I;
2012         }
2013     } else {
2014         if (cs->interrupt_request & CPU_INTERRUPT_HARD) {
2015             ret |= CPSR_I;
2016         }
2017     }
2018 
2019     if (hcr_el2 & HCR_FMO) {
2020         if (cs->interrupt_request & CPU_INTERRUPT_VFIQ) {
2021             ret |= CPSR_F;
2022         }
2023     } else {
2024         if (cs->interrupt_request & CPU_INTERRUPT_FIQ) {
2025             ret |= CPSR_F;
2026         }
2027     }
2028 
2029     if (hcr_el2 & HCR_AMO) {
2030         if (cs->interrupt_request & CPU_INTERRUPT_VSERR) {
2031             ret |= CPSR_A;
2032         }
2033     }
2034 
2035     return ret;
2036 }
2037 
2038 static CPAccessResult access_aa64_tid1(CPUARMState *env, const ARMCPRegInfo *ri,
2039                                        bool isread)
2040 {
2041     if (arm_current_el(env) == 1 && (arm_hcr_el2_eff(env) & HCR_TID1)) {
2042         return CP_ACCESS_TRAP_EL2;
2043     }
2044 
2045     return CP_ACCESS_OK;
2046 }
2047 
2048 static CPAccessResult access_aa32_tid1(CPUARMState *env, const ARMCPRegInfo *ri,
2049                                        bool isread)
2050 {
2051     if (arm_feature(env, ARM_FEATURE_V8)) {
2052         return access_aa64_tid1(env, ri, isread);
2053     }
2054 
2055     return CP_ACCESS_OK;
2056 }
2057 
2058 static const ARMCPRegInfo v7_cp_reginfo[] = {
2059     /* the old v6 WFI, UNPREDICTABLE in v7 but we choose to NOP */
2060     { .name = "NOP", .cp = 15, .crn = 7, .crm = 0, .opc1 = 0, .opc2 = 4,
2061       .access = PL1_W, .type = ARM_CP_NOP },
2062     /*
2063      * Performance monitors are implementation defined in v7,
2064      * but with an ARM recommended set of registers, which we
2065      * follow.
2066      *
2067      * Performance registers fall into three categories:
2068      *  (a) always UNDEF in PL0, RW in PL1 (PMINTENSET, PMINTENCLR)
2069      *  (b) RO in PL0 (ie UNDEF on write), RW in PL1 (PMUSERENR)
2070      *  (c) UNDEF in PL0 if PMUSERENR.EN==0, otherwise accessible (all others)
2071      * For the cases controlled by PMUSERENR we must set .access to PL0_RW
2072      * or PL0_RO as appropriate and then check PMUSERENR in the helper fn.
2073      */
2074     { .name = "PMCNTENSET", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 1,
2075       .access = PL0_RW, .type = ARM_CP_ALIAS | ARM_CP_IO,
2076       .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmcnten),
2077       .writefn = pmcntenset_write,
2078       .accessfn = pmreg_access,
2079       .fgt = FGT_PMCNTEN,
2080       .raw_writefn = raw_write },
2081     { .name = "PMCNTENSET_EL0", .state = ARM_CP_STATE_AA64, .type = ARM_CP_IO,
2082       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 1,
2083       .access = PL0_RW, .accessfn = pmreg_access,
2084       .fgt = FGT_PMCNTEN,
2085       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmcnten), .resetvalue = 0,
2086       .writefn = pmcntenset_write, .raw_writefn = raw_write },
2087     { .name = "PMCNTENCLR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 2,
2088       .access = PL0_RW,
2089       .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmcnten),
2090       .accessfn = pmreg_access,
2091       .fgt = FGT_PMCNTEN,
2092       .writefn = pmcntenclr_write,
2093       .type = ARM_CP_ALIAS | ARM_CP_IO },
2094     { .name = "PMCNTENCLR_EL0", .state = ARM_CP_STATE_AA64,
2095       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 2,
2096       .access = PL0_RW, .accessfn = pmreg_access,
2097       .fgt = FGT_PMCNTEN,
2098       .type = ARM_CP_ALIAS | ARM_CP_IO,
2099       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmcnten),
2100       .writefn = pmcntenclr_write },
2101     { .name = "PMOVSR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 3,
2102       .access = PL0_RW, .type = ARM_CP_IO,
2103       .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmovsr),
2104       .accessfn = pmreg_access,
2105       .fgt = FGT_PMOVS,
2106       .writefn = pmovsr_write,
2107       .raw_writefn = raw_write },
2108     { .name = "PMOVSCLR_EL0", .state = ARM_CP_STATE_AA64,
2109       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 3,
2110       .access = PL0_RW, .accessfn = pmreg_access,
2111       .fgt = FGT_PMOVS,
2112       .type = ARM_CP_ALIAS | ARM_CP_IO,
2113       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmovsr),
2114       .writefn = pmovsr_write,
2115       .raw_writefn = raw_write },
2116     { .name = "PMSWINC", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 4,
2117       .access = PL0_W, .accessfn = pmreg_access_swinc,
2118       .fgt = FGT_PMSWINC_EL0,
2119       .type = ARM_CP_NO_RAW | ARM_CP_IO,
2120       .writefn = pmswinc_write },
2121     { .name = "PMSWINC_EL0", .state = ARM_CP_STATE_AA64,
2122       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 4,
2123       .access = PL0_W, .accessfn = pmreg_access_swinc,
2124       .fgt = FGT_PMSWINC_EL0,
2125       .type = ARM_CP_NO_RAW | ARM_CP_IO,
2126       .writefn = pmswinc_write },
2127     { .name = "PMSELR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 5,
2128       .access = PL0_RW, .type = ARM_CP_ALIAS,
2129       .fgt = FGT_PMSELR_EL0,
2130       .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmselr),
2131       .accessfn = pmreg_access_selr, .writefn = pmselr_write,
2132       .raw_writefn = raw_write},
2133     { .name = "PMSELR_EL0", .state = ARM_CP_STATE_AA64,
2134       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 5,
2135       .access = PL0_RW, .accessfn = pmreg_access_selr,
2136       .fgt = FGT_PMSELR_EL0,
2137       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmselr),
2138       .writefn = pmselr_write, .raw_writefn = raw_write, },
2139     { .name = "PMCCNTR", .cp = 15, .crn = 9, .crm = 13, .opc1 = 0, .opc2 = 0,
2140       .access = PL0_RW, .resetvalue = 0, .type = ARM_CP_ALIAS | ARM_CP_IO,
2141       .fgt = FGT_PMCCNTR_EL0,
2142       .readfn = pmccntr_read, .writefn = pmccntr_write32,
2143       .accessfn = pmreg_access_ccntr },
2144     { .name = "PMCCNTR_EL0", .state = ARM_CP_STATE_AA64,
2145       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 13, .opc2 = 0,
2146       .access = PL0_RW, .accessfn = pmreg_access_ccntr,
2147       .fgt = FGT_PMCCNTR_EL0,
2148       .type = ARM_CP_IO,
2149       .fieldoffset = offsetof(CPUARMState, cp15.c15_ccnt),
2150       .readfn = pmccntr_read, .writefn = pmccntr_write,
2151       .raw_readfn = raw_read, .raw_writefn = raw_write, },
2152     { .name = "PMCCFILTR", .cp = 15, .opc1 = 0, .crn = 14, .crm = 15, .opc2 = 7,
2153       .writefn = pmccfiltr_write_a32, .readfn = pmccfiltr_read_a32,
2154       .access = PL0_RW, .accessfn = pmreg_access,
2155       .fgt = FGT_PMCCFILTR_EL0,
2156       .type = ARM_CP_ALIAS | ARM_CP_IO,
2157       .resetvalue = 0, },
2158     { .name = "PMCCFILTR_EL0", .state = ARM_CP_STATE_AA64,
2159       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 15, .opc2 = 7,
2160       .writefn = pmccfiltr_write, .raw_writefn = raw_write,
2161       .access = PL0_RW, .accessfn = pmreg_access,
2162       .fgt = FGT_PMCCFILTR_EL0,
2163       .type = ARM_CP_IO,
2164       .fieldoffset = offsetof(CPUARMState, cp15.pmccfiltr_el0),
2165       .resetvalue = 0, },
2166     { .name = "PMXEVTYPER", .cp = 15, .crn = 9, .crm = 13, .opc1 = 0, .opc2 = 1,
2167       .access = PL0_RW, .type = ARM_CP_NO_RAW | ARM_CP_IO,
2168       .accessfn = pmreg_access,
2169       .fgt = FGT_PMEVTYPERN_EL0,
2170       .writefn = pmxevtyper_write, .readfn = pmxevtyper_read },
2171     { .name = "PMXEVTYPER_EL0", .state = ARM_CP_STATE_AA64,
2172       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 13, .opc2 = 1,
2173       .access = PL0_RW, .type = ARM_CP_NO_RAW | ARM_CP_IO,
2174       .accessfn = pmreg_access,
2175       .fgt = FGT_PMEVTYPERN_EL0,
2176       .writefn = pmxevtyper_write, .readfn = pmxevtyper_read },
2177     { .name = "PMXEVCNTR", .cp = 15, .crn = 9, .crm = 13, .opc1 = 0, .opc2 = 2,
2178       .access = PL0_RW, .type = ARM_CP_NO_RAW | ARM_CP_IO,
2179       .accessfn = pmreg_access_xevcntr,
2180       .fgt = FGT_PMEVCNTRN_EL0,
2181       .writefn = pmxevcntr_write, .readfn = pmxevcntr_read },
2182     { .name = "PMXEVCNTR_EL0", .state = ARM_CP_STATE_AA64,
2183       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 13, .opc2 = 2,
2184       .access = PL0_RW, .type = ARM_CP_NO_RAW | ARM_CP_IO,
2185       .accessfn = pmreg_access_xevcntr,
2186       .fgt = FGT_PMEVCNTRN_EL0,
2187       .writefn = pmxevcntr_write, .readfn = pmxevcntr_read },
2188     { .name = "PMUSERENR", .cp = 15, .crn = 9, .crm = 14, .opc1 = 0, .opc2 = 0,
2189       .access = PL0_R | PL1_RW, .accessfn = access_tpm,
2190       .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmuserenr),
2191       .resetvalue = 0,
2192       .writefn = pmuserenr_write, .raw_writefn = raw_write },
2193     { .name = "PMUSERENR_EL0", .state = ARM_CP_STATE_AA64,
2194       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 14, .opc2 = 0,
2195       .access = PL0_R | PL1_RW, .accessfn = access_tpm, .type = ARM_CP_ALIAS,
2196       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmuserenr),
2197       .resetvalue = 0,
2198       .writefn = pmuserenr_write, .raw_writefn = raw_write },
2199     { .name = "PMINTENSET", .cp = 15, .crn = 9, .crm = 14, .opc1 = 0, .opc2 = 1,
2200       .access = PL1_RW, .accessfn = access_tpm,
2201       .fgt = FGT_PMINTEN,
2202       .type = ARM_CP_ALIAS | ARM_CP_IO,
2203       .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pminten),
2204       .resetvalue = 0,
2205       .writefn = pmintenset_write, .raw_writefn = raw_write },
2206     { .name = "PMINTENSET_EL1", .state = ARM_CP_STATE_AA64,
2207       .opc0 = 3, .opc1 = 0, .crn = 9, .crm = 14, .opc2 = 1,
2208       .access = PL1_RW, .accessfn = access_tpm,
2209       .fgt = FGT_PMINTEN,
2210       .type = ARM_CP_IO,
2211       .fieldoffset = offsetof(CPUARMState, cp15.c9_pminten),
2212       .writefn = pmintenset_write, .raw_writefn = raw_write,
2213       .resetvalue = 0x0 },
2214     { .name = "PMINTENCLR", .cp = 15, .crn = 9, .crm = 14, .opc1 = 0, .opc2 = 2,
2215       .access = PL1_RW, .accessfn = access_tpm,
2216       .fgt = FGT_PMINTEN,
2217       .type = ARM_CP_ALIAS | ARM_CP_IO | ARM_CP_NO_RAW,
2218       .fieldoffset = offsetof(CPUARMState, cp15.c9_pminten),
2219       .writefn = pmintenclr_write, },
2220     { .name = "PMINTENCLR_EL1", .state = ARM_CP_STATE_AA64,
2221       .opc0 = 3, .opc1 = 0, .crn = 9, .crm = 14, .opc2 = 2,
2222       .access = PL1_RW, .accessfn = access_tpm,
2223       .fgt = FGT_PMINTEN,
2224       .type = ARM_CP_ALIAS | ARM_CP_IO | ARM_CP_NO_RAW,
2225       .fieldoffset = offsetof(CPUARMState, cp15.c9_pminten),
2226       .writefn = pmintenclr_write },
2227     { .name = "CCSIDR", .state = ARM_CP_STATE_BOTH,
2228       .opc0 = 3, .crn = 0, .crm = 0, .opc1 = 1, .opc2 = 0,
2229       .access = PL1_R,
2230       .accessfn = access_tid4,
2231       .fgt = FGT_CCSIDR_EL1,
2232       .readfn = ccsidr_read, .type = ARM_CP_NO_RAW },
2233     { .name = "CSSELR", .state = ARM_CP_STATE_BOTH,
2234       .opc0 = 3, .crn = 0, .crm = 0, .opc1 = 2, .opc2 = 0,
2235       .access = PL1_RW,
2236       .accessfn = access_tid4,
2237       .fgt = FGT_CSSELR_EL1,
2238       .writefn = csselr_write, .resetvalue = 0,
2239       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.csselr_s),
2240                              offsetof(CPUARMState, cp15.csselr_ns) } },
2241     /*
2242      * Auxiliary ID register: this actually has an IMPDEF value but for now
2243      * just RAZ for all cores:
2244      */
2245     { .name = "AIDR", .state = ARM_CP_STATE_BOTH,
2246       .opc0 = 3, .opc1 = 1, .crn = 0, .crm = 0, .opc2 = 7,
2247       .access = PL1_R, .type = ARM_CP_CONST,
2248       .accessfn = access_aa64_tid1,
2249       .fgt = FGT_AIDR_EL1,
2250       .resetvalue = 0 },
2251     /*
2252      * Auxiliary fault status registers: these also are IMPDEF, and we
2253      * choose to RAZ/WI for all cores.
2254      */
2255     { .name = "AFSR0_EL1", .state = ARM_CP_STATE_BOTH,
2256       .opc0 = 3, .opc1 = 0, .crn = 5, .crm = 1, .opc2 = 0,
2257       .access = PL1_RW, .accessfn = access_tvm_trvm,
2258       .fgt = FGT_AFSR0_EL1,
2259       .nv2_redirect_offset = 0x128 | NV2_REDIR_NV1,
2260       .type = ARM_CP_CONST, .resetvalue = 0 },
2261     { .name = "AFSR1_EL1", .state = ARM_CP_STATE_BOTH,
2262       .opc0 = 3, .opc1 = 0, .crn = 5, .crm = 1, .opc2 = 1,
2263       .access = PL1_RW, .accessfn = access_tvm_trvm,
2264       .fgt = FGT_AFSR1_EL1,
2265       .nv2_redirect_offset = 0x130 | NV2_REDIR_NV1,
2266       .type = ARM_CP_CONST, .resetvalue = 0 },
2267     /*
2268      * MAIR can just read-as-written because we don't implement caches
2269      * and so don't need to care about memory attributes.
2270      */
2271     { .name = "MAIR_EL1", .state = ARM_CP_STATE_AA64,
2272       .opc0 = 3, .opc1 = 0, .crn = 10, .crm = 2, .opc2 = 0,
2273       .access = PL1_RW, .accessfn = access_tvm_trvm,
2274       .fgt = FGT_MAIR_EL1,
2275       .nv2_redirect_offset = 0x140 | NV2_REDIR_NV1,
2276       .fieldoffset = offsetof(CPUARMState, cp15.mair_el[1]),
2277       .resetvalue = 0 },
2278     { .name = "MAIR_EL3", .state = ARM_CP_STATE_AA64,
2279       .opc0 = 3, .opc1 = 6, .crn = 10, .crm = 2, .opc2 = 0,
2280       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.mair_el[3]),
2281       .resetvalue = 0 },
2282     /*
2283      * For non-long-descriptor page tables these are PRRR and NMRR;
2284      * regardless they still act as reads-as-written for QEMU.
2285      */
2286      /*
2287       * MAIR0/1 are defined separately from their 64-bit counterpart which
2288       * allows them to assign the correct fieldoffset based on the endianness
2289       * handled in the field definitions.
2290       */
2291     { .name = "MAIR0", .state = ARM_CP_STATE_AA32,
2292       .cp = 15, .opc1 = 0, .crn = 10, .crm = 2, .opc2 = 0,
2293       .access = PL1_RW, .accessfn = access_tvm_trvm,
2294       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.mair0_s),
2295                              offsetof(CPUARMState, cp15.mair0_ns) },
2296       .resetfn = arm_cp_reset_ignore },
2297     { .name = "MAIR1", .state = ARM_CP_STATE_AA32,
2298       .cp = 15, .opc1 = 0, .crn = 10, .crm = 2, .opc2 = 1,
2299       .access = PL1_RW, .accessfn = access_tvm_trvm,
2300       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.mair1_s),
2301                              offsetof(CPUARMState, cp15.mair1_ns) },
2302       .resetfn = arm_cp_reset_ignore },
2303     { .name = "ISR_EL1", .state = ARM_CP_STATE_BOTH,
2304       .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 1, .opc2 = 0,
2305       .fgt = FGT_ISR_EL1,
2306       .type = ARM_CP_NO_RAW, .access = PL1_R, .readfn = isr_read },
2307     /* 32 bit ITLB invalidates */
2308     { .name = "ITLBIALL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 0,
2309       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlb,
2310       .writefn = tlbiall_write },
2311     { .name = "ITLBIMVA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 1,
2312       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlb,
2313       .writefn = tlbimva_write },
2314     { .name = "ITLBIASID", .cp = 15, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 2,
2315       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlb,
2316       .writefn = tlbiasid_write },
2317     /* 32 bit DTLB invalidates */
2318     { .name = "DTLBIALL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 0,
2319       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlb,
2320       .writefn = tlbiall_write },
2321     { .name = "DTLBIMVA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 1,
2322       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlb,
2323       .writefn = tlbimva_write },
2324     { .name = "DTLBIASID", .cp = 15, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 2,
2325       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlb,
2326       .writefn = tlbiasid_write },
2327     /* 32 bit TLB invalidates */
2328     { .name = "TLBIALL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 0,
2329       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlb,
2330       .writefn = tlbiall_write },
2331     { .name = "TLBIMVA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 1,
2332       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlb,
2333       .writefn = tlbimva_write },
2334     { .name = "TLBIASID", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 2,
2335       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlb,
2336       .writefn = tlbiasid_write },
2337     { .name = "TLBIMVAA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 3,
2338       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlb,
2339       .writefn = tlbimvaa_write },
2340 };
2341 
2342 static const ARMCPRegInfo v7mp_cp_reginfo[] = {
2343     /* 32 bit TLB invalidates, Inner Shareable */
2344     { .name = "TLBIALLIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 0,
2345       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlbis,
2346       .writefn = tlbiall_is_write },
2347     { .name = "TLBIMVAIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 1,
2348       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlbis,
2349       .writefn = tlbimva_is_write },
2350     { .name = "TLBIASIDIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 2,
2351       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlbis,
2352       .writefn = tlbiasid_is_write },
2353     { .name = "TLBIMVAAIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 3,
2354       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlbis,
2355       .writefn = tlbimvaa_is_write },
2356 };
2357 
2358 static const ARMCPRegInfo pmovsset_cp_reginfo[] = {
2359     /* PMOVSSET is not implemented in v7 before v7ve */
2360     { .name = "PMOVSSET", .cp = 15, .opc1 = 0, .crn = 9, .crm = 14, .opc2 = 3,
2361       .access = PL0_RW, .accessfn = pmreg_access,
2362       .fgt = FGT_PMOVS,
2363       .type = ARM_CP_ALIAS | ARM_CP_IO,
2364       .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmovsr),
2365       .writefn = pmovsset_write,
2366       .raw_writefn = raw_write },
2367     { .name = "PMOVSSET_EL0", .state = ARM_CP_STATE_AA64,
2368       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 14, .opc2 = 3,
2369       .access = PL0_RW, .accessfn = pmreg_access,
2370       .fgt = FGT_PMOVS,
2371       .type = ARM_CP_ALIAS | ARM_CP_IO,
2372       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmovsr),
2373       .writefn = pmovsset_write,
2374       .raw_writefn = raw_write },
2375 };
2376 
2377 static void teecr_write(CPUARMState *env, const ARMCPRegInfo *ri,
2378                         uint64_t value)
2379 {
2380     value &= 1;
2381     env->teecr = value;
2382 }
2383 
2384 static CPAccessResult teecr_access(CPUARMState *env, const ARMCPRegInfo *ri,
2385                                    bool isread)
2386 {
2387     /*
2388      * HSTR.TTEE only exists in v7A, not v8A, but v8A doesn't have T2EE
2389      * at all, so we don't need to check whether we're v8A.
2390      */
2391     if (arm_current_el(env) < 2 && !arm_is_secure_below_el3(env) &&
2392         (env->cp15.hstr_el2 & HSTR_TTEE)) {
2393         return CP_ACCESS_TRAP_EL2;
2394     }
2395     return CP_ACCESS_OK;
2396 }
2397 
2398 static CPAccessResult teehbr_access(CPUARMState *env, const ARMCPRegInfo *ri,
2399                                     bool isread)
2400 {
2401     if (arm_current_el(env) == 0 && (env->teecr & 1)) {
2402         return CP_ACCESS_TRAP;
2403     }
2404     return teecr_access(env, ri, isread);
2405 }
2406 
2407 static const ARMCPRegInfo t2ee_cp_reginfo[] = {
2408     { .name = "TEECR", .cp = 14, .crn = 0, .crm = 0, .opc1 = 6, .opc2 = 0,
2409       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, teecr),
2410       .resetvalue = 0,
2411       .writefn = teecr_write, .accessfn = teecr_access },
2412     { .name = "TEEHBR", .cp = 14, .crn = 1, .crm = 0, .opc1 = 6, .opc2 = 0,
2413       .access = PL0_RW, .fieldoffset = offsetof(CPUARMState, teehbr),
2414       .accessfn = teehbr_access, .resetvalue = 0 },
2415 };
2416 
2417 static const ARMCPRegInfo v6k_cp_reginfo[] = {
2418     { .name = "TPIDR_EL0", .state = ARM_CP_STATE_AA64,
2419       .opc0 = 3, .opc1 = 3, .opc2 = 2, .crn = 13, .crm = 0,
2420       .access = PL0_RW,
2421       .fgt = FGT_TPIDR_EL0,
2422       .fieldoffset = offsetof(CPUARMState, cp15.tpidr_el[0]), .resetvalue = 0 },
2423     { .name = "TPIDRURW", .cp = 15, .crn = 13, .crm = 0, .opc1 = 0, .opc2 = 2,
2424       .access = PL0_RW,
2425       .fgt = FGT_TPIDR_EL0,
2426       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.tpidrurw_s),
2427                              offsetoflow32(CPUARMState, cp15.tpidrurw_ns) },
2428       .resetfn = arm_cp_reset_ignore },
2429     { .name = "TPIDRRO_EL0", .state = ARM_CP_STATE_AA64,
2430       .opc0 = 3, .opc1 = 3, .opc2 = 3, .crn = 13, .crm = 0,
2431       .access = PL0_R | PL1_W,
2432       .fgt = FGT_TPIDRRO_EL0,
2433       .fieldoffset = offsetof(CPUARMState, cp15.tpidrro_el[0]),
2434       .resetvalue = 0},
2435     { .name = "TPIDRURO", .cp = 15, .crn = 13, .crm = 0, .opc1 = 0, .opc2 = 3,
2436       .access = PL0_R | PL1_W,
2437       .fgt = FGT_TPIDRRO_EL0,
2438       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.tpidruro_s),
2439                              offsetoflow32(CPUARMState, cp15.tpidruro_ns) },
2440       .resetfn = arm_cp_reset_ignore },
2441     { .name = "TPIDR_EL1", .state = ARM_CP_STATE_AA64,
2442       .opc0 = 3, .opc1 = 0, .opc2 = 4, .crn = 13, .crm = 0,
2443       .access = PL1_RW,
2444       .fgt = FGT_TPIDR_EL1,
2445       .fieldoffset = offsetof(CPUARMState, cp15.tpidr_el[1]), .resetvalue = 0 },
2446     { .name = "TPIDRPRW", .opc1 = 0, .cp = 15, .crn = 13, .crm = 0, .opc2 = 4,
2447       .access = PL1_RW,
2448       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.tpidrprw_s),
2449                              offsetoflow32(CPUARMState, cp15.tpidrprw_ns) },
2450       .resetvalue = 0 },
2451 };
2452 
2453 #ifndef CONFIG_USER_ONLY
2454 
2455 static CPAccessResult gt_cntfrq_access(CPUARMState *env, const ARMCPRegInfo *ri,
2456                                        bool isread)
2457 {
2458     /*
2459      * CNTFRQ: not visible from PL0 if both PL0PCTEN and PL0VCTEN are zero.
2460      * Writable only at the highest implemented exception level.
2461      */
2462     int el = arm_current_el(env);
2463     uint64_t hcr;
2464     uint32_t cntkctl;
2465 
2466     switch (el) {
2467     case 0:
2468         hcr = arm_hcr_el2_eff(env);
2469         if ((hcr & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)) {
2470             cntkctl = env->cp15.cnthctl_el2;
2471         } else {
2472             cntkctl = env->cp15.c14_cntkctl;
2473         }
2474         if (!extract32(cntkctl, 0, 2)) {
2475             return CP_ACCESS_TRAP;
2476         }
2477         break;
2478     case 1:
2479         if (!isread && ri->state == ARM_CP_STATE_AA32 &&
2480             arm_is_secure_below_el3(env)) {
2481             /* Accesses from 32-bit Secure EL1 UNDEF (*not* trap to EL3!) */
2482             return CP_ACCESS_TRAP_UNCATEGORIZED;
2483         }
2484         break;
2485     case 2:
2486     case 3:
2487         break;
2488     }
2489 
2490     if (!isread && el < arm_highest_el(env)) {
2491         return CP_ACCESS_TRAP_UNCATEGORIZED;
2492     }
2493 
2494     return CP_ACCESS_OK;
2495 }
2496 
2497 static CPAccessResult gt_counter_access(CPUARMState *env, int timeridx,
2498                                         bool isread)
2499 {
2500     unsigned int cur_el = arm_current_el(env);
2501     bool has_el2 = arm_is_el2_enabled(env);
2502     uint64_t hcr = arm_hcr_el2_eff(env);
2503 
2504     switch (cur_el) {
2505     case 0:
2506         /* If HCR_EL2.<E2H,TGE> == '11': check CNTHCTL_EL2.EL0[PV]CTEN. */
2507         if ((hcr & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)) {
2508             return (extract32(env->cp15.cnthctl_el2, timeridx, 1)
2509                     ? CP_ACCESS_OK : CP_ACCESS_TRAP_EL2);
2510         }
2511 
2512         /* CNT[PV]CT: not visible from PL0 if EL0[PV]CTEN is zero */
2513         if (!extract32(env->cp15.c14_cntkctl, timeridx, 1)) {
2514             return CP_ACCESS_TRAP;
2515         }
2516         /* fall through */
2517     case 1:
2518         /* Check CNTHCTL_EL2.EL1PCTEN, which changes location based on E2H. */
2519         if (has_el2 && timeridx == GTIMER_PHYS &&
2520             (hcr & HCR_E2H
2521              ? !extract32(env->cp15.cnthctl_el2, 10, 1)
2522              : !extract32(env->cp15.cnthctl_el2, 0, 1))) {
2523             return CP_ACCESS_TRAP_EL2;
2524         }
2525         break;
2526     }
2527     return CP_ACCESS_OK;
2528 }
2529 
2530 static CPAccessResult gt_timer_access(CPUARMState *env, int timeridx,
2531                                       bool isread)
2532 {
2533     unsigned int cur_el = arm_current_el(env);
2534     bool has_el2 = arm_is_el2_enabled(env);
2535     uint64_t hcr = arm_hcr_el2_eff(env);
2536 
2537     switch (cur_el) {
2538     case 0:
2539         if ((hcr & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)) {
2540             /* If HCR_EL2.<E2H,TGE> == '11': check CNTHCTL_EL2.EL0[PV]TEN. */
2541             return (extract32(env->cp15.cnthctl_el2, 9 - timeridx, 1)
2542                     ? CP_ACCESS_OK : CP_ACCESS_TRAP_EL2);
2543         }
2544 
2545         /*
2546          * CNT[PV]_CVAL, CNT[PV]_CTL, CNT[PV]_TVAL: not visible from
2547          * EL0 if EL0[PV]TEN is zero.
2548          */
2549         if (!extract32(env->cp15.c14_cntkctl, 9 - timeridx, 1)) {
2550             return CP_ACCESS_TRAP;
2551         }
2552         /* fall through */
2553 
2554     case 1:
2555         if (has_el2 && timeridx == GTIMER_PHYS) {
2556             if (hcr & HCR_E2H) {
2557                 /* If HCR_EL2.<E2H,TGE> == '10': check CNTHCTL_EL2.EL1PTEN. */
2558                 if (!extract32(env->cp15.cnthctl_el2, 11, 1)) {
2559                     return CP_ACCESS_TRAP_EL2;
2560                 }
2561             } else {
2562                 /* If HCR_EL2.<E2H> == 0: check CNTHCTL_EL2.EL1PCEN. */
2563                 if (!extract32(env->cp15.cnthctl_el2, 1, 1)) {
2564                     return CP_ACCESS_TRAP_EL2;
2565                 }
2566             }
2567         }
2568         break;
2569     }
2570     return CP_ACCESS_OK;
2571 }
2572 
2573 static CPAccessResult gt_pct_access(CPUARMState *env,
2574                                     const ARMCPRegInfo *ri,
2575                                     bool isread)
2576 {
2577     return gt_counter_access(env, GTIMER_PHYS, isread);
2578 }
2579 
2580 static CPAccessResult gt_vct_access(CPUARMState *env,
2581                                     const ARMCPRegInfo *ri,
2582                                     bool isread)
2583 {
2584     return gt_counter_access(env, GTIMER_VIRT, isread);
2585 }
2586 
2587 static CPAccessResult gt_ptimer_access(CPUARMState *env, const ARMCPRegInfo *ri,
2588                                        bool isread)
2589 {
2590     return gt_timer_access(env, GTIMER_PHYS, isread);
2591 }
2592 
2593 static CPAccessResult gt_vtimer_access(CPUARMState *env, const ARMCPRegInfo *ri,
2594                                        bool isread)
2595 {
2596     return gt_timer_access(env, GTIMER_VIRT, isread);
2597 }
2598 
2599 static CPAccessResult gt_stimer_access(CPUARMState *env,
2600                                        const ARMCPRegInfo *ri,
2601                                        bool isread)
2602 {
2603     /*
2604      * The AArch64 register view of the secure physical timer is
2605      * always accessible from EL3, and configurably accessible from
2606      * Secure EL1.
2607      */
2608     switch (arm_current_el(env)) {
2609     case 1:
2610         if (!arm_is_secure(env)) {
2611             return CP_ACCESS_TRAP;
2612         }
2613         if (!(env->cp15.scr_el3 & SCR_ST)) {
2614             return CP_ACCESS_TRAP_EL3;
2615         }
2616         return CP_ACCESS_OK;
2617     case 0:
2618     case 2:
2619         return CP_ACCESS_TRAP;
2620     case 3:
2621         return CP_ACCESS_OK;
2622     default:
2623         g_assert_not_reached();
2624     }
2625 }
2626 
2627 static uint64_t gt_get_countervalue(CPUARMState *env)
2628 {
2629     ARMCPU *cpu = env_archcpu(env);
2630 
2631     return qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) / gt_cntfrq_period_ns(cpu);
2632 }
2633 
2634 static void gt_update_irq(ARMCPU *cpu, int timeridx)
2635 {
2636     CPUARMState *env = &cpu->env;
2637     uint64_t cnthctl = env->cp15.cnthctl_el2;
2638     ARMSecuritySpace ss = arm_security_space(env);
2639     /* ISTATUS && !IMASK */
2640     int irqstate = (env->cp15.c14_timer[timeridx].ctl & 6) == 4;
2641 
2642     /*
2643      * If bit CNTHCTL_EL2.CNT[VP]MASK is set, it overrides IMASK.
2644      * It is RES0 in Secure and NonSecure state.
2645      */
2646     if ((ss == ARMSS_Root || ss == ARMSS_Realm) &&
2647         ((timeridx == GTIMER_VIRT && (cnthctl & CNTHCTL_CNTVMASK)) ||
2648          (timeridx == GTIMER_PHYS && (cnthctl & CNTHCTL_CNTPMASK)))) {
2649         irqstate = 0;
2650     }
2651 
2652     qemu_set_irq(cpu->gt_timer_outputs[timeridx], irqstate);
2653     trace_arm_gt_update_irq(timeridx, irqstate);
2654 }
2655 
2656 void gt_rme_post_el_change(ARMCPU *cpu, void *ignored)
2657 {
2658     /*
2659      * Changing security state between Root and Secure/NonSecure, which may
2660      * happen when switching EL, can change the effective value of CNTHCTL_EL2
2661      * mask bits. Update the IRQ state accordingly.
2662      */
2663     gt_update_irq(cpu, GTIMER_VIRT);
2664     gt_update_irq(cpu, GTIMER_PHYS);
2665 }
2666 
2667 static void gt_recalc_timer(ARMCPU *cpu, int timeridx)
2668 {
2669     ARMGenericTimer *gt = &cpu->env.cp15.c14_timer[timeridx];
2670 
2671     if (gt->ctl & 1) {
2672         /*
2673          * Timer enabled: calculate and set current ISTATUS, irq, and
2674          * reset timer to when ISTATUS next has to change
2675          */
2676         uint64_t offset = timeridx == GTIMER_VIRT ?
2677                                       cpu->env.cp15.cntvoff_el2 : 0;
2678         uint64_t count = gt_get_countervalue(&cpu->env);
2679         /* Note that this must be unsigned 64 bit arithmetic: */
2680         int istatus = count - offset >= gt->cval;
2681         uint64_t nexttick;
2682 
2683         gt->ctl = deposit32(gt->ctl, 2, 1, istatus);
2684 
2685         if (istatus) {
2686             /*
2687              * Next transition is when (count - offset) rolls back over to 0.
2688              * If offset > count then this is when count == offset;
2689              * if offset <= count then this is when count == offset + 2^64
2690              * For the latter case we set nexttick to an "as far in future
2691              * as possible" value and let the code below handle it.
2692              */
2693             if (offset > count) {
2694                 nexttick = offset;
2695             } else {
2696                 nexttick = UINT64_MAX;
2697             }
2698         } else {
2699             /*
2700              * Next transition is when (count - offset) == cval, i.e.
2701              * when count == (cval + offset).
2702              * If that would overflow, then again we set up the next interrupt
2703              * for "as far in the future as possible" for the code below.
2704              */
2705             if (uadd64_overflow(gt->cval, offset, &nexttick)) {
2706                 nexttick = UINT64_MAX;
2707             }
2708         }
2709         /*
2710          * Note that the desired next expiry time might be beyond the
2711          * signed-64-bit range of a QEMUTimer -- in this case we just
2712          * set the timer for as far in the future as possible. When the
2713          * timer expires we will reset the timer for any remaining period.
2714          */
2715         if (nexttick > INT64_MAX / gt_cntfrq_period_ns(cpu)) {
2716             timer_mod_ns(cpu->gt_timer[timeridx], INT64_MAX);
2717         } else {
2718             timer_mod(cpu->gt_timer[timeridx], nexttick);
2719         }
2720         trace_arm_gt_recalc(timeridx, nexttick);
2721     } else {
2722         /* Timer disabled: ISTATUS and timer output always clear */
2723         gt->ctl &= ~4;
2724         timer_del(cpu->gt_timer[timeridx]);
2725         trace_arm_gt_recalc_disabled(timeridx);
2726     }
2727     gt_update_irq(cpu, timeridx);
2728 }
2729 
2730 static void gt_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri,
2731                            int timeridx)
2732 {
2733     ARMCPU *cpu = env_archcpu(env);
2734 
2735     timer_del(cpu->gt_timer[timeridx]);
2736 }
2737 
2738 static uint64_t gt_cnt_read(CPUARMState *env, const ARMCPRegInfo *ri)
2739 {
2740     return gt_get_countervalue(env);
2741 }
2742 
2743 static uint64_t gt_virt_cnt_offset(CPUARMState *env)
2744 {
2745     uint64_t hcr;
2746 
2747     switch (arm_current_el(env)) {
2748     case 2:
2749         hcr = arm_hcr_el2_eff(env);
2750         if (hcr & HCR_E2H) {
2751             return 0;
2752         }
2753         break;
2754     case 0:
2755         hcr = arm_hcr_el2_eff(env);
2756         if ((hcr & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)) {
2757             return 0;
2758         }
2759         break;
2760     }
2761 
2762     return env->cp15.cntvoff_el2;
2763 }
2764 
2765 static uint64_t gt_virt_cnt_read(CPUARMState *env, const ARMCPRegInfo *ri)
2766 {
2767     return gt_get_countervalue(env) - gt_virt_cnt_offset(env);
2768 }
2769 
2770 static void gt_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
2771                           int timeridx,
2772                           uint64_t value)
2773 {
2774     trace_arm_gt_cval_write(timeridx, value);
2775     env->cp15.c14_timer[timeridx].cval = value;
2776     gt_recalc_timer(env_archcpu(env), timeridx);
2777 }
2778 
2779 static uint64_t gt_tval_read(CPUARMState *env, const ARMCPRegInfo *ri,
2780                              int timeridx)
2781 {
2782     uint64_t offset = 0;
2783 
2784     switch (timeridx) {
2785     case GTIMER_VIRT:
2786     case GTIMER_HYPVIRT:
2787         offset = gt_virt_cnt_offset(env);
2788         break;
2789     }
2790 
2791     return (uint32_t)(env->cp15.c14_timer[timeridx].cval -
2792                       (gt_get_countervalue(env) - offset));
2793 }
2794 
2795 static void gt_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
2796                           int timeridx,
2797                           uint64_t value)
2798 {
2799     uint64_t offset = 0;
2800 
2801     switch (timeridx) {
2802     case GTIMER_VIRT:
2803     case GTIMER_HYPVIRT:
2804         offset = gt_virt_cnt_offset(env);
2805         break;
2806     }
2807 
2808     trace_arm_gt_tval_write(timeridx, value);
2809     env->cp15.c14_timer[timeridx].cval = gt_get_countervalue(env) - offset +
2810                                          sextract64(value, 0, 32);
2811     gt_recalc_timer(env_archcpu(env), timeridx);
2812 }
2813 
2814 static void gt_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
2815                          int timeridx,
2816                          uint64_t value)
2817 {
2818     ARMCPU *cpu = env_archcpu(env);
2819     uint32_t oldval = env->cp15.c14_timer[timeridx].ctl;
2820 
2821     trace_arm_gt_ctl_write(timeridx, value);
2822     env->cp15.c14_timer[timeridx].ctl = deposit64(oldval, 0, 2, value);
2823     if ((oldval ^ value) & 1) {
2824         /* Enable toggled */
2825         gt_recalc_timer(cpu, timeridx);
2826     } else if ((oldval ^ value) & 2) {
2827         /*
2828          * IMASK toggled: don't need to recalculate,
2829          * just set the interrupt line based on ISTATUS
2830          */
2831         trace_arm_gt_imask_toggle(timeridx);
2832         gt_update_irq(cpu, timeridx);
2833     }
2834 }
2835 
2836 static void gt_phys_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri)
2837 {
2838     gt_timer_reset(env, ri, GTIMER_PHYS);
2839 }
2840 
2841 static void gt_phys_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
2842                                uint64_t value)
2843 {
2844     gt_cval_write(env, ri, GTIMER_PHYS, value);
2845 }
2846 
2847 static uint64_t gt_phys_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
2848 {
2849     return gt_tval_read(env, ri, GTIMER_PHYS);
2850 }
2851 
2852 static void gt_phys_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
2853                                uint64_t value)
2854 {
2855     gt_tval_write(env, ri, GTIMER_PHYS, value);
2856 }
2857 
2858 static void gt_phys_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
2859                               uint64_t value)
2860 {
2861     gt_ctl_write(env, ri, GTIMER_PHYS, value);
2862 }
2863 
2864 static int gt_phys_redir_timeridx(CPUARMState *env)
2865 {
2866     switch (arm_mmu_idx(env)) {
2867     case ARMMMUIdx_E20_0:
2868     case ARMMMUIdx_E20_2:
2869     case ARMMMUIdx_E20_2_PAN:
2870         return GTIMER_HYP;
2871     default:
2872         return GTIMER_PHYS;
2873     }
2874 }
2875 
2876 static int gt_virt_redir_timeridx(CPUARMState *env)
2877 {
2878     switch (arm_mmu_idx(env)) {
2879     case ARMMMUIdx_E20_0:
2880     case ARMMMUIdx_E20_2:
2881     case ARMMMUIdx_E20_2_PAN:
2882         return GTIMER_HYPVIRT;
2883     default:
2884         return GTIMER_VIRT;
2885     }
2886 }
2887 
2888 static uint64_t gt_phys_redir_cval_read(CPUARMState *env,
2889                                         const ARMCPRegInfo *ri)
2890 {
2891     int timeridx = gt_phys_redir_timeridx(env);
2892     return env->cp15.c14_timer[timeridx].cval;
2893 }
2894 
2895 static void gt_phys_redir_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
2896                                      uint64_t value)
2897 {
2898     int timeridx = gt_phys_redir_timeridx(env);
2899     gt_cval_write(env, ri, timeridx, value);
2900 }
2901 
2902 static uint64_t gt_phys_redir_tval_read(CPUARMState *env,
2903                                         const ARMCPRegInfo *ri)
2904 {
2905     int timeridx = gt_phys_redir_timeridx(env);
2906     return gt_tval_read(env, ri, timeridx);
2907 }
2908 
2909 static void gt_phys_redir_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
2910                                      uint64_t value)
2911 {
2912     int timeridx = gt_phys_redir_timeridx(env);
2913     gt_tval_write(env, ri, timeridx, value);
2914 }
2915 
2916 static uint64_t gt_phys_redir_ctl_read(CPUARMState *env,
2917                                        const ARMCPRegInfo *ri)
2918 {
2919     int timeridx = gt_phys_redir_timeridx(env);
2920     return env->cp15.c14_timer[timeridx].ctl;
2921 }
2922 
2923 static void gt_phys_redir_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
2924                                     uint64_t value)
2925 {
2926     int timeridx = gt_phys_redir_timeridx(env);
2927     gt_ctl_write(env, ri, timeridx, value);
2928 }
2929 
2930 static void gt_virt_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri)
2931 {
2932     gt_timer_reset(env, ri, GTIMER_VIRT);
2933 }
2934 
2935 static void gt_virt_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
2936                                uint64_t value)
2937 {
2938     gt_cval_write(env, ri, GTIMER_VIRT, value);
2939 }
2940 
2941 static uint64_t gt_virt_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
2942 {
2943     return gt_tval_read(env, ri, GTIMER_VIRT);
2944 }
2945 
2946 static void gt_virt_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
2947                                uint64_t value)
2948 {
2949     gt_tval_write(env, ri, GTIMER_VIRT, value);
2950 }
2951 
2952 static void gt_virt_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
2953                               uint64_t value)
2954 {
2955     gt_ctl_write(env, ri, GTIMER_VIRT, value);
2956 }
2957 
2958 static void gt_cnthctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
2959                              uint64_t value)
2960 {
2961     ARMCPU *cpu = env_archcpu(env);
2962     uint32_t oldval = env->cp15.cnthctl_el2;
2963 
2964     raw_write(env, ri, value);
2965 
2966     if ((oldval ^ value) & CNTHCTL_CNTVMASK) {
2967         gt_update_irq(cpu, GTIMER_VIRT);
2968     } else if ((oldval ^ value) & CNTHCTL_CNTPMASK) {
2969         gt_update_irq(cpu, GTIMER_PHYS);
2970     }
2971 }
2972 
2973 static void gt_cntvoff_write(CPUARMState *env, const ARMCPRegInfo *ri,
2974                               uint64_t value)
2975 {
2976     ARMCPU *cpu = env_archcpu(env);
2977 
2978     trace_arm_gt_cntvoff_write(value);
2979     raw_write(env, ri, value);
2980     gt_recalc_timer(cpu, GTIMER_VIRT);
2981 }
2982 
2983 static uint64_t gt_virt_redir_cval_read(CPUARMState *env,
2984                                         const ARMCPRegInfo *ri)
2985 {
2986     int timeridx = gt_virt_redir_timeridx(env);
2987     return env->cp15.c14_timer[timeridx].cval;
2988 }
2989 
2990 static void gt_virt_redir_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
2991                                      uint64_t value)
2992 {
2993     int timeridx = gt_virt_redir_timeridx(env);
2994     gt_cval_write(env, ri, timeridx, value);
2995 }
2996 
2997 static uint64_t gt_virt_redir_tval_read(CPUARMState *env,
2998                                         const ARMCPRegInfo *ri)
2999 {
3000     int timeridx = gt_virt_redir_timeridx(env);
3001     return gt_tval_read(env, ri, timeridx);
3002 }
3003 
3004 static void gt_virt_redir_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
3005                                      uint64_t value)
3006 {
3007     int timeridx = gt_virt_redir_timeridx(env);
3008     gt_tval_write(env, ri, timeridx, value);
3009 }
3010 
3011 static uint64_t gt_virt_redir_ctl_read(CPUARMState *env,
3012                                        const ARMCPRegInfo *ri)
3013 {
3014     int timeridx = gt_virt_redir_timeridx(env);
3015     return env->cp15.c14_timer[timeridx].ctl;
3016 }
3017 
3018 static void gt_virt_redir_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
3019                                     uint64_t value)
3020 {
3021     int timeridx = gt_virt_redir_timeridx(env);
3022     gt_ctl_write(env, ri, timeridx, value);
3023 }
3024 
3025 static void gt_hyp_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri)
3026 {
3027     gt_timer_reset(env, ri, GTIMER_HYP);
3028 }
3029 
3030 static void gt_hyp_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
3031                               uint64_t value)
3032 {
3033     gt_cval_write(env, ri, GTIMER_HYP, value);
3034 }
3035 
3036 static uint64_t gt_hyp_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
3037 {
3038     return gt_tval_read(env, ri, GTIMER_HYP);
3039 }
3040 
3041 static void gt_hyp_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
3042                               uint64_t value)
3043 {
3044     gt_tval_write(env, ri, GTIMER_HYP, value);
3045 }
3046 
3047 static void gt_hyp_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
3048                               uint64_t value)
3049 {
3050     gt_ctl_write(env, ri, GTIMER_HYP, value);
3051 }
3052 
3053 static void gt_sec_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri)
3054 {
3055     gt_timer_reset(env, ri, GTIMER_SEC);
3056 }
3057 
3058 static void gt_sec_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
3059                               uint64_t value)
3060 {
3061     gt_cval_write(env, ri, GTIMER_SEC, value);
3062 }
3063 
3064 static uint64_t gt_sec_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
3065 {
3066     return gt_tval_read(env, ri, GTIMER_SEC);
3067 }
3068 
3069 static void gt_sec_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
3070                               uint64_t value)
3071 {
3072     gt_tval_write(env, ri, GTIMER_SEC, value);
3073 }
3074 
3075 static void gt_sec_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
3076                               uint64_t value)
3077 {
3078     gt_ctl_write(env, ri, GTIMER_SEC, value);
3079 }
3080 
3081 static void gt_hv_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri)
3082 {
3083     gt_timer_reset(env, ri, GTIMER_HYPVIRT);
3084 }
3085 
3086 static void gt_hv_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
3087                              uint64_t value)
3088 {
3089     gt_cval_write(env, ri, GTIMER_HYPVIRT, value);
3090 }
3091 
3092 static uint64_t gt_hv_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
3093 {
3094     return gt_tval_read(env, ri, GTIMER_HYPVIRT);
3095 }
3096 
3097 static void gt_hv_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
3098                              uint64_t value)
3099 {
3100     gt_tval_write(env, ri, GTIMER_HYPVIRT, value);
3101 }
3102 
3103 static void gt_hv_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
3104                             uint64_t value)
3105 {
3106     gt_ctl_write(env, ri, GTIMER_HYPVIRT, value);
3107 }
3108 
3109 void arm_gt_ptimer_cb(void *opaque)
3110 {
3111     ARMCPU *cpu = opaque;
3112 
3113     gt_recalc_timer(cpu, GTIMER_PHYS);
3114 }
3115 
3116 void arm_gt_vtimer_cb(void *opaque)
3117 {
3118     ARMCPU *cpu = opaque;
3119 
3120     gt_recalc_timer(cpu, GTIMER_VIRT);
3121 }
3122 
3123 void arm_gt_htimer_cb(void *opaque)
3124 {
3125     ARMCPU *cpu = opaque;
3126 
3127     gt_recalc_timer(cpu, GTIMER_HYP);
3128 }
3129 
3130 void arm_gt_stimer_cb(void *opaque)
3131 {
3132     ARMCPU *cpu = opaque;
3133 
3134     gt_recalc_timer(cpu, GTIMER_SEC);
3135 }
3136 
3137 void arm_gt_hvtimer_cb(void *opaque)
3138 {
3139     ARMCPU *cpu = opaque;
3140 
3141     gt_recalc_timer(cpu, GTIMER_HYPVIRT);
3142 }
3143 
3144 static void arm_gt_cntfrq_reset(CPUARMState *env, const ARMCPRegInfo *opaque)
3145 {
3146     ARMCPU *cpu = env_archcpu(env);
3147 
3148     cpu->env.cp15.c14_cntfrq = cpu->gt_cntfrq_hz;
3149 }
3150 
3151 static const ARMCPRegInfo generic_timer_cp_reginfo[] = {
3152     /*
3153      * Note that CNTFRQ is purely reads-as-written for the benefit
3154      * of software; writing it doesn't actually change the timer frequency.
3155      * Our reset value matches the fixed frequency we implement the timer at.
3156      */
3157     { .name = "CNTFRQ", .cp = 15, .crn = 14, .crm = 0, .opc1 = 0, .opc2 = 0,
3158       .type = ARM_CP_ALIAS,
3159       .access = PL1_RW | PL0_R, .accessfn = gt_cntfrq_access,
3160       .fieldoffset = offsetoflow32(CPUARMState, cp15.c14_cntfrq),
3161     },
3162     { .name = "CNTFRQ_EL0", .state = ARM_CP_STATE_AA64,
3163       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 0, .opc2 = 0,
3164       .access = PL1_RW | PL0_R, .accessfn = gt_cntfrq_access,
3165       .fieldoffset = offsetof(CPUARMState, cp15.c14_cntfrq),
3166       .resetfn = arm_gt_cntfrq_reset,
3167     },
3168     /* overall control: mostly access permissions */
3169     { .name = "CNTKCTL", .state = ARM_CP_STATE_BOTH,
3170       .opc0 = 3, .opc1 = 0, .crn = 14, .crm = 1, .opc2 = 0,
3171       .access = PL1_RW,
3172       .fieldoffset = offsetof(CPUARMState, cp15.c14_cntkctl),
3173       .resetvalue = 0,
3174     },
3175     /* per-timer control */
3176     { .name = "CNTP_CTL", .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 1,
3177       .secure = ARM_CP_SECSTATE_NS,
3178       .type = ARM_CP_IO | ARM_CP_ALIAS, .access = PL0_RW,
3179       .accessfn = gt_ptimer_access,
3180       .fieldoffset = offsetoflow32(CPUARMState,
3181                                    cp15.c14_timer[GTIMER_PHYS].ctl),
3182       .readfn = gt_phys_redir_ctl_read, .raw_readfn = raw_read,
3183       .writefn = gt_phys_redir_ctl_write, .raw_writefn = raw_write,
3184     },
3185     { .name = "CNTP_CTL_S",
3186       .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 1,
3187       .secure = ARM_CP_SECSTATE_S,
3188       .type = ARM_CP_IO | ARM_CP_ALIAS, .access = PL0_RW,
3189       .accessfn = gt_ptimer_access,
3190       .fieldoffset = offsetoflow32(CPUARMState,
3191                                    cp15.c14_timer[GTIMER_SEC].ctl),
3192       .writefn = gt_sec_ctl_write, .raw_writefn = raw_write,
3193     },
3194     { .name = "CNTP_CTL_EL0", .state = ARM_CP_STATE_AA64,
3195       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 2, .opc2 = 1,
3196       .type = ARM_CP_IO, .access = PL0_RW,
3197       .accessfn = gt_ptimer_access,
3198       .nv2_redirect_offset = 0x180 | NV2_REDIR_NV1,
3199       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].ctl),
3200       .resetvalue = 0,
3201       .readfn = gt_phys_redir_ctl_read, .raw_readfn = raw_read,
3202       .writefn = gt_phys_redir_ctl_write, .raw_writefn = raw_write,
3203     },
3204     { .name = "CNTV_CTL", .cp = 15, .crn = 14, .crm = 3, .opc1 = 0, .opc2 = 1,
3205       .type = ARM_CP_IO | ARM_CP_ALIAS, .access = PL0_RW,
3206       .accessfn = gt_vtimer_access,
3207       .fieldoffset = offsetoflow32(CPUARMState,
3208                                    cp15.c14_timer[GTIMER_VIRT].ctl),
3209       .readfn = gt_virt_redir_ctl_read, .raw_readfn = raw_read,
3210       .writefn = gt_virt_redir_ctl_write, .raw_writefn = raw_write,
3211     },
3212     { .name = "CNTV_CTL_EL0", .state = ARM_CP_STATE_AA64,
3213       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 1,
3214       .type = ARM_CP_IO, .access = PL0_RW,
3215       .accessfn = gt_vtimer_access,
3216       .nv2_redirect_offset = 0x170 | NV2_REDIR_NV1,
3217       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].ctl),
3218       .resetvalue = 0,
3219       .readfn = gt_virt_redir_ctl_read, .raw_readfn = raw_read,
3220       .writefn = gt_virt_redir_ctl_write, .raw_writefn = raw_write,
3221     },
3222     /* TimerValue views: a 32 bit downcounting view of the underlying state */
3223     { .name = "CNTP_TVAL", .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 0,
3224       .secure = ARM_CP_SECSTATE_NS,
3225       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL0_RW,
3226       .accessfn = gt_ptimer_access,
3227       .readfn = gt_phys_redir_tval_read, .writefn = gt_phys_redir_tval_write,
3228     },
3229     { .name = "CNTP_TVAL_S",
3230       .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 0,
3231       .secure = ARM_CP_SECSTATE_S,
3232       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL0_RW,
3233       .accessfn = gt_ptimer_access,
3234       .readfn = gt_sec_tval_read, .writefn = gt_sec_tval_write,
3235     },
3236     { .name = "CNTP_TVAL_EL0", .state = ARM_CP_STATE_AA64,
3237       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 2, .opc2 = 0,
3238       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL0_RW,
3239       .accessfn = gt_ptimer_access, .resetfn = gt_phys_timer_reset,
3240       .readfn = gt_phys_redir_tval_read, .writefn = gt_phys_redir_tval_write,
3241     },
3242     { .name = "CNTV_TVAL", .cp = 15, .crn = 14, .crm = 3, .opc1 = 0, .opc2 = 0,
3243       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL0_RW,
3244       .accessfn = gt_vtimer_access,
3245       .readfn = gt_virt_redir_tval_read, .writefn = gt_virt_redir_tval_write,
3246     },
3247     { .name = "CNTV_TVAL_EL0", .state = ARM_CP_STATE_AA64,
3248       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 0,
3249       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL0_RW,
3250       .accessfn = gt_vtimer_access, .resetfn = gt_virt_timer_reset,
3251       .readfn = gt_virt_redir_tval_read, .writefn = gt_virt_redir_tval_write,
3252     },
3253     /* The counter itself */
3254     { .name = "CNTPCT", .cp = 15, .crm = 14, .opc1 = 0,
3255       .access = PL0_R, .type = ARM_CP_64BIT | ARM_CP_NO_RAW | ARM_CP_IO,
3256       .accessfn = gt_pct_access,
3257       .readfn = gt_cnt_read, .resetfn = arm_cp_reset_ignore,
3258     },
3259     { .name = "CNTPCT_EL0", .state = ARM_CP_STATE_AA64,
3260       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 0, .opc2 = 1,
3261       .access = PL0_R, .type = ARM_CP_NO_RAW | ARM_CP_IO,
3262       .accessfn = gt_pct_access, .readfn = gt_cnt_read,
3263     },
3264     { .name = "CNTVCT", .cp = 15, .crm = 14, .opc1 = 1,
3265       .access = PL0_R, .type = ARM_CP_64BIT | ARM_CP_NO_RAW | ARM_CP_IO,
3266       .accessfn = gt_vct_access,
3267       .readfn = gt_virt_cnt_read, .resetfn = arm_cp_reset_ignore,
3268     },
3269     { .name = "CNTVCT_EL0", .state = ARM_CP_STATE_AA64,
3270       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 0, .opc2 = 2,
3271       .access = PL0_R, .type = ARM_CP_NO_RAW | ARM_CP_IO,
3272       .accessfn = gt_vct_access, .readfn = gt_virt_cnt_read,
3273     },
3274     /* Comparison value, indicating when the timer goes off */
3275     { .name = "CNTP_CVAL", .cp = 15, .crm = 14, .opc1 = 2,
3276       .secure = ARM_CP_SECSTATE_NS,
3277       .access = PL0_RW,
3278       .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_ALIAS,
3279       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].cval),
3280       .accessfn = gt_ptimer_access,
3281       .readfn = gt_phys_redir_cval_read, .raw_readfn = raw_read,
3282       .writefn = gt_phys_redir_cval_write, .raw_writefn = raw_write,
3283     },
3284     { .name = "CNTP_CVAL_S", .cp = 15, .crm = 14, .opc1 = 2,
3285       .secure = ARM_CP_SECSTATE_S,
3286       .access = PL0_RW,
3287       .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_ALIAS,
3288       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_SEC].cval),
3289       .accessfn = gt_ptimer_access,
3290       .writefn = gt_sec_cval_write, .raw_writefn = raw_write,
3291     },
3292     { .name = "CNTP_CVAL_EL0", .state = ARM_CP_STATE_AA64,
3293       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 2, .opc2 = 2,
3294       .access = PL0_RW,
3295       .type = ARM_CP_IO,
3296       .nv2_redirect_offset = 0x178 | NV2_REDIR_NV1,
3297       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].cval),
3298       .resetvalue = 0, .accessfn = gt_ptimer_access,
3299       .readfn = gt_phys_redir_cval_read, .raw_readfn = raw_read,
3300       .writefn = gt_phys_redir_cval_write, .raw_writefn = raw_write,
3301     },
3302     { .name = "CNTV_CVAL", .cp = 15, .crm = 14, .opc1 = 3,
3303       .access = PL0_RW,
3304       .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_ALIAS,
3305       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].cval),
3306       .accessfn = gt_vtimer_access,
3307       .readfn = gt_virt_redir_cval_read, .raw_readfn = raw_read,
3308       .writefn = gt_virt_redir_cval_write, .raw_writefn = raw_write,
3309     },
3310     { .name = "CNTV_CVAL_EL0", .state = ARM_CP_STATE_AA64,
3311       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 2,
3312       .access = PL0_RW,
3313       .type = ARM_CP_IO,
3314       .nv2_redirect_offset = 0x168 | NV2_REDIR_NV1,
3315       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].cval),
3316       .resetvalue = 0, .accessfn = gt_vtimer_access,
3317       .readfn = gt_virt_redir_cval_read, .raw_readfn = raw_read,
3318       .writefn = gt_virt_redir_cval_write, .raw_writefn = raw_write,
3319     },
3320     /*
3321      * Secure timer -- this is actually restricted to only EL3
3322      * and configurably Secure-EL1 via the accessfn.
3323      */
3324     { .name = "CNTPS_TVAL_EL1", .state = ARM_CP_STATE_AA64,
3325       .opc0 = 3, .opc1 = 7, .crn = 14, .crm = 2, .opc2 = 0,
3326       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW,
3327       .accessfn = gt_stimer_access,
3328       .readfn = gt_sec_tval_read,
3329       .writefn = gt_sec_tval_write,
3330       .resetfn = gt_sec_timer_reset,
3331     },
3332     { .name = "CNTPS_CTL_EL1", .state = ARM_CP_STATE_AA64,
3333       .opc0 = 3, .opc1 = 7, .crn = 14, .crm = 2, .opc2 = 1,
3334       .type = ARM_CP_IO, .access = PL1_RW,
3335       .accessfn = gt_stimer_access,
3336       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_SEC].ctl),
3337       .resetvalue = 0,
3338       .writefn = gt_sec_ctl_write, .raw_writefn = raw_write,
3339     },
3340     { .name = "CNTPS_CVAL_EL1", .state = ARM_CP_STATE_AA64,
3341       .opc0 = 3, .opc1 = 7, .crn = 14, .crm = 2, .opc2 = 2,
3342       .type = ARM_CP_IO, .access = PL1_RW,
3343       .accessfn = gt_stimer_access,
3344       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_SEC].cval),
3345       .writefn = gt_sec_cval_write, .raw_writefn = raw_write,
3346     },
3347 };
3348 
3349 #else
3350 
3351 /*
3352  * In user-mode most of the generic timer registers are inaccessible
3353  * however modern kernels (4.12+) allow access to cntvct_el0
3354  */
3355 
3356 static uint64_t gt_virt_cnt_read(CPUARMState *env, const ARMCPRegInfo *ri)
3357 {
3358     ARMCPU *cpu = env_archcpu(env);
3359 
3360     /*
3361      * Currently we have no support for QEMUTimer in linux-user so we
3362      * can't call gt_get_countervalue(env), instead we directly
3363      * call the lower level functions.
3364      */
3365     return cpu_get_clock() / gt_cntfrq_period_ns(cpu);
3366 }
3367 
3368 static const ARMCPRegInfo generic_timer_cp_reginfo[] = {
3369     { .name = "CNTFRQ_EL0", .state = ARM_CP_STATE_AA64,
3370       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 0, .opc2 = 0,
3371       .type = ARM_CP_CONST, .access = PL0_R /* no PL1_RW in linux-user */,
3372       .fieldoffset = offsetof(CPUARMState, cp15.c14_cntfrq),
3373       .resetvalue = NANOSECONDS_PER_SECOND / GTIMER_SCALE,
3374     },
3375     { .name = "CNTVCT_EL0", .state = ARM_CP_STATE_AA64,
3376       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 0, .opc2 = 2,
3377       .access = PL0_R, .type = ARM_CP_NO_RAW | ARM_CP_IO,
3378       .readfn = gt_virt_cnt_read,
3379     },
3380 };
3381 
3382 #endif
3383 
3384 static void par_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
3385 {
3386     if (arm_feature(env, ARM_FEATURE_LPAE)) {
3387         raw_write(env, ri, value);
3388     } else if (arm_feature(env, ARM_FEATURE_V7)) {
3389         raw_write(env, ri, value & 0xfffff6ff);
3390     } else {
3391         raw_write(env, ri, value & 0xfffff1ff);
3392     }
3393 }
3394 
3395 #ifndef CONFIG_USER_ONLY
3396 /* get_phys_addr() isn't present for user-mode-only targets */
3397 
3398 static CPAccessResult ats_access(CPUARMState *env, const ARMCPRegInfo *ri,
3399                                  bool isread)
3400 {
3401     if (ri->opc2 & 4) {
3402         /*
3403          * The ATS12NSO* operations must trap to EL3 or EL2 if executed in
3404          * Secure EL1 (which can only happen if EL3 is AArch64).
3405          * They are simply UNDEF if executed from NS EL1.
3406          * They function normally from EL2 or EL3.
3407          */
3408         if (arm_current_el(env) == 1) {
3409             if (arm_is_secure_below_el3(env)) {
3410                 if (env->cp15.scr_el3 & SCR_EEL2) {
3411                     return CP_ACCESS_TRAP_EL2;
3412                 }
3413                 return CP_ACCESS_TRAP_EL3;
3414             }
3415             return CP_ACCESS_TRAP_UNCATEGORIZED;
3416         }
3417     }
3418     return CP_ACCESS_OK;
3419 }
3420 
3421 #ifdef CONFIG_TCG
3422 static int par_el1_shareability(GetPhysAddrResult *res)
3423 {
3424     /*
3425      * The PAR_EL1.SH field must be 0b10 for Device or Normal-NC
3426      * memory -- see pseudocode PAREncodeShareability().
3427      */
3428     if (((res->cacheattrs.attrs & 0xf0) == 0) ||
3429         res->cacheattrs.attrs == 0x44 || res->cacheattrs.attrs == 0x40) {
3430         return 2;
3431     }
3432     return res->cacheattrs.shareability;
3433 }
3434 
3435 static uint64_t do_ats_write(CPUARMState *env, uint64_t value,
3436                              MMUAccessType access_type, ARMMMUIdx mmu_idx,
3437                              ARMSecuritySpace ss)
3438 {
3439     bool ret;
3440     uint64_t par64;
3441     bool format64 = false;
3442     ARMMMUFaultInfo fi = {};
3443     GetPhysAddrResult res = {};
3444 
3445     /*
3446      * I_MXTJT: Granule protection checks are not performed on the final address
3447      * of a successful translation.
3448      */
3449     ret = get_phys_addr_with_space_nogpc(env, value, access_type, mmu_idx, ss,
3450                                          &res, &fi);
3451 
3452     /*
3453      * ATS operations only do S1 or S1+S2 translations, so we never
3454      * have to deal with the ARMCacheAttrs format for S2 only.
3455      */
3456     assert(!res.cacheattrs.is_s2_format);
3457 
3458     if (ret) {
3459         /*
3460          * Some kinds of translation fault must cause exceptions rather
3461          * than being reported in the PAR.
3462          */
3463         int current_el = arm_current_el(env);
3464         int target_el;
3465         uint32_t syn, fsr, fsc;
3466         bool take_exc = false;
3467 
3468         if (fi.s1ptw && current_el == 1
3469             && arm_mmu_idx_is_stage1_of_2(mmu_idx)) {
3470             /*
3471              * Synchronous stage 2 fault on an access made as part of the
3472              * translation table walk for AT S1E0* or AT S1E1* insn
3473              * executed from NS EL1. If this is a synchronous external abort
3474              * and SCR_EL3.EA == 1, then we take a synchronous external abort
3475              * to EL3. Otherwise the fault is taken as an exception to EL2,
3476              * and HPFAR_EL2 holds the faulting IPA.
3477              */
3478             if (fi.type == ARMFault_SyncExternalOnWalk &&
3479                 (env->cp15.scr_el3 & SCR_EA)) {
3480                 target_el = 3;
3481             } else {
3482                 env->cp15.hpfar_el2 = extract64(fi.s2addr, 12, 47) << 4;
3483                 if (arm_is_secure_below_el3(env) && fi.s1ns) {
3484                     env->cp15.hpfar_el2 |= HPFAR_NS;
3485                 }
3486                 target_el = 2;
3487             }
3488             take_exc = true;
3489         } else if (fi.type == ARMFault_SyncExternalOnWalk) {
3490             /*
3491              * Synchronous external aborts during a translation table walk
3492              * are taken as Data Abort exceptions.
3493              */
3494             if (fi.stage2) {
3495                 if (current_el == 3) {
3496                     target_el = 3;
3497                 } else {
3498                     target_el = 2;
3499                 }
3500             } else {
3501                 target_el = exception_target_el(env);
3502             }
3503             take_exc = true;
3504         }
3505 
3506         if (take_exc) {
3507             /* Construct FSR and FSC using same logic as arm_deliver_fault() */
3508             if (target_el == 2 || arm_el_is_aa64(env, target_el) ||
3509                 arm_s1_regime_using_lpae_format(env, mmu_idx)) {
3510                 fsr = arm_fi_to_lfsc(&fi);
3511                 fsc = extract32(fsr, 0, 6);
3512             } else {
3513                 fsr = arm_fi_to_sfsc(&fi);
3514                 fsc = 0x3f;
3515             }
3516             /*
3517              * Report exception with ESR indicating a fault due to a
3518              * translation table walk for a cache maintenance instruction.
3519              */
3520             syn = syn_data_abort_no_iss(current_el == target_el, 0,
3521                                         fi.ea, 1, fi.s1ptw, 1, fsc);
3522             env->exception.vaddress = value;
3523             env->exception.fsr = fsr;
3524             raise_exception(env, EXCP_DATA_ABORT, syn, target_el);
3525         }
3526     }
3527 
3528     if (is_a64(env)) {
3529         format64 = true;
3530     } else if (arm_feature(env, ARM_FEATURE_LPAE)) {
3531         /*
3532          * ATS1Cxx:
3533          * * TTBCR.EAE determines whether the result is returned using the
3534          *   32-bit or the 64-bit PAR format
3535          * * Instructions executed in Hyp mode always use the 64bit format
3536          *
3537          * ATS1S2NSOxx uses the 64bit format if any of the following is true:
3538          * * The Non-secure TTBCR.EAE bit is set to 1
3539          * * The implementation includes EL2, and the value of HCR.VM is 1
3540          *
3541          * (Note that HCR.DC makes HCR.VM behave as if it is 1.)
3542          *
3543          * ATS1Hx always uses the 64bit format.
3544          */
3545         format64 = arm_s1_regime_using_lpae_format(env, mmu_idx);
3546 
3547         if (arm_feature(env, ARM_FEATURE_EL2)) {
3548             if (mmu_idx == ARMMMUIdx_E10_0 ||
3549                 mmu_idx == ARMMMUIdx_E10_1 ||
3550                 mmu_idx == ARMMMUIdx_E10_1_PAN) {
3551                 format64 |= env->cp15.hcr_el2 & (HCR_VM | HCR_DC);
3552             } else {
3553                 format64 |= arm_current_el(env) == 2;
3554             }
3555         }
3556     }
3557 
3558     if (format64) {
3559         /* Create a 64-bit PAR */
3560         par64 = (1 << 11); /* LPAE bit always set */
3561         if (!ret) {
3562             par64 |= res.f.phys_addr & ~0xfffULL;
3563             if (!res.f.attrs.secure) {
3564                 par64 |= (1 << 9); /* NS */
3565             }
3566             par64 |= (uint64_t)res.cacheattrs.attrs << 56; /* ATTR */
3567             par64 |= par_el1_shareability(&res) << 7; /* SH */
3568         } else {
3569             uint32_t fsr = arm_fi_to_lfsc(&fi);
3570 
3571             par64 |= 1; /* F */
3572             par64 |= (fsr & 0x3f) << 1; /* FS */
3573             if (fi.stage2) {
3574                 par64 |= (1 << 9); /* S */
3575             }
3576             if (fi.s1ptw) {
3577                 par64 |= (1 << 8); /* PTW */
3578             }
3579         }
3580     } else {
3581         /*
3582          * fsr is a DFSR/IFSR value for the short descriptor
3583          * translation table format (with WnR always clear).
3584          * Convert it to a 32-bit PAR.
3585          */
3586         if (!ret) {
3587             /* We do not set any attribute bits in the PAR */
3588             if (res.f.lg_page_size == 24
3589                 && arm_feature(env, ARM_FEATURE_V7)) {
3590                 par64 = (res.f.phys_addr & 0xff000000) | (1 << 1);
3591             } else {
3592                 par64 = res.f.phys_addr & 0xfffff000;
3593             }
3594             if (!res.f.attrs.secure) {
3595                 par64 |= (1 << 9); /* NS */
3596             }
3597         } else {
3598             uint32_t fsr = arm_fi_to_sfsc(&fi);
3599 
3600             par64 = ((fsr & (1 << 10)) >> 5) | ((fsr & (1 << 12)) >> 6) |
3601                     ((fsr & 0xf) << 1) | 1;
3602         }
3603     }
3604     return par64;
3605 }
3606 #endif /* CONFIG_TCG */
3607 
3608 static void ats_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
3609 {
3610 #ifdef CONFIG_TCG
3611     MMUAccessType access_type = ri->opc2 & 1 ? MMU_DATA_STORE : MMU_DATA_LOAD;
3612     uint64_t par64;
3613     ARMMMUIdx mmu_idx;
3614     int el = arm_current_el(env);
3615     ARMSecuritySpace ss = arm_security_space(env);
3616 
3617     switch (ri->opc2 & 6) {
3618     case 0:
3619         /* stage 1 current state PL1: ATS1CPR, ATS1CPW, ATS1CPRP, ATS1CPWP */
3620         switch (el) {
3621         case 3:
3622             mmu_idx = ARMMMUIdx_E3;
3623             break;
3624         case 2:
3625             g_assert(ss != ARMSS_Secure);  /* ARMv8.4-SecEL2 is 64-bit only */
3626             /* fall through */
3627         case 1:
3628             if (ri->crm == 9 && arm_pan_enabled(env)) {
3629                 mmu_idx = ARMMMUIdx_Stage1_E1_PAN;
3630             } else {
3631                 mmu_idx = ARMMMUIdx_Stage1_E1;
3632             }
3633             break;
3634         default:
3635             g_assert_not_reached();
3636         }
3637         break;
3638     case 2:
3639         /* stage 1 current state PL0: ATS1CUR, ATS1CUW */
3640         switch (el) {
3641         case 3:
3642             mmu_idx = ARMMMUIdx_E10_0;
3643             break;
3644         case 2:
3645             g_assert(ss != ARMSS_Secure);  /* ARMv8.4-SecEL2 is 64-bit only */
3646             mmu_idx = ARMMMUIdx_Stage1_E0;
3647             break;
3648         case 1:
3649             mmu_idx = ARMMMUIdx_Stage1_E0;
3650             break;
3651         default:
3652             g_assert_not_reached();
3653         }
3654         break;
3655     case 4:
3656         /* stage 1+2 NonSecure PL1: ATS12NSOPR, ATS12NSOPW */
3657         mmu_idx = ARMMMUIdx_E10_1;
3658         ss = ARMSS_NonSecure;
3659         break;
3660     case 6:
3661         /* stage 1+2 NonSecure PL0: ATS12NSOUR, ATS12NSOUW */
3662         mmu_idx = ARMMMUIdx_E10_0;
3663         ss = ARMSS_NonSecure;
3664         break;
3665     default:
3666         g_assert_not_reached();
3667     }
3668 
3669     par64 = do_ats_write(env, value, access_type, mmu_idx, ss);
3670 
3671     A32_BANKED_CURRENT_REG_SET(env, par, par64);
3672 #else
3673     /* Handled by hardware accelerator. */
3674     g_assert_not_reached();
3675 #endif /* CONFIG_TCG */
3676 }
3677 
3678 static void ats1h_write(CPUARMState *env, const ARMCPRegInfo *ri,
3679                         uint64_t value)
3680 {
3681 #ifdef CONFIG_TCG
3682     MMUAccessType access_type = ri->opc2 & 1 ? MMU_DATA_STORE : MMU_DATA_LOAD;
3683     uint64_t par64;
3684 
3685     /* There is no SecureEL2 for AArch32. */
3686     par64 = do_ats_write(env, value, access_type, ARMMMUIdx_E2,
3687                          ARMSS_NonSecure);
3688 
3689     A32_BANKED_CURRENT_REG_SET(env, par, par64);
3690 #else
3691     /* Handled by hardware accelerator. */
3692     g_assert_not_reached();
3693 #endif /* CONFIG_TCG */
3694 }
3695 
3696 static CPAccessResult at_e012_access(CPUARMState *env, const ARMCPRegInfo *ri,
3697                                      bool isread)
3698 {
3699     /*
3700      * R_NYXTL: instruction is UNDEFINED if it applies to an Exception level
3701      * lower than EL3 and the combination SCR_EL3.{NSE,NS} is reserved. This can
3702      * only happen when executing at EL3 because that combination also causes an
3703      * illegal exception return. We don't need to check FEAT_RME either, because
3704      * scr_write() ensures that the NSE bit is not set otherwise.
3705      */
3706     if ((env->cp15.scr_el3 & (SCR_NSE | SCR_NS)) == SCR_NSE) {
3707         return CP_ACCESS_TRAP;
3708     }
3709     return CP_ACCESS_OK;
3710 }
3711 
3712 static CPAccessResult at_s1e2_access(CPUARMState *env, const ARMCPRegInfo *ri,
3713                                      bool isread)
3714 {
3715     if (arm_current_el(env) == 3 &&
3716         !(env->cp15.scr_el3 & (SCR_NS | SCR_EEL2))) {
3717         return CP_ACCESS_TRAP;
3718     }
3719     return at_e012_access(env, ri, isread);
3720 }
3721 
3722 static CPAccessResult at_s1e01_access(CPUARMState *env, const ARMCPRegInfo *ri,
3723                                       bool isread)
3724 {
3725     if (arm_current_el(env) == 1 && (arm_hcr_el2_eff(env) & HCR_AT)) {
3726         return CP_ACCESS_TRAP_EL2;
3727     }
3728     return at_e012_access(env, ri, isread);
3729 }
3730 
3731 static void ats_write64(CPUARMState *env, const ARMCPRegInfo *ri,
3732                         uint64_t value)
3733 {
3734 #ifdef CONFIG_TCG
3735     MMUAccessType access_type = ri->opc2 & 1 ? MMU_DATA_STORE : MMU_DATA_LOAD;
3736     ARMMMUIdx mmu_idx;
3737     uint64_t hcr_el2 = arm_hcr_el2_eff(env);
3738     bool regime_e20 = (hcr_el2 & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE);
3739 
3740     switch (ri->opc2 & 6) {
3741     case 0:
3742         switch (ri->opc1) {
3743         case 0: /* AT S1E1R, AT S1E1W, AT S1E1RP, AT S1E1WP */
3744             if (ri->crm == 9 && arm_pan_enabled(env)) {
3745                 mmu_idx = regime_e20 ?
3746                           ARMMMUIdx_E20_2_PAN : ARMMMUIdx_Stage1_E1_PAN;
3747             } else {
3748                 mmu_idx = regime_e20 ? ARMMMUIdx_E20_2 : ARMMMUIdx_Stage1_E1;
3749             }
3750             break;
3751         case 4: /* AT S1E2R, AT S1E2W */
3752             mmu_idx = hcr_el2 & HCR_E2H ? ARMMMUIdx_E20_2 : ARMMMUIdx_E2;
3753             break;
3754         case 6: /* AT S1E3R, AT S1E3W */
3755             mmu_idx = ARMMMUIdx_E3;
3756             break;
3757         default:
3758             g_assert_not_reached();
3759         }
3760         break;
3761     case 2: /* AT S1E0R, AT S1E0W */
3762         mmu_idx = regime_e20 ? ARMMMUIdx_E20_0 : ARMMMUIdx_Stage1_E0;
3763         break;
3764     case 4: /* AT S12E1R, AT S12E1W */
3765         mmu_idx = regime_e20 ? ARMMMUIdx_E20_2 : ARMMMUIdx_E10_1;
3766         break;
3767     case 6: /* AT S12E0R, AT S12E0W */
3768         mmu_idx = regime_e20 ? ARMMMUIdx_E20_0 : ARMMMUIdx_E10_0;
3769         break;
3770     default:
3771         g_assert_not_reached();
3772     }
3773 
3774     env->cp15.par_el[1] = do_ats_write(env, value, access_type,
3775                                        mmu_idx, arm_security_space(env));
3776 #else
3777     /* Handled by hardware accelerator. */
3778     g_assert_not_reached();
3779 #endif /* CONFIG_TCG */
3780 }
3781 #endif
3782 
3783 /* Return basic MPU access permission bits.  */
3784 static uint32_t simple_mpu_ap_bits(uint32_t val)
3785 {
3786     uint32_t ret;
3787     uint32_t mask;
3788     int i;
3789     ret = 0;
3790     mask = 3;
3791     for (i = 0; i < 16; i += 2) {
3792         ret |= (val >> i) & mask;
3793         mask <<= 2;
3794     }
3795     return ret;
3796 }
3797 
3798 /* Pad basic MPU access permission bits to extended format.  */
3799 static uint32_t extended_mpu_ap_bits(uint32_t val)
3800 {
3801     uint32_t ret;
3802     uint32_t mask;
3803     int i;
3804     ret = 0;
3805     mask = 3;
3806     for (i = 0; i < 16; i += 2) {
3807         ret |= (val & mask) << i;
3808         mask <<= 2;
3809     }
3810     return ret;
3811 }
3812 
3813 static void pmsav5_data_ap_write(CPUARMState *env, const ARMCPRegInfo *ri,
3814                                  uint64_t value)
3815 {
3816     env->cp15.pmsav5_data_ap = extended_mpu_ap_bits(value);
3817 }
3818 
3819 static uint64_t pmsav5_data_ap_read(CPUARMState *env, const ARMCPRegInfo *ri)
3820 {
3821     return simple_mpu_ap_bits(env->cp15.pmsav5_data_ap);
3822 }
3823 
3824 static void pmsav5_insn_ap_write(CPUARMState *env, const ARMCPRegInfo *ri,
3825                                  uint64_t value)
3826 {
3827     env->cp15.pmsav5_insn_ap = extended_mpu_ap_bits(value);
3828 }
3829 
3830 static uint64_t pmsav5_insn_ap_read(CPUARMState *env, const ARMCPRegInfo *ri)
3831 {
3832     return simple_mpu_ap_bits(env->cp15.pmsav5_insn_ap);
3833 }
3834 
3835 static uint64_t pmsav7_read(CPUARMState *env, const ARMCPRegInfo *ri)
3836 {
3837     uint32_t *u32p = *(uint32_t **)raw_ptr(env, ri);
3838 
3839     if (!u32p) {
3840         return 0;
3841     }
3842 
3843     u32p += env->pmsav7.rnr[M_REG_NS];
3844     return *u32p;
3845 }
3846 
3847 static void pmsav7_write(CPUARMState *env, const ARMCPRegInfo *ri,
3848                          uint64_t value)
3849 {
3850     ARMCPU *cpu = env_archcpu(env);
3851     uint32_t *u32p = *(uint32_t **)raw_ptr(env, ri);
3852 
3853     if (!u32p) {
3854         return;
3855     }
3856 
3857     u32p += env->pmsav7.rnr[M_REG_NS];
3858     tlb_flush(CPU(cpu)); /* Mappings may have changed - purge! */
3859     *u32p = value;
3860 }
3861 
3862 static void pmsav7_rgnr_write(CPUARMState *env, const ARMCPRegInfo *ri,
3863                               uint64_t value)
3864 {
3865     ARMCPU *cpu = env_archcpu(env);
3866     uint32_t nrgs = cpu->pmsav7_dregion;
3867 
3868     if (value >= nrgs) {
3869         qemu_log_mask(LOG_GUEST_ERROR,
3870                       "PMSAv7 RGNR write >= # supported regions, %" PRIu32
3871                       " > %" PRIu32 "\n", (uint32_t)value, nrgs);
3872         return;
3873     }
3874 
3875     raw_write(env, ri, value);
3876 }
3877 
3878 static void prbar_write(CPUARMState *env, const ARMCPRegInfo *ri,
3879                           uint64_t value)
3880 {
3881     ARMCPU *cpu = env_archcpu(env);
3882 
3883     tlb_flush(CPU(cpu)); /* Mappings may have changed - purge! */
3884     env->pmsav8.rbar[M_REG_NS][env->pmsav7.rnr[M_REG_NS]] = value;
3885 }
3886 
3887 static uint64_t prbar_read(CPUARMState *env, const ARMCPRegInfo *ri)
3888 {
3889     return env->pmsav8.rbar[M_REG_NS][env->pmsav7.rnr[M_REG_NS]];
3890 }
3891 
3892 static void prlar_write(CPUARMState *env, const ARMCPRegInfo *ri,
3893                           uint64_t value)
3894 {
3895     ARMCPU *cpu = env_archcpu(env);
3896 
3897     tlb_flush(CPU(cpu)); /* Mappings may have changed - purge! */
3898     env->pmsav8.rlar[M_REG_NS][env->pmsav7.rnr[M_REG_NS]] = value;
3899 }
3900 
3901 static uint64_t prlar_read(CPUARMState *env, const ARMCPRegInfo *ri)
3902 {
3903     return env->pmsav8.rlar[M_REG_NS][env->pmsav7.rnr[M_REG_NS]];
3904 }
3905 
3906 static void prselr_write(CPUARMState *env, const ARMCPRegInfo *ri,
3907                            uint64_t value)
3908 {
3909     ARMCPU *cpu = env_archcpu(env);
3910 
3911     /*
3912      * Ignore writes that would select not implemented region.
3913      * This is architecturally UNPREDICTABLE.
3914      */
3915     if (value >= cpu->pmsav7_dregion) {
3916         return;
3917     }
3918 
3919     env->pmsav7.rnr[M_REG_NS] = value;
3920 }
3921 
3922 static void hprbar_write(CPUARMState *env, const ARMCPRegInfo *ri,
3923                           uint64_t value)
3924 {
3925     ARMCPU *cpu = env_archcpu(env);
3926 
3927     tlb_flush(CPU(cpu)); /* Mappings may have changed - purge! */
3928     env->pmsav8.hprbar[env->pmsav8.hprselr] = value;
3929 }
3930 
3931 static uint64_t hprbar_read(CPUARMState *env, const ARMCPRegInfo *ri)
3932 {
3933     return env->pmsav8.hprbar[env->pmsav8.hprselr];
3934 }
3935 
3936 static void hprlar_write(CPUARMState *env, const ARMCPRegInfo *ri,
3937                           uint64_t value)
3938 {
3939     ARMCPU *cpu = env_archcpu(env);
3940 
3941     tlb_flush(CPU(cpu)); /* Mappings may have changed - purge! */
3942     env->pmsav8.hprlar[env->pmsav8.hprselr] = value;
3943 }
3944 
3945 static uint64_t hprlar_read(CPUARMState *env, const ARMCPRegInfo *ri)
3946 {
3947     return env->pmsav8.hprlar[env->pmsav8.hprselr];
3948 }
3949 
3950 static void hprenr_write(CPUARMState *env, const ARMCPRegInfo *ri,
3951                           uint64_t value)
3952 {
3953     uint32_t n;
3954     uint32_t bit;
3955     ARMCPU *cpu = env_archcpu(env);
3956 
3957     /* Ignore writes to unimplemented regions */
3958     int rmax = MIN(cpu->pmsav8r_hdregion, 32);
3959     value &= MAKE_64BIT_MASK(0, rmax);
3960 
3961     tlb_flush(CPU(cpu)); /* Mappings may have changed - purge! */
3962 
3963     /* Register alias is only valid for first 32 indexes */
3964     for (n = 0; n < rmax; ++n) {
3965         bit = extract32(value, n, 1);
3966         env->pmsav8.hprlar[n] = deposit32(
3967                     env->pmsav8.hprlar[n], 0, 1, bit);
3968     }
3969 }
3970 
3971 static uint64_t hprenr_read(CPUARMState *env, const ARMCPRegInfo *ri)
3972 {
3973     uint32_t n;
3974     uint32_t result = 0x0;
3975     ARMCPU *cpu = env_archcpu(env);
3976 
3977     /* Register alias is only valid for first 32 indexes */
3978     for (n = 0; n < MIN(cpu->pmsav8r_hdregion, 32); ++n) {
3979         if (env->pmsav8.hprlar[n] & 0x1) {
3980             result |= (0x1 << n);
3981         }
3982     }
3983     return result;
3984 }
3985 
3986 static void hprselr_write(CPUARMState *env, const ARMCPRegInfo *ri,
3987                            uint64_t value)
3988 {
3989     ARMCPU *cpu = env_archcpu(env);
3990 
3991     /*
3992      * Ignore writes that would select not implemented region.
3993      * This is architecturally UNPREDICTABLE.
3994      */
3995     if (value >= cpu->pmsav8r_hdregion) {
3996         return;
3997     }
3998 
3999     env->pmsav8.hprselr = value;
4000 }
4001 
4002 static void pmsav8r_regn_write(CPUARMState *env, const ARMCPRegInfo *ri,
4003                           uint64_t value)
4004 {
4005     ARMCPU *cpu = env_archcpu(env);
4006     uint8_t index = (extract32(ri->opc0, 0, 1) << 4) |
4007                     (extract32(ri->crm, 0, 3) << 1) | extract32(ri->opc2, 2, 1);
4008 
4009     tlb_flush(CPU(cpu)); /* Mappings may have changed - purge! */
4010 
4011     if (ri->opc1 & 4) {
4012         if (index >= cpu->pmsav8r_hdregion) {
4013             return;
4014         }
4015         if (ri->opc2 & 0x1) {
4016             env->pmsav8.hprlar[index] = value;
4017         } else {
4018             env->pmsav8.hprbar[index] = value;
4019         }
4020     } else {
4021         if (index >= cpu->pmsav7_dregion) {
4022             return;
4023         }
4024         if (ri->opc2 & 0x1) {
4025             env->pmsav8.rlar[M_REG_NS][index] = value;
4026         } else {
4027             env->pmsav8.rbar[M_REG_NS][index] = value;
4028         }
4029     }
4030 }
4031 
4032 static uint64_t pmsav8r_regn_read(CPUARMState *env, const ARMCPRegInfo *ri)
4033 {
4034     ARMCPU *cpu = env_archcpu(env);
4035     uint8_t index = (extract32(ri->opc0, 0, 1) << 4) |
4036                     (extract32(ri->crm, 0, 3) << 1) | extract32(ri->opc2, 2, 1);
4037 
4038     if (ri->opc1 & 4) {
4039         if (index >= cpu->pmsav8r_hdregion) {
4040             return 0x0;
4041         }
4042         if (ri->opc2 & 0x1) {
4043             return env->pmsav8.hprlar[index];
4044         } else {
4045             return env->pmsav8.hprbar[index];
4046         }
4047     } else {
4048         if (index >= cpu->pmsav7_dregion) {
4049             return 0x0;
4050         }
4051         if (ri->opc2 & 0x1) {
4052             return env->pmsav8.rlar[M_REG_NS][index];
4053         } else {
4054             return env->pmsav8.rbar[M_REG_NS][index];
4055         }
4056     }
4057 }
4058 
4059 static const ARMCPRegInfo pmsav8r_cp_reginfo[] = {
4060     { .name = "PRBAR",
4061       .cp = 15, .opc1 = 0, .crn = 6, .crm = 3, .opc2 = 0,
4062       .access = PL1_RW, .type = ARM_CP_NO_RAW,
4063       .accessfn = access_tvm_trvm,
4064       .readfn = prbar_read, .writefn = prbar_write },
4065     { .name = "PRLAR",
4066       .cp = 15, .opc1 = 0, .crn = 6, .crm = 3, .opc2 = 1,
4067       .access = PL1_RW, .type = ARM_CP_NO_RAW,
4068       .accessfn = access_tvm_trvm,
4069       .readfn = prlar_read, .writefn = prlar_write },
4070     { .name = "PRSELR", .resetvalue = 0,
4071       .cp = 15, .opc1 = 0, .crn = 6, .crm = 2, .opc2 = 1,
4072       .access = PL1_RW, .accessfn = access_tvm_trvm,
4073       .writefn = prselr_write,
4074       .fieldoffset = offsetof(CPUARMState, pmsav7.rnr[M_REG_NS]) },
4075     { .name = "HPRBAR", .resetvalue = 0,
4076       .cp = 15, .opc1 = 4, .crn = 6, .crm = 3, .opc2 = 0,
4077       .access = PL2_RW, .type = ARM_CP_NO_RAW,
4078       .readfn = hprbar_read, .writefn = hprbar_write },
4079     { .name = "HPRLAR",
4080       .cp = 15, .opc1 = 4, .crn = 6, .crm = 3, .opc2 = 1,
4081       .access = PL2_RW, .type = ARM_CP_NO_RAW,
4082       .readfn = hprlar_read, .writefn = hprlar_write },
4083     { .name = "HPRSELR", .resetvalue = 0,
4084       .cp = 15, .opc1 = 4, .crn = 6, .crm = 2, .opc2 = 1,
4085       .access = PL2_RW,
4086       .writefn = hprselr_write,
4087       .fieldoffset = offsetof(CPUARMState, pmsav8.hprselr) },
4088     { .name = "HPRENR",
4089       .cp = 15, .opc1 = 4, .crn = 6, .crm = 1, .opc2 = 1,
4090       .access = PL2_RW, .type = ARM_CP_NO_RAW,
4091       .readfn = hprenr_read, .writefn = hprenr_write },
4092 };
4093 
4094 static const ARMCPRegInfo pmsav7_cp_reginfo[] = {
4095     /*
4096      * Reset for all these registers is handled in arm_cpu_reset(),
4097      * because the PMSAv7 is also used by M-profile CPUs, which do
4098      * not register cpregs but still need the state to be reset.
4099      */
4100     { .name = "DRBAR", .cp = 15, .crn = 6, .opc1 = 0, .crm = 1, .opc2 = 0,
4101       .access = PL1_RW, .type = ARM_CP_NO_RAW,
4102       .fieldoffset = offsetof(CPUARMState, pmsav7.drbar),
4103       .readfn = pmsav7_read, .writefn = pmsav7_write,
4104       .resetfn = arm_cp_reset_ignore },
4105     { .name = "DRSR", .cp = 15, .crn = 6, .opc1 = 0, .crm = 1, .opc2 = 2,
4106       .access = PL1_RW, .type = ARM_CP_NO_RAW,
4107       .fieldoffset = offsetof(CPUARMState, pmsav7.drsr),
4108       .readfn = pmsav7_read, .writefn = pmsav7_write,
4109       .resetfn = arm_cp_reset_ignore },
4110     { .name = "DRACR", .cp = 15, .crn = 6, .opc1 = 0, .crm = 1, .opc2 = 4,
4111       .access = PL1_RW, .type = ARM_CP_NO_RAW,
4112       .fieldoffset = offsetof(CPUARMState, pmsav7.dracr),
4113       .readfn = pmsav7_read, .writefn = pmsav7_write,
4114       .resetfn = arm_cp_reset_ignore },
4115     { .name = "RGNR", .cp = 15, .crn = 6, .opc1 = 0, .crm = 2, .opc2 = 0,
4116       .access = PL1_RW,
4117       .fieldoffset = offsetof(CPUARMState, pmsav7.rnr[M_REG_NS]),
4118       .writefn = pmsav7_rgnr_write,
4119       .resetfn = arm_cp_reset_ignore },
4120 };
4121 
4122 static const ARMCPRegInfo pmsav5_cp_reginfo[] = {
4123     { .name = "DATA_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 0,
4124       .access = PL1_RW, .type = ARM_CP_ALIAS,
4125       .fieldoffset = offsetof(CPUARMState, cp15.pmsav5_data_ap),
4126       .readfn = pmsav5_data_ap_read, .writefn = pmsav5_data_ap_write, },
4127     { .name = "INSN_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 1,
4128       .access = PL1_RW, .type = ARM_CP_ALIAS,
4129       .fieldoffset = offsetof(CPUARMState, cp15.pmsav5_insn_ap),
4130       .readfn = pmsav5_insn_ap_read, .writefn = pmsav5_insn_ap_write, },
4131     { .name = "DATA_EXT_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 2,
4132       .access = PL1_RW,
4133       .fieldoffset = offsetof(CPUARMState, cp15.pmsav5_data_ap),
4134       .resetvalue = 0, },
4135     { .name = "INSN_EXT_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 3,
4136       .access = PL1_RW,
4137       .fieldoffset = offsetof(CPUARMState, cp15.pmsav5_insn_ap),
4138       .resetvalue = 0, },
4139     { .name = "DCACHE_CFG", .cp = 15, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 0,
4140       .access = PL1_RW,
4141       .fieldoffset = offsetof(CPUARMState, cp15.c2_data), .resetvalue = 0, },
4142     { .name = "ICACHE_CFG", .cp = 15, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 1,
4143       .access = PL1_RW,
4144       .fieldoffset = offsetof(CPUARMState, cp15.c2_insn), .resetvalue = 0, },
4145     /* Protection region base and size registers */
4146     { .name = "946_PRBS0", .cp = 15, .crn = 6, .crm = 0, .opc1 = 0,
4147       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
4148       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[0]) },
4149     { .name = "946_PRBS1", .cp = 15, .crn = 6, .crm = 1, .opc1 = 0,
4150       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
4151       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[1]) },
4152     { .name = "946_PRBS2", .cp = 15, .crn = 6, .crm = 2, .opc1 = 0,
4153       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
4154       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[2]) },
4155     { .name = "946_PRBS3", .cp = 15, .crn = 6, .crm = 3, .opc1 = 0,
4156       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
4157       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[3]) },
4158     { .name = "946_PRBS4", .cp = 15, .crn = 6, .crm = 4, .opc1 = 0,
4159       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
4160       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[4]) },
4161     { .name = "946_PRBS5", .cp = 15, .crn = 6, .crm = 5, .opc1 = 0,
4162       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
4163       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[5]) },
4164     { .name = "946_PRBS6", .cp = 15, .crn = 6, .crm = 6, .opc1 = 0,
4165       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
4166       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[6]) },
4167     { .name = "946_PRBS7", .cp = 15, .crn = 6, .crm = 7, .opc1 = 0,
4168       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
4169       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[7]) },
4170 };
4171 
4172 static void vmsa_ttbcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
4173                              uint64_t value)
4174 {
4175     ARMCPU *cpu = env_archcpu(env);
4176 
4177     if (!arm_feature(env, ARM_FEATURE_V8)) {
4178         if (arm_feature(env, ARM_FEATURE_LPAE) && (value & TTBCR_EAE)) {
4179             /*
4180              * Pre ARMv8 bits [21:19], [15:14] and [6:3] are UNK/SBZP when
4181              * using Long-descriptor translation table format
4182              */
4183             value &= ~((7 << 19) | (3 << 14) | (0xf << 3));
4184         } else if (arm_feature(env, ARM_FEATURE_EL3)) {
4185             /*
4186              * In an implementation that includes the Security Extensions
4187              * TTBCR has additional fields PD0 [4] and PD1 [5] for
4188              * Short-descriptor translation table format.
4189              */
4190             value &= TTBCR_PD1 | TTBCR_PD0 | TTBCR_N;
4191         } else {
4192             value &= TTBCR_N;
4193         }
4194     }
4195 
4196     if (arm_feature(env, ARM_FEATURE_LPAE)) {
4197         /*
4198          * With LPAE the TTBCR could result in a change of ASID
4199          * via the TTBCR.A1 bit, so do a TLB flush.
4200          */
4201         tlb_flush(CPU(cpu));
4202     }
4203     raw_write(env, ri, value);
4204 }
4205 
4206 static void vmsa_tcr_el12_write(CPUARMState *env, const ARMCPRegInfo *ri,
4207                                uint64_t value)
4208 {
4209     ARMCPU *cpu = env_archcpu(env);
4210 
4211     /* For AArch64 the A1 bit could result in a change of ASID, so TLB flush. */
4212     tlb_flush(CPU(cpu));
4213     raw_write(env, ri, value);
4214 }
4215 
4216 static void vmsa_ttbr_write(CPUARMState *env, const ARMCPRegInfo *ri,
4217                             uint64_t value)
4218 {
4219     /* If the ASID changes (with a 64-bit write), we must flush the TLB.  */
4220     if (cpreg_field_is_64bit(ri) &&
4221         extract64(raw_read(env, ri) ^ value, 48, 16) != 0) {
4222         ARMCPU *cpu = env_archcpu(env);
4223         tlb_flush(CPU(cpu));
4224     }
4225     raw_write(env, ri, value);
4226 }
4227 
4228 static void vmsa_tcr_ttbr_el2_write(CPUARMState *env, const ARMCPRegInfo *ri,
4229                                     uint64_t value)
4230 {
4231     /*
4232      * If we are running with E2&0 regime, then an ASID is active.
4233      * Flush if that might be changing.  Note we're not checking
4234      * TCR_EL2.A1 to know if this is really the TTBRx_EL2 that
4235      * holds the active ASID, only checking the field that might.
4236      */
4237     if (extract64(raw_read(env, ri) ^ value, 48, 16) &&
4238         (arm_hcr_el2_eff(env) & HCR_E2H)) {
4239         uint16_t mask = ARMMMUIdxBit_E20_2 |
4240                         ARMMMUIdxBit_E20_2_PAN |
4241                         ARMMMUIdxBit_E20_0;
4242         tlb_flush_by_mmuidx(env_cpu(env), mask);
4243     }
4244     raw_write(env, ri, value);
4245 }
4246 
4247 static void vttbr_write(CPUARMState *env, const ARMCPRegInfo *ri,
4248                         uint64_t value)
4249 {
4250     ARMCPU *cpu = env_archcpu(env);
4251     CPUState *cs = CPU(cpu);
4252 
4253     /*
4254      * A change in VMID to the stage2 page table (Stage2) invalidates
4255      * the stage2 and combined stage 1&2 tlbs (EL10_1 and EL10_0).
4256      */
4257     if (extract64(raw_read(env, ri) ^ value, 48, 16) != 0) {
4258         tlb_flush_by_mmuidx(cs, alle1_tlbmask(env));
4259     }
4260     raw_write(env, ri, value);
4261 }
4262 
4263 static const ARMCPRegInfo vmsa_pmsa_cp_reginfo[] = {
4264     { .name = "DFSR", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 0,
4265       .access = PL1_RW, .accessfn = access_tvm_trvm, .type = ARM_CP_ALIAS,
4266       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.dfsr_s),
4267                              offsetoflow32(CPUARMState, cp15.dfsr_ns) }, },
4268     { .name = "IFSR", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 1,
4269       .access = PL1_RW, .accessfn = access_tvm_trvm, .resetvalue = 0,
4270       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.ifsr_s),
4271                              offsetoflow32(CPUARMState, cp15.ifsr_ns) } },
4272     { .name = "DFAR", .cp = 15, .opc1 = 0, .crn = 6, .crm = 0, .opc2 = 0,
4273       .access = PL1_RW, .accessfn = access_tvm_trvm, .resetvalue = 0,
4274       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.dfar_s),
4275                              offsetof(CPUARMState, cp15.dfar_ns) } },
4276     { .name = "FAR_EL1", .state = ARM_CP_STATE_AA64,
4277       .opc0 = 3, .crn = 6, .crm = 0, .opc1 = 0, .opc2 = 0,
4278       .access = PL1_RW, .accessfn = access_tvm_trvm,
4279       .fgt = FGT_FAR_EL1,
4280       .nv2_redirect_offset = 0x220 | NV2_REDIR_NV1,
4281       .fieldoffset = offsetof(CPUARMState, cp15.far_el[1]),
4282       .resetvalue = 0, },
4283 };
4284 
4285 static const ARMCPRegInfo vmsa_cp_reginfo[] = {
4286     { .name = "ESR_EL1", .state = ARM_CP_STATE_AA64,
4287       .opc0 = 3, .crn = 5, .crm = 2, .opc1 = 0, .opc2 = 0,
4288       .access = PL1_RW, .accessfn = access_tvm_trvm,
4289       .fgt = FGT_ESR_EL1,
4290       .nv2_redirect_offset = 0x138 | NV2_REDIR_NV1,
4291       .fieldoffset = offsetof(CPUARMState, cp15.esr_el[1]), .resetvalue = 0, },
4292     { .name = "TTBR0_EL1", .state = ARM_CP_STATE_BOTH,
4293       .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 0, .opc2 = 0,
4294       .access = PL1_RW, .accessfn = access_tvm_trvm,
4295       .fgt = FGT_TTBR0_EL1,
4296       .nv2_redirect_offset = 0x200 | NV2_REDIR_NV1,
4297       .writefn = vmsa_ttbr_write, .resetvalue = 0, .raw_writefn = raw_write,
4298       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr0_s),
4299                              offsetof(CPUARMState, cp15.ttbr0_ns) } },
4300     { .name = "TTBR1_EL1", .state = ARM_CP_STATE_BOTH,
4301       .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 0, .opc2 = 1,
4302       .access = PL1_RW, .accessfn = access_tvm_trvm,
4303       .fgt = FGT_TTBR1_EL1,
4304       .nv2_redirect_offset = 0x210 | NV2_REDIR_NV1,
4305       .writefn = vmsa_ttbr_write, .resetvalue = 0, .raw_writefn = raw_write,
4306       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr1_s),
4307                              offsetof(CPUARMState, cp15.ttbr1_ns) } },
4308     { .name = "TCR_EL1", .state = ARM_CP_STATE_AA64,
4309       .opc0 = 3, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 2,
4310       .access = PL1_RW, .accessfn = access_tvm_trvm,
4311       .fgt = FGT_TCR_EL1,
4312       .nv2_redirect_offset = 0x120 | NV2_REDIR_NV1,
4313       .writefn = vmsa_tcr_el12_write,
4314       .raw_writefn = raw_write,
4315       .resetvalue = 0,
4316       .fieldoffset = offsetof(CPUARMState, cp15.tcr_el[1]) },
4317     { .name = "TTBCR", .cp = 15, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 2,
4318       .access = PL1_RW, .accessfn = access_tvm_trvm,
4319       .type = ARM_CP_ALIAS, .writefn = vmsa_ttbcr_write,
4320       .raw_writefn = raw_write,
4321       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.tcr_el[3]),
4322                              offsetoflow32(CPUARMState, cp15.tcr_el[1])} },
4323 };
4324 
4325 /*
4326  * Note that unlike TTBCR, writing to TTBCR2 does not require flushing
4327  * qemu tlbs nor adjusting cached masks.
4328  */
4329 static const ARMCPRegInfo ttbcr2_reginfo = {
4330     .name = "TTBCR2", .cp = 15, .opc1 = 0, .crn = 2, .crm = 0, .opc2 = 3,
4331     .access = PL1_RW, .accessfn = access_tvm_trvm,
4332     .type = ARM_CP_ALIAS,
4333     .bank_fieldoffsets = {
4334         offsetofhigh32(CPUARMState, cp15.tcr_el[3]),
4335         offsetofhigh32(CPUARMState, cp15.tcr_el[1]),
4336     },
4337 };
4338 
4339 static void omap_ticonfig_write(CPUARMState *env, const ARMCPRegInfo *ri,
4340                                 uint64_t value)
4341 {
4342     env->cp15.c15_ticonfig = value & 0xe7;
4343     /* The OS_TYPE bit in this register changes the reported CPUID! */
4344     env->cp15.c0_cpuid = (value & (1 << 5)) ?
4345         ARM_CPUID_TI915T : ARM_CPUID_TI925T;
4346 }
4347 
4348 static void omap_threadid_write(CPUARMState *env, const ARMCPRegInfo *ri,
4349                                 uint64_t value)
4350 {
4351     env->cp15.c15_threadid = value & 0xffff;
4352 }
4353 
4354 static void omap_wfi_write(CPUARMState *env, const ARMCPRegInfo *ri,
4355                            uint64_t value)
4356 {
4357     /* Wait-for-interrupt (deprecated) */
4358     cpu_interrupt(env_cpu(env), CPU_INTERRUPT_HALT);
4359 }
4360 
4361 static void omap_cachemaint_write(CPUARMState *env, const ARMCPRegInfo *ri,
4362                                   uint64_t value)
4363 {
4364     /*
4365      * On OMAP there are registers indicating the max/min index of dcache lines
4366      * containing a dirty line; cache flush operations have to reset these.
4367      */
4368     env->cp15.c15_i_max = 0x000;
4369     env->cp15.c15_i_min = 0xff0;
4370 }
4371 
4372 static const ARMCPRegInfo omap_cp_reginfo[] = {
4373     { .name = "DFSR", .cp = 15, .crn = 5, .crm = CP_ANY,
4374       .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_OVERRIDE,
4375       .fieldoffset = offsetoflow32(CPUARMState, cp15.esr_el[1]),
4376       .resetvalue = 0, },
4377     { .name = "", .cp = 15, .crn = 15, .crm = 0, .opc1 = 0, .opc2 = 0,
4378       .access = PL1_RW, .type = ARM_CP_NOP },
4379     { .name = "TICONFIG", .cp = 15, .crn = 15, .crm = 1, .opc1 = 0, .opc2 = 0,
4380       .access = PL1_RW,
4381       .fieldoffset = offsetof(CPUARMState, cp15.c15_ticonfig), .resetvalue = 0,
4382       .writefn = omap_ticonfig_write },
4383     { .name = "IMAX", .cp = 15, .crn = 15, .crm = 2, .opc1 = 0, .opc2 = 0,
4384       .access = PL1_RW,
4385       .fieldoffset = offsetof(CPUARMState, cp15.c15_i_max), .resetvalue = 0, },
4386     { .name = "IMIN", .cp = 15, .crn = 15, .crm = 3, .opc1 = 0, .opc2 = 0,
4387       .access = PL1_RW, .resetvalue = 0xff0,
4388       .fieldoffset = offsetof(CPUARMState, cp15.c15_i_min) },
4389     { .name = "THREADID", .cp = 15, .crn = 15, .crm = 4, .opc1 = 0, .opc2 = 0,
4390       .access = PL1_RW,
4391       .fieldoffset = offsetof(CPUARMState, cp15.c15_threadid), .resetvalue = 0,
4392       .writefn = omap_threadid_write },
4393     { .name = "TI925T_STATUS", .cp = 15, .crn = 15,
4394       .crm = 8, .opc1 = 0, .opc2 = 0, .access = PL1_RW,
4395       .type = ARM_CP_NO_RAW,
4396       .readfn = arm_cp_read_zero, .writefn = omap_wfi_write, },
4397     /*
4398      * TODO: Peripheral port remap register:
4399      * On OMAP2 mcr p15, 0, rn, c15, c2, 4 sets up the interrupt controller
4400      * base address at $rn & ~0xfff and map size of 0x200 << ($rn & 0xfff),
4401      * when MMU is off.
4402      */
4403     { .name = "OMAP_CACHEMAINT", .cp = 15, .crn = 7, .crm = CP_ANY,
4404       .opc1 = 0, .opc2 = CP_ANY, .access = PL1_W,
4405       .type = ARM_CP_OVERRIDE | ARM_CP_NO_RAW,
4406       .writefn = omap_cachemaint_write },
4407     { .name = "C9", .cp = 15, .crn = 9,
4408       .crm = CP_ANY, .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW,
4409       .type = ARM_CP_CONST | ARM_CP_OVERRIDE, .resetvalue = 0 },
4410 };
4411 
4412 static void xscale_cpar_write(CPUARMState *env, const ARMCPRegInfo *ri,
4413                               uint64_t value)
4414 {
4415     env->cp15.c15_cpar = value & 0x3fff;
4416 }
4417 
4418 static const ARMCPRegInfo xscale_cp_reginfo[] = {
4419     { .name = "XSCALE_CPAR",
4420       .cp = 15, .crn = 15, .crm = 1, .opc1 = 0, .opc2 = 0, .access = PL1_RW,
4421       .fieldoffset = offsetof(CPUARMState, cp15.c15_cpar), .resetvalue = 0,
4422       .writefn = xscale_cpar_write, },
4423     { .name = "XSCALE_AUXCR",
4424       .cp = 15, .crn = 1, .crm = 0, .opc1 = 0, .opc2 = 1, .access = PL1_RW,
4425       .fieldoffset = offsetof(CPUARMState, cp15.c1_xscaleauxcr),
4426       .resetvalue = 0, },
4427     /*
4428      * XScale specific cache-lockdown: since we have no cache we NOP these
4429      * and hope the guest does not really rely on cache behaviour.
4430      */
4431     { .name = "XSCALE_LOCK_ICACHE_LINE",
4432       .cp = 15, .opc1 = 0, .crn = 9, .crm = 1, .opc2 = 0,
4433       .access = PL1_W, .type = ARM_CP_NOP },
4434     { .name = "XSCALE_UNLOCK_ICACHE",
4435       .cp = 15, .opc1 = 0, .crn = 9, .crm = 1, .opc2 = 1,
4436       .access = PL1_W, .type = ARM_CP_NOP },
4437     { .name = "XSCALE_DCACHE_LOCK",
4438       .cp = 15, .opc1 = 0, .crn = 9, .crm = 2, .opc2 = 0,
4439       .access = PL1_RW, .type = ARM_CP_NOP },
4440     { .name = "XSCALE_UNLOCK_DCACHE",
4441       .cp = 15, .opc1 = 0, .crn = 9, .crm = 2, .opc2 = 1,
4442       .access = PL1_W, .type = ARM_CP_NOP },
4443 };
4444 
4445 static const ARMCPRegInfo dummy_c15_cp_reginfo[] = {
4446     /*
4447      * RAZ/WI the whole crn=15 space, when we don't have a more specific
4448      * implementation of this implementation-defined space.
4449      * Ideally this should eventually disappear in favour of actually
4450      * implementing the correct behaviour for all cores.
4451      */
4452     { .name = "C15_IMPDEF", .cp = 15, .crn = 15,
4453       .crm = CP_ANY, .opc1 = CP_ANY, .opc2 = CP_ANY,
4454       .access = PL1_RW,
4455       .type = ARM_CP_CONST | ARM_CP_NO_RAW | ARM_CP_OVERRIDE,
4456       .resetvalue = 0 },
4457 };
4458 
4459 static const ARMCPRegInfo cache_dirty_status_cp_reginfo[] = {
4460     /* Cache status: RAZ because we have no cache so it's always clean */
4461     { .name = "CDSR", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 6,
4462       .access = PL1_R, .type = ARM_CP_CONST | ARM_CP_NO_RAW,
4463       .resetvalue = 0 },
4464 };
4465 
4466 static const ARMCPRegInfo cache_block_ops_cp_reginfo[] = {
4467     /* We never have a block transfer operation in progress */
4468     { .name = "BXSR", .cp = 15, .crn = 7, .crm = 12, .opc1 = 0, .opc2 = 4,
4469       .access = PL0_R, .type = ARM_CP_CONST | ARM_CP_NO_RAW,
4470       .resetvalue = 0 },
4471     /* The cache ops themselves: these all NOP for QEMU */
4472     { .name = "IICR", .cp = 15, .crm = 5, .opc1 = 0,
4473       .access = PL1_W, .type = ARM_CP_NOP | ARM_CP_64BIT },
4474     { .name = "IDCR", .cp = 15, .crm = 6, .opc1 = 0,
4475       .access = PL1_W, .type = ARM_CP_NOP | ARM_CP_64BIT },
4476     { .name = "CDCR", .cp = 15, .crm = 12, .opc1 = 0,
4477       .access = PL0_W, .type = ARM_CP_NOP | ARM_CP_64BIT },
4478     { .name = "PIR", .cp = 15, .crm = 12, .opc1 = 1,
4479       .access = PL0_W, .type = ARM_CP_NOP | ARM_CP_64BIT },
4480     { .name = "PDR", .cp = 15, .crm = 12, .opc1 = 2,
4481       .access = PL0_W, .type = ARM_CP_NOP | ARM_CP_64BIT },
4482     { .name = "CIDCR", .cp = 15, .crm = 14, .opc1 = 0,
4483       .access = PL1_W, .type = ARM_CP_NOP | ARM_CP_64BIT },
4484 };
4485 
4486 static const ARMCPRegInfo cache_test_clean_cp_reginfo[] = {
4487     /*
4488      * The cache test-and-clean instructions always return (1 << 30)
4489      * to indicate that there are no dirty cache lines.
4490      */
4491     { .name = "TC_DCACHE", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 3,
4492       .access = PL0_R, .type = ARM_CP_CONST | ARM_CP_NO_RAW,
4493       .resetvalue = (1 << 30) },
4494     { .name = "TCI_DCACHE", .cp = 15, .crn = 7, .crm = 14, .opc1 = 0, .opc2 = 3,
4495       .access = PL0_R, .type = ARM_CP_CONST | ARM_CP_NO_RAW,
4496       .resetvalue = (1 << 30) },
4497 };
4498 
4499 static const ARMCPRegInfo strongarm_cp_reginfo[] = {
4500     /* Ignore ReadBuffer accesses */
4501     { .name = "C9_READBUFFER", .cp = 15, .crn = 9,
4502       .crm = CP_ANY, .opc1 = CP_ANY, .opc2 = CP_ANY,
4503       .access = PL1_RW, .resetvalue = 0,
4504       .type = ARM_CP_CONST | ARM_CP_OVERRIDE | ARM_CP_NO_RAW },
4505 };
4506 
4507 static uint64_t midr_read(CPUARMState *env, const ARMCPRegInfo *ri)
4508 {
4509     unsigned int cur_el = arm_current_el(env);
4510 
4511     if (arm_is_el2_enabled(env) && cur_el == 1) {
4512         return env->cp15.vpidr_el2;
4513     }
4514     return raw_read(env, ri);
4515 }
4516 
4517 static uint64_t mpidr_read_val(CPUARMState *env)
4518 {
4519     ARMCPU *cpu = env_archcpu(env);
4520     uint64_t mpidr = cpu->mp_affinity;
4521 
4522     if (arm_feature(env, ARM_FEATURE_V7MP)) {
4523         mpidr |= (1U << 31);
4524         /*
4525          * Cores which are uniprocessor (non-coherent)
4526          * but still implement the MP extensions set
4527          * bit 30. (For instance, Cortex-R5).
4528          */
4529         if (cpu->mp_is_up) {
4530             mpidr |= (1u << 30);
4531         }
4532     }
4533     return mpidr;
4534 }
4535 
4536 static uint64_t mpidr_read(CPUARMState *env, const ARMCPRegInfo *ri)
4537 {
4538     unsigned int cur_el = arm_current_el(env);
4539 
4540     if (arm_is_el2_enabled(env) && cur_el == 1) {
4541         return env->cp15.vmpidr_el2;
4542     }
4543     return mpidr_read_val(env);
4544 }
4545 
4546 static const ARMCPRegInfo lpae_cp_reginfo[] = {
4547     /* NOP AMAIR0/1 */
4548     { .name = "AMAIR0", .state = ARM_CP_STATE_BOTH,
4549       .opc0 = 3, .crn = 10, .crm = 3, .opc1 = 0, .opc2 = 0,
4550       .access = PL1_RW, .accessfn = access_tvm_trvm,
4551       .fgt = FGT_AMAIR_EL1,
4552       .nv2_redirect_offset = 0x148 | NV2_REDIR_NV1,
4553       .type = ARM_CP_CONST, .resetvalue = 0 },
4554     /* AMAIR1 is mapped to AMAIR_EL1[63:32] */
4555     { .name = "AMAIR1", .cp = 15, .crn = 10, .crm = 3, .opc1 = 0, .opc2 = 1,
4556       .access = PL1_RW, .accessfn = access_tvm_trvm,
4557       .type = ARM_CP_CONST, .resetvalue = 0 },
4558     { .name = "PAR", .cp = 15, .crm = 7, .opc1 = 0,
4559       .access = PL1_RW, .type = ARM_CP_64BIT, .resetvalue = 0,
4560       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.par_s),
4561                              offsetof(CPUARMState, cp15.par_ns)} },
4562     { .name = "TTBR0", .cp = 15, .crm = 2, .opc1 = 0,
4563       .access = PL1_RW, .accessfn = access_tvm_trvm,
4564       .type = ARM_CP_64BIT | ARM_CP_ALIAS,
4565       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr0_s),
4566                              offsetof(CPUARMState, cp15.ttbr0_ns) },
4567       .writefn = vmsa_ttbr_write, .raw_writefn = raw_write },
4568     { .name = "TTBR1", .cp = 15, .crm = 2, .opc1 = 1,
4569       .access = PL1_RW, .accessfn = access_tvm_trvm,
4570       .type = ARM_CP_64BIT | ARM_CP_ALIAS,
4571       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr1_s),
4572                              offsetof(CPUARMState, cp15.ttbr1_ns) },
4573       .writefn = vmsa_ttbr_write, .raw_writefn = raw_write },
4574 };
4575 
4576 static uint64_t aa64_fpcr_read(CPUARMState *env, const ARMCPRegInfo *ri)
4577 {
4578     return vfp_get_fpcr(env);
4579 }
4580 
4581 static void aa64_fpcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
4582                             uint64_t value)
4583 {
4584     vfp_set_fpcr(env, value);
4585 }
4586 
4587 static uint64_t aa64_fpsr_read(CPUARMState *env, const ARMCPRegInfo *ri)
4588 {
4589     return vfp_get_fpsr(env);
4590 }
4591 
4592 static void aa64_fpsr_write(CPUARMState *env, const ARMCPRegInfo *ri,
4593                             uint64_t value)
4594 {
4595     vfp_set_fpsr(env, value);
4596 }
4597 
4598 static CPAccessResult aa64_daif_access(CPUARMState *env, const ARMCPRegInfo *ri,
4599                                        bool isread)
4600 {
4601     if (arm_current_el(env) == 0 && !(arm_sctlr(env, 0) & SCTLR_UMA)) {
4602         return CP_ACCESS_TRAP;
4603     }
4604     return CP_ACCESS_OK;
4605 }
4606 
4607 static void aa64_daif_write(CPUARMState *env, const ARMCPRegInfo *ri,
4608                             uint64_t value)
4609 {
4610     env->daif = value & PSTATE_DAIF;
4611 }
4612 
4613 static uint64_t aa64_pan_read(CPUARMState *env, const ARMCPRegInfo *ri)
4614 {
4615     return env->pstate & PSTATE_PAN;
4616 }
4617 
4618 static void aa64_pan_write(CPUARMState *env, const ARMCPRegInfo *ri,
4619                            uint64_t value)
4620 {
4621     env->pstate = (env->pstate & ~PSTATE_PAN) | (value & PSTATE_PAN);
4622 }
4623 
4624 static const ARMCPRegInfo pan_reginfo = {
4625     .name = "PAN", .state = ARM_CP_STATE_AA64,
4626     .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 2, .opc2 = 3,
4627     .type = ARM_CP_NO_RAW, .access = PL1_RW,
4628     .readfn = aa64_pan_read, .writefn = aa64_pan_write
4629 };
4630 
4631 static uint64_t aa64_uao_read(CPUARMState *env, const ARMCPRegInfo *ri)
4632 {
4633     return env->pstate & PSTATE_UAO;
4634 }
4635 
4636 static void aa64_uao_write(CPUARMState *env, const ARMCPRegInfo *ri,
4637                            uint64_t value)
4638 {
4639     env->pstate = (env->pstate & ~PSTATE_UAO) | (value & PSTATE_UAO);
4640 }
4641 
4642 static const ARMCPRegInfo uao_reginfo = {
4643     .name = "UAO", .state = ARM_CP_STATE_AA64,
4644     .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 2, .opc2 = 4,
4645     .type = ARM_CP_NO_RAW, .access = PL1_RW,
4646     .readfn = aa64_uao_read, .writefn = aa64_uao_write
4647 };
4648 
4649 static uint64_t aa64_dit_read(CPUARMState *env, const ARMCPRegInfo *ri)
4650 {
4651     return env->pstate & PSTATE_DIT;
4652 }
4653 
4654 static void aa64_dit_write(CPUARMState *env, const ARMCPRegInfo *ri,
4655                            uint64_t value)
4656 {
4657     env->pstate = (env->pstate & ~PSTATE_DIT) | (value & PSTATE_DIT);
4658 }
4659 
4660 static const ARMCPRegInfo dit_reginfo = {
4661     .name = "DIT", .state = ARM_CP_STATE_AA64,
4662     .opc0 = 3, .opc1 = 3, .crn = 4, .crm = 2, .opc2 = 5,
4663     .type = ARM_CP_NO_RAW, .access = PL0_RW,
4664     .readfn = aa64_dit_read, .writefn = aa64_dit_write
4665 };
4666 
4667 static uint64_t aa64_ssbs_read(CPUARMState *env, const ARMCPRegInfo *ri)
4668 {
4669     return env->pstate & PSTATE_SSBS;
4670 }
4671 
4672 static void aa64_ssbs_write(CPUARMState *env, const ARMCPRegInfo *ri,
4673                            uint64_t value)
4674 {
4675     env->pstate = (env->pstate & ~PSTATE_SSBS) | (value & PSTATE_SSBS);
4676 }
4677 
4678 static const ARMCPRegInfo ssbs_reginfo = {
4679     .name = "SSBS", .state = ARM_CP_STATE_AA64,
4680     .opc0 = 3, .opc1 = 3, .crn = 4, .crm = 2, .opc2 = 6,
4681     .type = ARM_CP_NO_RAW, .access = PL0_RW,
4682     .readfn = aa64_ssbs_read, .writefn = aa64_ssbs_write
4683 };
4684 
4685 static CPAccessResult aa64_cacheop_poc_access(CPUARMState *env,
4686                                               const ARMCPRegInfo *ri,
4687                                               bool isread)
4688 {
4689     /* Cache invalidate/clean to Point of Coherency or Persistence...  */
4690     switch (arm_current_el(env)) {
4691     case 0:
4692         /* ... EL0 must UNDEF unless SCTLR_EL1.UCI is set.  */
4693         if (!(arm_sctlr(env, 0) & SCTLR_UCI)) {
4694             return CP_ACCESS_TRAP;
4695         }
4696         /* fall through */
4697     case 1:
4698         /* ... EL1 must trap to EL2 if HCR_EL2.TPCP is set.  */
4699         if (arm_hcr_el2_eff(env) & HCR_TPCP) {
4700             return CP_ACCESS_TRAP_EL2;
4701         }
4702         break;
4703     }
4704     return CP_ACCESS_OK;
4705 }
4706 
4707 static CPAccessResult do_cacheop_pou_access(CPUARMState *env, uint64_t hcrflags)
4708 {
4709     /* Cache invalidate/clean to Point of Unification... */
4710     switch (arm_current_el(env)) {
4711     case 0:
4712         /* ... EL0 must UNDEF unless SCTLR_EL1.UCI is set.  */
4713         if (!(arm_sctlr(env, 0) & SCTLR_UCI)) {
4714             return CP_ACCESS_TRAP;
4715         }
4716         /* fall through */
4717     case 1:
4718         /* ... EL1 must trap to EL2 if relevant HCR_EL2 flags are set.  */
4719         if (arm_hcr_el2_eff(env) & hcrflags) {
4720             return CP_ACCESS_TRAP_EL2;
4721         }
4722         break;
4723     }
4724     return CP_ACCESS_OK;
4725 }
4726 
4727 static CPAccessResult access_ticab(CPUARMState *env, const ARMCPRegInfo *ri,
4728                                    bool isread)
4729 {
4730     return do_cacheop_pou_access(env, HCR_TICAB | HCR_TPU);
4731 }
4732 
4733 static CPAccessResult access_tocu(CPUARMState *env, const ARMCPRegInfo *ri,
4734                                   bool isread)
4735 {
4736     return do_cacheop_pou_access(env, HCR_TOCU | HCR_TPU);
4737 }
4738 
4739 /*
4740  * See: D4.7.2 TLB maintenance requirements and the TLB maintenance instructions
4741  * Page D4-1736 (DDI0487A.b)
4742  */
4743 
4744 static int vae1_tlbmask(CPUARMState *env)
4745 {
4746     uint64_t hcr = arm_hcr_el2_eff(env);
4747     uint16_t mask;
4748 
4749     if ((hcr & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)) {
4750         mask = ARMMMUIdxBit_E20_2 |
4751                ARMMMUIdxBit_E20_2_PAN |
4752                ARMMMUIdxBit_E20_0;
4753     } else {
4754         mask = ARMMMUIdxBit_E10_1 |
4755                ARMMMUIdxBit_E10_1_PAN |
4756                ARMMMUIdxBit_E10_0;
4757     }
4758     return mask;
4759 }
4760 
4761 static int vae2_tlbmask(CPUARMState *env)
4762 {
4763     uint64_t hcr = arm_hcr_el2_eff(env);
4764     uint16_t mask;
4765 
4766     if (hcr & HCR_E2H) {
4767         mask = ARMMMUIdxBit_E20_2 |
4768                ARMMMUIdxBit_E20_2_PAN |
4769                ARMMMUIdxBit_E20_0;
4770     } else {
4771         mask = ARMMMUIdxBit_E2;
4772     }
4773     return mask;
4774 }
4775 
4776 /* Return 56 if TBI is enabled, 64 otherwise. */
4777 static int tlbbits_for_regime(CPUARMState *env, ARMMMUIdx mmu_idx,
4778                               uint64_t addr)
4779 {
4780     uint64_t tcr = regime_tcr(env, mmu_idx);
4781     int tbi = aa64_va_parameter_tbi(tcr, mmu_idx);
4782     int select = extract64(addr, 55, 1);
4783 
4784     return (tbi >> select) & 1 ? 56 : 64;
4785 }
4786 
4787 static int vae1_tlbbits(CPUARMState *env, uint64_t addr)
4788 {
4789     uint64_t hcr = arm_hcr_el2_eff(env);
4790     ARMMMUIdx mmu_idx;
4791 
4792     /* Only the regime of the mmu_idx below is significant. */
4793     if ((hcr & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)) {
4794         mmu_idx = ARMMMUIdx_E20_0;
4795     } else {
4796         mmu_idx = ARMMMUIdx_E10_0;
4797     }
4798 
4799     return tlbbits_for_regime(env, mmu_idx, addr);
4800 }
4801 
4802 static int vae2_tlbbits(CPUARMState *env, uint64_t addr)
4803 {
4804     uint64_t hcr = arm_hcr_el2_eff(env);
4805     ARMMMUIdx mmu_idx;
4806 
4807     /*
4808      * Only the regime of the mmu_idx below is significant.
4809      * Regime EL2&0 has two ranges with separate TBI configuration, while EL2
4810      * only has one.
4811      */
4812     if (hcr & HCR_E2H) {
4813         mmu_idx = ARMMMUIdx_E20_2;
4814     } else {
4815         mmu_idx = ARMMMUIdx_E2;
4816     }
4817 
4818     return tlbbits_for_regime(env, mmu_idx, addr);
4819 }
4820 
4821 static void tlbi_aa64_vmalle1is_write(CPUARMState *env, const ARMCPRegInfo *ri,
4822                                       uint64_t value)
4823 {
4824     CPUState *cs = env_cpu(env);
4825     int mask = vae1_tlbmask(env);
4826 
4827     tlb_flush_by_mmuidx_all_cpus_synced(cs, mask);
4828 }
4829 
4830 static void tlbi_aa64_vmalle1_write(CPUARMState *env, const ARMCPRegInfo *ri,
4831                                     uint64_t value)
4832 {
4833     CPUState *cs = env_cpu(env);
4834     int mask = vae1_tlbmask(env);
4835 
4836     if (tlb_force_broadcast(env)) {
4837         tlb_flush_by_mmuidx_all_cpus_synced(cs, mask);
4838     } else {
4839         tlb_flush_by_mmuidx(cs, mask);
4840     }
4841 }
4842 
4843 static int e2_tlbmask(CPUARMState *env)
4844 {
4845     return (ARMMMUIdxBit_E20_0 |
4846             ARMMMUIdxBit_E20_2 |
4847             ARMMMUIdxBit_E20_2_PAN |
4848             ARMMMUIdxBit_E2);
4849 }
4850 
4851 static void tlbi_aa64_alle1_write(CPUARMState *env, const ARMCPRegInfo *ri,
4852                                   uint64_t value)
4853 {
4854     CPUState *cs = env_cpu(env);
4855     int mask = alle1_tlbmask(env);
4856 
4857     tlb_flush_by_mmuidx(cs, mask);
4858 }
4859 
4860 static void tlbi_aa64_alle2_write(CPUARMState *env, const ARMCPRegInfo *ri,
4861                                   uint64_t value)
4862 {
4863     CPUState *cs = env_cpu(env);
4864     int mask = e2_tlbmask(env);
4865 
4866     tlb_flush_by_mmuidx(cs, mask);
4867 }
4868 
4869 static void tlbi_aa64_alle3_write(CPUARMState *env, const ARMCPRegInfo *ri,
4870                                   uint64_t value)
4871 {
4872     ARMCPU *cpu = env_archcpu(env);
4873     CPUState *cs = CPU(cpu);
4874 
4875     tlb_flush_by_mmuidx(cs, ARMMMUIdxBit_E3);
4876 }
4877 
4878 static void tlbi_aa64_alle1is_write(CPUARMState *env, const ARMCPRegInfo *ri,
4879                                     uint64_t value)
4880 {
4881     CPUState *cs = env_cpu(env);
4882     int mask = alle1_tlbmask(env);
4883 
4884     tlb_flush_by_mmuidx_all_cpus_synced(cs, mask);
4885 }
4886 
4887 static void tlbi_aa64_alle2is_write(CPUARMState *env, const ARMCPRegInfo *ri,
4888                                     uint64_t value)
4889 {
4890     CPUState *cs = env_cpu(env);
4891     int mask = e2_tlbmask(env);
4892 
4893     tlb_flush_by_mmuidx_all_cpus_synced(cs, mask);
4894 }
4895 
4896 static void tlbi_aa64_alle3is_write(CPUARMState *env, const ARMCPRegInfo *ri,
4897                                     uint64_t value)
4898 {
4899     CPUState *cs = env_cpu(env);
4900 
4901     tlb_flush_by_mmuidx_all_cpus_synced(cs, ARMMMUIdxBit_E3);
4902 }
4903 
4904 static void tlbi_aa64_vae2_write(CPUARMState *env, const ARMCPRegInfo *ri,
4905                                  uint64_t value)
4906 {
4907     /*
4908      * Invalidate by VA, EL2
4909      * Currently handles both VAE2 and VALE2, since we don't support
4910      * flush-last-level-only.
4911      */
4912     CPUState *cs = env_cpu(env);
4913     int mask = vae2_tlbmask(env);
4914     uint64_t pageaddr = sextract64(value << 12, 0, 56);
4915     int bits = vae2_tlbbits(env, pageaddr);
4916 
4917     tlb_flush_page_bits_by_mmuidx(cs, pageaddr, mask, bits);
4918 }
4919 
4920 static void tlbi_aa64_vae3_write(CPUARMState *env, const ARMCPRegInfo *ri,
4921                                  uint64_t value)
4922 {
4923     /*
4924      * Invalidate by VA, EL3
4925      * Currently handles both VAE3 and VALE3, since we don't support
4926      * flush-last-level-only.
4927      */
4928     ARMCPU *cpu = env_archcpu(env);
4929     CPUState *cs = CPU(cpu);
4930     uint64_t pageaddr = sextract64(value << 12, 0, 56);
4931 
4932     tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdxBit_E3);
4933 }
4934 
4935 static void tlbi_aa64_vae1is_write(CPUARMState *env, const ARMCPRegInfo *ri,
4936                                    uint64_t value)
4937 {
4938     CPUState *cs = env_cpu(env);
4939     int mask = vae1_tlbmask(env);
4940     uint64_t pageaddr = sextract64(value << 12, 0, 56);
4941     int bits = vae1_tlbbits(env, pageaddr);
4942 
4943     tlb_flush_page_bits_by_mmuidx_all_cpus_synced(cs, pageaddr, mask, bits);
4944 }
4945 
4946 static void tlbi_aa64_vae1_write(CPUARMState *env, const ARMCPRegInfo *ri,
4947                                  uint64_t value)
4948 {
4949     /*
4950      * Invalidate by VA, EL1&0 (AArch64 version).
4951      * Currently handles all of VAE1, VAAE1, VAALE1 and VALE1,
4952      * since we don't support flush-for-specific-ASID-only or
4953      * flush-last-level-only.
4954      */
4955     CPUState *cs = env_cpu(env);
4956     int mask = vae1_tlbmask(env);
4957     uint64_t pageaddr = sextract64(value << 12, 0, 56);
4958     int bits = vae1_tlbbits(env, pageaddr);
4959 
4960     if (tlb_force_broadcast(env)) {
4961         tlb_flush_page_bits_by_mmuidx_all_cpus_synced(cs, pageaddr, mask, bits);
4962     } else {
4963         tlb_flush_page_bits_by_mmuidx(cs, pageaddr, mask, bits);
4964     }
4965 }
4966 
4967 static void tlbi_aa64_vae2is_write(CPUARMState *env, const ARMCPRegInfo *ri,
4968                                    uint64_t value)
4969 {
4970     CPUState *cs = env_cpu(env);
4971     int mask = vae2_tlbmask(env);
4972     uint64_t pageaddr = sextract64(value << 12, 0, 56);
4973     int bits = vae2_tlbbits(env, pageaddr);
4974 
4975     tlb_flush_page_bits_by_mmuidx_all_cpus_synced(cs, pageaddr, mask, bits);
4976 }
4977 
4978 static void tlbi_aa64_vae3is_write(CPUARMState *env, const ARMCPRegInfo *ri,
4979                                    uint64_t value)
4980 {
4981     CPUState *cs = env_cpu(env);
4982     uint64_t pageaddr = sextract64(value << 12, 0, 56);
4983     int bits = tlbbits_for_regime(env, ARMMMUIdx_E3, pageaddr);
4984 
4985     tlb_flush_page_bits_by_mmuidx_all_cpus_synced(cs, pageaddr,
4986                                                   ARMMMUIdxBit_E3, bits);
4987 }
4988 
4989 static int ipas2e1_tlbmask(CPUARMState *env, int64_t value)
4990 {
4991     /*
4992      * The MSB of value is the NS field, which only applies if SEL2
4993      * is implemented and SCR_EL3.NS is not set (i.e. in secure mode).
4994      */
4995     return (value >= 0
4996             && cpu_isar_feature(aa64_sel2, env_archcpu(env))
4997             && arm_is_secure_below_el3(env)
4998             ? ARMMMUIdxBit_Stage2_S
4999             : ARMMMUIdxBit_Stage2);
5000 }
5001 
5002 static void tlbi_aa64_ipas2e1_write(CPUARMState *env, const ARMCPRegInfo *ri,
5003                                     uint64_t value)
5004 {
5005     CPUState *cs = env_cpu(env);
5006     int mask = ipas2e1_tlbmask(env, value);
5007     uint64_t pageaddr = sextract64(value << 12, 0, 56);
5008 
5009     if (tlb_force_broadcast(env)) {
5010         tlb_flush_page_by_mmuidx_all_cpus_synced(cs, pageaddr, mask);
5011     } else {
5012         tlb_flush_page_by_mmuidx(cs, pageaddr, mask);
5013     }
5014 }
5015 
5016 static void tlbi_aa64_ipas2e1is_write(CPUARMState *env, const ARMCPRegInfo *ri,
5017                                       uint64_t value)
5018 {
5019     CPUState *cs = env_cpu(env);
5020     int mask = ipas2e1_tlbmask(env, value);
5021     uint64_t pageaddr = sextract64(value << 12, 0, 56);
5022 
5023     tlb_flush_page_by_mmuidx_all_cpus_synced(cs, pageaddr, mask);
5024 }
5025 
5026 #ifdef TARGET_AARCH64
5027 typedef struct {
5028     uint64_t base;
5029     uint64_t length;
5030 } TLBIRange;
5031 
5032 static ARMGranuleSize tlbi_range_tg_to_gran_size(int tg)
5033 {
5034     /*
5035      * Note that the TLBI range TG field encoding differs from both
5036      * TG0 and TG1 encodings.
5037      */
5038     switch (tg) {
5039     case 1:
5040         return Gran4K;
5041     case 2:
5042         return Gran16K;
5043     case 3:
5044         return Gran64K;
5045     default:
5046         return GranInvalid;
5047     }
5048 }
5049 
5050 static TLBIRange tlbi_aa64_get_range(CPUARMState *env, ARMMMUIdx mmuidx,
5051                                      uint64_t value)
5052 {
5053     unsigned int page_size_granule, page_shift, num, scale, exponent;
5054     /* Extract one bit to represent the va selector in use. */
5055     uint64_t select = sextract64(value, 36, 1);
5056     ARMVAParameters param = aa64_va_parameters(env, select, mmuidx, true, false);
5057     TLBIRange ret = { };
5058     ARMGranuleSize gran;
5059 
5060     page_size_granule = extract64(value, 46, 2);
5061     gran = tlbi_range_tg_to_gran_size(page_size_granule);
5062 
5063     /* The granule encoded in value must match the granule in use. */
5064     if (gran != param.gran) {
5065         qemu_log_mask(LOG_GUEST_ERROR, "Invalid tlbi page size granule %d\n",
5066                       page_size_granule);
5067         return ret;
5068     }
5069 
5070     page_shift = arm_granule_bits(gran);
5071     num = extract64(value, 39, 5);
5072     scale = extract64(value, 44, 2);
5073     exponent = (5 * scale) + 1;
5074 
5075     ret.length = (num + 1) << (exponent + page_shift);
5076 
5077     if (param.select) {
5078         ret.base = sextract64(value, 0, 37);
5079     } else {
5080         ret.base = extract64(value, 0, 37);
5081     }
5082     if (param.ds) {
5083         /*
5084          * With DS=1, BaseADDR is always shifted 16 so that it is able
5085          * to address all 52 va bits.  The input address is perforce
5086          * aligned on a 64k boundary regardless of translation granule.
5087          */
5088         page_shift = 16;
5089     }
5090     ret.base <<= page_shift;
5091 
5092     return ret;
5093 }
5094 
5095 static void do_rvae_write(CPUARMState *env, uint64_t value,
5096                           int idxmap, bool synced)
5097 {
5098     ARMMMUIdx one_idx = ARM_MMU_IDX_A | ctz32(idxmap);
5099     TLBIRange range;
5100     int bits;
5101 
5102     range = tlbi_aa64_get_range(env, one_idx, value);
5103     bits = tlbbits_for_regime(env, one_idx, range.base);
5104 
5105     if (synced) {
5106         tlb_flush_range_by_mmuidx_all_cpus_synced(env_cpu(env),
5107                                                   range.base,
5108                                                   range.length,
5109                                                   idxmap,
5110                                                   bits);
5111     } else {
5112         tlb_flush_range_by_mmuidx(env_cpu(env), range.base,
5113                                   range.length, idxmap, bits);
5114     }
5115 }
5116 
5117 static void tlbi_aa64_rvae1_write(CPUARMState *env,
5118                                   const ARMCPRegInfo *ri,
5119                                   uint64_t value)
5120 {
5121     /*
5122      * Invalidate by VA range, EL1&0.
5123      * Currently handles all of RVAE1, RVAAE1, RVAALE1 and RVALE1,
5124      * since we don't support flush-for-specific-ASID-only or
5125      * flush-last-level-only.
5126      */
5127 
5128     do_rvae_write(env, value, vae1_tlbmask(env),
5129                   tlb_force_broadcast(env));
5130 }
5131 
5132 static void tlbi_aa64_rvae1is_write(CPUARMState *env,
5133                                     const ARMCPRegInfo *ri,
5134                                     uint64_t value)
5135 {
5136     /*
5137      * Invalidate by VA range, Inner/Outer Shareable EL1&0.
5138      * Currently handles all of RVAE1IS, RVAE1OS, RVAAE1IS, RVAAE1OS,
5139      * RVAALE1IS, RVAALE1OS, RVALE1IS and RVALE1OS, since we don't support
5140      * flush-for-specific-ASID-only, flush-last-level-only or inner/outer
5141      * shareable specific flushes.
5142      */
5143 
5144     do_rvae_write(env, value, vae1_tlbmask(env), true);
5145 }
5146 
5147 static void tlbi_aa64_rvae2_write(CPUARMState *env,
5148                                   const ARMCPRegInfo *ri,
5149                                   uint64_t value)
5150 {
5151     /*
5152      * Invalidate by VA range, EL2.
5153      * Currently handles all of RVAE2 and RVALE2,
5154      * since we don't support flush-for-specific-ASID-only or
5155      * flush-last-level-only.
5156      */
5157 
5158     do_rvae_write(env, value, vae2_tlbmask(env),
5159                   tlb_force_broadcast(env));
5160 
5161 
5162 }
5163 
5164 static void tlbi_aa64_rvae2is_write(CPUARMState *env,
5165                                     const ARMCPRegInfo *ri,
5166                                     uint64_t value)
5167 {
5168     /*
5169      * Invalidate by VA range, Inner/Outer Shareable, EL2.
5170      * Currently handles all of RVAE2IS, RVAE2OS, RVALE2IS and RVALE2OS,
5171      * since we don't support flush-for-specific-ASID-only,
5172      * flush-last-level-only or inner/outer shareable specific flushes.
5173      */
5174 
5175     do_rvae_write(env, value, vae2_tlbmask(env), true);
5176 
5177 }
5178 
5179 static void tlbi_aa64_rvae3_write(CPUARMState *env,
5180                                   const ARMCPRegInfo *ri,
5181                                   uint64_t value)
5182 {
5183     /*
5184      * Invalidate by VA range, EL3.
5185      * Currently handles all of RVAE3 and RVALE3,
5186      * since we don't support flush-for-specific-ASID-only or
5187      * flush-last-level-only.
5188      */
5189 
5190     do_rvae_write(env, value, ARMMMUIdxBit_E3, tlb_force_broadcast(env));
5191 }
5192 
5193 static void tlbi_aa64_rvae3is_write(CPUARMState *env,
5194                                     const ARMCPRegInfo *ri,
5195                                     uint64_t value)
5196 {
5197     /*
5198      * Invalidate by VA range, EL3, Inner/Outer Shareable.
5199      * Currently handles all of RVAE3IS, RVAE3OS, RVALE3IS and RVALE3OS,
5200      * since we don't support flush-for-specific-ASID-only,
5201      * flush-last-level-only or inner/outer specific flushes.
5202      */
5203 
5204     do_rvae_write(env, value, ARMMMUIdxBit_E3, true);
5205 }
5206 
5207 static void tlbi_aa64_ripas2e1_write(CPUARMState *env, const ARMCPRegInfo *ri,
5208                                      uint64_t value)
5209 {
5210     do_rvae_write(env, value, ipas2e1_tlbmask(env, value),
5211                   tlb_force_broadcast(env));
5212 }
5213 
5214 static void tlbi_aa64_ripas2e1is_write(CPUARMState *env,
5215                                        const ARMCPRegInfo *ri,
5216                                        uint64_t value)
5217 {
5218     do_rvae_write(env, value, ipas2e1_tlbmask(env, value), true);
5219 }
5220 #endif
5221 
5222 static CPAccessResult aa64_zva_access(CPUARMState *env, const ARMCPRegInfo *ri,
5223                                       bool isread)
5224 {
5225     int cur_el = arm_current_el(env);
5226 
5227     if (cur_el < 2) {
5228         uint64_t hcr = arm_hcr_el2_eff(env);
5229 
5230         if (cur_el == 0) {
5231             if ((hcr & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)) {
5232                 if (!(env->cp15.sctlr_el[2] & SCTLR_DZE)) {
5233                     return CP_ACCESS_TRAP_EL2;
5234                 }
5235             } else {
5236                 if (!(env->cp15.sctlr_el[1] & SCTLR_DZE)) {
5237                     return CP_ACCESS_TRAP;
5238                 }
5239                 if (hcr & HCR_TDZ) {
5240                     return CP_ACCESS_TRAP_EL2;
5241                 }
5242             }
5243         } else if (hcr & HCR_TDZ) {
5244             return CP_ACCESS_TRAP_EL2;
5245         }
5246     }
5247     return CP_ACCESS_OK;
5248 }
5249 
5250 static uint64_t aa64_dczid_read(CPUARMState *env, const ARMCPRegInfo *ri)
5251 {
5252     ARMCPU *cpu = env_archcpu(env);
5253     int dzp_bit = 1 << 4;
5254 
5255     /* DZP indicates whether DC ZVA access is allowed */
5256     if (aa64_zva_access(env, NULL, false) == CP_ACCESS_OK) {
5257         dzp_bit = 0;
5258     }
5259     return cpu->dcz_blocksize | dzp_bit;
5260 }
5261 
5262 static CPAccessResult sp_el0_access(CPUARMState *env, const ARMCPRegInfo *ri,
5263                                     bool isread)
5264 {
5265     if (!(env->pstate & PSTATE_SP)) {
5266         /*
5267          * Access to SP_EL0 is undefined if it's being used as
5268          * the stack pointer.
5269          */
5270         return CP_ACCESS_TRAP_UNCATEGORIZED;
5271     }
5272     return CP_ACCESS_OK;
5273 }
5274 
5275 static uint64_t spsel_read(CPUARMState *env, const ARMCPRegInfo *ri)
5276 {
5277     return env->pstate & PSTATE_SP;
5278 }
5279 
5280 static void spsel_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t val)
5281 {
5282     update_spsel(env, val);
5283 }
5284 
5285 static void sctlr_write(CPUARMState *env, const ARMCPRegInfo *ri,
5286                         uint64_t value)
5287 {
5288     ARMCPU *cpu = env_archcpu(env);
5289 
5290     if (arm_feature(env, ARM_FEATURE_PMSA) && !cpu->has_mpu) {
5291         /* M bit is RAZ/WI for PMSA with no MPU implemented */
5292         value &= ~SCTLR_M;
5293     }
5294 
5295     /* ??? Lots of these bits are not implemented.  */
5296 
5297     if (ri->state == ARM_CP_STATE_AA64 && !cpu_isar_feature(aa64_mte, cpu)) {
5298         if (ri->opc1 == 6) { /* SCTLR_EL3 */
5299             value &= ~(SCTLR_ITFSB | SCTLR_TCF | SCTLR_ATA);
5300         } else {
5301             value &= ~(SCTLR_ITFSB | SCTLR_TCF0 | SCTLR_TCF |
5302                        SCTLR_ATA0 | SCTLR_ATA);
5303         }
5304     }
5305 
5306     if (raw_read(env, ri) == value) {
5307         /*
5308          * Skip the TLB flush if nothing actually changed; Linux likes
5309          * to do a lot of pointless SCTLR writes.
5310          */
5311         return;
5312     }
5313 
5314     raw_write(env, ri, value);
5315 
5316     /* This may enable/disable the MMU, so do a TLB flush.  */
5317     tlb_flush(CPU(cpu));
5318 
5319     if (tcg_enabled() && ri->type & ARM_CP_SUPPRESS_TB_END) {
5320         /*
5321          * Normally we would always end the TB on an SCTLR write; see the
5322          * comment in ARMCPRegInfo sctlr initialization below for why Xscale
5323          * is special.  Setting ARM_CP_SUPPRESS_TB_END also stops the rebuild
5324          * of hflags from the translator, so do it here.
5325          */
5326         arm_rebuild_hflags(env);
5327     }
5328 }
5329 
5330 static void mdcr_el3_write(CPUARMState *env, const ARMCPRegInfo *ri,
5331                            uint64_t value)
5332 {
5333     /*
5334      * Some MDCR_EL3 bits affect whether PMU counters are running:
5335      * if we are trying to change any of those then we must
5336      * bracket this update with PMU start/finish calls.
5337      */
5338     bool pmu_op = (env->cp15.mdcr_el3 ^ value) & MDCR_EL3_PMU_ENABLE_BITS;
5339 
5340     if (pmu_op) {
5341         pmu_op_start(env);
5342     }
5343     env->cp15.mdcr_el3 = value;
5344     if (pmu_op) {
5345         pmu_op_finish(env);
5346     }
5347 }
5348 
5349 static void sdcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
5350                        uint64_t value)
5351 {
5352     /* Not all bits defined for MDCR_EL3 exist in the AArch32 SDCR */
5353     mdcr_el3_write(env, ri, value & SDCR_VALID_MASK);
5354 }
5355 
5356 static void mdcr_el2_write(CPUARMState *env, const ARMCPRegInfo *ri,
5357                            uint64_t value)
5358 {
5359     /*
5360      * Some MDCR_EL2 bits affect whether PMU counters are running:
5361      * if we are trying to change any of those then we must
5362      * bracket this update with PMU start/finish calls.
5363      */
5364     bool pmu_op = (env->cp15.mdcr_el2 ^ value) & MDCR_EL2_PMU_ENABLE_BITS;
5365 
5366     if (pmu_op) {
5367         pmu_op_start(env);
5368     }
5369     env->cp15.mdcr_el2 = value;
5370     if (pmu_op) {
5371         pmu_op_finish(env);
5372     }
5373 }
5374 
5375 static CPAccessResult access_nv1(CPUARMState *env, const ARMCPRegInfo *ri,
5376                                  bool isread)
5377 {
5378     if (arm_current_el(env) == 1) {
5379         uint64_t hcr_nv = arm_hcr_el2_eff(env) & (HCR_NV | HCR_NV1 | HCR_NV2);
5380 
5381         if (hcr_nv == (HCR_NV | HCR_NV1)) {
5382             return CP_ACCESS_TRAP_EL2;
5383         }
5384     }
5385     return CP_ACCESS_OK;
5386 }
5387 
5388 #ifdef CONFIG_USER_ONLY
5389 /*
5390  * `IC IVAU` is handled to improve compatibility with JITs that dual-map their
5391  * code to get around W^X restrictions, where one region is writable and the
5392  * other is executable.
5393  *
5394  * Since the executable region is never written to we cannot detect code
5395  * changes when running in user mode, and rely on the emulated JIT telling us
5396  * that the code has changed by executing this instruction.
5397  */
5398 static void ic_ivau_write(CPUARMState *env, const ARMCPRegInfo *ri,
5399                           uint64_t value)
5400 {
5401     uint64_t icache_line_mask, start_address, end_address;
5402     const ARMCPU *cpu;
5403 
5404     cpu = env_archcpu(env);
5405 
5406     icache_line_mask = (4 << extract32(cpu->ctr, 0, 4)) - 1;
5407     start_address = value & ~icache_line_mask;
5408     end_address = value | icache_line_mask;
5409 
5410     mmap_lock();
5411 
5412     tb_invalidate_phys_range(start_address, end_address);
5413 
5414     mmap_unlock();
5415 }
5416 #endif
5417 
5418 static const ARMCPRegInfo v8_cp_reginfo[] = {
5419     /*
5420      * Minimal set of EL0-visible registers. This will need to be expanded
5421      * significantly for system emulation of AArch64 CPUs.
5422      */
5423     { .name = "NZCV", .state = ARM_CP_STATE_AA64,
5424       .opc0 = 3, .opc1 = 3, .opc2 = 0, .crn = 4, .crm = 2,
5425       .access = PL0_RW, .type = ARM_CP_NZCV },
5426     { .name = "DAIF", .state = ARM_CP_STATE_AA64,
5427       .opc0 = 3, .opc1 = 3, .opc2 = 1, .crn = 4, .crm = 2,
5428       .type = ARM_CP_NO_RAW,
5429       .access = PL0_RW, .accessfn = aa64_daif_access,
5430       .fieldoffset = offsetof(CPUARMState, daif),
5431       .writefn = aa64_daif_write, .resetfn = arm_cp_reset_ignore },
5432     { .name = "FPCR", .state = ARM_CP_STATE_AA64,
5433       .opc0 = 3, .opc1 = 3, .opc2 = 0, .crn = 4, .crm = 4,
5434       .access = PL0_RW, .type = ARM_CP_FPU | ARM_CP_SUPPRESS_TB_END,
5435       .readfn = aa64_fpcr_read, .writefn = aa64_fpcr_write },
5436     { .name = "FPSR", .state = ARM_CP_STATE_AA64,
5437       .opc0 = 3, .opc1 = 3, .opc2 = 1, .crn = 4, .crm = 4,
5438       .access = PL0_RW, .type = ARM_CP_FPU | ARM_CP_SUPPRESS_TB_END,
5439       .readfn = aa64_fpsr_read, .writefn = aa64_fpsr_write },
5440     { .name = "DCZID_EL0", .state = ARM_CP_STATE_AA64,
5441       .opc0 = 3, .opc1 = 3, .opc2 = 7, .crn = 0, .crm = 0,
5442       .access = PL0_R, .type = ARM_CP_NO_RAW,
5443       .fgt = FGT_DCZID_EL0,
5444       .readfn = aa64_dczid_read },
5445     { .name = "DC_ZVA", .state = ARM_CP_STATE_AA64,
5446       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 4, .opc2 = 1,
5447       .access = PL0_W, .type = ARM_CP_DC_ZVA,
5448 #ifndef CONFIG_USER_ONLY
5449       /* Avoid overhead of an access check that always passes in user-mode */
5450       .accessfn = aa64_zva_access,
5451       .fgt = FGT_DCZVA,
5452 #endif
5453     },
5454     { .name = "CURRENTEL", .state = ARM_CP_STATE_AA64,
5455       .opc0 = 3, .opc1 = 0, .opc2 = 2, .crn = 4, .crm = 2,
5456       .access = PL1_R, .type = ARM_CP_CURRENTEL },
5457     /*
5458      * Instruction cache ops. All of these except `IC IVAU` NOP because we
5459      * don't emulate caches.
5460      */
5461     { .name = "IC_IALLUIS", .state = ARM_CP_STATE_AA64,
5462       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 1, .opc2 = 0,
5463       .access = PL1_W, .type = ARM_CP_NOP,
5464       .fgt = FGT_ICIALLUIS,
5465       .accessfn = access_ticab },
5466     { .name = "IC_IALLU", .state = ARM_CP_STATE_AA64,
5467       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 0,
5468       .access = PL1_W, .type = ARM_CP_NOP,
5469       .fgt = FGT_ICIALLU,
5470       .accessfn = access_tocu },
5471     { .name = "IC_IVAU", .state = ARM_CP_STATE_AA64,
5472       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 5, .opc2 = 1,
5473       .access = PL0_W,
5474       .fgt = FGT_ICIVAU,
5475       .accessfn = access_tocu,
5476 #ifdef CONFIG_USER_ONLY
5477       .type = ARM_CP_NO_RAW,
5478       .writefn = ic_ivau_write
5479 #else
5480       .type = ARM_CP_NOP
5481 #endif
5482     },
5483     /* Cache ops: all NOPs since we don't emulate caches */
5484     { .name = "DC_IVAC", .state = ARM_CP_STATE_AA64,
5485       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 1,
5486       .access = PL1_W, .accessfn = aa64_cacheop_poc_access,
5487       .fgt = FGT_DCIVAC,
5488       .type = ARM_CP_NOP },
5489     { .name = "DC_ISW", .state = ARM_CP_STATE_AA64,
5490       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 2,
5491       .fgt = FGT_DCISW,
5492       .access = PL1_W, .accessfn = access_tsw, .type = ARM_CP_NOP },
5493     { .name = "DC_CVAC", .state = ARM_CP_STATE_AA64,
5494       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 10, .opc2 = 1,
5495       .access = PL0_W, .type = ARM_CP_NOP,
5496       .fgt = FGT_DCCVAC,
5497       .accessfn = aa64_cacheop_poc_access },
5498     { .name = "DC_CSW", .state = ARM_CP_STATE_AA64,
5499       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 10, .opc2 = 2,
5500       .fgt = FGT_DCCSW,
5501       .access = PL1_W, .accessfn = access_tsw, .type = ARM_CP_NOP },
5502     { .name = "DC_CVAU", .state = ARM_CP_STATE_AA64,
5503       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 11, .opc2 = 1,
5504       .access = PL0_W, .type = ARM_CP_NOP,
5505       .fgt = FGT_DCCVAU,
5506       .accessfn = access_tocu },
5507     { .name = "DC_CIVAC", .state = ARM_CP_STATE_AA64,
5508       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 14, .opc2 = 1,
5509       .access = PL0_W, .type = ARM_CP_NOP,
5510       .fgt = FGT_DCCIVAC,
5511       .accessfn = aa64_cacheop_poc_access },
5512     { .name = "DC_CISW", .state = ARM_CP_STATE_AA64,
5513       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 14, .opc2 = 2,
5514       .fgt = FGT_DCCISW,
5515       .access = PL1_W, .accessfn = access_tsw, .type = ARM_CP_NOP },
5516     /* TLBI operations */
5517     { .name = "TLBI_VMALLE1IS", .state = ARM_CP_STATE_AA64,
5518       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 0,
5519       .access = PL1_W, .accessfn = access_ttlbis, .type = ARM_CP_NO_RAW,
5520       .fgt = FGT_TLBIVMALLE1IS,
5521       .writefn = tlbi_aa64_vmalle1is_write },
5522     { .name = "TLBI_VAE1IS", .state = ARM_CP_STATE_AA64,
5523       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 1,
5524       .access = PL1_W, .accessfn = access_ttlbis, .type = ARM_CP_NO_RAW,
5525       .fgt = FGT_TLBIVAE1IS,
5526       .writefn = tlbi_aa64_vae1is_write },
5527     { .name = "TLBI_ASIDE1IS", .state = ARM_CP_STATE_AA64,
5528       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 2,
5529       .access = PL1_W, .accessfn = access_ttlbis, .type = ARM_CP_NO_RAW,
5530       .fgt = FGT_TLBIASIDE1IS,
5531       .writefn = tlbi_aa64_vmalle1is_write },
5532     { .name = "TLBI_VAAE1IS", .state = ARM_CP_STATE_AA64,
5533       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 3,
5534       .access = PL1_W, .accessfn = access_ttlbis, .type = ARM_CP_NO_RAW,
5535       .fgt = FGT_TLBIVAAE1IS,
5536       .writefn = tlbi_aa64_vae1is_write },
5537     { .name = "TLBI_VALE1IS", .state = ARM_CP_STATE_AA64,
5538       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 5,
5539       .access = PL1_W, .accessfn = access_ttlbis, .type = ARM_CP_NO_RAW,
5540       .fgt = FGT_TLBIVALE1IS,
5541       .writefn = tlbi_aa64_vae1is_write },
5542     { .name = "TLBI_VAALE1IS", .state = ARM_CP_STATE_AA64,
5543       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 7,
5544       .access = PL1_W, .accessfn = access_ttlbis, .type = ARM_CP_NO_RAW,
5545       .fgt = FGT_TLBIVAALE1IS,
5546       .writefn = tlbi_aa64_vae1is_write },
5547     { .name = "TLBI_VMALLE1", .state = ARM_CP_STATE_AA64,
5548       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 0,
5549       .access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
5550       .fgt = FGT_TLBIVMALLE1,
5551       .writefn = tlbi_aa64_vmalle1_write },
5552     { .name = "TLBI_VAE1", .state = ARM_CP_STATE_AA64,
5553       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 1,
5554       .access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
5555       .fgt = FGT_TLBIVAE1,
5556       .writefn = tlbi_aa64_vae1_write },
5557     { .name = "TLBI_ASIDE1", .state = ARM_CP_STATE_AA64,
5558       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 2,
5559       .access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
5560       .fgt = FGT_TLBIASIDE1,
5561       .writefn = tlbi_aa64_vmalle1_write },
5562     { .name = "TLBI_VAAE1", .state = ARM_CP_STATE_AA64,
5563       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 3,
5564       .access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
5565       .fgt = FGT_TLBIVAAE1,
5566       .writefn = tlbi_aa64_vae1_write },
5567     { .name = "TLBI_VALE1", .state = ARM_CP_STATE_AA64,
5568       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 5,
5569       .access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
5570       .fgt = FGT_TLBIVALE1,
5571       .writefn = tlbi_aa64_vae1_write },
5572     { .name = "TLBI_VAALE1", .state = ARM_CP_STATE_AA64,
5573       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 7,
5574       .access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
5575       .fgt = FGT_TLBIVAALE1,
5576       .writefn = tlbi_aa64_vae1_write },
5577     { .name = "TLBI_IPAS2E1IS", .state = ARM_CP_STATE_AA64,
5578       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 1,
5579       .access = PL2_W, .type = ARM_CP_NO_RAW,
5580       .writefn = tlbi_aa64_ipas2e1is_write },
5581     { .name = "TLBI_IPAS2LE1IS", .state = ARM_CP_STATE_AA64,
5582       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 5,
5583       .access = PL2_W, .type = ARM_CP_NO_RAW,
5584       .writefn = tlbi_aa64_ipas2e1is_write },
5585     { .name = "TLBI_ALLE1IS", .state = ARM_CP_STATE_AA64,
5586       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 4,
5587       .access = PL2_W, .type = ARM_CP_NO_RAW,
5588       .writefn = tlbi_aa64_alle1is_write },
5589     { .name = "TLBI_VMALLS12E1IS", .state = ARM_CP_STATE_AA64,
5590       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 6,
5591       .access = PL2_W, .type = ARM_CP_NO_RAW,
5592       .writefn = tlbi_aa64_alle1is_write },
5593     { .name = "TLBI_IPAS2E1", .state = ARM_CP_STATE_AA64,
5594       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 1,
5595       .access = PL2_W, .type = ARM_CP_NO_RAW,
5596       .writefn = tlbi_aa64_ipas2e1_write },
5597     { .name = "TLBI_IPAS2LE1", .state = ARM_CP_STATE_AA64,
5598       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 5,
5599       .access = PL2_W, .type = ARM_CP_NO_RAW,
5600       .writefn = tlbi_aa64_ipas2e1_write },
5601     { .name = "TLBI_ALLE1", .state = ARM_CP_STATE_AA64,
5602       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 4,
5603       .access = PL2_W, .type = ARM_CP_NO_RAW,
5604       .writefn = tlbi_aa64_alle1_write },
5605     { .name = "TLBI_VMALLS12E1", .state = ARM_CP_STATE_AA64,
5606       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 6,
5607       .access = PL2_W, .type = ARM_CP_NO_RAW,
5608       .writefn = tlbi_aa64_alle1is_write },
5609 #ifndef CONFIG_USER_ONLY
5610     /* 64 bit address translation operations */
5611     { .name = "AT_S1E1R", .state = ARM_CP_STATE_AA64,
5612       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 0,
5613       .access = PL1_W, .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC,
5614       .fgt = FGT_ATS1E1R,
5615       .accessfn = at_s1e01_access, .writefn = ats_write64 },
5616     { .name = "AT_S1E1W", .state = ARM_CP_STATE_AA64,
5617       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 1,
5618       .access = PL1_W, .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC,
5619       .fgt = FGT_ATS1E1W,
5620       .accessfn = at_s1e01_access, .writefn = ats_write64 },
5621     { .name = "AT_S1E0R", .state = ARM_CP_STATE_AA64,
5622       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 2,
5623       .access = PL1_W, .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC,
5624       .fgt = FGT_ATS1E0R,
5625       .accessfn = at_s1e01_access, .writefn = ats_write64 },
5626     { .name = "AT_S1E0W", .state = ARM_CP_STATE_AA64,
5627       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 3,
5628       .access = PL1_W, .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC,
5629       .fgt = FGT_ATS1E0W,
5630       .accessfn = at_s1e01_access, .writefn = ats_write64 },
5631     { .name = "AT_S12E1R", .state = ARM_CP_STATE_AA64,
5632       .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 4,
5633       .access = PL2_W, .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC,
5634       .accessfn = at_e012_access, .writefn = ats_write64 },
5635     { .name = "AT_S12E1W", .state = ARM_CP_STATE_AA64,
5636       .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 5,
5637       .access = PL2_W, .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC,
5638       .accessfn = at_e012_access, .writefn = ats_write64 },
5639     { .name = "AT_S12E0R", .state = ARM_CP_STATE_AA64,
5640       .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 6,
5641       .access = PL2_W, .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC,
5642       .accessfn = at_e012_access, .writefn = ats_write64 },
5643     { .name = "AT_S12E0W", .state = ARM_CP_STATE_AA64,
5644       .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 7,
5645       .access = PL2_W, .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC,
5646       .accessfn = at_e012_access, .writefn = ats_write64 },
5647     /* AT S1E2* are elsewhere as they UNDEF from EL3 if EL2 is not present */
5648     { .name = "AT_S1E3R", .state = ARM_CP_STATE_AA64,
5649       .opc0 = 1, .opc1 = 6, .crn = 7, .crm = 8, .opc2 = 0,
5650       .access = PL3_W, .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC,
5651       .writefn = ats_write64 },
5652     { .name = "AT_S1E3W", .state = ARM_CP_STATE_AA64,
5653       .opc0 = 1, .opc1 = 6, .crn = 7, .crm = 8, .opc2 = 1,
5654       .access = PL3_W, .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC,
5655       .writefn = ats_write64 },
5656     { .name = "PAR_EL1", .state = ARM_CP_STATE_AA64,
5657       .type = ARM_CP_ALIAS,
5658       .opc0 = 3, .opc1 = 0, .crn = 7, .crm = 4, .opc2 = 0,
5659       .access = PL1_RW, .resetvalue = 0,
5660       .fgt = FGT_PAR_EL1,
5661       .fieldoffset = offsetof(CPUARMState, cp15.par_el[1]),
5662       .writefn = par_write },
5663 #endif
5664     /* TLB invalidate last level of translation table walk */
5665     { .name = "TLBIMVALIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 5,
5666       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlbis,
5667       .writefn = tlbimva_is_write },
5668     { .name = "TLBIMVAALIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 7,
5669       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlbis,
5670       .writefn = tlbimvaa_is_write },
5671     { .name = "TLBIMVAL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 5,
5672       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlb,
5673       .writefn = tlbimva_write },
5674     { .name = "TLBIMVAAL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 7,
5675       .type = ARM_CP_NO_RAW, .access = PL1_W, .accessfn = access_ttlb,
5676       .writefn = tlbimvaa_write },
5677     { .name = "TLBIMVALH", .cp = 15, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 5,
5678       .type = ARM_CP_NO_RAW, .access = PL2_W,
5679       .writefn = tlbimva_hyp_write },
5680     { .name = "TLBIMVALHIS",
5681       .cp = 15, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 5,
5682       .type = ARM_CP_NO_RAW, .access = PL2_W,
5683       .writefn = tlbimva_hyp_is_write },
5684     { .name = "TLBIIPAS2",
5685       .cp = 15, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 1,
5686       .type = ARM_CP_NO_RAW, .access = PL2_W,
5687       .writefn = tlbiipas2_hyp_write },
5688     { .name = "TLBIIPAS2IS",
5689       .cp = 15, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 1,
5690       .type = ARM_CP_NO_RAW, .access = PL2_W,
5691       .writefn = tlbiipas2is_hyp_write },
5692     { .name = "TLBIIPAS2L",
5693       .cp = 15, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 5,
5694       .type = ARM_CP_NO_RAW, .access = PL2_W,
5695       .writefn = tlbiipas2_hyp_write },
5696     { .name = "TLBIIPAS2LIS",
5697       .cp = 15, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 5,
5698       .type = ARM_CP_NO_RAW, .access = PL2_W,
5699       .writefn = tlbiipas2is_hyp_write },
5700     /* 32 bit cache operations */
5701     { .name = "ICIALLUIS", .cp = 15, .opc1 = 0, .crn = 7, .crm = 1, .opc2 = 0,
5702       .type = ARM_CP_NOP, .access = PL1_W, .accessfn = access_ticab },
5703     { .name = "BPIALLUIS", .cp = 15, .opc1 = 0, .crn = 7, .crm = 1, .opc2 = 6,
5704       .type = ARM_CP_NOP, .access = PL1_W },
5705     { .name = "ICIALLU", .cp = 15, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 0,
5706       .type = ARM_CP_NOP, .access = PL1_W, .accessfn = access_tocu },
5707     { .name = "ICIMVAU", .cp = 15, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 1,
5708       .type = ARM_CP_NOP, .access = PL1_W, .accessfn = access_tocu },
5709     { .name = "BPIALL", .cp = 15, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 6,
5710       .type = ARM_CP_NOP, .access = PL1_W },
5711     { .name = "BPIMVA", .cp = 15, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 7,
5712       .type = ARM_CP_NOP, .access = PL1_W },
5713     { .name = "DCIMVAC", .cp = 15, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 1,
5714       .type = ARM_CP_NOP, .access = PL1_W, .accessfn = aa64_cacheop_poc_access },
5715     { .name = "DCISW", .cp = 15, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 2,
5716       .type = ARM_CP_NOP, .access = PL1_W, .accessfn = access_tsw },
5717     { .name = "DCCMVAC", .cp = 15, .opc1 = 0, .crn = 7, .crm = 10, .opc2 = 1,
5718       .type = ARM_CP_NOP, .access = PL1_W, .accessfn = aa64_cacheop_poc_access },
5719     { .name = "DCCSW", .cp = 15, .opc1 = 0, .crn = 7, .crm = 10, .opc2 = 2,
5720       .type = ARM_CP_NOP, .access = PL1_W, .accessfn = access_tsw },
5721     { .name = "DCCMVAU", .cp = 15, .opc1 = 0, .crn = 7, .crm = 11, .opc2 = 1,
5722       .type = ARM_CP_NOP, .access = PL1_W, .accessfn = access_tocu },
5723     { .name = "DCCIMVAC", .cp = 15, .opc1 = 0, .crn = 7, .crm = 14, .opc2 = 1,
5724       .type = ARM_CP_NOP, .access = PL1_W, .accessfn = aa64_cacheop_poc_access },
5725     { .name = "DCCISW", .cp = 15, .opc1 = 0, .crn = 7, .crm = 14, .opc2 = 2,
5726       .type = ARM_CP_NOP, .access = PL1_W, .accessfn = access_tsw },
5727     /* MMU Domain access control / MPU write buffer control */
5728     { .name = "DACR", .cp = 15, .opc1 = 0, .crn = 3, .crm = 0, .opc2 = 0,
5729       .access = PL1_RW, .accessfn = access_tvm_trvm, .resetvalue = 0,
5730       .writefn = dacr_write, .raw_writefn = raw_write,
5731       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.dacr_s),
5732                              offsetoflow32(CPUARMState, cp15.dacr_ns) } },
5733     { .name = "ELR_EL1", .state = ARM_CP_STATE_AA64,
5734       .type = ARM_CP_ALIAS,
5735       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 0, .opc2 = 1,
5736       .access = PL1_RW, .accessfn = access_nv1,
5737       .nv2_redirect_offset = 0x230 | NV2_REDIR_NV1,
5738       .fieldoffset = offsetof(CPUARMState, elr_el[1]) },
5739     { .name = "SPSR_EL1", .state = ARM_CP_STATE_AA64,
5740       .type = ARM_CP_ALIAS,
5741       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 0, .opc2 = 0,
5742       .access = PL1_RW, .accessfn = access_nv1,
5743       .nv2_redirect_offset = 0x160 | NV2_REDIR_NV1,
5744       .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_SVC]) },
5745     /*
5746      * We rely on the access checks not allowing the guest to write to the
5747      * state field when SPSel indicates that it's being used as the stack
5748      * pointer.
5749      */
5750     { .name = "SP_EL0", .state = ARM_CP_STATE_AA64,
5751       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 1, .opc2 = 0,
5752       .access = PL1_RW, .accessfn = sp_el0_access,
5753       .type = ARM_CP_ALIAS,
5754       .fieldoffset = offsetof(CPUARMState, sp_el[0]) },
5755     { .name = "SP_EL1", .state = ARM_CP_STATE_AA64,
5756       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 1, .opc2 = 0,
5757       .nv2_redirect_offset = 0x240,
5758       .access = PL2_RW, .type = ARM_CP_ALIAS | ARM_CP_EL3_NO_EL2_KEEP,
5759       .fieldoffset = offsetof(CPUARMState, sp_el[1]) },
5760     { .name = "SPSel", .state = ARM_CP_STATE_AA64,
5761       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 2, .opc2 = 0,
5762       .type = ARM_CP_NO_RAW,
5763       .access = PL1_RW, .readfn = spsel_read, .writefn = spsel_write },
5764     { .name = "SPSR_IRQ", .state = ARM_CP_STATE_AA64,
5765       .type = ARM_CP_ALIAS,
5766       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 3, .opc2 = 0,
5767       .access = PL2_RW,
5768       .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_IRQ]) },
5769     { .name = "SPSR_ABT", .state = ARM_CP_STATE_AA64,
5770       .type = ARM_CP_ALIAS,
5771       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 3, .opc2 = 1,
5772       .access = PL2_RW,
5773       .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_ABT]) },
5774     { .name = "SPSR_UND", .state = ARM_CP_STATE_AA64,
5775       .type = ARM_CP_ALIAS,
5776       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 3, .opc2 = 2,
5777       .access = PL2_RW,
5778       .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_UND]) },
5779     { .name = "SPSR_FIQ", .state = ARM_CP_STATE_AA64,
5780       .type = ARM_CP_ALIAS,
5781       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 3, .opc2 = 3,
5782       .access = PL2_RW,
5783       .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_FIQ]) },
5784     { .name = "MDCR_EL3", .state = ARM_CP_STATE_AA64,
5785       .type = ARM_CP_IO,
5786       .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 3, .opc2 = 1,
5787       .resetvalue = 0,
5788       .access = PL3_RW,
5789       .writefn = mdcr_el3_write,
5790       .fieldoffset = offsetof(CPUARMState, cp15.mdcr_el3) },
5791     { .name = "SDCR", .type = ARM_CP_ALIAS | ARM_CP_IO,
5792       .cp = 15, .opc1 = 0, .crn = 1, .crm = 3, .opc2 = 1,
5793       .access = PL1_RW, .accessfn = access_trap_aa32s_el1,
5794       .writefn = sdcr_write,
5795       .fieldoffset = offsetoflow32(CPUARMState, cp15.mdcr_el3) },
5796 };
5797 
5798 /* These are present only when EL1 supports AArch32 */
5799 static const ARMCPRegInfo v8_aa32_el1_reginfo[] = {
5800     { .name = "FPEXC32_EL2", .state = ARM_CP_STATE_AA64,
5801       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 3, .opc2 = 0,
5802       .access = PL2_RW,
5803       .type = ARM_CP_ALIAS | ARM_CP_FPU | ARM_CP_EL3_NO_EL2_KEEP,
5804       .fieldoffset = offsetof(CPUARMState, vfp.xregs[ARM_VFP_FPEXC]) },
5805     { .name = "DACR32_EL2", .state = ARM_CP_STATE_AA64,
5806       .opc0 = 3, .opc1 = 4, .crn = 3, .crm = 0, .opc2 = 0,
5807       .access = PL2_RW, .resetvalue = 0, .type = ARM_CP_EL3_NO_EL2_KEEP,
5808       .writefn = dacr_write, .raw_writefn = raw_write,
5809       .fieldoffset = offsetof(CPUARMState, cp15.dacr32_el2) },
5810     { .name = "IFSR32_EL2", .state = ARM_CP_STATE_AA64,
5811       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 0, .opc2 = 1,
5812       .access = PL2_RW, .resetvalue = 0, .type = ARM_CP_EL3_NO_EL2_KEEP,
5813       .fieldoffset = offsetof(CPUARMState, cp15.ifsr32_el2) },
5814 };
5815 
5816 static void do_hcr_write(CPUARMState *env, uint64_t value, uint64_t valid_mask)
5817 {
5818     ARMCPU *cpu = env_archcpu(env);
5819 
5820     if (arm_feature(env, ARM_FEATURE_V8)) {
5821         valid_mask |= MAKE_64BIT_MASK(0, 34);  /* ARMv8.0 */
5822     } else {
5823         valid_mask |= MAKE_64BIT_MASK(0, 28);  /* ARMv7VE */
5824     }
5825 
5826     if (arm_feature(env, ARM_FEATURE_EL3)) {
5827         valid_mask &= ~HCR_HCD;
5828     } else if (cpu->psci_conduit != QEMU_PSCI_CONDUIT_SMC) {
5829         /*
5830          * Architecturally HCR.TSC is RES0 if EL3 is not implemented.
5831          * However, if we're using the SMC PSCI conduit then QEMU is
5832          * effectively acting like EL3 firmware and so the guest at
5833          * EL2 should retain the ability to prevent EL1 from being
5834          * able to make SMC calls into the ersatz firmware, so in
5835          * that case HCR.TSC should be read/write.
5836          */
5837         valid_mask &= ~HCR_TSC;
5838     }
5839 
5840     if (arm_feature(env, ARM_FEATURE_AARCH64)) {
5841         if (cpu_isar_feature(aa64_vh, cpu)) {
5842             valid_mask |= HCR_E2H;
5843         }
5844         if (cpu_isar_feature(aa64_ras, cpu)) {
5845             valid_mask |= HCR_TERR | HCR_TEA;
5846         }
5847         if (cpu_isar_feature(aa64_lor, cpu)) {
5848             valid_mask |= HCR_TLOR;
5849         }
5850         if (cpu_isar_feature(aa64_pauth, cpu)) {
5851             valid_mask |= HCR_API | HCR_APK;
5852         }
5853         if (cpu_isar_feature(aa64_mte, cpu)) {
5854             valid_mask |= HCR_ATA | HCR_DCT | HCR_TID5;
5855         }
5856         if (cpu_isar_feature(aa64_scxtnum, cpu)) {
5857             valid_mask |= HCR_ENSCXT;
5858         }
5859         if (cpu_isar_feature(aa64_fwb, cpu)) {
5860             valid_mask |= HCR_FWB;
5861         }
5862         if (cpu_isar_feature(aa64_rme, cpu)) {
5863             valid_mask |= HCR_GPF;
5864         }
5865         if (cpu_isar_feature(aa64_nv, cpu)) {
5866             valid_mask |= HCR_NV | HCR_NV1 | HCR_AT;
5867         }
5868         if (cpu_isar_feature(aa64_nv2, cpu)) {
5869             valid_mask |= HCR_NV2;
5870         }
5871     }
5872 
5873     if (cpu_isar_feature(any_evt, cpu)) {
5874         valid_mask |= HCR_TTLBIS | HCR_TTLBOS | HCR_TICAB | HCR_TOCU | HCR_TID4;
5875     } else if (cpu_isar_feature(any_half_evt, cpu)) {
5876         valid_mask |= HCR_TICAB | HCR_TOCU | HCR_TID4;
5877     }
5878 
5879     /* Clear RES0 bits.  */
5880     value &= valid_mask;
5881 
5882     /*
5883      * These bits change the MMU setup:
5884      * HCR_VM enables stage 2 translation
5885      * HCR_PTW forbids certain page-table setups
5886      * HCR_DC disables stage1 and enables stage2 translation
5887      * HCR_DCT enables tagging on (disabled) stage1 translation
5888      * HCR_FWB changes the interpretation of stage2 descriptor bits
5889      * HCR_NV and HCR_NV1 affect interpretation of descriptor bits
5890      */
5891     if ((env->cp15.hcr_el2 ^ value) &
5892         (HCR_VM | HCR_PTW | HCR_DC | HCR_DCT | HCR_FWB | HCR_NV | HCR_NV1)) {
5893         tlb_flush(CPU(cpu));
5894     }
5895     env->cp15.hcr_el2 = value;
5896 
5897     /*
5898      * Updates to VI and VF require us to update the status of
5899      * virtual interrupts, which are the logical OR of these bits
5900      * and the state of the input lines from the GIC. (This requires
5901      * that we have the BQL, which is done by marking the
5902      * reginfo structs as ARM_CP_IO.)
5903      * Note that if a write to HCR pends a VIRQ or VFIQ it is never
5904      * possible for it to be taken immediately, because VIRQ and
5905      * VFIQ are masked unless running at EL0 or EL1, and HCR
5906      * can only be written at EL2.
5907      */
5908     g_assert(bql_locked());
5909     arm_cpu_update_virq(cpu);
5910     arm_cpu_update_vfiq(cpu);
5911     arm_cpu_update_vserr(cpu);
5912 }
5913 
5914 static void hcr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
5915 {
5916     do_hcr_write(env, value, 0);
5917 }
5918 
5919 static void hcr_writehigh(CPUARMState *env, const ARMCPRegInfo *ri,
5920                           uint64_t value)
5921 {
5922     /* Handle HCR2 write, i.e. write to high half of HCR_EL2 */
5923     value = deposit64(env->cp15.hcr_el2, 32, 32, value);
5924     do_hcr_write(env, value, MAKE_64BIT_MASK(0, 32));
5925 }
5926 
5927 static void hcr_writelow(CPUARMState *env, const ARMCPRegInfo *ri,
5928                          uint64_t value)
5929 {
5930     /* Handle HCR write, i.e. write to low half of HCR_EL2 */
5931     value = deposit64(env->cp15.hcr_el2, 0, 32, value);
5932     do_hcr_write(env, value, MAKE_64BIT_MASK(32, 32));
5933 }
5934 
5935 /*
5936  * Return the effective value of HCR_EL2, at the given security state.
5937  * Bits that are not included here:
5938  * RW       (read from SCR_EL3.RW as needed)
5939  */
5940 uint64_t arm_hcr_el2_eff_secstate(CPUARMState *env, ARMSecuritySpace space)
5941 {
5942     uint64_t ret = env->cp15.hcr_el2;
5943 
5944     assert(space != ARMSS_Root);
5945 
5946     if (!arm_is_el2_enabled_secstate(env, space)) {
5947         /*
5948          * "This register has no effect if EL2 is not enabled in the
5949          * current Security state".  This is ARMv8.4-SecEL2 speak for
5950          * !(SCR_EL3.NS==1 || SCR_EL3.EEL2==1).
5951          *
5952          * Prior to that, the language was "In an implementation that
5953          * includes EL3, when the value of SCR_EL3.NS is 0 the PE behaves
5954          * as if this field is 0 for all purposes other than a direct
5955          * read or write access of HCR_EL2".  With lots of enumeration
5956          * on a per-field basis.  In current QEMU, this is condition
5957          * is arm_is_secure_below_el3.
5958          *
5959          * Since the v8.4 language applies to the entire register, and
5960          * appears to be backward compatible, use that.
5961          */
5962         return 0;
5963     }
5964 
5965     /*
5966      * For a cpu that supports both aarch64 and aarch32, we can set bits
5967      * in HCR_EL2 (e.g. via EL3) that are RES0 when we enter EL2 as aa32.
5968      * Ignore all of the bits in HCR+HCR2 that are not valid for aarch32.
5969      */
5970     if (!arm_el_is_aa64(env, 2)) {
5971         uint64_t aa32_valid;
5972 
5973         /*
5974          * These bits are up-to-date as of ARMv8.6.
5975          * For HCR, it's easiest to list just the 2 bits that are invalid.
5976          * For HCR2, list those that are valid.
5977          */
5978         aa32_valid = MAKE_64BIT_MASK(0, 32) & ~(HCR_RW | HCR_TDZ);
5979         aa32_valid |= (HCR_CD | HCR_ID | HCR_TERR | HCR_TEA | HCR_MIOCNCE |
5980                        HCR_TID4 | HCR_TICAB | HCR_TOCU | HCR_TTLBIS);
5981         ret &= aa32_valid;
5982     }
5983 
5984     if (ret & HCR_TGE) {
5985         /* These bits are up-to-date as of ARMv8.6.  */
5986         if (ret & HCR_E2H) {
5987             ret &= ~(HCR_VM | HCR_FMO | HCR_IMO | HCR_AMO |
5988                      HCR_BSU_MASK | HCR_DC | HCR_TWI | HCR_TWE |
5989                      HCR_TID0 | HCR_TID2 | HCR_TPCP | HCR_TPU |
5990                      HCR_TDZ | HCR_CD | HCR_ID | HCR_MIOCNCE |
5991                      HCR_TID4 | HCR_TICAB | HCR_TOCU | HCR_ENSCXT |
5992                      HCR_TTLBIS | HCR_TTLBOS | HCR_TID5);
5993         } else {
5994             ret |= HCR_FMO | HCR_IMO | HCR_AMO;
5995         }
5996         ret &= ~(HCR_SWIO | HCR_PTW | HCR_VF | HCR_VI | HCR_VSE |
5997                  HCR_FB | HCR_TID1 | HCR_TID3 | HCR_TSC | HCR_TACR |
5998                  HCR_TSW | HCR_TTLB | HCR_TVM | HCR_HCD | HCR_TRVM |
5999                  HCR_TLOR);
6000     }
6001 
6002     return ret;
6003 }
6004 
6005 uint64_t arm_hcr_el2_eff(CPUARMState *env)
6006 {
6007     if (arm_feature(env, ARM_FEATURE_M)) {
6008         return 0;
6009     }
6010     return arm_hcr_el2_eff_secstate(env, arm_security_space_below_el3(env));
6011 }
6012 
6013 /*
6014  * Corresponds to ARM pseudocode function ELIsInHost().
6015  */
6016 bool el_is_in_host(CPUARMState *env, int el)
6017 {
6018     uint64_t mask;
6019 
6020     /*
6021      * Since we only care about E2H and TGE, we can skip arm_hcr_el2_eff().
6022      * Perform the simplest bit tests first, and validate EL2 afterward.
6023      */
6024     if (el & 1) {
6025         return false; /* EL1 or EL3 */
6026     }
6027 
6028     /*
6029      * Note that hcr_write() checks isar_feature_aa64_vh(),
6030      * aka HaveVirtHostExt(), in allowing HCR_E2H to be set.
6031      */
6032     mask = el ? HCR_E2H : HCR_E2H | HCR_TGE;
6033     if ((env->cp15.hcr_el2 & mask) != mask) {
6034         return false;
6035     }
6036 
6037     /* TGE and/or E2H set: double check those bits are currently legal. */
6038     return arm_is_el2_enabled(env) && arm_el_is_aa64(env, 2);
6039 }
6040 
6041 static void hcrx_write(CPUARMState *env, const ARMCPRegInfo *ri,
6042                        uint64_t value)
6043 {
6044     uint64_t valid_mask = 0;
6045 
6046     /* FEAT_MOPS adds MSCEn and MCE2 */
6047     if (cpu_isar_feature(aa64_mops, env_archcpu(env))) {
6048         valid_mask |= HCRX_MSCEN | HCRX_MCE2;
6049     }
6050 
6051     /* Clear RES0 bits.  */
6052     env->cp15.hcrx_el2 = value & valid_mask;
6053 }
6054 
6055 static CPAccessResult access_hxen(CPUARMState *env, const ARMCPRegInfo *ri,
6056                                   bool isread)
6057 {
6058     if (arm_current_el(env) == 2
6059         && arm_feature(env, ARM_FEATURE_EL3)
6060         && !(env->cp15.scr_el3 & SCR_HXEN)) {
6061         return CP_ACCESS_TRAP_EL3;
6062     }
6063     return CP_ACCESS_OK;
6064 }
6065 
6066 static const ARMCPRegInfo hcrx_el2_reginfo = {
6067     .name = "HCRX_EL2", .state = ARM_CP_STATE_AA64,
6068     .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 2, .opc2 = 2,
6069     .access = PL2_RW, .writefn = hcrx_write, .accessfn = access_hxen,
6070     .nv2_redirect_offset = 0xa0,
6071     .fieldoffset = offsetof(CPUARMState, cp15.hcrx_el2),
6072 };
6073 
6074 /* Return the effective value of HCRX_EL2.  */
6075 uint64_t arm_hcrx_el2_eff(CPUARMState *env)
6076 {
6077     /*
6078      * The bits in this register behave as 0 for all purposes other than
6079      * direct reads of the register if SCR_EL3.HXEn is 0.
6080      * If EL2 is not enabled in the current security state, then the
6081      * bit may behave as if 0, or as if 1, depending on the bit.
6082      * For the moment, we treat the EL2-disabled case as taking
6083      * priority over the HXEn-disabled case. This is true for the only
6084      * bit for a feature which we implement where the answer is different
6085      * for the two cases (MSCEn for FEAT_MOPS).
6086      * This may need to be revisited for future bits.
6087      */
6088     if (!arm_is_el2_enabled(env)) {
6089         uint64_t hcrx = 0;
6090         if (cpu_isar_feature(aa64_mops, env_archcpu(env))) {
6091             /* MSCEn behaves as 1 if EL2 is not enabled */
6092             hcrx |= HCRX_MSCEN;
6093         }
6094         return hcrx;
6095     }
6096     if (arm_feature(env, ARM_FEATURE_EL3) && !(env->cp15.scr_el3 & SCR_HXEN)) {
6097         return 0;
6098     }
6099     return env->cp15.hcrx_el2;
6100 }
6101 
6102 static void cptr_el2_write(CPUARMState *env, const ARMCPRegInfo *ri,
6103                            uint64_t value)
6104 {
6105     /*
6106      * For A-profile AArch32 EL3, if NSACR.CP10
6107      * is 0 then HCPTR.{TCP11,TCP10} ignore writes and read as 1.
6108      */
6109     if (arm_feature(env, ARM_FEATURE_EL3) && !arm_el_is_aa64(env, 3) &&
6110         !arm_is_secure(env) && !extract32(env->cp15.nsacr, 10, 1)) {
6111         uint64_t mask = R_HCPTR_TCP11_MASK | R_HCPTR_TCP10_MASK;
6112         value = (value & ~mask) | (env->cp15.cptr_el[2] & mask);
6113     }
6114     env->cp15.cptr_el[2] = value;
6115 }
6116 
6117 static uint64_t cptr_el2_read(CPUARMState *env, const ARMCPRegInfo *ri)
6118 {
6119     /*
6120      * For A-profile AArch32 EL3, if NSACR.CP10
6121      * is 0 then HCPTR.{TCP11,TCP10} ignore writes and read as 1.
6122      */
6123     uint64_t value = env->cp15.cptr_el[2];
6124 
6125     if (arm_feature(env, ARM_FEATURE_EL3) && !arm_el_is_aa64(env, 3) &&
6126         !arm_is_secure(env) && !extract32(env->cp15.nsacr, 10, 1)) {
6127         value |= R_HCPTR_TCP11_MASK | R_HCPTR_TCP10_MASK;
6128     }
6129     return value;
6130 }
6131 
6132 static const ARMCPRegInfo el2_cp_reginfo[] = {
6133     { .name = "HCR_EL2", .state = ARM_CP_STATE_AA64,
6134       .type = ARM_CP_IO,
6135       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 0,
6136       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.hcr_el2),
6137       .nv2_redirect_offset = 0x78,
6138       .writefn = hcr_write, .raw_writefn = raw_write },
6139     { .name = "HCR", .state = ARM_CP_STATE_AA32,
6140       .type = ARM_CP_ALIAS | ARM_CP_IO,
6141       .cp = 15, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 0,
6142       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.hcr_el2),
6143       .writefn = hcr_writelow },
6144     { .name = "HACR_EL2", .state = ARM_CP_STATE_BOTH,
6145       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 7,
6146       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
6147     { .name = "ELR_EL2", .state = ARM_CP_STATE_AA64,
6148       .type = ARM_CP_ALIAS | ARM_CP_NV2_REDIRECT,
6149       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 0, .opc2 = 1,
6150       .access = PL2_RW,
6151       .fieldoffset = offsetof(CPUARMState, elr_el[2]) },
6152     { .name = "ESR_EL2", .state = ARM_CP_STATE_BOTH,
6153       .type = ARM_CP_NV2_REDIRECT,
6154       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 2, .opc2 = 0,
6155       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.esr_el[2]) },
6156     { .name = "FAR_EL2", .state = ARM_CP_STATE_BOTH,
6157       .type = ARM_CP_NV2_REDIRECT,
6158       .opc0 = 3, .opc1 = 4, .crn = 6, .crm = 0, .opc2 = 0,
6159       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.far_el[2]) },
6160     { .name = "HIFAR", .state = ARM_CP_STATE_AA32,
6161       .type = ARM_CP_ALIAS,
6162       .cp = 15, .opc1 = 4, .crn = 6, .crm = 0, .opc2 = 2,
6163       .access = PL2_RW,
6164       .fieldoffset = offsetofhigh32(CPUARMState, cp15.far_el[2]) },
6165     { .name = "SPSR_EL2", .state = ARM_CP_STATE_AA64,
6166       .type = ARM_CP_ALIAS | ARM_CP_NV2_REDIRECT,
6167       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 0, .opc2 = 0,
6168       .access = PL2_RW,
6169       .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_HYP]) },
6170     { .name = "VBAR_EL2", .state = ARM_CP_STATE_BOTH,
6171       .opc0 = 3, .opc1 = 4, .crn = 12, .crm = 0, .opc2 = 0,
6172       .access = PL2_RW, .writefn = vbar_write,
6173       .fieldoffset = offsetof(CPUARMState, cp15.vbar_el[2]),
6174       .resetvalue = 0 },
6175     { .name = "SP_EL2", .state = ARM_CP_STATE_AA64,
6176       .opc0 = 3, .opc1 = 6, .crn = 4, .crm = 1, .opc2 = 0,
6177       .access = PL3_RW, .type = ARM_CP_ALIAS,
6178       .fieldoffset = offsetof(CPUARMState, sp_el[2]) },
6179     { .name = "CPTR_EL2", .state = ARM_CP_STATE_BOTH,
6180       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 2,
6181       .access = PL2_RW, .accessfn = cptr_access, .resetvalue = 0,
6182       .fieldoffset = offsetof(CPUARMState, cp15.cptr_el[2]),
6183       .readfn = cptr_el2_read, .writefn = cptr_el2_write },
6184     { .name = "MAIR_EL2", .state = ARM_CP_STATE_BOTH,
6185       .opc0 = 3, .opc1 = 4, .crn = 10, .crm = 2, .opc2 = 0,
6186       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.mair_el[2]),
6187       .resetvalue = 0 },
6188     { .name = "HMAIR1", .state = ARM_CP_STATE_AA32,
6189       .cp = 15, .opc1 = 4, .crn = 10, .crm = 2, .opc2 = 1,
6190       .access = PL2_RW, .type = ARM_CP_ALIAS,
6191       .fieldoffset = offsetofhigh32(CPUARMState, cp15.mair_el[2]) },
6192     { .name = "AMAIR_EL2", .state = ARM_CP_STATE_BOTH,
6193       .opc0 = 3, .opc1 = 4, .crn = 10, .crm = 3, .opc2 = 0,
6194       .access = PL2_RW, .type = ARM_CP_CONST,
6195       .resetvalue = 0 },
6196     /* HAMAIR1 is mapped to AMAIR_EL2[63:32] */
6197     { .name = "HAMAIR1", .state = ARM_CP_STATE_AA32,
6198       .cp = 15, .opc1 = 4, .crn = 10, .crm = 3, .opc2 = 1,
6199       .access = PL2_RW, .type = ARM_CP_CONST,
6200       .resetvalue = 0 },
6201     { .name = "AFSR0_EL2", .state = ARM_CP_STATE_BOTH,
6202       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 1, .opc2 = 0,
6203       .access = PL2_RW, .type = ARM_CP_CONST,
6204       .resetvalue = 0 },
6205     { .name = "AFSR1_EL2", .state = ARM_CP_STATE_BOTH,
6206       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 1, .opc2 = 1,
6207       .access = PL2_RW, .type = ARM_CP_CONST,
6208       .resetvalue = 0 },
6209     { .name = "TCR_EL2", .state = ARM_CP_STATE_BOTH,
6210       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 2,
6211       .access = PL2_RW, .writefn = vmsa_tcr_el12_write,
6212       .raw_writefn = raw_write,
6213       .fieldoffset = offsetof(CPUARMState, cp15.tcr_el[2]) },
6214     { .name = "VTCR", .state = ARM_CP_STATE_AA32,
6215       .cp = 15, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 2,
6216       .type = ARM_CP_ALIAS,
6217       .access = PL2_RW, .accessfn = access_el3_aa32ns,
6218       .fieldoffset = offsetoflow32(CPUARMState, cp15.vtcr_el2) },
6219     { .name = "VTCR_EL2", .state = ARM_CP_STATE_AA64,
6220       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 2,
6221       .access = PL2_RW,
6222       .nv2_redirect_offset = 0x40,
6223       /* no .writefn needed as this can't cause an ASID change */
6224       .fieldoffset = offsetof(CPUARMState, cp15.vtcr_el2) },
6225     { .name = "VTTBR", .state = ARM_CP_STATE_AA32,
6226       .cp = 15, .opc1 = 6, .crm = 2,
6227       .type = ARM_CP_64BIT | ARM_CP_ALIAS,
6228       .access = PL2_RW, .accessfn = access_el3_aa32ns,
6229       .fieldoffset = offsetof(CPUARMState, cp15.vttbr_el2),
6230       .writefn = vttbr_write, .raw_writefn = raw_write },
6231     { .name = "VTTBR_EL2", .state = ARM_CP_STATE_AA64,
6232       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 0,
6233       .access = PL2_RW, .writefn = vttbr_write, .raw_writefn = raw_write,
6234       .nv2_redirect_offset = 0x20,
6235       .fieldoffset = offsetof(CPUARMState, cp15.vttbr_el2) },
6236     { .name = "SCTLR_EL2", .state = ARM_CP_STATE_BOTH,
6237       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 0, .opc2 = 0,
6238       .access = PL2_RW, .raw_writefn = raw_write, .writefn = sctlr_write,
6239       .fieldoffset = offsetof(CPUARMState, cp15.sctlr_el[2]) },
6240     { .name = "TPIDR_EL2", .state = ARM_CP_STATE_BOTH,
6241       .opc0 = 3, .opc1 = 4, .crn = 13, .crm = 0, .opc2 = 2,
6242       .access = PL2_RW, .resetvalue = 0,
6243       .nv2_redirect_offset = 0x90,
6244       .fieldoffset = offsetof(CPUARMState, cp15.tpidr_el[2]) },
6245     { .name = "TTBR0_EL2", .state = ARM_CP_STATE_AA64,
6246       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 0,
6247       .access = PL2_RW, .resetvalue = 0,
6248       .writefn = vmsa_tcr_ttbr_el2_write, .raw_writefn = raw_write,
6249       .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el[2]) },
6250     { .name = "HTTBR", .cp = 15, .opc1 = 4, .crm = 2,
6251       .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_ALIAS,
6252       .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el[2]) },
6253     { .name = "TLBIALLNSNH",
6254       .cp = 15, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 4,
6255       .type = ARM_CP_NO_RAW, .access = PL2_W,
6256       .writefn = tlbiall_nsnh_write },
6257     { .name = "TLBIALLNSNHIS",
6258       .cp = 15, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 4,
6259       .type = ARM_CP_NO_RAW, .access = PL2_W,
6260       .writefn = tlbiall_nsnh_is_write },
6261     { .name = "TLBIALLH", .cp = 15, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 0,
6262       .type = ARM_CP_NO_RAW, .access = PL2_W,
6263       .writefn = tlbiall_hyp_write },
6264     { .name = "TLBIALLHIS", .cp = 15, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 0,
6265       .type = ARM_CP_NO_RAW, .access = PL2_W,
6266       .writefn = tlbiall_hyp_is_write },
6267     { .name = "TLBIMVAH", .cp = 15, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 1,
6268       .type = ARM_CP_NO_RAW, .access = PL2_W,
6269       .writefn = tlbimva_hyp_write },
6270     { .name = "TLBIMVAHIS", .cp = 15, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 1,
6271       .type = ARM_CP_NO_RAW, .access = PL2_W,
6272       .writefn = tlbimva_hyp_is_write },
6273     { .name = "TLBI_ALLE2", .state = ARM_CP_STATE_AA64,
6274       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 0,
6275       .access = PL2_W, .type = ARM_CP_NO_RAW | ARM_CP_EL3_NO_EL2_UNDEF,
6276       .writefn = tlbi_aa64_alle2_write },
6277     { .name = "TLBI_VAE2", .state = ARM_CP_STATE_AA64,
6278       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 1,
6279       .access = PL2_W, .type = ARM_CP_NO_RAW | ARM_CP_EL3_NO_EL2_UNDEF,
6280       .writefn = tlbi_aa64_vae2_write },
6281     { .name = "TLBI_VALE2", .state = ARM_CP_STATE_AA64,
6282       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 5,
6283       .access = PL2_W, .type = ARM_CP_NO_RAW | ARM_CP_EL3_NO_EL2_UNDEF,
6284       .writefn = tlbi_aa64_vae2_write },
6285     { .name = "TLBI_ALLE2IS", .state = ARM_CP_STATE_AA64,
6286       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 0,
6287       .access = PL2_W, .type = ARM_CP_NO_RAW | ARM_CP_EL3_NO_EL2_UNDEF,
6288       .writefn = tlbi_aa64_alle2is_write },
6289     { .name = "TLBI_VAE2IS", .state = ARM_CP_STATE_AA64,
6290       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 1,
6291       .access = PL2_W, .type = ARM_CP_NO_RAW | ARM_CP_EL3_NO_EL2_UNDEF,
6292       .writefn = tlbi_aa64_vae2is_write },
6293     { .name = "TLBI_VALE2IS", .state = ARM_CP_STATE_AA64,
6294       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 5,
6295       .access = PL2_W, .type = ARM_CP_NO_RAW | ARM_CP_EL3_NO_EL2_UNDEF,
6296       .writefn = tlbi_aa64_vae2is_write },
6297 #ifndef CONFIG_USER_ONLY
6298     /*
6299      * Unlike the other EL2-related AT operations, these must
6300      * UNDEF from EL3 if EL2 is not implemented, which is why we
6301      * define them here rather than with the rest of the AT ops.
6302      */
6303     { .name = "AT_S1E2R", .state = ARM_CP_STATE_AA64,
6304       .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 0,
6305       .access = PL2_W, .accessfn = at_s1e2_access,
6306       .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC | ARM_CP_EL3_NO_EL2_UNDEF,
6307       .writefn = ats_write64 },
6308     { .name = "AT_S1E2W", .state = ARM_CP_STATE_AA64,
6309       .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 1,
6310       .access = PL2_W, .accessfn = at_s1e2_access,
6311       .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC | ARM_CP_EL3_NO_EL2_UNDEF,
6312       .writefn = ats_write64 },
6313     /*
6314      * The AArch32 ATS1H* operations are CONSTRAINED UNPREDICTABLE
6315      * if EL2 is not implemented; we choose to UNDEF. Behaviour at EL3
6316      * with SCR.NS == 0 outside Monitor mode is UNPREDICTABLE; we choose
6317      * to behave as if SCR.NS was 1.
6318      */
6319     { .name = "ATS1HR", .cp = 15, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 0,
6320       .access = PL2_W,
6321       .writefn = ats1h_write, .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC },
6322     { .name = "ATS1HW", .cp = 15, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 1,
6323       .access = PL2_W,
6324       .writefn = ats1h_write, .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC },
6325     { .name = "CNTHCTL_EL2", .state = ARM_CP_STATE_BOTH,
6326       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 1, .opc2 = 0,
6327       /*
6328        * ARMv7 requires bit 0 and 1 to reset to 1. ARMv8 defines the
6329        * reset values as IMPDEF. We choose to reset to 3 to comply with
6330        * both ARMv7 and ARMv8.
6331        */
6332       .access = PL2_RW, .type = ARM_CP_IO, .resetvalue = 3,
6333       .writefn = gt_cnthctl_write, .raw_writefn = raw_write,
6334       .fieldoffset = offsetof(CPUARMState, cp15.cnthctl_el2) },
6335     { .name = "CNTVOFF_EL2", .state = ARM_CP_STATE_AA64,
6336       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 0, .opc2 = 3,
6337       .access = PL2_RW, .type = ARM_CP_IO, .resetvalue = 0,
6338       .writefn = gt_cntvoff_write,
6339       .nv2_redirect_offset = 0x60,
6340       .fieldoffset = offsetof(CPUARMState, cp15.cntvoff_el2) },
6341     { .name = "CNTVOFF", .cp = 15, .opc1 = 4, .crm = 14,
6342       .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_ALIAS | ARM_CP_IO,
6343       .writefn = gt_cntvoff_write,
6344       .fieldoffset = offsetof(CPUARMState, cp15.cntvoff_el2) },
6345     { .name = "CNTHP_CVAL_EL2", .state = ARM_CP_STATE_AA64,
6346       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 2,
6347       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_HYP].cval),
6348       .type = ARM_CP_IO, .access = PL2_RW,
6349       .writefn = gt_hyp_cval_write, .raw_writefn = raw_write },
6350     { .name = "CNTHP_CVAL", .cp = 15, .opc1 = 6, .crm = 14,
6351       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_HYP].cval),
6352       .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_IO,
6353       .writefn = gt_hyp_cval_write, .raw_writefn = raw_write },
6354     { .name = "CNTHP_TVAL_EL2", .state = ARM_CP_STATE_BOTH,
6355       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 0,
6356       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL2_RW,
6357       .resetfn = gt_hyp_timer_reset,
6358       .readfn = gt_hyp_tval_read, .writefn = gt_hyp_tval_write },
6359     { .name = "CNTHP_CTL_EL2", .state = ARM_CP_STATE_BOTH,
6360       .type = ARM_CP_IO,
6361       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 1,
6362       .access = PL2_RW,
6363       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_HYP].ctl),
6364       .resetvalue = 0,
6365       .writefn = gt_hyp_ctl_write, .raw_writefn = raw_write },
6366 #endif
6367     { .name = "HPFAR", .state = ARM_CP_STATE_AA32,
6368       .cp = 15, .opc1 = 4, .crn = 6, .crm = 0, .opc2 = 4,
6369       .access = PL2_RW, .accessfn = access_el3_aa32ns,
6370       .fieldoffset = offsetof(CPUARMState, cp15.hpfar_el2) },
6371     { .name = "HPFAR_EL2", .state = ARM_CP_STATE_AA64,
6372       .opc0 = 3, .opc1 = 4, .crn = 6, .crm = 0, .opc2 = 4,
6373       .access = PL2_RW,
6374       .fieldoffset = offsetof(CPUARMState, cp15.hpfar_el2) },
6375     { .name = "HSTR_EL2", .state = ARM_CP_STATE_BOTH,
6376       .cp = 15, .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 3,
6377       .access = PL2_RW,
6378       .nv2_redirect_offset = 0x80,
6379       .fieldoffset = offsetof(CPUARMState, cp15.hstr_el2) },
6380 };
6381 
6382 static const ARMCPRegInfo el2_v8_cp_reginfo[] = {
6383     { .name = "HCR2", .state = ARM_CP_STATE_AA32,
6384       .type = ARM_CP_ALIAS | ARM_CP_IO,
6385       .cp = 15, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 4,
6386       .access = PL2_RW,
6387       .fieldoffset = offsetofhigh32(CPUARMState, cp15.hcr_el2),
6388       .writefn = hcr_writehigh },
6389 };
6390 
6391 static CPAccessResult sel2_access(CPUARMState *env, const ARMCPRegInfo *ri,
6392                                   bool isread)
6393 {
6394     if (arm_current_el(env) == 3 || arm_is_secure_below_el3(env)) {
6395         return CP_ACCESS_OK;
6396     }
6397     return CP_ACCESS_TRAP_UNCATEGORIZED;
6398 }
6399 
6400 static const ARMCPRegInfo el2_sec_cp_reginfo[] = {
6401     { .name = "VSTTBR_EL2", .state = ARM_CP_STATE_AA64,
6402       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 6, .opc2 = 0,
6403       .access = PL2_RW, .accessfn = sel2_access,
6404       .nv2_redirect_offset = 0x30,
6405       .fieldoffset = offsetof(CPUARMState, cp15.vsttbr_el2) },
6406     { .name = "VSTCR_EL2", .state = ARM_CP_STATE_AA64,
6407       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 6, .opc2 = 2,
6408       .access = PL2_RW, .accessfn = sel2_access,
6409       .nv2_redirect_offset = 0x48,
6410       .fieldoffset = offsetof(CPUARMState, cp15.vstcr_el2) },
6411 };
6412 
6413 static CPAccessResult nsacr_access(CPUARMState *env, const ARMCPRegInfo *ri,
6414                                    bool isread)
6415 {
6416     /*
6417      * The NSACR is RW at EL3, and RO for NS EL1 and NS EL2.
6418      * At Secure EL1 it traps to EL3 or EL2.
6419      */
6420     if (arm_current_el(env) == 3) {
6421         return CP_ACCESS_OK;
6422     }
6423     if (arm_is_secure_below_el3(env)) {
6424         if (env->cp15.scr_el3 & SCR_EEL2) {
6425             return CP_ACCESS_TRAP_EL2;
6426         }
6427         return CP_ACCESS_TRAP_EL3;
6428     }
6429     /* Accesses from EL1 NS and EL2 NS are UNDEF for write but allow reads. */
6430     if (isread) {
6431         return CP_ACCESS_OK;
6432     }
6433     return CP_ACCESS_TRAP_UNCATEGORIZED;
6434 }
6435 
6436 static const ARMCPRegInfo el3_cp_reginfo[] = {
6437     { .name = "SCR_EL3", .state = ARM_CP_STATE_AA64,
6438       .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 1, .opc2 = 0,
6439       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.scr_el3),
6440       .resetfn = scr_reset, .writefn = scr_write, .raw_writefn = raw_write },
6441     { .name = "SCR",  .type = ARM_CP_ALIAS | ARM_CP_NEWEL,
6442       .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 0,
6443       .access = PL1_RW, .accessfn = access_trap_aa32s_el1,
6444       .fieldoffset = offsetoflow32(CPUARMState, cp15.scr_el3),
6445       .writefn = scr_write, .raw_writefn = raw_write },
6446     { .name = "SDER32_EL3", .state = ARM_CP_STATE_AA64,
6447       .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 1, .opc2 = 1,
6448       .access = PL3_RW, .resetvalue = 0,
6449       .fieldoffset = offsetof(CPUARMState, cp15.sder) },
6450     { .name = "SDER",
6451       .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 1,
6452       .access = PL3_RW, .resetvalue = 0,
6453       .fieldoffset = offsetoflow32(CPUARMState, cp15.sder) },
6454     { .name = "MVBAR", .cp = 15, .opc1 = 0, .crn = 12, .crm = 0, .opc2 = 1,
6455       .access = PL1_RW, .accessfn = access_trap_aa32s_el1,
6456       .writefn = vbar_write, .resetvalue = 0,
6457       .fieldoffset = offsetof(CPUARMState, cp15.mvbar) },
6458     { .name = "TTBR0_EL3", .state = ARM_CP_STATE_AA64,
6459       .opc0 = 3, .opc1 = 6, .crn = 2, .crm = 0, .opc2 = 0,
6460       .access = PL3_RW, .resetvalue = 0,
6461       .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el[3]) },
6462     { .name = "TCR_EL3", .state = ARM_CP_STATE_AA64,
6463       .opc0 = 3, .opc1 = 6, .crn = 2, .crm = 0, .opc2 = 2,
6464       .access = PL3_RW,
6465       /* no .writefn needed as this can't cause an ASID change */
6466       .resetvalue = 0,
6467       .fieldoffset = offsetof(CPUARMState, cp15.tcr_el[3]) },
6468     { .name = "ELR_EL3", .state = ARM_CP_STATE_AA64,
6469       .type = ARM_CP_ALIAS,
6470       .opc0 = 3, .opc1 = 6, .crn = 4, .crm = 0, .opc2 = 1,
6471       .access = PL3_RW,
6472       .fieldoffset = offsetof(CPUARMState, elr_el[3]) },
6473     { .name = "ESR_EL3", .state = ARM_CP_STATE_AA64,
6474       .opc0 = 3, .opc1 = 6, .crn = 5, .crm = 2, .opc2 = 0,
6475       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.esr_el[3]) },
6476     { .name = "FAR_EL3", .state = ARM_CP_STATE_AA64,
6477       .opc0 = 3, .opc1 = 6, .crn = 6, .crm = 0, .opc2 = 0,
6478       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.far_el[3]) },
6479     { .name = "SPSR_EL3", .state = ARM_CP_STATE_AA64,
6480       .type = ARM_CP_ALIAS,
6481       .opc0 = 3, .opc1 = 6, .crn = 4, .crm = 0, .opc2 = 0,
6482       .access = PL3_RW,
6483       .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_MON]) },
6484     { .name = "VBAR_EL3", .state = ARM_CP_STATE_AA64,
6485       .opc0 = 3, .opc1 = 6, .crn = 12, .crm = 0, .opc2 = 0,
6486       .access = PL3_RW, .writefn = vbar_write,
6487       .fieldoffset = offsetof(CPUARMState, cp15.vbar_el[3]),
6488       .resetvalue = 0 },
6489     { .name = "CPTR_EL3", .state = ARM_CP_STATE_AA64,
6490       .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 1, .opc2 = 2,
6491       .access = PL3_RW, .accessfn = cptr_access, .resetvalue = 0,
6492       .fieldoffset = offsetof(CPUARMState, cp15.cptr_el[3]) },
6493     { .name = "TPIDR_EL3", .state = ARM_CP_STATE_AA64,
6494       .opc0 = 3, .opc1 = 6, .crn = 13, .crm = 0, .opc2 = 2,
6495       .access = PL3_RW, .resetvalue = 0,
6496       .fieldoffset = offsetof(CPUARMState, cp15.tpidr_el[3]) },
6497     { .name = "AMAIR_EL3", .state = ARM_CP_STATE_AA64,
6498       .opc0 = 3, .opc1 = 6, .crn = 10, .crm = 3, .opc2 = 0,
6499       .access = PL3_RW, .type = ARM_CP_CONST,
6500       .resetvalue = 0 },
6501     { .name = "AFSR0_EL3", .state = ARM_CP_STATE_BOTH,
6502       .opc0 = 3, .opc1 = 6, .crn = 5, .crm = 1, .opc2 = 0,
6503       .access = PL3_RW, .type = ARM_CP_CONST,
6504       .resetvalue = 0 },
6505     { .name = "AFSR1_EL3", .state = ARM_CP_STATE_BOTH,
6506       .opc0 = 3, .opc1 = 6, .crn = 5, .crm = 1, .opc2 = 1,
6507       .access = PL3_RW, .type = ARM_CP_CONST,
6508       .resetvalue = 0 },
6509     { .name = "TLBI_ALLE3IS", .state = ARM_CP_STATE_AA64,
6510       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 3, .opc2 = 0,
6511       .access = PL3_W, .type = ARM_CP_NO_RAW,
6512       .writefn = tlbi_aa64_alle3is_write },
6513     { .name = "TLBI_VAE3IS", .state = ARM_CP_STATE_AA64,
6514       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 3, .opc2 = 1,
6515       .access = PL3_W, .type = ARM_CP_NO_RAW,
6516       .writefn = tlbi_aa64_vae3is_write },
6517     { .name = "TLBI_VALE3IS", .state = ARM_CP_STATE_AA64,
6518       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 3, .opc2 = 5,
6519       .access = PL3_W, .type = ARM_CP_NO_RAW,
6520       .writefn = tlbi_aa64_vae3is_write },
6521     { .name = "TLBI_ALLE3", .state = ARM_CP_STATE_AA64,
6522       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 7, .opc2 = 0,
6523       .access = PL3_W, .type = ARM_CP_NO_RAW,
6524       .writefn = tlbi_aa64_alle3_write },
6525     { .name = "TLBI_VAE3", .state = ARM_CP_STATE_AA64,
6526       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 7, .opc2 = 1,
6527       .access = PL3_W, .type = ARM_CP_NO_RAW,
6528       .writefn = tlbi_aa64_vae3_write },
6529     { .name = "TLBI_VALE3", .state = ARM_CP_STATE_AA64,
6530       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 7, .opc2 = 5,
6531       .access = PL3_W, .type = ARM_CP_NO_RAW,
6532       .writefn = tlbi_aa64_vae3_write },
6533 };
6534 
6535 #ifndef CONFIG_USER_ONLY
6536 
6537 static CPAccessResult e2h_access(CPUARMState *env, const ARMCPRegInfo *ri,
6538                                  bool isread)
6539 {
6540     if (arm_current_el(env) == 1) {
6541         /* This must be a FEAT_NV access */
6542         /* TODO: FEAT_ECV will need to check CNTHCTL_EL2 here */
6543         return CP_ACCESS_OK;
6544     }
6545     if (!(arm_hcr_el2_eff(env) & HCR_E2H)) {
6546         return CP_ACCESS_TRAP;
6547     }
6548     return CP_ACCESS_OK;
6549 }
6550 
6551 /* Test if system register redirection is to occur in the current state.  */
6552 static bool redirect_for_e2h(CPUARMState *env)
6553 {
6554     return arm_current_el(env) == 2 && (arm_hcr_el2_eff(env) & HCR_E2H);
6555 }
6556 
6557 static uint64_t el2_e2h_read(CPUARMState *env, const ARMCPRegInfo *ri)
6558 {
6559     CPReadFn *readfn;
6560 
6561     if (redirect_for_e2h(env)) {
6562         /* Switch to the saved EL2 version of the register.  */
6563         ri = ri->opaque;
6564         readfn = ri->readfn;
6565     } else {
6566         readfn = ri->orig_readfn;
6567     }
6568     if (readfn == NULL) {
6569         readfn = raw_read;
6570     }
6571     return readfn(env, ri);
6572 }
6573 
6574 static void el2_e2h_write(CPUARMState *env, const ARMCPRegInfo *ri,
6575                           uint64_t value)
6576 {
6577     CPWriteFn *writefn;
6578 
6579     if (redirect_for_e2h(env)) {
6580         /* Switch to the saved EL2 version of the register.  */
6581         ri = ri->opaque;
6582         writefn = ri->writefn;
6583     } else {
6584         writefn = ri->orig_writefn;
6585     }
6586     if (writefn == NULL) {
6587         writefn = raw_write;
6588     }
6589     writefn(env, ri, value);
6590 }
6591 
6592 static uint64_t el2_e2h_e12_read(CPUARMState *env, const ARMCPRegInfo *ri)
6593 {
6594     /* Pass the EL1 register accessor its ri, not the EL12 alias ri */
6595     return ri->orig_readfn(env, ri->opaque);
6596 }
6597 
6598 static void el2_e2h_e12_write(CPUARMState *env, const ARMCPRegInfo *ri,
6599                               uint64_t value)
6600 {
6601     /* Pass the EL1 register accessor its ri, not the EL12 alias ri */
6602     return ri->orig_writefn(env, ri->opaque, value);
6603 }
6604 
6605 static CPAccessResult el2_e2h_e12_access(CPUARMState *env,
6606                                          const ARMCPRegInfo *ri,
6607                                          bool isread)
6608 {
6609     if (arm_current_el(env) == 1) {
6610         /*
6611          * This must be a FEAT_NV access (will either trap or redirect
6612          * to memory). None of the registers with _EL12 aliases want to
6613          * apply their trap controls for this kind of access, so don't
6614          * call the orig_accessfn or do the "UNDEF when E2H is 0" check.
6615          */
6616         return CP_ACCESS_OK;
6617     }
6618     /* FOO_EL12 aliases only exist when E2H is 1; otherwise they UNDEF */
6619     if (!(arm_hcr_el2_eff(env) & HCR_E2H)) {
6620         return CP_ACCESS_TRAP_UNCATEGORIZED;
6621     }
6622     if (ri->orig_accessfn) {
6623         return ri->orig_accessfn(env, ri->opaque, isread);
6624     }
6625     return CP_ACCESS_OK;
6626 }
6627 
6628 static void define_arm_vh_e2h_redirects_aliases(ARMCPU *cpu)
6629 {
6630     struct E2HAlias {
6631         uint32_t src_key, dst_key, new_key;
6632         const char *src_name, *dst_name, *new_name;
6633         bool (*feature)(const ARMISARegisters *id);
6634     };
6635 
6636 #define K(op0, op1, crn, crm, op2) \
6637     ENCODE_AA64_CP_REG(CP_REG_ARM64_SYSREG_CP, crn, crm, op0, op1, op2)
6638 
6639     static const struct E2HAlias aliases[] = {
6640         { K(3, 0,  1, 0, 0), K(3, 4,  1, 0, 0), K(3, 5, 1, 0, 0),
6641           "SCTLR", "SCTLR_EL2", "SCTLR_EL12" },
6642         { K(3, 0,  1, 0, 2), K(3, 4,  1, 1, 2), K(3, 5, 1, 0, 2),
6643           "CPACR", "CPTR_EL2", "CPACR_EL12" },
6644         { K(3, 0,  2, 0, 0), K(3, 4,  2, 0, 0), K(3, 5, 2, 0, 0),
6645           "TTBR0_EL1", "TTBR0_EL2", "TTBR0_EL12" },
6646         { K(3, 0,  2, 0, 1), K(3, 4,  2, 0, 1), K(3, 5, 2, 0, 1),
6647           "TTBR1_EL1", "TTBR1_EL2", "TTBR1_EL12" },
6648         { K(3, 0,  2, 0, 2), K(3, 4,  2, 0, 2), K(3, 5, 2, 0, 2),
6649           "TCR_EL1", "TCR_EL2", "TCR_EL12" },
6650         { K(3, 0,  4, 0, 0), K(3, 4,  4, 0, 0), K(3, 5, 4, 0, 0),
6651           "SPSR_EL1", "SPSR_EL2", "SPSR_EL12" },
6652         { K(3, 0,  4, 0, 1), K(3, 4,  4, 0, 1), K(3, 5, 4, 0, 1),
6653           "ELR_EL1", "ELR_EL2", "ELR_EL12" },
6654         { K(3, 0,  5, 1, 0), K(3, 4,  5, 1, 0), K(3, 5, 5, 1, 0),
6655           "AFSR0_EL1", "AFSR0_EL2", "AFSR0_EL12" },
6656         { K(3, 0,  5, 1, 1), K(3, 4,  5, 1, 1), K(3, 5, 5, 1, 1),
6657           "AFSR1_EL1", "AFSR1_EL2", "AFSR1_EL12" },
6658         { K(3, 0,  5, 2, 0), K(3, 4,  5, 2, 0), K(3, 5, 5, 2, 0),
6659           "ESR_EL1", "ESR_EL2", "ESR_EL12" },
6660         { K(3, 0,  6, 0, 0), K(3, 4,  6, 0, 0), K(3, 5, 6, 0, 0),
6661           "FAR_EL1", "FAR_EL2", "FAR_EL12" },
6662         { K(3, 0, 10, 2, 0), K(3, 4, 10, 2, 0), K(3, 5, 10, 2, 0),
6663           "MAIR_EL1", "MAIR_EL2", "MAIR_EL12" },
6664         { K(3, 0, 10, 3, 0), K(3, 4, 10, 3, 0), K(3, 5, 10, 3, 0),
6665           "AMAIR0", "AMAIR_EL2", "AMAIR_EL12" },
6666         { K(3, 0, 12, 0, 0), K(3, 4, 12, 0, 0), K(3, 5, 12, 0, 0),
6667           "VBAR", "VBAR_EL2", "VBAR_EL12" },
6668         { K(3, 0, 13, 0, 1), K(3, 4, 13, 0, 1), K(3, 5, 13, 0, 1),
6669           "CONTEXTIDR_EL1", "CONTEXTIDR_EL2", "CONTEXTIDR_EL12" },
6670         { K(3, 0, 14, 1, 0), K(3, 4, 14, 1, 0), K(3, 5, 14, 1, 0),
6671           "CNTKCTL", "CNTHCTL_EL2", "CNTKCTL_EL12" },
6672 
6673         /*
6674          * Note that redirection of ZCR is mentioned in the description
6675          * of ZCR_EL2, and aliasing in the description of ZCR_EL1, but
6676          * not in the summary table.
6677          */
6678         { K(3, 0,  1, 2, 0), K(3, 4,  1, 2, 0), K(3, 5, 1, 2, 0),
6679           "ZCR_EL1", "ZCR_EL2", "ZCR_EL12", isar_feature_aa64_sve },
6680         { K(3, 0,  1, 2, 6), K(3, 4,  1, 2, 6), K(3, 5, 1, 2, 6),
6681           "SMCR_EL1", "SMCR_EL2", "SMCR_EL12", isar_feature_aa64_sme },
6682 
6683         { K(3, 0,  5, 6, 0), K(3, 4,  5, 6, 0), K(3, 5, 5, 6, 0),
6684           "TFSR_EL1", "TFSR_EL2", "TFSR_EL12", isar_feature_aa64_mte },
6685 
6686         { K(3, 0, 13, 0, 7), K(3, 4, 13, 0, 7), K(3, 5, 13, 0, 7),
6687           "SCXTNUM_EL1", "SCXTNUM_EL2", "SCXTNUM_EL12",
6688           isar_feature_aa64_scxtnum },
6689 
6690         /* TODO: ARMv8.2-SPE -- PMSCR_EL2 */
6691         /* TODO: ARMv8.4-Trace -- TRFCR_EL2 */
6692     };
6693 #undef K
6694 
6695     size_t i;
6696 
6697     for (i = 0; i < ARRAY_SIZE(aliases); i++) {
6698         const struct E2HAlias *a = &aliases[i];
6699         ARMCPRegInfo *src_reg, *dst_reg, *new_reg;
6700         bool ok;
6701 
6702         if (a->feature && !a->feature(&cpu->isar)) {
6703             continue;
6704         }
6705 
6706         src_reg = g_hash_table_lookup(cpu->cp_regs,
6707                                       (gpointer)(uintptr_t)a->src_key);
6708         dst_reg = g_hash_table_lookup(cpu->cp_regs,
6709                                       (gpointer)(uintptr_t)a->dst_key);
6710         g_assert(src_reg != NULL);
6711         g_assert(dst_reg != NULL);
6712 
6713         /* Cross-compare names to detect typos in the keys.  */
6714         g_assert(strcmp(src_reg->name, a->src_name) == 0);
6715         g_assert(strcmp(dst_reg->name, a->dst_name) == 0);
6716 
6717         /* None of the core system registers use opaque; we will.  */
6718         g_assert(src_reg->opaque == NULL);
6719 
6720         /* Create alias before redirection so we dup the right data. */
6721         new_reg = g_memdup(src_reg, sizeof(ARMCPRegInfo));
6722 
6723         new_reg->name = a->new_name;
6724         new_reg->type |= ARM_CP_ALIAS;
6725         /* Remove PL1/PL0 access, leaving PL2/PL3 R/W in place.  */
6726         new_reg->access &= PL2_RW | PL3_RW;
6727         /* The new_reg op fields are as per new_key, not the target reg */
6728         new_reg->crn = (a->new_key & CP_REG_ARM64_SYSREG_CRN_MASK)
6729             >> CP_REG_ARM64_SYSREG_CRN_SHIFT;
6730         new_reg->crm = (a->new_key & CP_REG_ARM64_SYSREG_CRM_MASK)
6731             >> CP_REG_ARM64_SYSREG_CRM_SHIFT;
6732         new_reg->opc0 = (a->new_key & CP_REG_ARM64_SYSREG_OP0_MASK)
6733             >> CP_REG_ARM64_SYSREG_OP0_SHIFT;
6734         new_reg->opc1 = (a->new_key & CP_REG_ARM64_SYSREG_OP1_MASK)
6735             >> CP_REG_ARM64_SYSREG_OP1_SHIFT;
6736         new_reg->opc2 = (a->new_key & CP_REG_ARM64_SYSREG_OP2_MASK)
6737             >> CP_REG_ARM64_SYSREG_OP2_SHIFT;
6738         new_reg->opaque = src_reg;
6739         new_reg->orig_readfn = src_reg->readfn ?: raw_read;
6740         new_reg->orig_writefn = src_reg->writefn ?: raw_write;
6741         new_reg->orig_accessfn = src_reg->accessfn;
6742         if (!new_reg->raw_readfn) {
6743             new_reg->raw_readfn = raw_read;
6744         }
6745         if (!new_reg->raw_writefn) {
6746             new_reg->raw_writefn = raw_write;
6747         }
6748         new_reg->readfn = el2_e2h_e12_read;
6749         new_reg->writefn = el2_e2h_e12_write;
6750         new_reg->accessfn = el2_e2h_e12_access;
6751 
6752         /*
6753          * If the _EL1 register is redirected to memory by FEAT_NV2,
6754          * then it shares the offset with the _EL12 register,
6755          * and which one is redirected depends on HCR_EL2.NV1.
6756          */
6757         if (new_reg->nv2_redirect_offset) {
6758             assert(new_reg->nv2_redirect_offset & NV2_REDIR_NV1);
6759             new_reg->nv2_redirect_offset &= ~NV2_REDIR_NV1;
6760             new_reg->nv2_redirect_offset |= NV2_REDIR_NO_NV1;
6761         }
6762 
6763         ok = g_hash_table_insert(cpu->cp_regs,
6764                                  (gpointer)(uintptr_t)a->new_key, new_reg);
6765         g_assert(ok);
6766 
6767         src_reg->opaque = dst_reg;
6768         src_reg->orig_readfn = src_reg->readfn ?: raw_read;
6769         src_reg->orig_writefn = src_reg->writefn ?: raw_write;
6770         if (!src_reg->raw_readfn) {
6771             src_reg->raw_readfn = raw_read;
6772         }
6773         if (!src_reg->raw_writefn) {
6774             src_reg->raw_writefn = raw_write;
6775         }
6776         src_reg->readfn = el2_e2h_read;
6777         src_reg->writefn = el2_e2h_write;
6778     }
6779 }
6780 #endif
6781 
6782 static CPAccessResult ctr_el0_access(CPUARMState *env, const ARMCPRegInfo *ri,
6783                                      bool isread)
6784 {
6785     int cur_el = arm_current_el(env);
6786 
6787     if (cur_el < 2) {
6788         uint64_t hcr = arm_hcr_el2_eff(env);
6789 
6790         if (cur_el == 0) {
6791             if ((hcr & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)) {
6792                 if (!(env->cp15.sctlr_el[2] & SCTLR_UCT)) {
6793                     return CP_ACCESS_TRAP_EL2;
6794                 }
6795             } else {
6796                 if (!(env->cp15.sctlr_el[1] & SCTLR_UCT)) {
6797                     return CP_ACCESS_TRAP;
6798                 }
6799                 if (hcr & HCR_TID2) {
6800                     return CP_ACCESS_TRAP_EL2;
6801                 }
6802             }
6803         } else if (hcr & HCR_TID2) {
6804             return CP_ACCESS_TRAP_EL2;
6805         }
6806     }
6807 
6808     if (arm_current_el(env) < 2 && arm_hcr_el2_eff(env) & HCR_TID2) {
6809         return CP_ACCESS_TRAP_EL2;
6810     }
6811 
6812     return CP_ACCESS_OK;
6813 }
6814 
6815 /*
6816  * Check for traps to RAS registers, which are controlled
6817  * by HCR_EL2.TERR and SCR_EL3.TERR.
6818  */
6819 static CPAccessResult access_terr(CPUARMState *env, const ARMCPRegInfo *ri,
6820                                   bool isread)
6821 {
6822     int el = arm_current_el(env);
6823 
6824     if (el < 2 && (arm_hcr_el2_eff(env) & HCR_TERR)) {
6825         return CP_ACCESS_TRAP_EL2;
6826     }
6827     if (el < 3 && (env->cp15.scr_el3 & SCR_TERR)) {
6828         return CP_ACCESS_TRAP_EL3;
6829     }
6830     return CP_ACCESS_OK;
6831 }
6832 
6833 static uint64_t disr_read(CPUARMState *env, const ARMCPRegInfo *ri)
6834 {
6835     int el = arm_current_el(env);
6836 
6837     if (el < 2 && (arm_hcr_el2_eff(env) & HCR_AMO)) {
6838         return env->cp15.vdisr_el2;
6839     }
6840     if (el < 3 && (env->cp15.scr_el3 & SCR_EA)) {
6841         return 0; /* RAZ/WI */
6842     }
6843     return env->cp15.disr_el1;
6844 }
6845 
6846 static void disr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t val)
6847 {
6848     int el = arm_current_el(env);
6849 
6850     if (el < 2 && (arm_hcr_el2_eff(env) & HCR_AMO)) {
6851         env->cp15.vdisr_el2 = val;
6852         return;
6853     }
6854     if (el < 3 && (env->cp15.scr_el3 & SCR_EA)) {
6855         return; /* RAZ/WI */
6856     }
6857     env->cp15.disr_el1 = val;
6858 }
6859 
6860 /*
6861  * Minimal RAS implementation with no Error Records.
6862  * Which means that all of the Error Record registers:
6863  *   ERXADDR_EL1
6864  *   ERXCTLR_EL1
6865  *   ERXFR_EL1
6866  *   ERXMISC0_EL1
6867  *   ERXMISC1_EL1
6868  *   ERXMISC2_EL1
6869  *   ERXMISC3_EL1
6870  *   ERXPFGCDN_EL1  (RASv1p1)
6871  *   ERXPFGCTL_EL1  (RASv1p1)
6872  *   ERXPFGF_EL1    (RASv1p1)
6873  *   ERXSTATUS_EL1
6874  * and
6875  *   ERRSELR_EL1
6876  * may generate UNDEFINED, which is the effect we get by not
6877  * listing them at all.
6878  *
6879  * These registers have fine-grained trap bits, but UNDEF-to-EL1
6880  * is higher priority than FGT-to-EL2 so we do not need to list them
6881  * in order to check for an FGT.
6882  */
6883 static const ARMCPRegInfo minimal_ras_reginfo[] = {
6884     { .name = "DISR_EL1", .state = ARM_CP_STATE_BOTH,
6885       .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 1, .opc2 = 1,
6886       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.disr_el1),
6887       .readfn = disr_read, .writefn = disr_write, .raw_writefn = raw_write },
6888     { .name = "ERRIDR_EL1", .state = ARM_CP_STATE_BOTH,
6889       .opc0 = 3, .opc1 = 0, .crn = 5, .crm = 3, .opc2 = 0,
6890       .access = PL1_R, .accessfn = access_terr,
6891       .fgt = FGT_ERRIDR_EL1,
6892       .type = ARM_CP_CONST, .resetvalue = 0 },
6893     { .name = "VDISR_EL2", .state = ARM_CP_STATE_BOTH,
6894       .opc0 = 3, .opc1 = 4, .crn = 12, .crm = 1, .opc2 = 1,
6895       .nv2_redirect_offset = 0x500,
6896       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.vdisr_el2) },
6897     { .name = "VSESR_EL2", .state = ARM_CP_STATE_BOTH,
6898       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 2, .opc2 = 3,
6899       .nv2_redirect_offset = 0x508,
6900       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.vsesr_el2) },
6901 };
6902 
6903 /*
6904  * Return the exception level to which exceptions should be taken
6905  * via SVEAccessTrap.  This excludes the check for whether the exception
6906  * should be routed through AArch64.AdvSIMDFPAccessTrap.  That can easily
6907  * be found by testing 0 < fp_exception_el < sve_exception_el.
6908  *
6909  * C.f. the ARM pseudocode function CheckSVEEnabled.  Note that the
6910  * pseudocode does *not* separate out the FP trap checks, but has them
6911  * all in one function.
6912  */
6913 int sve_exception_el(CPUARMState *env, int el)
6914 {
6915 #ifndef CONFIG_USER_ONLY
6916     if (el <= 1 && !el_is_in_host(env, el)) {
6917         switch (FIELD_EX64(env->cp15.cpacr_el1, CPACR_EL1, ZEN)) {
6918         case 1:
6919             if (el != 0) {
6920                 break;
6921             }
6922             /* fall through */
6923         case 0:
6924         case 2:
6925             return 1;
6926         }
6927     }
6928 
6929     if (el <= 2 && arm_is_el2_enabled(env)) {
6930         /* CPTR_EL2 changes format with HCR_EL2.E2H (regardless of TGE). */
6931         if (env->cp15.hcr_el2 & HCR_E2H) {
6932             switch (FIELD_EX64(env->cp15.cptr_el[2], CPTR_EL2, ZEN)) {
6933             case 1:
6934                 if (el != 0 || !(env->cp15.hcr_el2 & HCR_TGE)) {
6935                     break;
6936                 }
6937                 /* fall through */
6938             case 0:
6939             case 2:
6940                 return 2;
6941             }
6942         } else {
6943             if (FIELD_EX64(env->cp15.cptr_el[2], CPTR_EL2, TZ)) {
6944                 return 2;
6945             }
6946         }
6947     }
6948 
6949     /* CPTR_EL3.  Since EZ is negative we must check for EL3.  */
6950     if (arm_feature(env, ARM_FEATURE_EL3)
6951         && !FIELD_EX64(env->cp15.cptr_el[3], CPTR_EL3, EZ)) {
6952         return 3;
6953     }
6954 #endif
6955     return 0;
6956 }
6957 
6958 /*
6959  * Return the exception level to which exceptions should be taken for SME.
6960  * C.f. the ARM pseudocode function CheckSMEAccess.
6961  */
6962 int sme_exception_el(CPUARMState *env, int el)
6963 {
6964 #ifndef CONFIG_USER_ONLY
6965     if (el <= 1 && !el_is_in_host(env, el)) {
6966         switch (FIELD_EX64(env->cp15.cpacr_el1, CPACR_EL1, SMEN)) {
6967         case 1:
6968             if (el != 0) {
6969                 break;
6970             }
6971             /* fall through */
6972         case 0:
6973         case 2:
6974             return 1;
6975         }
6976     }
6977 
6978     if (el <= 2 && arm_is_el2_enabled(env)) {
6979         /* CPTR_EL2 changes format with HCR_EL2.E2H (regardless of TGE). */
6980         if (env->cp15.hcr_el2 & HCR_E2H) {
6981             switch (FIELD_EX64(env->cp15.cptr_el[2], CPTR_EL2, SMEN)) {
6982             case 1:
6983                 if (el != 0 || !(env->cp15.hcr_el2 & HCR_TGE)) {
6984                     break;
6985                 }
6986                 /* fall through */
6987             case 0:
6988             case 2:
6989                 return 2;
6990             }
6991         } else {
6992             if (FIELD_EX64(env->cp15.cptr_el[2], CPTR_EL2, TSM)) {
6993                 return 2;
6994             }
6995         }
6996     }
6997 
6998     /* CPTR_EL3.  Since ESM is negative we must check for EL3.  */
6999     if (arm_feature(env, ARM_FEATURE_EL3)
7000         && !FIELD_EX64(env->cp15.cptr_el[3], CPTR_EL3, ESM)) {
7001         return 3;
7002     }
7003 #endif
7004     return 0;
7005 }
7006 
7007 /*
7008  * Given that SVE is enabled, return the vector length for EL.
7009  */
7010 uint32_t sve_vqm1_for_el_sm(CPUARMState *env, int el, bool sm)
7011 {
7012     ARMCPU *cpu = env_archcpu(env);
7013     uint64_t *cr = env->vfp.zcr_el;
7014     uint32_t map = cpu->sve_vq.map;
7015     uint32_t len = ARM_MAX_VQ - 1;
7016 
7017     if (sm) {
7018         cr = env->vfp.smcr_el;
7019         map = cpu->sme_vq.map;
7020     }
7021 
7022     if (el <= 1 && !el_is_in_host(env, el)) {
7023         len = MIN(len, 0xf & (uint32_t)cr[1]);
7024     }
7025     if (el <= 2 && arm_feature(env, ARM_FEATURE_EL2)) {
7026         len = MIN(len, 0xf & (uint32_t)cr[2]);
7027     }
7028     if (arm_feature(env, ARM_FEATURE_EL3)) {
7029         len = MIN(len, 0xf & (uint32_t)cr[3]);
7030     }
7031 
7032     map &= MAKE_64BIT_MASK(0, len + 1);
7033     if (map != 0) {
7034         return 31 - clz32(map);
7035     }
7036 
7037     /* Bit 0 is always set for Normal SVE -- not so for Streaming SVE. */
7038     assert(sm);
7039     return ctz32(cpu->sme_vq.map);
7040 }
7041 
7042 uint32_t sve_vqm1_for_el(CPUARMState *env, int el)
7043 {
7044     return sve_vqm1_for_el_sm(env, el, FIELD_EX64(env->svcr, SVCR, SM));
7045 }
7046 
7047 static void zcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
7048                       uint64_t value)
7049 {
7050     int cur_el = arm_current_el(env);
7051     int old_len = sve_vqm1_for_el(env, cur_el);
7052     int new_len;
7053 
7054     /* Bits other than [3:0] are RAZ/WI.  */
7055     QEMU_BUILD_BUG_ON(ARM_MAX_VQ > 16);
7056     raw_write(env, ri, value & 0xf);
7057 
7058     /*
7059      * Because we arrived here, we know both FP and SVE are enabled;
7060      * otherwise we would have trapped access to the ZCR_ELn register.
7061      */
7062     new_len = sve_vqm1_for_el(env, cur_el);
7063     if (new_len < old_len) {
7064         aarch64_sve_narrow_vq(env, new_len + 1);
7065     }
7066 }
7067 
7068 static const ARMCPRegInfo zcr_reginfo[] = {
7069     { .name = "ZCR_EL1", .state = ARM_CP_STATE_AA64,
7070       .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 2, .opc2 = 0,
7071       .nv2_redirect_offset = 0x1e0 | NV2_REDIR_NV1,
7072       .access = PL1_RW, .type = ARM_CP_SVE,
7073       .fieldoffset = offsetof(CPUARMState, vfp.zcr_el[1]),
7074       .writefn = zcr_write, .raw_writefn = raw_write },
7075     { .name = "ZCR_EL2", .state = ARM_CP_STATE_AA64,
7076       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 2, .opc2 = 0,
7077       .access = PL2_RW, .type = ARM_CP_SVE,
7078       .fieldoffset = offsetof(CPUARMState, vfp.zcr_el[2]),
7079       .writefn = zcr_write, .raw_writefn = raw_write },
7080     { .name = "ZCR_EL3", .state = ARM_CP_STATE_AA64,
7081       .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 2, .opc2 = 0,
7082       .access = PL3_RW, .type = ARM_CP_SVE,
7083       .fieldoffset = offsetof(CPUARMState, vfp.zcr_el[3]),
7084       .writefn = zcr_write, .raw_writefn = raw_write },
7085 };
7086 
7087 #ifdef TARGET_AARCH64
7088 static CPAccessResult access_tpidr2(CPUARMState *env, const ARMCPRegInfo *ri,
7089                                     bool isread)
7090 {
7091     int el = arm_current_el(env);
7092 
7093     if (el == 0) {
7094         uint64_t sctlr = arm_sctlr(env, el);
7095         if (!(sctlr & SCTLR_EnTP2)) {
7096             return CP_ACCESS_TRAP;
7097         }
7098     }
7099     /* TODO: FEAT_FGT */
7100     if (el < 3
7101         && arm_feature(env, ARM_FEATURE_EL3)
7102         && !(env->cp15.scr_el3 & SCR_ENTP2)) {
7103         return CP_ACCESS_TRAP_EL3;
7104     }
7105     return CP_ACCESS_OK;
7106 }
7107 
7108 static CPAccessResult access_smprimap(CPUARMState *env, const ARMCPRegInfo *ri,
7109                                       bool isread)
7110 {
7111     /* If EL1 this is a FEAT_NV access and CPTR_EL3.ESM doesn't apply */
7112     if (arm_current_el(env) == 2
7113         && arm_feature(env, ARM_FEATURE_EL3)
7114         && !FIELD_EX64(env->cp15.cptr_el[3], CPTR_EL3, ESM)) {
7115         return CP_ACCESS_TRAP_EL3;
7116     }
7117     return CP_ACCESS_OK;
7118 }
7119 
7120 static CPAccessResult access_smpri(CPUARMState *env, const ARMCPRegInfo *ri,
7121                                    bool isread)
7122 {
7123     if (arm_current_el(env) < 3
7124         && arm_feature(env, ARM_FEATURE_EL3)
7125         && !FIELD_EX64(env->cp15.cptr_el[3], CPTR_EL3, ESM)) {
7126         return CP_ACCESS_TRAP_EL3;
7127     }
7128     return CP_ACCESS_OK;
7129 }
7130 
7131 /* ResetSVEState */
7132 static void arm_reset_sve_state(CPUARMState *env)
7133 {
7134     memset(env->vfp.zregs, 0, sizeof(env->vfp.zregs));
7135     /* Recall that FFR is stored as pregs[16]. */
7136     memset(env->vfp.pregs, 0, sizeof(env->vfp.pregs));
7137     vfp_set_fpcr(env, 0x0800009f);
7138 }
7139 
7140 void aarch64_set_svcr(CPUARMState *env, uint64_t new, uint64_t mask)
7141 {
7142     uint64_t change = (env->svcr ^ new) & mask;
7143 
7144     if (change == 0) {
7145         return;
7146     }
7147     env->svcr ^= change;
7148 
7149     if (change & R_SVCR_SM_MASK) {
7150         arm_reset_sve_state(env);
7151     }
7152 
7153     /*
7154      * ResetSMEState.
7155      *
7156      * SetPSTATE_ZA zeros on enable and disable.  We can zero this only
7157      * on enable: while disabled, the storage is inaccessible and the
7158      * value does not matter.  We're not saving the storage in vmstate
7159      * when disabled either.
7160      */
7161     if (change & new & R_SVCR_ZA_MASK) {
7162         memset(env->zarray, 0, sizeof(env->zarray));
7163     }
7164 
7165     if (tcg_enabled()) {
7166         arm_rebuild_hflags(env);
7167     }
7168 }
7169 
7170 static void svcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
7171                        uint64_t value)
7172 {
7173     aarch64_set_svcr(env, value, -1);
7174 }
7175 
7176 static void smcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
7177                        uint64_t value)
7178 {
7179     int cur_el = arm_current_el(env);
7180     int old_len = sve_vqm1_for_el(env, cur_el);
7181     int new_len;
7182 
7183     QEMU_BUILD_BUG_ON(ARM_MAX_VQ > R_SMCR_LEN_MASK + 1);
7184     value &= R_SMCR_LEN_MASK | R_SMCR_FA64_MASK;
7185     raw_write(env, ri, value);
7186 
7187     /*
7188      * Note that it is CONSTRAINED UNPREDICTABLE what happens to ZA storage
7189      * when SVL is widened (old values kept, or zeros).  Choose to keep the
7190      * current values for simplicity.  But for QEMU internals, we must still
7191      * apply the narrower SVL to the Zregs and Pregs -- see the comment
7192      * above aarch64_sve_narrow_vq.
7193      */
7194     new_len = sve_vqm1_for_el(env, cur_el);
7195     if (new_len < old_len) {
7196         aarch64_sve_narrow_vq(env, new_len + 1);
7197     }
7198 }
7199 
7200 static const ARMCPRegInfo sme_reginfo[] = {
7201     { .name = "TPIDR2_EL0", .state = ARM_CP_STATE_AA64,
7202       .opc0 = 3, .opc1 = 3, .crn = 13, .crm = 0, .opc2 = 5,
7203       .access = PL0_RW, .accessfn = access_tpidr2,
7204       .fgt = FGT_NTPIDR2_EL0,
7205       .fieldoffset = offsetof(CPUARMState, cp15.tpidr2_el0) },
7206     { .name = "SVCR", .state = ARM_CP_STATE_AA64,
7207       .opc0 = 3, .opc1 = 3, .crn = 4, .crm = 2, .opc2 = 2,
7208       .access = PL0_RW, .type = ARM_CP_SME,
7209       .fieldoffset = offsetof(CPUARMState, svcr),
7210       .writefn = svcr_write, .raw_writefn = raw_write },
7211     { .name = "SMCR_EL1", .state = ARM_CP_STATE_AA64,
7212       .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 2, .opc2 = 6,
7213       .nv2_redirect_offset = 0x1f0 | NV2_REDIR_NV1,
7214       .access = PL1_RW, .type = ARM_CP_SME,
7215       .fieldoffset = offsetof(CPUARMState, vfp.smcr_el[1]),
7216       .writefn = smcr_write, .raw_writefn = raw_write },
7217     { .name = "SMCR_EL2", .state = ARM_CP_STATE_AA64,
7218       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 2, .opc2 = 6,
7219       .access = PL2_RW, .type = ARM_CP_SME,
7220       .fieldoffset = offsetof(CPUARMState, vfp.smcr_el[2]),
7221       .writefn = smcr_write, .raw_writefn = raw_write },
7222     { .name = "SMCR_EL3", .state = ARM_CP_STATE_AA64,
7223       .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 2, .opc2 = 6,
7224       .access = PL3_RW, .type = ARM_CP_SME,
7225       .fieldoffset = offsetof(CPUARMState, vfp.smcr_el[3]),
7226       .writefn = smcr_write, .raw_writefn = raw_write },
7227     { .name = "SMIDR_EL1", .state = ARM_CP_STATE_AA64,
7228       .opc0 = 3, .opc1 = 1, .crn = 0, .crm = 0, .opc2 = 6,
7229       .access = PL1_R, .accessfn = access_aa64_tid1,
7230       /*
7231        * IMPLEMENTOR = 0 (software)
7232        * REVISION    = 0 (implementation defined)
7233        * SMPS        = 0 (no streaming execution priority in QEMU)
7234        * AFFINITY    = 0 (streaming sve mode not shared with other PEs)
7235        */
7236       .type = ARM_CP_CONST, .resetvalue = 0, },
7237     /*
7238      * Because SMIDR_EL1.SMPS is 0, SMPRI_EL1 and SMPRIMAP_EL2 are RES 0.
7239      */
7240     { .name = "SMPRI_EL1", .state = ARM_CP_STATE_AA64,
7241       .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 2, .opc2 = 4,
7242       .access = PL1_RW, .accessfn = access_smpri,
7243       .fgt = FGT_NSMPRI_EL1,
7244       .type = ARM_CP_CONST, .resetvalue = 0 },
7245     { .name = "SMPRIMAP_EL2", .state = ARM_CP_STATE_AA64,
7246       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 2, .opc2 = 5,
7247       .nv2_redirect_offset = 0x1f8,
7248       .access = PL2_RW, .accessfn = access_smprimap,
7249       .type = ARM_CP_CONST, .resetvalue = 0 },
7250 };
7251 
7252 static void tlbi_aa64_paall_write(CPUARMState *env, const ARMCPRegInfo *ri,
7253                                   uint64_t value)
7254 {
7255     CPUState *cs = env_cpu(env);
7256 
7257     tlb_flush(cs);
7258 }
7259 
7260 static void gpccr_write(CPUARMState *env, const ARMCPRegInfo *ri,
7261                         uint64_t value)
7262 {
7263     /* L0GPTSZ is RO; other bits not mentioned are RES0. */
7264     uint64_t rw_mask = R_GPCCR_PPS_MASK | R_GPCCR_IRGN_MASK |
7265         R_GPCCR_ORGN_MASK | R_GPCCR_SH_MASK | R_GPCCR_PGS_MASK |
7266         R_GPCCR_GPC_MASK | R_GPCCR_GPCP_MASK;
7267 
7268     env->cp15.gpccr_el3 = (value & rw_mask) | (env->cp15.gpccr_el3 & ~rw_mask);
7269 }
7270 
7271 static void gpccr_reset(CPUARMState *env, const ARMCPRegInfo *ri)
7272 {
7273     env->cp15.gpccr_el3 = FIELD_DP64(0, GPCCR, L0GPTSZ,
7274                                      env_archcpu(env)->reset_l0gptsz);
7275 }
7276 
7277 static void tlbi_aa64_paallos_write(CPUARMState *env, const ARMCPRegInfo *ri,
7278                                     uint64_t value)
7279 {
7280     CPUState *cs = env_cpu(env);
7281 
7282     tlb_flush_all_cpus_synced(cs);
7283 }
7284 
7285 static const ARMCPRegInfo rme_reginfo[] = {
7286     { .name = "GPCCR_EL3", .state = ARM_CP_STATE_AA64,
7287       .opc0 = 3, .opc1 = 6, .crn = 2, .crm = 1, .opc2 = 6,
7288       .access = PL3_RW, .writefn = gpccr_write, .resetfn = gpccr_reset,
7289       .fieldoffset = offsetof(CPUARMState, cp15.gpccr_el3) },
7290     { .name = "GPTBR_EL3", .state = ARM_CP_STATE_AA64,
7291       .opc0 = 3, .opc1 = 6, .crn = 2, .crm = 1, .opc2 = 4,
7292       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.gptbr_el3) },
7293     { .name = "MFAR_EL3", .state = ARM_CP_STATE_AA64,
7294       .opc0 = 3, .opc1 = 6, .crn = 6, .crm = 0, .opc2 = 5,
7295       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.mfar_el3) },
7296     { .name = "TLBI_PAALL", .state = ARM_CP_STATE_AA64,
7297       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 7, .opc2 = 4,
7298       .access = PL3_W, .type = ARM_CP_NO_RAW,
7299       .writefn = tlbi_aa64_paall_write },
7300     { .name = "TLBI_PAALLOS", .state = ARM_CP_STATE_AA64,
7301       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 1, .opc2 = 4,
7302       .access = PL3_W, .type = ARM_CP_NO_RAW,
7303       .writefn = tlbi_aa64_paallos_write },
7304     /*
7305      * QEMU does not have a way to invalidate by physical address, thus
7306      * invalidating a range of physical addresses is accomplished by
7307      * flushing all tlb entries in the outer shareable domain,
7308      * just like PAALLOS.
7309      */
7310     { .name = "TLBI_RPALOS", .state = ARM_CP_STATE_AA64,
7311       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 4, .opc2 = 7,
7312       .access = PL3_W, .type = ARM_CP_NO_RAW,
7313       .writefn = tlbi_aa64_paallos_write },
7314     { .name = "TLBI_RPAOS", .state = ARM_CP_STATE_AA64,
7315       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 4, .opc2 = 3,
7316       .access = PL3_W, .type = ARM_CP_NO_RAW,
7317       .writefn = tlbi_aa64_paallos_write },
7318     { .name = "DC_CIPAPA", .state = ARM_CP_STATE_AA64,
7319       .opc0 = 1, .opc1 = 6, .crn = 7, .crm = 14, .opc2 = 1,
7320       .access = PL3_W, .type = ARM_CP_NOP },
7321 };
7322 
7323 static const ARMCPRegInfo rme_mte_reginfo[] = {
7324     { .name = "DC_CIGDPAPA", .state = ARM_CP_STATE_AA64,
7325       .opc0 = 1, .opc1 = 6, .crn = 7, .crm = 14, .opc2 = 5,
7326       .access = PL3_W, .type = ARM_CP_NOP },
7327 };
7328 #endif /* TARGET_AARCH64 */
7329 
7330 static void define_pmu_regs(ARMCPU *cpu)
7331 {
7332     /*
7333      * v7 performance monitor control register: same implementor
7334      * field as main ID register, and we implement four counters in
7335      * addition to the cycle count register.
7336      */
7337     unsigned int i, pmcrn = pmu_num_counters(&cpu->env);
7338     ARMCPRegInfo pmcr = {
7339         .name = "PMCR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 0,
7340         .access = PL0_RW,
7341         .fgt = FGT_PMCR_EL0,
7342         .type = ARM_CP_IO | ARM_CP_ALIAS,
7343         .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmcr),
7344         .accessfn = pmreg_access,
7345         .readfn = pmcr_read, .raw_readfn = raw_read,
7346         .writefn = pmcr_write, .raw_writefn = raw_write,
7347     };
7348     ARMCPRegInfo pmcr64 = {
7349         .name = "PMCR_EL0", .state = ARM_CP_STATE_AA64,
7350         .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 0,
7351         .access = PL0_RW, .accessfn = pmreg_access,
7352         .fgt = FGT_PMCR_EL0,
7353         .type = ARM_CP_IO,
7354         .fieldoffset = offsetof(CPUARMState, cp15.c9_pmcr),
7355         .resetvalue = cpu->isar.reset_pmcr_el0,
7356         .readfn = pmcr_read, .raw_readfn = raw_read,
7357         .writefn = pmcr_write, .raw_writefn = raw_write,
7358     };
7359 
7360     define_one_arm_cp_reg(cpu, &pmcr);
7361     define_one_arm_cp_reg(cpu, &pmcr64);
7362     for (i = 0; i < pmcrn; i++) {
7363         char *pmevcntr_name = g_strdup_printf("PMEVCNTR%d", i);
7364         char *pmevcntr_el0_name = g_strdup_printf("PMEVCNTR%d_EL0", i);
7365         char *pmevtyper_name = g_strdup_printf("PMEVTYPER%d", i);
7366         char *pmevtyper_el0_name = g_strdup_printf("PMEVTYPER%d_EL0", i);
7367         ARMCPRegInfo pmev_regs[] = {
7368             { .name = pmevcntr_name, .cp = 15, .crn = 14,
7369               .crm = 8 | (3 & (i >> 3)), .opc1 = 0, .opc2 = i & 7,
7370               .access = PL0_RW, .type = ARM_CP_IO | ARM_CP_ALIAS,
7371               .fgt = FGT_PMEVCNTRN_EL0,
7372               .readfn = pmevcntr_readfn, .writefn = pmevcntr_writefn,
7373               .accessfn = pmreg_access_xevcntr },
7374             { .name = pmevcntr_el0_name, .state = ARM_CP_STATE_AA64,
7375               .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 8 | (3 & (i >> 3)),
7376               .opc2 = i & 7, .access = PL0_RW, .accessfn = pmreg_access_xevcntr,
7377               .type = ARM_CP_IO,
7378               .fgt = FGT_PMEVCNTRN_EL0,
7379               .readfn = pmevcntr_readfn, .writefn = pmevcntr_writefn,
7380               .raw_readfn = pmevcntr_rawread,
7381               .raw_writefn = pmevcntr_rawwrite },
7382             { .name = pmevtyper_name, .cp = 15, .crn = 14,
7383               .crm = 12 | (3 & (i >> 3)), .opc1 = 0, .opc2 = i & 7,
7384               .access = PL0_RW, .type = ARM_CP_IO | ARM_CP_ALIAS,
7385               .fgt = FGT_PMEVTYPERN_EL0,
7386               .readfn = pmevtyper_readfn, .writefn = pmevtyper_writefn,
7387               .accessfn = pmreg_access },
7388             { .name = pmevtyper_el0_name, .state = ARM_CP_STATE_AA64,
7389               .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 12 | (3 & (i >> 3)),
7390               .opc2 = i & 7, .access = PL0_RW, .accessfn = pmreg_access,
7391               .fgt = FGT_PMEVTYPERN_EL0,
7392               .type = ARM_CP_IO,
7393               .readfn = pmevtyper_readfn, .writefn = pmevtyper_writefn,
7394               .raw_writefn = pmevtyper_rawwrite },
7395         };
7396         define_arm_cp_regs(cpu, pmev_regs);
7397         g_free(pmevcntr_name);
7398         g_free(pmevcntr_el0_name);
7399         g_free(pmevtyper_name);
7400         g_free(pmevtyper_el0_name);
7401     }
7402     if (cpu_isar_feature(aa32_pmuv3p1, cpu)) {
7403         ARMCPRegInfo v81_pmu_regs[] = {
7404             { .name = "PMCEID2", .state = ARM_CP_STATE_AA32,
7405               .cp = 15, .opc1 = 0, .crn = 9, .crm = 14, .opc2 = 4,
7406               .access = PL0_R, .accessfn = pmreg_access, .type = ARM_CP_CONST,
7407               .fgt = FGT_PMCEIDN_EL0,
7408               .resetvalue = extract64(cpu->pmceid0, 32, 32) },
7409             { .name = "PMCEID3", .state = ARM_CP_STATE_AA32,
7410               .cp = 15, .opc1 = 0, .crn = 9, .crm = 14, .opc2 = 5,
7411               .access = PL0_R, .accessfn = pmreg_access, .type = ARM_CP_CONST,
7412               .fgt = FGT_PMCEIDN_EL0,
7413               .resetvalue = extract64(cpu->pmceid1, 32, 32) },
7414         };
7415         define_arm_cp_regs(cpu, v81_pmu_regs);
7416     }
7417     if (cpu_isar_feature(any_pmuv3p4, cpu)) {
7418         static const ARMCPRegInfo v84_pmmir = {
7419             .name = "PMMIR_EL1", .state = ARM_CP_STATE_BOTH,
7420             .opc0 = 3, .opc1 = 0, .crn = 9, .crm = 14, .opc2 = 6,
7421             .access = PL1_R, .accessfn = pmreg_access, .type = ARM_CP_CONST,
7422             .fgt = FGT_PMMIR_EL1,
7423             .resetvalue = 0
7424         };
7425         define_one_arm_cp_reg(cpu, &v84_pmmir);
7426     }
7427 }
7428 
7429 #ifndef CONFIG_USER_ONLY
7430 /*
7431  * We don't know until after realize whether there's a GICv3
7432  * attached, and that is what registers the gicv3 sysregs.
7433  * So we have to fill in the GIC fields in ID_PFR/ID_PFR1_EL1/ID_AA64PFR0_EL1
7434  * at runtime.
7435  */
7436 static uint64_t id_pfr1_read(CPUARMState *env, const ARMCPRegInfo *ri)
7437 {
7438     ARMCPU *cpu = env_archcpu(env);
7439     uint64_t pfr1 = cpu->isar.id_pfr1;
7440 
7441     if (env->gicv3state) {
7442         pfr1 |= 1 << 28;
7443     }
7444     return pfr1;
7445 }
7446 
7447 static uint64_t id_aa64pfr0_read(CPUARMState *env, const ARMCPRegInfo *ri)
7448 {
7449     ARMCPU *cpu = env_archcpu(env);
7450     uint64_t pfr0 = cpu->isar.id_aa64pfr0;
7451 
7452     if (env->gicv3state) {
7453         pfr0 |= 1 << 24;
7454     }
7455     return pfr0;
7456 }
7457 #endif
7458 
7459 /*
7460  * Shared logic between LORID and the rest of the LOR* registers.
7461  * Secure state exclusion has already been dealt with.
7462  */
7463 static CPAccessResult access_lor_ns(CPUARMState *env,
7464                                     const ARMCPRegInfo *ri, bool isread)
7465 {
7466     int el = arm_current_el(env);
7467 
7468     if (el < 2 && (arm_hcr_el2_eff(env) & HCR_TLOR)) {
7469         return CP_ACCESS_TRAP_EL2;
7470     }
7471     if (el < 3 && (env->cp15.scr_el3 & SCR_TLOR)) {
7472         return CP_ACCESS_TRAP_EL3;
7473     }
7474     return CP_ACCESS_OK;
7475 }
7476 
7477 static CPAccessResult access_lor_other(CPUARMState *env,
7478                                        const ARMCPRegInfo *ri, bool isread)
7479 {
7480     if (arm_is_secure_below_el3(env)) {
7481         /* Access denied in secure mode.  */
7482         return CP_ACCESS_TRAP;
7483     }
7484     return access_lor_ns(env, ri, isread);
7485 }
7486 
7487 /*
7488  * A trivial implementation of ARMv8.1-LOR leaves all of these
7489  * registers fixed at 0, which indicates that there are zero
7490  * supported Limited Ordering regions.
7491  */
7492 static const ARMCPRegInfo lor_reginfo[] = {
7493     { .name = "LORSA_EL1", .state = ARM_CP_STATE_AA64,
7494       .opc0 = 3, .opc1 = 0, .crn = 10, .crm = 4, .opc2 = 0,
7495       .access = PL1_RW, .accessfn = access_lor_other,
7496       .fgt = FGT_LORSA_EL1,
7497       .type = ARM_CP_CONST, .resetvalue = 0 },
7498     { .name = "LOREA_EL1", .state = ARM_CP_STATE_AA64,
7499       .opc0 = 3, .opc1 = 0, .crn = 10, .crm = 4, .opc2 = 1,
7500       .access = PL1_RW, .accessfn = access_lor_other,
7501       .fgt = FGT_LOREA_EL1,
7502       .type = ARM_CP_CONST, .resetvalue = 0 },
7503     { .name = "LORN_EL1", .state = ARM_CP_STATE_AA64,
7504       .opc0 = 3, .opc1 = 0, .crn = 10, .crm = 4, .opc2 = 2,
7505       .access = PL1_RW, .accessfn = access_lor_other,
7506       .fgt = FGT_LORN_EL1,
7507       .type = ARM_CP_CONST, .resetvalue = 0 },
7508     { .name = "LORC_EL1", .state = ARM_CP_STATE_AA64,
7509       .opc0 = 3, .opc1 = 0, .crn = 10, .crm = 4, .opc2 = 3,
7510       .access = PL1_RW, .accessfn = access_lor_other,
7511       .fgt = FGT_LORC_EL1,
7512       .type = ARM_CP_CONST, .resetvalue = 0 },
7513     { .name = "LORID_EL1", .state = ARM_CP_STATE_AA64,
7514       .opc0 = 3, .opc1 = 0, .crn = 10, .crm = 4, .opc2 = 7,
7515       .access = PL1_R, .accessfn = access_lor_ns,
7516       .fgt = FGT_LORID_EL1,
7517       .type = ARM_CP_CONST, .resetvalue = 0 },
7518 };
7519 
7520 #ifdef TARGET_AARCH64
7521 static CPAccessResult access_pauth(CPUARMState *env, const ARMCPRegInfo *ri,
7522                                    bool isread)
7523 {
7524     int el = arm_current_el(env);
7525 
7526     if (el < 2 &&
7527         arm_is_el2_enabled(env) &&
7528         !(arm_hcr_el2_eff(env) & HCR_APK)) {
7529         return CP_ACCESS_TRAP_EL2;
7530     }
7531     if (el < 3 &&
7532         arm_feature(env, ARM_FEATURE_EL3) &&
7533         !(env->cp15.scr_el3 & SCR_APK)) {
7534         return CP_ACCESS_TRAP_EL3;
7535     }
7536     return CP_ACCESS_OK;
7537 }
7538 
7539 static const ARMCPRegInfo pauth_reginfo[] = {
7540     { .name = "APDAKEYLO_EL1", .state = ARM_CP_STATE_AA64,
7541       .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 2, .opc2 = 0,
7542       .access = PL1_RW, .accessfn = access_pauth,
7543       .fgt = FGT_APDAKEY,
7544       .fieldoffset = offsetof(CPUARMState, keys.apda.lo) },
7545     { .name = "APDAKEYHI_EL1", .state = ARM_CP_STATE_AA64,
7546       .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 2, .opc2 = 1,
7547       .access = PL1_RW, .accessfn = access_pauth,
7548       .fgt = FGT_APDAKEY,
7549       .fieldoffset = offsetof(CPUARMState, keys.apda.hi) },
7550     { .name = "APDBKEYLO_EL1", .state = ARM_CP_STATE_AA64,
7551       .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 2, .opc2 = 2,
7552       .access = PL1_RW, .accessfn = access_pauth,
7553       .fgt = FGT_APDBKEY,
7554       .fieldoffset = offsetof(CPUARMState, keys.apdb.lo) },
7555     { .name = "APDBKEYHI_EL1", .state = ARM_CP_STATE_AA64,
7556       .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 2, .opc2 = 3,
7557       .access = PL1_RW, .accessfn = access_pauth,
7558       .fgt = FGT_APDBKEY,
7559       .fieldoffset = offsetof(CPUARMState, keys.apdb.hi) },
7560     { .name = "APGAKEYLO_EL1", .state = ARM_CP_STATE_AA64,
7561       .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 3, .opc2 = 0,
7562       .access = PL1_RW, .accessfn = access_pauth,
7563       .fgt = FGT_APGAKEY,
7564       .fieldoffset = offsetof(CPUARMState, keys.apga.lo) },
7565     { .name = "APGAKEYHI_EL1", .state = ARM_CP_STATE_AA64,
7566       .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 3, .opc2 = 1,
7567       .access = PL1_RW, .accessfn = access_pauth,
7568       .fgt = FGT_APGAKEY,
7569       .fieldoffset = offsetof(CPUARMState, keys.apga.hi) },
7570     { .name = "APIAKEYLO_EL1", .state = ARM_CP_STATE_AA64,
7571       .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 1, .opc2 = 0,
7572       .access = PL1_RW, .accessfn = access_pauth,
7573       .fgt = FGT_APIAKEY,
7574       .fieldoffset = offsetof(CPUARMState, keys.apia.lo) },
7575     { .name = "APIAKEYHI_EL1", .state = ARM_CP_STATE_AA64,
7576       .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 1, .opc2 = 1,
7577       .access = PL1_RW, .accessfn = access_pauth,
7578       .fgt = FGT_APIAKEY,
7579       .fieldoffset = offsetof(CPUARMState, keys.apia.hi) },
7580     { .name = "APIBKEYLO_EL1", .state = ARM_CP_STATE_AA64,
7581       .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 1, .opc2 = 2,
7582       .access = PL1_RW, .accessfn = access_pauth,
7583       .fgt = FGT_APIBKEY,
7584       .fieldoffset = offsetof(CPUARMState, keys.apib.lo) },
7585     { .name = "APIBKEYHI_EL1", .state = ARM_CP_STATE_AA64,
7586       .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 1, .opc2 = 3,
7587       .access = PL1_RW, .accessfn = access_pauth,
7588       .fgt = FGT_APIBKEY,
7589       .fieldoffset = offsetof(CPUARMState, keys.apib.hi) },
7590 };
7591 
7592 static const ARMCPRegInfo tlbirange_reginfo[] = {
7593     { .name = "TLBI_RVAE1IS", .state = ARM_CP_STATE_AA64,
7594       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 2, .opc2 = 1,
7595       .access = PL1_W, .accessfn = access_ttlbis, .type = ARM_CP_NO_RAW,
7596       .fgt = FGT_TLBIRVAE1IS,
7597       .writefn = tlbi_aa64_rvae1is_write },
7598     { .name = "TLBI_RVAAE1IS", .state = ARM_CP_STATE_AA64,
7599       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 2, .opc2 = 3,
7600       .access = PL1_W, .accessfn = access_ttlbis, .type = ARM_CP_NO_RAW,
7601       .fgt = FGT_TLBIRVAAE1IS,
7602       .writefn = tlbi_aa64_rvae1is_write },
7603    { .name = "TLBI_RVALE1IS", .state = ARM_CP_STATE_AA64,
7604       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 2, .opc2 = 5,
7605       .access = PL1_W, .accessfn = access_ttlbis, .type = ARM_CP_NO_RAW,
7606       .fgt = FGT_TLBIRVALE1IS,
7607       .writefn = tlbi_aa64_rvae1is_write },
7608     { .name = "TLBI_RVAALE1IS", .state = ARM_CP_STATE_AA64,
7609       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 2, .opc2 = 7,
7610       .access = PL1_W, .accessfn = access_ttlbis, .type = ARM_CP_NO_RAW,
7611       .fgt = FGT_TLBIRVAALE1IS,
7612       .writefn = tlbi_aa64_rvae1is_write },
7613     { .name = "TLBI_RVAE1OS", .state = ARM_CP_STATE_AA64,
7614       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 1,
7615       .access = PL1_W, .accessfn = access_ttlbos, .type = ARM_CP_NO_RAW,
7616       .fgt = FGT_TLBIRVAE1OS,
7617       .writefn = tlbi_aa64_rvae1is_write },
7618     { .name = "TLBI_RVAAE1OS", .state = ARM_CP_STATE_AA64,
7619       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 3,
7620       .access = PL1_W, .accessfn = access_ttlbos, .type = ARM_CP_NO_RAW,
7621       .fgt = FGT_TLBIRVAAE1OS,
7622       .writefn = tlbi_aa64_rvae1is_write },
7623    { .name = "TLBI_RVALE1OS", .state = ARM_CP_STATE_AA64,
7624       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 5,
7625       .access = PL1_W, .accessfn = access_ttlbos, .type = ARM_CP_NO_RAW,
7626       .fgt = FGT_TLBIRVALE1OS,
7627       .writefn = tlbi_aa64_rvae1is_write },
7628     { .name = "TLBI_RVAALE1OS", .state = ARM_CP_STATE_AA64,
7629       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 7,
7630       .access = PL1_W, .accessfn = access_ttlbos, .type = ARM_CP_NO_RAW,
7631       .fgt = FGT_TLBIRVAALE1OS,
7632       .writefn = tlbi_aa64_rvae1is_write },
7633     { .name = "TLBI_RVAE1", .state = ARM_CP_STATE_AA64,
7634       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 1,
7635       .access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
7636       .fgt = FGT_TLBIRVAE1,
7637       .writefn = tlbi_aa64_rvae1_write },
7638     { .name = "TLBI_RVAAE1", .state = ARM_CP_STATE_AA64,
7639       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 3,
7640       .access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
7641       .fgt = FGT_TLBIRVAAE1,
7642       .writefn = tlbi_aa64_rvae1_write },
7643    { .name = "TLBI_RVALE1", .state = ARM_CP_STATE_AA64,
7644       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 5,
7645       .access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
7646       .fgt = FGT_TLBIRVALE1,
7647       .writefn = tlbi_aa64_rvae1_write },
7648     { .name = "TLBI_RVAALE1", .state = ARM_CP_STATE_AA64,
7649       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 7,
7650       .access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
7651       .fgt = FGT_TLBIRVAALE1,
7652       .writefn = tlbi_aa64_rvae1_write },
7653     { .name = "TLBI_RIPAS2E1IS", .state = ARM_CP_STATE_AA64,
7654       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 2,
7655       .access = PL2_W, .type = ARM_CP_NO_RAW,
7656       .writefn = tlbi_aa64_ripas2e1is_write },
7657     { .name = "TLBI_RIPAS2LE1IS", .state = ARM_CP_STATE_AA64,
7658       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 6,
7659       .access = PL2_W, .type = ARM_CP_NO_RAW,
7660       .writefn = tlbi_aa64_ripas2e1is_write },
7661     { .name = "TLBI_RVAE2IS", .state = ARM_CP_STATE_AA64,
7662       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 2, .opc2 = 1,
7663       .access = PL2_W, .type = ARM_CP_NO_RAW | ARM_CP_EL3_NO_EL2_UNDEF,
7664       .writefn = tlbi_aa64_rvae2is_write },
7665    { .name = "TLBI_RVALE2IS", .state = ARM_CP_STATE_AA64,
7666       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 2, .opc2 = 5,
7667       .access = PL2_W, .type = ARM_CP_NO_RAW | ARM_CP_EL3_NO_EL2_UNDEF,
7668       .writefn = tlbi_aa64_rvae2is_write },
7669     { .name = "TLBI_RIPAS2E1", .state = ARM_CP_STATE_AA64,
7670       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 2,
7671       .access = PL2_W, .type = ARM_CP_NO_RAW,
7672       .writefn = tlbi_aa64_ripas2e1_write },
7673     { .name = "TLBI_RIPAS2LE1", .state = ARM_CP_STATE_AA64,
7674       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 6,
7675       .access = PL2_W, .type = ARM_CP_NO_RAW,
7676       .writefn = tlbi_aa64_ripas2e1_write },
7677    { .name = "TLBI_RVAE2OS", .state = ARM_CP_STATE_AA64,
7678       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 5, .opc2 = 1,
7679       .access = PL2_W, .type = ARM_CP_NO_RAW | ARM_CP_EL3_NO_EL2_UNDEF,
7680       .writefn = tlbi_aa64_rvae2is_write },
7681    { .name = "TLBI_RVALE2OS", .state = ARM_CP_STATE_AA64,
7682       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 5, .opc2 = 5,
7683       .access = PL2_W, .type = ARM_CP_NO_RAW | ARM_CP_EL3_NO_EL2_UNDEF,
7684       .writefn = tlbi_aa64_rvae2is_write },
7685     { .name = "TLBI_RVAE2", .state = ARM_CP_STATE_AA64,
7686       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 6, .opc2 = 1,
7687       .access = PL2_W, .type = ARM_CP_NO_RAW | ARM_CP_EL3_NO_EL2_UNDEF,
7688       .writefn = tlbi_aa64_rvae2_write },
7689    { .name = "TLBI_RVALE2", .state = ARM_CP_STATE_AA64,
7690       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 6, .opc2 = 5,
7691       .access = PL2_W, .type = ARM_CP_NO_RAW | ARM_CP_EL3_NO_EL2_UNDEF,
7692       .writefn = tlbi_aa64_rvae2_write },
7693    { .name = "TLBI_RVAE3IS", .state = ARM_CP_STATE_AA64,
7694       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 2, .opc2 = 1,
7695       .access = PL3_W, .type = ARM_CP_NO_RAW,
7696       .writefn = tlbi_aa64_rvae3is_write },
7697    { .name = "TLBI_RVALE3IS", .state = ARM_CP_STATE_AA64,
7698       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 2, .opc2 = 5,
7699       .access = PL3_W, .type = ARM_CP_NO_RAW,
7700       .writefn = tlbi_aa64_rvae3is_write },
7701    { .name = "TLBI_RVAE3OS", .state = ARM_CP_STATE_AA64,
7702       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 5, .opc2 = 1,
7703       .access = PL3_W, .type = ARM_CP_NO_RAW,
7704       .writefn = tlbi_aa64_rvae3is_write },
7705    { .name = "TLBI_RVALE3OS", .state = ARM_CP_STATE_AA64,
7706       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 5, .opc2 = 5,
7707       .access = PL3_W, .type = ARM_CP_NO_RAW,
7708       .writefn = tlbi_aa64_rvae3is_write },
7709    { .name = "TLBI_RVAE3", .state = ARM_CP_STATE_AA64,
7710       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 6, .opc2 = 1,
7711       .access = PL3_W, .type = ARM_CP_NO_RAW,
7712       .writefn = tlbi_aa64_rvae3_write },
7713    { .name = "TLBI_RVALE3", .state = ARM_CP_STATE_AA64,
7714       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 6, .opc2 = 5,
7715       .access = PL3_W, .type = ARM_CP_NO_RAW,
7716       .writefn = tlbi_aa64_rvae3_write },
7717 };
7718 
7719 static const ARMCPRegInfo tlbios_reginfo[] = {
7720     { .name = "TLBI_VMALLE1OS", .state = ARM_CP_STATE_AA64,
7721       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 1, .opc2 = 0,
7722       .access = PL1_W, .accessfn = access_ttlbos, .type = ARM_CP_NO_RAW,
7723       .fgt = FGT_TLBIVMALLE1OS,
7724       .writefn = tlbi_aa64_vmalle1is_write },
7725     { .name = "TLBI_VAE1OS", .state = ARM_CP_STATE_AA64,
7726       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 1, .opc2 = 1,
7727       .fgt = FGT_TLBIVAE1OS,
7728       .access = PL1_W, .accessfn = access_ttlbos, .type = ARM_CP_NO_RAW,
7729       .writefn = tlbi_aa64_vae1is_write },
7730     { .name = "TLBI_ASIDE1OS", .state = ARM_CP_STATE_AA64,
7731       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 1, .opc2 = 2,
7732       .access = PL1_W, .accessfn = access_ttlbos, .type = ARM_CP_NO_RAW,
7733       .fgt = FGT_TLBIASIDE1OS,
7734       .writefn = tlbi_aa64_vmalle1is_write },
7735     { .name = "TLBI_VAAE1OS", .state = ARM_CP_STATE_AA64,
7736       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 1, .opc2 = 3,
7737       .access = PL1_W, .accessfn = access_ttlbos, .type = ARM_CP_NO_RAW,
7738       .fgt = FGT_TLBIVAAE1OS,
7739       .writefn = tlbi_aa64_vae1is_write },
7740     { .name = "TLBI_VALE1OS", .state = ARM_CP_STATE_AA64,
7741       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 1, .opc2 = 5,
7742       .access = PL1_W, .accessfn = access_ttlbos, .type = ARM_CP_NO_RAW,
7743       .fgt = FGT_TLBIVALE1OS,
7744       .writefn = tlbi_aa64_vae1is_write },
7745     { .name = "TLBI_VAALE1OS", .state = ARM_CP_STATE_AA64,
7746       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 1, .opc2 = 7,
7747       .access = PL1_W, .accessfn = access_ttlbos, .type = ARM_CP_NO_RAW,
7748       .fgt = FGT_TLBIVAALE1OS,
7749       .writefn = tlbi_aa64_vae1is_write },
7750     { .name = "TLBI_ALLE2OS", .state = ARM_CP_STATE_AA64,
7751       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 1, .opc2 = 0,
7752       .access = PL2_W, .type = ARM_CP_NO_RAW | ARM_CP_EL3_NO_EL2_UNDEF,
7753       .writefn = tlbi_aa64_alle2is_write },
7754     { .name = "TLBI_VAE2OS", .state = ARM_CP_STATE_AA64,
7755       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 1, .opc2 = 1,
7756       .access = PL2_W, .type = ARM_CP_NO_RAW | ARM_CP_EL3_NO_EL2_UNDEF,
7757       .writefn = tlbi_aa64_vae2is_write },
7758    { .name = "TLBI_ALLE1OS", .state = ARM_CP_STATE_AA64,
7759       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 1, .opc2 = 4,
7760       .access = PL2_W, .type = ARM_CP_NO_RAW,
7761       .writefn = tlbi_aa64_alle1is_write },
7762     { .name = "TLBI_VALE2OS", .state = ARM_CP_STATE_AA64,
7763       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 1, .opc2 = 5,
7764       .access = PL2_W, .type = ARM_CP_NO_RAW | ARM_CP_EL3_NO_EL2_UNDEF,
7765       .writefn = tlbi_aa64_vae2is_write },
7766     { .name = "TLBI_VMALLS12E1OS", .state = ARM_CP_STATE_AA64,
7767       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 1, .opc2 = 6,
7768       .access = PL2_W, .type = ARM_CP_NO_RAW,
7769       .writefn = tlbi_aa64_alle1is_write },
7770     { .name = "TLBI_IPAS2E1OS", .state = ARM_CP_STATE_AA64,
7771       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 0,
7772       .access = PL2_W, .type = ARM_CP_NOP },
7773     { .name = "TLBI_RIPAS2E1OS", .state = ARM_CP_STATE_AA64,
7774       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 3,
7775       .access = PL2_W, .type = ARM_CP_NOP },
7776     { .name = "TLBI_IPAS2LE1OS", .state = ARM_CP_STATE_AA64,
7777       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 4,
7778       .access = PL2_W, .type = ARM_CP_NOP },
7779     { .name = "TLBI_RIPAS2LE1OS", .state = ARM_CP_STATE_AA64,
7780       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 7,
7781       .access = PL2_W, .type = ARM_CP_NOP },
7782     { .name = "TLBI_ALLE3OS", .state = ARM_CP_STATE_AA64,
7783       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 1, .opc2 = 0,
7784       .access = PL3_W, .type = ARM_CP_NO_RAW,
7785       .writefn = tlbi_aa64_alle3is_write },
7786     { .name = "TLBI_VAE3OS", .state = ARM_CP_STATE_AA64,
7787       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 1, .opc2 = 1,
7788       .access = PL3_W, .type = ARM_CP_NO_RAW,
7789       .writefn = tlbi_aa64_vae3is_write },
7790     { .name = "TLBI_VALE3OS", .state = ARM_CP_STATE_AA64,
7791       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 1, .opc2 = 5,
7792       .access = PL3_W, .type = ARM_CP_NO_RAW,
7793       .writefn = tlbi_aa64_vae3is_write },
7794 };
7795 
7796 static uint64_t rndr_readfn(CPUARMState *env, const ARMCPRegInfo *ri)
7797 {
7798     Error *err = NULL;
7799     uint64_t ret;
7800 
7801     /* Success sets NZCV = 0000.  */
7802     env->NF = env->CF = env->VF = 0, env->ZF = 1;
7803 
7804     if (qemu_guest_getrandom(&ret, sizeof(ret), &err) < 0) {
7805         /*
7806          * ??? Failed, for unknown reasons in the crypto subsystem.
7807          * The best we can do is log the reason and return the
7808          * timed-out indication to the guest.  There is no reason
7809          * we know to expect this failure to be transitory, so the
7810          * guest may well hang retrying the operation.
7811          */
7812         qemu_log_mask(LOG_UNIMP, "%s: Crypto failure: %s",
7813                       ri->name, error_get_pretty(err));
7814         error_free(err);
7815 
7816         env->ZF = 0; /* NZCF = 0100 */
7817         return 0;
7818     }
7819     return ret;
7820 }
7821 
7822 /* We do not support re-seeding, so the two registers operate the same.  */
7823 static const ARMCPRegInfo rndr_reginfo[] = {
7824     { .name = "RNDR", .state = ARM_CP_STATE_AA64,
7825       .type = ARM_CP_NO_RAW | ARM_CP_SUPPRESS_TB_END | ARM_CP_IO,
7826       .opc0 = 3, .opc1 = 3, .crn = 2, .crm = 4, .opc2 = 0,
7827       .access = PL0_R, .readfn = rndr_readfn },
7828     { .name = "RNDRRS", .state = ARM_CP_STATE_AA64,
7829       .type = ARM_CP_NO_RAW | ARM_CP_SUPPRESS_TB_END | ARM_CP_IO,
7830       .opc0 = 3, .opc1 = 3, .crn = 2, .crm = 4, .opc2 = 1,
7831       .access = PL0_R, .readfn = rndr_readfn },
7832 };
7833 
7834 static void dccvap_writefn(CPUARMState *env, const ARMCPRegInfo *opaque,
7835                           uint64_t value)
7836 {
7837 #ifdef CONFIG_TCG
7838     ARMCPU *cpu = env_archcpu(env);
7839     /* CTR_EL0 System register -> DminLine, bits [19:16] */
7840     uint64_t dline_size = 4 << ((cpu->ctr >> 16) & 0xF);
7841     uint64_t vaddr_in = (uint64_t) value;
7842     uint64_t vaddr = vaddr_in & ~(dline_size - 1);
7843     void *haddr;
7844     int mem_idx = cpu_mmu_index(env, false);
7845 
7846     /* This won't be crossing page boundaries */
7847     haddr = probe_read(env, vaddr, dline_size, mem_idx, GETPC());
7848     if (haddr) {
7849 #ifndef CONFIG_USER_ONLY
7850 
7851         ram_addr_t offset;
7852         MemoryRegion *mr;
7853 
7854         /* RCU lock is already being held */
7855         mr = memory_region_from_host(haddr, &offset);
7856 
7857         if (mr) {
7858             memory_region_writeback(mr, offset, dline_size);
7859         }
7860 #endif /*CONFIG_USER_ONLY*/
7861     }
7862 #else
7863     /* Handled by hardware accelerator. */
7864     g_assert_not_reached();
7865 #endif /* CONFIG_TCG */
7866 }
7867 
7868 static const ARMCPRegInfo dcpop_reg[] = {
7869     { .name = "DC_CVAP", .state = ARM_CP_STATE_AA64,
7870       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 12, .opc2 = 1,
7871       .access = PL0_W, .type = ARM_CP_NO_RAW | ARM_CP_SUPPRESS_TB_END,
7872       .fgt = FGT_DCCVAP,
7873       .accessfn = aa64_cacheop_poc_access, .writefn = dccvap_writefn },
7874 };
7875 
7876 static const ARMCPRegInfo dcpodp_reg[] = {
7877     { .name = "DC_CVADP", .state = ARM_CP_STATE_AA64,
7878       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 13, .opc2 = 1,
7879       .access = PL0_W, .type = ARM_CP_NO_RAW | ARM_CP_SUPPRESS_TB_END,
7880       .fgt = FGT_DCCVADP,
7881       .accessfn = aa64_cacheop_poc_access, .writefn = dccvap_writefn },
7882 };
7883 
7884 static CPAccessResult access_aa64_tid5(CPUARMState *env, const ARMCPRegInfo *ri,
7885                                        bool isread)
7886 {
7887     if ((arm_current_el(env) < 2) && (arm_hcr_el2_eff(env) & HCR_TID5)) {
7888         return CP_ACCESS_TRAP_EL2;
7889     }
7890 
7891     return CP_ACCESS_OK;
7892 }
7893 
7894 static CPAccessResult access_mte(CPUARMState *env, const ARMCPRegInfo *ri,
7895                                  bool isread)
7896 {
7897     int el = arm_current_el(env);
7898     if (el < 2 && arm_is_el2_enabled(env)) {
7899         uint64_t hcr = arm_hcr_el2_eff(env);
7900         if (!(hcr & HCR_ATA) && (!(hcr & HCR_E2H) || !(hcr & HCR_TGE))) {
7901             return CP_ACCESS_TRAP_EL2;
7902         }
7903     }
7904     if (el < 3 &&
7905         arm_feature(env, ARM_FEATURE_EL3) &&
7906         !(env->cp15.scr_el3 & SCR_ATA)) {
7907         return CP_ACCESS_TRAP_EL3;
7908     }
7909     return CP_ACCESS_OK;
7910 }
7911 
7912 static CPAccessResult access_tfsr_el1(CPUARMState *env, const ARMCPRegInfo *ri,
7913                                       bool isread)
7914 {
7915     CPAccessResult nv1 = access_nv1(env, ri, isread);
7916 
7917     if (nv1 != CP_ACCESS_OK) {
7918         return nv1;
7919     }
7920     return access_mte(env, ri, isread);
7921 }
7922 
7923 static CPAccessResult access_tfsr_el2(CPUARMState *env, const ARMCPRegInfo *ri,
7924                                       bool isread)
7925 {
7926     /*
7927      * TFSR_EL2: similar to generic access_mte(), but we need to
7928      * account for FEAT_NV. At EL1 this must be a FEAT_NV access;
7929      * if NV2 is enabled then we will redirect this to TFSR_EL1
7930      * after doing the HCR and SCR ATA traps; otherwise this will
7931      * be a trap to EL2 and the HCR/SCR traps do not apply.
7932      */
7933     int el = arm_current_el(env);
7934 
7935     if (el == 1 && (arm_hcr_el2_eff(env) & HCR_NV2)) {
7936         return CP_ACCESS_OK;
7937     }
7938     if (el < 2 && arm_is_el2_enabled(env)) {
7939         uint64_t hcr = arm_hcr_el2_eff(env);
7940         if (!(hcr & HCR_ATA) && (!(hcr & HCR_E2H) || !(hcr & HCR_TGE))) {
7941             return CP_ACCESS_TRAP_EL2;
7942         }
7943     }
7944     if (el < 3 &&
7945         arm_feature(env, ARM_FEATURE_EL3) &&
7946         !(env->cp15.scr_el3 & SCR_ATA)) {
7947         return CP_ACCESS_TRAP_EL3;
7948     }
7949     return CP_ACCESS_OK;
7950 }
7951 
7952 static uint64_t tco_read(CPUARMState *env, const ARMCPRegInfo *ri)
7953 {
7954     return env->pstate & PSTATE_TCO;
7955 }
7956 
7957 static void tco_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t val)
7958 {
7959     env->pstate = (env->pstate & ~PSTATE_TCO) | (val & PSTATE_TCO);
7960 }
7961 
7962 static const ARMCPRegInfo mte_reginfo[] = {
7963     { .name = "TFSRE0_EL1", .state = ARM_CP_STATE_AA64,
7964       .opc0 = 3, .opc1 = 0, .crn = 5, .crm = 6, .opc2 = 1,
7965       .access = PL1_RW, .accessfn = access_mte,
7966       .fieldoffset = offsetof(CPUARMState, cp15.tfsr_el[0]) },
7967     { .name = "TFSR_EL1", .state = ARM_CP_STATE_AA64,
7968       .opc0 = 3, .opc1 = 0, .crn = 5, .crm = 6, .opc2 = 0,
7969       .access = PL1_RW, .accessfn = access_tfsr_el1,
7970       .nv2_redirect_offset = 0x190 | NV2_REDIR_NV1,
7971       .fieldoffset = offsetof(CPUARMState, cp15.tfsr_el[1]) },
7972     { .name = "TFSR_EL2", .state = ARM_CP_STATE_AA64,
7973       .type = ARM_CP_NV2_REDIRECT,
7974       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 6, .opc2 = 0,
7975       .access = PL2_RW, .accessfn = access_tfsr_el2,
7976       .fieldoffset = offsetof(CPUARMState, cp15.tfsr_el[2]) },
7977     { .name = "TFSR_EL3", .state = ARM_CP_STATE_AA64,
7978       .opc0 = 3, .opc1 = 6, .crn = 5, .crm = 6, .opc2 = 0,
7979       .access = PL3_RW,
7980       .fieldoffset = offsetof(CPUARMState, cp15.tfsr_el[3]) },
7981     { .name = "RGSR_EL1", .state = ARM_CP_STATE_AA64,
7982       .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 5,
7983       .access = PL1_RW, .accessfn = access_mte,
7984       .fieldoffset = offsetof(CPUARMState, cp15.rgsr_el1) },
7985     { .name = "GCR_EL1", .state = ARM_CP_STATE_AA64,
7986       .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 6,
7987       .access = PL1_RW, .accessfn = access_mte,
7988       .fieldoffset = offsetof(CPUARMState, cp15.gcr_el1) },
7989     { .name = "TCO", .state = ARM_CP_STATE_AA64,
7990       .opc0 = 3, .opc1 = 3, .crn = 4, .crm = 2, .opc2 = 7,
7991       .type = ARM_CP_NO_RAW,
7992       .access = PL0_RW, .readfn = tco_read, .writefn = tco_write },
7993     { .name = "DC_IGVAC", .state = ARM_CP_STATE_AA64,
7994       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 3,
7995       .type = ARM_CP_NOP, .access = PL1_W,
7996       .fgt = FGT_DCIVAC,
7997       .accessfn = aa64_cacheop_poc_access },
7998     { .name = "DC_IGSW", .state = ARM_CP_STATE_AA64,
7999       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 4,
8000       .fgt = FGT_DCISW,
8001       .type = ARM_CP_NOP, .access = PL1_W, .accessfn = access_tsw },
8002     { .name = "DC_IGDVAC", .state = ARM_CP_STATE_AA64,
8003       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 5,
8004       .type = ARM_CP_NOP, .access = PL1_W,
8005       .fgt = FGT_DCIVAC,
8006       .accessfn = aa64_cacheop_poc_access },
8007     { .name = "DC_IGDSW", .state = ARM_CP_STATE_AA64,
8008       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 6,
8009       .fgt = FGT_DCISW,
8010       .type = ARM_CP_NOP, .access = PL1_W, .accessfn = access_tsw },
8011     { .name = "DC_CGSW", .state = ARM_CP_STATE_AA64,
8012       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 10, .opc2 = 4,
8013       .fgt = FGT_DCCSW,
8014       .type = ARM_CP_NOP, .access = PL1_W, .accessfn = access_tsw },
8015     { .name = "DC_CGDSW", .state = ARM_CP_STATE_AA64,
8016       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 10, .opc2 = 6,
8017       .fgt = FGT_DCCSW,
8018       .type = ARM_CP_NOP, .access = PL1_W, .accessfn = access_tsw },
8019     { .name = "DC_CIGSW", .state = ARM_CP_STATE_AA64,
8020       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 14, .opc2 = 4,
8021       .fgt = FGT_DCCISW,
8022       .type = ARM_CP_NOP, .access = PL1_W, .accessfn = access_tsw },
8023     { .name = "DC_CIGDSW", .state = ARM_CP_STATE_AA64,
8024       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 14, .opc2 = 6,
8025       .fgt = FGT_DCCISW,
8026       .type = ARM_CP_NOP, .access = PL1_W, .accessfn = access_tsw },
8027 };
8028 
8029 static const ARMCPRegInfo mte_tco_ro_reginfo[] = {
8030     { .name = "TCO", .state = ARM_CP_STATE_AA64,
8031       .opc0 = 3, .opc1 = 3, .crn = 4, .crm = 2, .opc2 = 7,
8032       .type = ARM_CP_CONST, .access = PL0_RW, },
8033 };
8034 
8035 static const ARMCPRegInfo mte_el0_cacheop_reginfo[] = {
8036     { .name = "DC_CGVAC", .state = ARM_CP_STATE_AA64,
8037       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 10, .opc2 = 3,
8038       .type = ARM_CP_NOP, .access = PL0_W,
8039       .fgt = FGT_DCCVAC,
8040       .accessfn = aa64_cacheop_poc_access },
8041     { .name = "DC_CGDVAC", .state = ARM_CP_STATE_AA64,
8042       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 10, .opc2 = 5,
8043       .type = ARM_CP_NOP, .access = PL0_W,
8044       .fgt = FGT_DCCVAC,
8045       .accessfn = aa64_cacheop_poc_access },
8046     { .name = "DC_CGVAP", .state = ARM_CP_STATE_AA64,
8047       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 12, .opc2 = 3,
8048       .type = ARM_CP_NOP, .access = PL0_W,
8049       .fgt = FGT_DCCVAP,
8050       .accessfn = aa64_cacheop_poc_access },
8051     { .name = "DC_CGDVAP", .state = ARM_CP_STATE_AA64,
8052       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 12, .opc2 = 5,
8053       .type = ARM_CP_NOP, .access = PL0_W,
8054       .fgt = FGT_DCCVAP,
8055       .accessfn = aa64_cacheop_poc_access },
8056     { .name = "DC_CGVADP", .state = ARM_CP_STATE_AA64,
8057       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 13, .opc2 = 3,
8058       .type = ARM_CP_NOP, .access = PL0_W,
8059       .fgt = FGT_DCCVADP,
8060       .accessfn = aa64_cacheop_poc_access },
8061     { .name = "DC_CGDVADP", .state = ARM_CP_STATE_AA64,
8062       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 13, .opc2 = 5,
8063       .type = ARM_CP_NOP, .access = PL0_W,
8064       .fgt = FGT_DCCVADP,
8065       .accessfn = aa64_cacheop_poc_access },
8066     { .name = "DC_CIGVAC", .state = ARM_CP_STATE_AA64,
8067       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 14, .opc2 = 3,
8068       .type = ARM_CP_NOP, .access = PL0_W,
8069       .fgt = FGT_DCCIVAC,
8070       .accessfn = aa64_cacheop_poc_access },
8071     { .name = "DC_CIGDVAC", .state = ARM_CP_STATE_AA64,
8072       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 14, .opc2 = 5,
8073       .type = ARM_CP_NOP, .access = PL0_W,
8074       .fgt = FGT_DCCIVAC,
8075       .accessfn = aa64_cacheop_poc_access },
8076     { .name = "DC_GVA", .state = ARM_CP_STATE_AA64,
8077       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 4, .opc2 = 3,
8078       .access = PL0_W, .type = ARM_CP_DC_GVA,
8079 #ifndef CONFIG_USER_ONLY
8080       /* Avoid overhead of an access check that always passes in user-mode */
8081       .accessfn = aa64_zva_access,
8082       .fgt = FGT_DCZVA,
8083 #endif
8084     },
8085     { .name = "DC_GZVA", .state = ARM_CP_STATE_AA64,
8086       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 4, .opc2 = 4,
8087       .access = PL0_W, .type = ARM_CP_DC_GZVA,
8088 #ifndef CONFIG_USER_ONLY
8089       /* Avoid overhead of an access check that always passes in user-mode */
8090       .accessfn = aa64_zva_access,
8091       .fgt = FGT_DCZVA,
8092 #endif
8093     },
8094 };
8095 
8096 static CPAccessResult access_scxtnum(CPUARMState *env, const ARMCPRegInfo *ri,
8097                                      bool isread)
8098 {
8099     uint64_t hcr = arm_hcr_el2_eff(env);
8100     int el = arm_current_el(env);
8101 
8102     if (el == 0 && !((hcr & HCR_E2H) && (hcr & HCR_TGE))) {
8103         if (env->cp15.sctlr_el[1] & SCTLR_TSCXT) {
8104             if (hcr & HCR_TGE) {
8105                 return CP_ACCESS_TRAP_EL2;
8106             }
8107             return CP_ACCESS_TRAP;
8108         }
8109     } else if (el < 2 && (env->cp15.sctlr_el[2] & SCTLR_TSCXT)) {
8110         return CP_ACCESS_TRAP_EL2;
8111     }
8112     if (el < 2 && arm_is_el2_enabled(env) && !(hcr & HCR_ENSCXT)) {
8113         return CP_ACCESS_TRAP_EL2;
8114     }
8115     if (el < 3
8116         && arm_feature(env, ARM_FEATURE_EL3)
8117         && !(env->cp15.scr_el3 & SCR_ENSCXT)) {
8118         return CP_ACCESS_TRAP_EL3;
8119     }
8120     return CP_ACCESS_OK;
8121 }
8122 
8123 static CPAccessResult access_scxtnum_el1(CPUARMState *env,
8124                                          const ARMCPRegInfo *ri,
8125                                          bool isread)
8126 {
8127     CPAccessResult nv1 = access_nv1(env, ri, isread);
8128 
8129     if (nv1 != CP_ACCESS_OK) {
8130         return nv1;
8131     }
8132     return access_scxtnum(env, ri, isread);
8133 }
8134 
8135 static const ARMCPRegInfo scxtnum_reginfo[] = {
8136     { .name = "SCXTNUM_EL0", .state = ARM_CP_STATE_AA64,
8137       .opc0 = 3, .opc1 = 3, .crn = 13, .crm = 0, .opc2 = 7,
8138       .access = PL0_RW, .accessfn = access_scxtnum,
8139       .fgt = FGT_SCXTNUM_EL0,
8140       .fieldoffset = offsetof(CPUARMState, scxtnum_el[0]) },
8141     { .name = "SCXTNUM_EL1", .state = ARM_CP_STATE_AA64,
8142       .opc0 = 3, .opc1 = 0, .crn = 13, .crm = 0, .opc2 = 7,
8143       .access = PL1_RW, .accessfn = access_scxtnum_el1,
8144       .fgt = FGT_SCXTNUM_EL1,
8145       .nv2_redirect_offset = 0x188 | NV2_REDIR_NV1,
8146       .fieldoffset = offsetof(CPUARMState, scxtnum_el[1]) },
8147     { .name = "SCXTNUM_EL2", .state = ARM_CP_STATE_AA64,
8148       .opc0 = 3, .opc1 = 4, .crn = 13, .crm = 0, .opc2 = 7,
8149       .access = PL2_RW, .accessfn = access_scxtnum,
8150       .fieldoffset = offsetof(CPUARMState, scxtnum_el[2]) },
8151     { .name = "SCXTNUM_EL3", .state = ARM_CP_STATE_AA64,
8152       .opc0 = 3, .opc1 = 6, .crn = 13, .crm = 0, .opc2 = 7,
8153       .access = PL3_RW,
8154       .fieldoffset = offsetof(CPUARMState, scxtnum_el[3]) },
8155 };
8156 
8157 static CPAccessResult access_fgt(CPUARMState *env, const ARMCPRegInfo *ri,
8158                                  bool isread)
8159 {
8160     if (arm_current_el(env) == 2 &&
8161         arm_feature(env, ARM_FEATURE_EL3) && !(env->cp15.scr_el3 & SCR_FGTEN)) {
8162         return CP_ACCESS_TRAP_EL3;
8163     }
8164     return CP_ACCESS_OK;
8165 }
8166 
8167 static const ARMCPRegInfo fgt_reginfo[] = {
8168     { .name = "HFGRTR_EL2", .state = ARM_CP_STATE_AA64,
8169       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 4,
8170       .nv2_redirect_offset = 0x1b8,
8171       .access = PL2_RW, .accessfn = access_fgt,
8172       .fieldoffset = offsetof(CPUARMState, cp15.fgt_read[FGTREG_HFGRTR]) },
8173     { .name = "HFGWTR_EL2", .state = ARM_CP_STATE_AA64,
8174       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 5,
8175       .nv2_redirect_offset = 0x1c0,
8176       .access = PL2_RW, .accessfn = access_fgt,
8177       .fieldoffset = offsetof(CPUARMState, cp15.fgt_write[FGTREG_HFGWTR]) },
8178     { .name = "HDFGRTR_EL2", .state = ARM_CP_STATE_AA64,
8179       .opc0 = 3, .opc1 = 4, .crn = 3, .crm = 1, .opc2 = 4,
8180       .nv2_redirect_offset = 0x1d0,
8181       .access = PL2_RW, .accessfn = access_fgt,
8182       .fieldoffset = offsetof(CPUARMState, cp15.fgt_read[FGTREG_HDFGRTR]) },
8183     { .name = "HDFGWTR_EL2", .state = ARM_CP_STATE_AA64,
8184       .opc0 = 3, .opc1 = 4, .crn = 3, .crm = 1, .opc2 = 5,
8185       .nv2_redirect_offset = 0x1d8,
8186       .access = PL2_RW, .accessfn = access_fgt,
8187       .fieldoffset = offsetof(CPUARMState, cp15.fgt_write[FGTREG_HDFGWTR]) },
8188     { .name = "HFGITR_EL2", .state = ARM_CP_STATE_AA64,
8189       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 6,
8190       .nv2_redirect_offset = 0x1c8,
8191       .access = PL2_RW, .accessfn = access_fgt,
8192       .fieldoffset = offsetof(CPUARMState, cp15.fgt_exec[FGTREG_HFGITR]) },
8193 };
8194 
8195 static void vncr_write(CPUARMState *env, const ARMCPRegInfo *ri,
8196                        uint64_t value)
8197 {
8198     /*
8199      * Clear the RES0 bottom 12 bits; this means at runtime we can guarantee
8200      * that VNCR_EL2 + offset is 64-bit aligned. We don't need to do anything
8201      * about the RESS bits at the top -- we choose the "generate an EL2
8202      * translation abort on use" CONSTRAINED UNPREDICTABLE option (i.e. let
8203      * the ptw.c code detect the resulting invalid address).
8204      */
8205     env->cp15.vncr_el2 = value & ~0xfffULL;
8206 }
8207 
8208 static const ARMCPRegInfo nv2_reginfo[] = {
8209     { .name = "VNCR_EL2", .state = ARM_CP_STATE_AA64,
8210       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 2, .opc2 = 0,
8211       .access = PL2_RW,
8212       .writefn = vncr_write,
8213       .nv2_redirect_offset = 0xb0,
8214       .fieldoffset = offsetof(CPUARMState, cp15.vncr_el2) },
8215 };
8216 
8217 #endif /* TARGET_AARCH64 */
8218 
8219 static CPAccessResult access_predinv(CPUARMState *env, const ARMCPRegInfo *ri,
8220                                      bool isread)
8221 {
8222     int el = arm_current_el(env);
8223 
8224     if (el == 0) {
8225         uint64_t sctlr = arm_sctlr(env, el);
8226         if (!(sctlr & SCTLR_EnRCTX)) {
8227             return CP_ACCESS_TRAP;
8228         }
8229     } else if (el == 1) {
8230         uint64_t hcr = arm_hcr_el2_eff(env);
8231         if (hcr & HCR_NV) {
8232             return CP_ACCESS_TRAP_EL2;
8233         }
8234     }
8235     return CP_ACCESS_OK;
8236 }
8237 
8238 static const ARMCPRegInfo predinv_reginfo[] = {
8239     { .name = "CFP_RCTX", .state = ARM_CP_STATE_AA64,
8240       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 3, .opc2 = 4,
8241       .fgt = FGT_CFPRCTX,
8242       .type = ARM_CP_NOP, .access = PL0_W, .accessfn = access_predinv },
8243     { .name = "DVP_RCTX", .state = ARM_CP_STATE_AA64,
8244       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 3, .opc2 = 5,
8245       .fgt = FGT_DVPRCTX,
8246       .type = ARM_CP_NOP, .access = PL0_W, .accessfn = access_predinv },
8247     { .name = "CPP_RCTX", .state = ARM_CP_STATE_AA64,
8248       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 3, .opc2 = 7,
8249       .fgt = FGT_CPPRCTX,
8250       .type = ARM_CP_NOP, .access = PL0_W, .accessfn = access_predinv },
8251     /*
8252      * Note the AArch32 opcodes have a different OPC1.
8253      */
8254     { .name = "CFPRCTX", .state = ARM_CP_STATE_AA32,
8255       .cp = 15, .opc1 = 0, .crn = 7, .crm = 3, .opc2 = 4,
8256       .fgt = FGT_CFPRCTX,
8257       .type = ARM_CP_NOP, .access = PL0_W, .accessfn = access_predinv },
8258     { .name = "DVPRCTX", .state = ARM_CP_STATE_AA32,
8259       .cp = 15, .opc1 = 0, .crn = 7, .crm = 3, .opc2 = 5,
8260       .fgt = FGT_DVPRCTX,
8261       .type = ARM_CP_NOP, .access = PL0_W, .accessfn = access_predinv },
8262     { .name = "CPPRCTX", .state = ARM_CP_STATE_AA32,
8263       .cp = 15, .opc1 = 0, .crn = 7, .crm = 3, .opc2 = 7,
8264       .fgt = FGT_CPPRCTX,
8265       .type = ARM_CP_NOP, .access = PL0_W, .accessfn = access_predinv },
8266 };
8267 
8268 static uint64_t ccsidr2_read(CPUARMState *env, const ARMCPRegInfo *ri)
8269 {
8270     /* Read the high 32 bits of the current CCSIDR */
8271     return extract64(ccsidr_read(env, ri), 32, 32);
8272 }
8273 
8274 static const ARMCPRegInfo ccsidr2_reginfo[] = {
8275     { .name = "CCSIDR2", .state = ARM_CP_STATE_BOTH,
8276       .opc0 = 3, .opc1 = 1, .crn = 0, .crm = 0, .opc2 = 2,
8277       .access = PL1_R,
8278       .accessfn = access_tid4,
8279       .readfn = ccsidr2_read, .type = ARM_CP_NO_RAW },
8280 };
8281 
8282 static CPAccessResult access_aa64_tid3(CPUARMState *env, const ARMCPRegInfo *ri,
8283                                        bool isread)
8284 {
8285     if ((arm_current_el(env) < 2) && (arm_hcr_el2_eff(env) & HCR_TID3)) {
8286         return CP_ACCESS_TRAP_EL2;
8287     }
8288 
8289     return CP_ACCESS_OK;
8290 }
8291 
8292 static CPAccessResult access_aa32_tid3(CPUARMState *env, const ARMCPRegInfo *ri,
8293                                        bool isread)
8294 {
8295     if (arm_feature(env, ARM_FEATURE_V8)) {
8296         return access_aa64_tid3(env, ri, isread);
8297     }
8298 
8299     return CP_ACCESS_OK;
8300 }
8301 
8302 static CPAccessResult access_jazelle(CPUARMState *env, const ARMCPRegInfo *ri,
8303                                      bool isread)
8304 {
8305     if (arm_current_el(env) == 1 && (arm_hcr_el2_eff(env) & HCR_TID0)) {
8306         return CP_ACCESS_TRAP_EL2;
8307     }
8308 
8309     return CP_ACCESS_OK;
8310 }
8311 
8312 static CPAccessResult access_joscr_jmcr(CPUARMState *env,
8313                                         const ARMCPRegInfo *ri, bool isread)
8314 {
8315     /*
8316      * HSTR.TJDBX traps JOSCR and JMCR accesses, but it exists only
8317      * in v7A, not in v8A.
8318      */
8319     if (!arm_feature(env, ARM_FEATURE_V8) &&
8320         arm_current_el(env) < 2 && !arm_is_secure_below_el3(env) &&
8321         (env->cp15.hstr_el2 & HSTR_TJDBX)) {
8322         return CP_ACCESS_TRAP_EL2;
8323     }
8324     return CP_ACCESS_OK;
8325 }
8326 
8327 static const ARMCPRegInfo jazelle_regs[] = {
8328     { .name = "JIDR",
8329       .cp = 14, .crn = 0, .crm = 0, .opc1 = 7, .opc2 = 0,
8330       .access = PL1_R, .accessfn = access_jazelle,
8331       .type = ARM_CP_CONST, .resetvalue = 0 },
8332     { .name = "JOSCR",
8333       .cp = 14, .crn = 1, .crm = 0, .opc1 = 7, .opc2 = 0,
8334       .accessfn = access_joscr_jmcr,
8335       .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
8336     { .name = "JMCR",
8337       .cp = 14, .crn = 2, .crm = 0, .opc1 = 7, .opc2 = 0,
8338       .accessfn = access_joscr_jmcr,
8339       .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
8340 };
8341 
8342 static const ARMCPRegInfo contextidr_el2 = {
8343     .name = "CONTEXTIDR_EL2", .state = ARM_CP_STATE_AA64,
8344     .opc0 = 3, .opc1 = 4, .crn = 13, .crm = 0, .opc2 = 1,
8345     .access = PL2_RW,
8346     .fieldoffset = offsetof(CPUARMState, cp15.contextidr_el[2])
8347 };
8348 
8349 static const ARMCPRegInfo vhe_reginfo[] = {
8350     { .name = "TTBR1_EL2", .state = ARM_CP_STATE_AA64,
8351       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 1,
8352       .access = PL2_RW, .writefn = vmsa_tcr_ttbr_el2_write,
8353       .raw_writefn = raw_write,
8354       .fieldoffset = offsetof(CPUARMState, cp15.ttbr1_el[2]) },
8355 #ifndef CONFIG_USER_ONLY
8356     { .name = "CNTHV_CVAL_EL2", .state = ARM_CP_STATE_AA64,
8357       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 3, .opc2 = 2,
8358       .fieldoffset =
8359         offsetof(CPUARMState, cp15.c14_timer[GTIMER_HYPVIRT].cval),
8360       .type = ARM_CP_IO, .access = PL2_RW,
8361       .writefn = gt_hv_cval_write, .raw_writefn = raw_write },
8362     { .name = "CNTHV_TVAL_EL2", .state = ARM_CP_STATE_BOTH,
8363       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 3, .opc2 = 0,
8364       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL2_RW,
8365       .resetfn = gt_hv_timer_reset,
8366       .readfn = gt_hv_tval_read, .writefn = gt_hv_tval_write },
8367     { .name = "CNTHV_CTL_EL2", .state = ARM_CP_STATE_BOTH,
8368       .type = ARM_CP_IO,
8369       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 3, .opc2 = 1,
8370       .access = PL2_RW,
8371       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_HYPVIRT].ctl),
8372       .writefn = gt_hv_ctl_write, .raw_writefn = raw_write },
8373     { .name = "CNTP_CTL_EL02", .state = ARM_CP_STATE_AA64,
8374       .opc0 = 3, .opc1 = 5, .crn = 14, .crm = 2, .opc2 = 1,
8375       .type = ARM_CP_IO | ARM_CP_ALIAS,
8376       .access = PL2_RW, .accessfn = e2h_access,
8377       .nv2_redirect_offset = 0x180 | NV2_REDIR_NO_NV1,
8378       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].ctl),
8379       .writefn = gt_phys_ctl_write, .raw_writefn = raw_write },
8380     { .name = "CNTV_CTL_EL02", .state = ARM_CP_STATE_AA64,
8381       .opc0 = 3, .opc1 = 5, .crn = 14, .crm = 3, .opc2 = 1,
8382       .type = ARM_CP_IO | ARM_CP_ALIAS,
8383       .access = PL2_RW, .accessfn = e2h_access,
8384       .nv2_redirect_offset = 0x170 | NV2_REDIR_NO_NV1,
8385       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].ctl),
8386       .writefn = gt_virt_ctl_write, .raw_writefn = raw_write },
8387     { .name = "CNTP_TVAL_EL02", .state = ARM_CP_STATE_AA64,
8388       .opc0 = 3, .opc1 = 5, .crn = 14, .crm = 2, .opc2 = 0,
8389       .type = ARM_CP_NO_RAW | ARM_CP_IO | ARM_CP_ALIAS,
8390       .access = PL2_RW, .accessfn = e2h_access,
8391       .readfn = gt_phys_tval_read, .writefn = gt_phys_tval_write },
8392     { .name = "CNTV_TVAL_EL02", .state = ARM_CP_STATE_AA64,
8393       .opc0 = 3, .opc1 = 5, .crn = 14, .crm = 3, .opc2 = 0,
8394       .type = ARM_CP_NO_RAW | ARM_CP_IO | ARM_CP_ALIAS,
8395       .access = PL2_RW, .accessfn = e2h_access,
8396       .readfn = gt_virt_tval_read, .writefn = gt_virt_tval_write },
8397     { .name = "CNTP_CVAL_EL02", .state = ARM_CP_STATE_AA64,
8398       .opc0 = 3, .opc1 = 5, .crn = 14, .crm = 2, .opc2 = 2,
8399       .type = ARM_CP_IO | ARM_CP_ALIAS,
8400       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].cval),
8401       .nv2_redirect_offset = 0x178 | NV2_REDIR_NO_NV1,
8402       .access = PL2_RW, .accessfn = e2h_access,
8403       .writefn = gt_phys_cval_write, .raw_writefn = raw_write },
8404     { .name = "CNTV_CVAL_EL02", .state = ARM_CP_STATE_AA64,
8405       .opc0 = 3, .opc1 = 5, .crn = 14, .crm = 3, .opc2 = 2,
8406       .type = ARM_CP_IO | ARM_CP_ALIAS,
8407       .nv2_redirect_offset = 0x168 | NV2_REDIR_NO_NV1,
8408       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].cval),
8409       .access = PL2_RW, .accessfn = e2h_access,
8410       .writefn = gt_virt_cval_write, .raw_writefn = raw_write },
8411 #endif
8412 };
8413 
8414 #ifndef CONFIG_USER_ONLY
8415 static const ARMCPRegInfo ats1e1_reginfo[] = {
8416     { .name = "AT_S1E1RP", .state = ARM_CP_STATE_AA64,
8417       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 9, .opc2 = 0,
8418       .access = PL1_W, .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC,
8419       .fgt = FGT_ATS1E1RP,
8420       .accessfn = at_s1e01_access, .writefn = ats_write64 },
8421     { .name = "AT_S1E1WP", .state = ARM_CP_STATE_AA64,
8422       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 9, .opc2 = 1,
8423       .access = PL1_W, .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC,
8424       .fgt = FGT_ATS1E1WP,
8425       .accessfn = at_s1e01_access, .writefn = ats_write64 },
8426 };
8427 
8428 static const ARMCPRegInfo ats1cp_reginfo[] = {
8429     { .name = "ATS1CPRP",
8430       .cp = 15, .opc1 = 0, .crn = 7, .crm = 9, .opc2 = 0,
8431       .access = PL1_W, .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC,
8432       .writefn = ats_write },
8433     { .name = "ATS1CPWP",
8434       .cp = 15, .opc1 = 0, .crn = 7, .crm = 9, .opc2 = 1,
8435       .access = PL1_W, .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC,
8436       .writefn = ats_write },
8437 };
8438 #endif
8439 
8440 /*
8441  * ACTLR2 and HACTLR2 map to ACTLR_EL1[63:32] and
8442  * ACTLR_EL2[63:32]. They exist only if the ID_MMFR4.AC2 field
8443  * is non-zero, which is never for ARMv7, optionally in ARMv8
8444  * and mandatorily for ARMv8.2 and up.
8445  * ACTLR2 is banked for S and NS if EL3 is AArch32. Since QEMU's
8446  * implementation is RAZ/WI we can ignore this detail, as we
8447  * do for ACTLR.
8448  */
8449 static const ARMCPRegInfo actlr2_hactlr2_reginfo[] = {
8450     { .name = "ACTLR2", .state = ARM_CP_STATE_AA32,
8451       .cp = 15, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 3,
8452       .access = PL1_RW, .accessfn = access_tacr,
8453       .type = ARM_CP_CONST, .resetvalue = 0 },
8454     { .name = "HACTLR2", .state = ARM_CP_STATE_AA32,
8455       .cp = 15, .opc1 = 4, .crn = 1, .crm = 0, .opc2 = 3,
8456       .access = PL2_RW, .type = ARM_CP_CONST,
8457       .resetvalue = 0 },
8458 };
8459 
8460 void register_cp_regs_for_features(ARMCPU *cpu)
8461 {
8462     /* Register all the coprocessor registers based on feature bits */
8463     CPUARMState *env = &cpu->env;
8464     if (arm_feature(env, ARM_FEATURE_M)) {
8465         /* M profile has no coprocessor registers */
8466         return;
8467     }
8468 
8469     define_arm_cp_regs(cpu, cp_reginfo);
8470     if (!arm_feature(env, ARM_FEATURE_V8)) {
8471         /*
8472          * Must go early as it is full of wildcards that may be
8473          * overridden by later definitions.
8474          */
8475         define_arm_cp_regs(cpu, not_v8_cp_reginfo);
8476     }
8477 
8478     if (arm_feature(env, ARM_FEATURE_V6)) {
8479         /* The ID registers all have impdef reset values */
8480         ARMCPRegInfo v6_idregs[] = {
8481             { .name = "ID_PFR0", .state = ARM_CP_STATE_BOTH,
8482               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 0,
8483               .access = PL1_R, .type = ARM_CP_CONST,
8484               .accessfn = access_aa32_tid3,
8485               .resetvalue = cpu->isar.id_pfr0 },
8486             /*
8487              * ID_PFR1 is not a plain ARM_CP_CONST because we don't know
8488              * the value of the GIC field until after we define these regs.
8489              */
8490             { .name = "ID_PFR1", .state = ARM_CP_STATE_BOTH,
8491               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 1,
8492               .access = PL1_R, .type = ARM_CP_NO_RAW,
8493               .accessfn = access_aa32_tid3,
8494 #ifdef CONFIG_USER_ONLY
8495               .type = ARM_CP_CONST,
8496               .resetvalue = cpu->isar.id_pfr1,
8497 #else
8498               .type = ARM_CP_NO_RAW,
8499               .accessfn = access_aa32_tid3,
8500               .readfn = id_pfr1_read,
8501               .writefn = arm_cp_write_ignore
8502 #endif
8503             },
8504             { .name = "ID_DFR0", .state = ARM_CP_STATE_BOTH,
8505               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 2,
8506               .access = PL1_R, .type = ARM_CP_CONST,
8507               .accessfn = access_aa32_tid3,
8508               .resetvalue = cpu->isar.id_dfr0 },
8509             { .name = "ID_AFR0", .state = ARM_CP_STATE_BOTH,
8510               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 3,
8511               .access = PL1_R, .type = ARM_CP_CONST,
8512               .accessfn = access_aa32_tid3,
8513               .resetvalue = cpu->id_afr0 },
8514             { .name = "ID_MMFR0", .state = ARM_CP_STATE_BOTH,
8515               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 4,
8516               .access = PL1_R, .type = ARM_CP_CONST,
8517               .accessfn = access_aa32_tid3,
8518               .resetvalue = cpu->isar.id_mmfr0 },
8519             { .name = "ID_MMFR1", .state = ARM_CP_STATE_BOTH,
8520               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 5,
8521               .access = PL1_R, .type = ARM_CP_CONST,
8522               .accessfn = access_aa32_tid3,
8523               .resetvalue = cpu->isar.id_mmfr1 },
8524             { .name = "ID_MMFR2", .state = ARM_CP_STATE_BOTH,
8525               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 6,
8526               .access = PL1_R, .type = ARM_CP_CONST,
8527               .accessfn = access_aa32_tid3,
8528               .resetvalue = cpu->isar.id_mmfr2 },
8529             { .name = "ID_MMFR3", .state = ARM_CP_STATE_BOTH,
8530               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 7,
8531               .access = PL1_R, .type = ARM_CP_CONST,
8532               .accessfn = access_aa32_tid3,
8533               .resetvalue = cpu->isar.id_mmfr3 },
8534             { .name = "ID_ISAR0", .state = ARM_CP_STATE_BOTH,
8535               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 0,
8536               .access = PL1_R, .type = ARM_CP_CONST,
8537               .accessfn = access_aa32_tid3,
8538               .resetvalue = cpu->isar.id_isar0 },
8539             { .name = "ID_ISAR1", .state = ARM_CP_STATE_BOTH,
8540               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 1,
8541               .access = PL1_R, .type = ARM_CP_CONST,
8542               .accessfn = access_aa32_tid3,
8543               .resetvalue = cpu->isar.id_isar1 },
8544             { .name = "ID_ISAR2", .state = ARM_CP_STATE_BOTH,
8545               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 2,
8546               .access = PL1_R, .type = ARM_CP_CONST,
8547               .accessfn = access_aa32_tid3,
8548               .resetvalue = cpu->isar.id_isar2 },
8549             { .name = "ID_ISAR3", .state = ARM_CP_STATE_BOTH,
8550               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 3,
8551               .access = PL1_R, .type = ARM_CP_CONST,
8552               .accessfn = access_aa32_tid3,
8553               .resetvalue = cpu->isar.id_isar3 },
8554             { .name = "ID_ISAR4", .state = ARM_CP_STATE_BOTH,
8555               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 4,
8556               .access = PL1_R, .type = ARM_CP_CONST,
8557               .accessfn = access_aa32_tid3,
8558               .resetvalue = cpu->isar.id_isar4 },
8559             { .name = "ID_ISAR5", .state = ARM_CP_STATE_BOTH,
8560               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 5,
8561               .access = PL1_R, .type = ARM_CP_CONST,
8562               .accessfn = access_aa32_tid3,
8563               .resetvalue = cpu->isar.id_isar5 },
8564             { .name = "ID_MMFR4", .state = ARM_CP_STATE_BOTH,
8565               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 6,
8566               .access = PL1_R, .type = ARM_CP_CONST,
8567               .accessfn = access_aa32_tid3,
8568               .resetvalue = cpu->isar.id_mmfr4 },
8569             { .name = "ID_ISAR6", .state = ARM_CP_STATE_BOTH,
8570               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 7,
8571               .access = PL1_R, .type = ARM_CP_CONST,
8572               .accessfn = access_aa32_tid3,
8573               .resetvalue = cpu->isar.id_isar6 },
8574         };
8575         define_arm_cp_regs(cpu, v6_idregs);
8576         define_arm_cp_regs(cpu, v6_cp_reginfo);
8577     } else {
8578         define_arm_cp_regs(cpu, not_v6_cp_reginfo);
8579     }
8580     if (arm_feature(env, ARM_FEATURE_V6K)) {
8581         define_arm_cp_regs(cpu, v6k_cp_reginfo);
8582     }
8583     if (arm_feature(env, ARM_FEATURE_V7MP) &&
8584         !arm_feature(env, ARM_FEATURE_PMSA)) {
8585         define_arm_cp_regs(cpu, v7mp_cp_reginfo);
8586     }
8587     if (arm_feature(env, ARM_FEATURE_V7VE)) {
8588         define_arm_cp_regs(cpu, pmovsset_cp_reginfo);
8589     }
8590     if (arm_feature(env, ARM_FEATURE_V7)) {
8591         ARMCPRegInfo clidr = {
8592             .name = "CLIDR", .state = ARM_CP_STATE_BOTH,
8593             .opc0 = 3, .crn = 0, .crm = 0, .opc1 = 1, .opc2 = 1,
8594             .access = PL1_R, .type = ARM_CP_CONST,
8595             .accessfn = access_tid4,
8596             .fgt = FGT_CLIDR_EL1,
8597             .resetvalue = cpu->clidr
8598         };
8599         define_one_arm_cp_reg(cpu, &clidr);
8600         define_arm_cp_regs(cpu, v7_cp_reginfo);
8601         define_debug_regs(cpu);
8602         define_pmu_regs(cpu);
8603     } else {
8604         define_arm_cp_regs(cpu, not_v7_cp_reginfo);
8605     }
8606     if (arm_feature(env, ARM_FEATURE_V8)) {
8607         /*
8608          * v8 ID registers, which all have impdef reset values.
8609          * Note that within the ID register ranges the unused slots
8610          * must all RAZ, not UNDEF; future architecture versions may
8611          * define new registers here.
8612          * ID registers which are AArch64 views of the AArch32 ID registers
8613          * which already existed in v6 and v7 are handled elsewhere,
8614          * in v6_idregs[].
8615          */
8616         int i;
8617         ARMCPRegInfo v8_idregs[] = {
8618             /*
8619              * ID_AA64PFR0_EL1 is not a plain ARM_CP_CONST in system
8620              * emulation because we don't know the right value for the
8621              * GIC field until after we define these regs.
8622              */
8623             { .name = "ID_AA64PFR0_EL1", .state = ARM_CP_STATE_AA64,
8624               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 0,
8625               .access = PL1_R,
8626 #ifdef CONFIG_USER_ONLY
8627               .type = ARM_CP_CONST,
8628               .resetvalue = cpu->isar.id_aa64pfr0
8629 #else
8630               .type = ARM_CP_NO_RAW,
8631               .accessfn = access_aa64_tid3,
8632               .readfn = id_aa64pfr0_read,
8633               .writefn = arm_cp_write_ignore
8634 #endif
8635             },
8636             { .name = "ID_AA64PFR1_EL1", .state = ARM_CP_STATE_AA64,
8637               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 1,
8638               .access = PL1_R, .type = ARM_CP_CONST,
8639               .accessfn = access_aa64_tid3,
8640               .resetvalue = cpu->isar.id_aa64pfr1},
8641             { .name = "ID_AA64PFR2_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
8642               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 2,
8643               .access = PL1_R, .type = ARM_CP_CONST,
8644               .accessfn = access_aa64_tid3,
8645               .resetvalue = 0 },
8646             { .name = "ID_AA64PFR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
8647               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 3,
8648               .access = PL1_R, .type = ARM_CP_CONST,
8649               .accessfn = access_aa64_tid3,
8650               .resetvalue = 0 },
8651             { .name = "ID_AA64ZFR0_EL1", .state = ARM_CP_STATE_AA64,
8652               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 4,
8653               .access = PL1_R, .type = ARM_CP_CONST,
8654               .accessfn = access_aa64_tid3,
8655               .resetvalue = cpu->isar.id_aa64zfr0 },
8656             { .name = "ID_AA64SMFR0_EL1", .state = ARM_CP_STATE_AA64,
8657               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 5,
8658               .access = PL1_R, .type = ARM_CP_CONST,
8659               .accessfn = access_aa64_tid3,
8660               .resetvalue = cpu->isar.id_aa64smfr0 },
8661             { .name = "ID_AA64PFR6_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
8662               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 6,
8663               .access = PL1_R, .type = ARM_CP_CONST,
8664               .accessfn = access_aa64_tid3,
8665               .resetvalue = 0 },
8666             { .name = "ID_AA64PFR7_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
8667               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 7,
8668               .access = PL1_R, .type = ARM_CP_CONST,
8669               .accessfn = access_aa64_tid3,
8670               .resetvalue = 0 },
8671             { .name = "ID_AA64DFR0_EL1", .state = ARM_CP_STATE_AA64,
8672               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 0,
8673               .access = PL1_R, .type = ARM_CP_CONST,
8674               .accessfn = access_aa64_tid3,
8675               .resetvalue = cpu->isar.id_aa64dfr0 },
8676             { .name = "ID_AA64DFR1_EL1", .state = ARM_CP_STATE_AA64,
8677               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 1,
8678               .access = PL1_R, .type = ARM_CP_CONST,
8679               .accessfn = access_aa64_tid3,
8680               .resetvalue = cpu->isar.id_aa64dfr1 },
8681             { .name = "ID_AA64DFR2_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
8682               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 2,
8683               .access = PL1_R, .type = ARM_CP_CONST,
8684               .accessfn = access_aa64_tid3,
8685               .resetvalue = 0 },
8686             { .name = "ID_AA64DFR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
8687               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 3,
8688               .access = PL1_R, .type = ARM_CP_CONST,
8689               .accessfn = access_aa64_tid3,
8690               .resetvalue = 0 },
8691             { .name = "ID_AA64AFR0_EL1", .state = ARM_CP_STATE_AA64,
8692               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 4,
8693               .access = PL1_R, .type = ARM_CP_CONST,
8694               .accessfn = access_aa64_tid3,
8695               .resetvalue = cpu->id_aa64afr0 },
8696             { .name = "ID_AA64AFR1_EL1", .state = ARM_CP_STATE_AA64,
8697               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 5,
8698               .access = PL1_R, .type = ARM_CP_CONST,
8699               .accessfn = access_aa64_tid3,
8700               .resetvalue = cpu->id_aa64afr1 },
8701             { .name = "ID_AA64AFR2_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
8702               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 6,
8703               .access = PL1_R, .type = ARM_CP_CONST,
8704               .accessfn = access_aa64_tid3,
8705               .resetvalue = 0 },
8706             { .name = "ID_AA64AFR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
8707               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 7,
8708               .access = PL1_R, .type = ARM_CP_CONST,
8709               .accessfn = access_aa64_tid3,
8710               .resetvalue = 0 },
8711             { .name = "ID_AA64ISAR0_EL1", .state = ARM_CP_STATE_AA64,
8712               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 0,
8713               .access = PL1_R, .type = ARM_CP_CONST,
8714               .accessfn = access_aa64_tid3,
8715               .resetvalue = cpu->isar.id_aa64isar0 },
8716             { .name = "ID_AA64ISAR1_EL1", .state = ARM_CP_STATE_AA64,
8717               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 1,
8718               .access = PL1_R, .type = ARM_CP_CONST,
8719               .accessfn = access_aa64_tid3,
8720               .resetvalue = cpu->isar.id_aa64isar1 },
8721             { .name = "ID_AA64ISAR2_EL1", .state = ARM_CP_STATE_AA64,
8722               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 2,
8723               .access = PL1_R, .type = ARM_CP_CONST,
8724               .accessfn = access_aa64_tid3,
8725               .resetvalue = cpu->isar.id_aa64isar2 },
8726             { .name = "ID_AA64ISAR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
8727               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 3,
8728               .access = PL1_R, .type = ARM_CP_CONST,
8729               .accessfn = access_aa64_tid3,
8730               .resetvalue = 0 },
8731             { .name = "ID_AA64ISAR4_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
8732               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 4,
8733               .access = PL1_R, .type = ARM_CP_CONST,
8734               .accessfn = access_aa64_tid3,
8735               .resetvalue = 0 },
8736             { .name = "ID_AA64ISAR5_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
8737               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 5,
8738               .access = PL1_R, .type = ARM_CP_CONST,
8739               .accessfn = access_aa64_tid3,
8740               .resetvalue = 0 },
8741             { .name = "ID_AA64ISAR6_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
8742               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 6,
8743               .access = PL1_R, .type = ARM_CP_CONST,
8744               .accessfn = access_aa64_tid3,
8745               .resetvalue = 0 },
8746             { .name = "ID_AA64ISAR7_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
8747               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 7,
8748               .access = PL1_R, .type = ARM_CP_CONST,
8749               .accessfn = access_aa64_tid3,
8750               .resetvalue = 0 },
8751             { .name = "ID_AA64MMFR0_EL1", .state = ARM_CP_STATE_AA64,
8752               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 0,
8753               .access = PL1_R, .type = ARM_CP_CONST,
8754               .accessfn = access_aa64_tid3,
8755               .resetvalue = cpu->isar.id_aa64mmfr0 },
8756             { .name = "ID_AA64MMFR1_EL1", .state = ARM_CP_STATE_AA64,
8757               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 1,
8758               .access = PL1_R, .type = ARM_CP_CONST,
8759               .accessfn = access_aa64_tid3,
8760               .resetvalue = cpu->isar.id_aa64mmfr1 },
8761             { .name = "ID_AA64MMFR2_EL1", .state = ARM_CP_STATE_AA64,
8762               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 2,
8763               .access = PL1_R, .type = ARM_CP_CONST,
8764               .accessfn = access_aa64_tid3,
8765               .resetvalue = cpu->isar.id_aa64mmfr2 },
8766             { .name = "ID_AA64MMFR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
8767               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 3,
8768               .access = PL1_R, .type = ARM_CP_CONST,
8769               .accessfn = access_aa64_tid3,
8770               .resetvalue = 0 },
8771             { .name = "ID_AA64MMFR4_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
8772               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 4,
8773               .access = PL1_R, .type = ARM_CP_CONST,
8774               .accessfn = access_aa64_tid3,
8775               .resetvalue = 0 },
8776             { .name = "ID_AA64MMFR5_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
8777               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 5,
8778               .access = PL1_R, .type = ARM_CP_CONST,
8779               .accessfn = access_aa64_tid3,
8780               .resetvalue = 0 },
8781             { .name = "ID_AA64MMFR6_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
8782               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 6,
8783               .access = PL1_R, .type = ARM_CP_CONST,
8784               .accessfn = access_aa64_tid3,
8785               .resetvalue = 0 },
8786             { .name = "ID_AA64MMFR7_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
8787               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 7,
8788               .access = PL1_R, .type = ARM_CP_CONST,
8789               .accessfn = access_aa64_tid3,
8790               .resetvalue = 0 },
8791             { .name = "MVFR0_EL1", .state = ARM_CP_STATE_AA64,
8792               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 0,
8793               .access = PL1_R, .type = ARM_CP_CONST,
8794               .accessfn = access_aa64_tid3,
8795               .resetvalue = cpu->isar.mvfr0 },
8796             { .name = "MVFR1_EL1", .state = ARM_CP_STATE_AA64,
8797               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 1,
8798               .access = PL1_R, .type = ARM_CP_CONST,
8799               .accessfn = access_aa64_tid3,
8800               .resetvalue = cpu->isar.mvfr1 },
8801             { .name = "MVFR2_EL1", .state = ARM_CP_STATE_AA64,
8802               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 2,
8803               .access = PL1_R, .type = ARM_CP_CONST,
8804               .accessfn = access_aa64_tid3,
8805               .resetvalue = cpu->isar.mvfr2 },
8806             /*
8807              * "0, c0, c3, {0,1,2}" are the encodings corresponding to
8808              * AArch64 MVFR[012]_EL1. Define the STATE_AA32 encoding
8809              * as RAZ, since it is in the "reserved for future ID
8810              * registers, RAZ" part of the AArch32 encoding space.
8811              */
8812             { .name = "RES_0_C0_C3_0", .state = ARM_CP_STATE_AA32,
8813               .cp = 15, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 0,
8814               .access = PL1_R, .type = ARM_CP_CONST,
8815               .accessfn = access_aa64_tid3,
8816               .resetvalue = 0 },
8817             { .name = "RES_0_C0_C3_1", .state = ARM_CP_STATE_AA32,
8818               .cp = 15, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 1,
8819               .access = PL1_R, .type = ARM_CP_CONST,
8820               .accessfn = access_aa64_tid3,
8821               .resetvalue = 0 },
8822             { .name = "RES_0_C0_C3_2", .state = ARM_CP_STATE_AA32,
8823               .cp = 15, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 2,
8824               .access = PL1_R, .type = ARM_CP_CONST,
8825               .accessfn = access_aa64_tid3,
8826               .resetvalue = 0 },
8827             /*
8828              * Other encodings in "0, c0, c3, ..." are STATE_BOTH because
8829              * they're also RAZ for AArch64, and in v8 are gradually
8830              * being filled with AArch64-view-of-AArch32-ID-register
8831              * for new ID registers.
8832              */
8833             { .name = "RES_0_C0_C3_3", .state = ARM_CP_STATE_BOTH,
8834               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 3,
8835               .access = PL1_R, .type = ARM_CP_CONST,
8836               .accessfn = access_aa64_tid3,
8837               .resetvalue = 0 },
8838             { .name = "ID_PFR2", .state = ARM_CP_STATE_BOTH,
8839               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 4,
8840               .access = PL1_R, .type = ARM_CP_CONST,
8841               .accessfn = access_aa64_tid3,
8842               .resetvalue = cpu->isar.id_pfr2 },
8843             { .name = "ID_DFR1", .state = ARM_CP_STATE_BOTH,
8844               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 5,
8845               .access = PL1_R, .type = ARM_CP_CONST,
8846               .accessfn = access_aa64_tid3,
8847               .resetvalue = cpu->isar.id_dfr1 },
8848             { .name = "ID_MMFR5", .state = ARM_CP_STATE_BOTH,
8849               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 6,
8850               .access = PL1_R, .type = ARM_CP_CONST,
8851               .accessfn = access_aa64_tid3,
8852               .resetvalue = cpu->isar.id_mmfr5 },
8853             { .name = "RES_0_C0_C3_7", .state = ARM_CP_STATE_BOTH,
8854               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 7,
8855               .access = PL1_R, .type = ARM_CP_CONST,
8856               .accessfn = access_aa64_tid3,
8857               .resetvalue = 0 },
8858             { .name = "PMCEID0", .state = ARM_CP_STATE_AA32,
8859               .cp = 15, .opc1 = 0, .crn = 9, .crm = 12, .opc2 = 6,
8860               .access = PL0_R, .accessfn = pmreg_access, .type = ARM_CP_CONST,
8861               .fgt = FGT_PMCEIDN_EL0,
8862               .resetvalue = extract64(cpu->pmceid0, 0, 32) },
8863             { .name = "PMCEID0_EL0", .state = ARM_CP_STATE_AA64,
8864               .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 6,
8865               .access = PL0_R, .accessfn = pmreg_access, .type = ARM_CP_CONST,
8866               .fgt = FGT_PMCEIDN_EL0,
8867               .resetvalue = cpu->pmceid0 },
8868             { .name = "PMCEID1", .state = ARM_CP_STATE_AA32,
8869               .cp = 15, .opc1 = 0, .crn = 9, .crm = 12, .opc2 = 7,
8870               .access = PL0_R, .accessfn = pmreg_access, .type = ARM_CP_CONST,
8871               .fgt = FGT_PMCEIDN_EL0,
8872               .resetvalue = extract64(cpu->pmceid1, 0, 32) },
8873             { .name = "PMCEID1_EL0", .state = ARM_CP_STATE_AA64,
8874               .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 7,
8875               .access = PL0_R, .accessfn = pmreg_access, .type = ARM_CP_CONST,
8876               .fgt = FGT_PMCEIDN_EL0,
8877               .resetvalue = cpu->pmceid1 },
8878         };
8879 #ifdef CONFIG_USER_ONLY
8880         static const ARMCPRegUserSpaceInfo v8_user_idregs[] = {
8881             { .name = "ID_AA64PFR0_EL1",
8882               .exported_bits = R_ID_AA64PFR0_FP_MASK |
8883                                R_ID_AA64PFR0_ADVSIMD_MASK |
8884                                R_ID_AA64PFR0_SVE_MASK |
8885                                R_ID_AA64PFR0_DIT_MASK,
8886               .fixed_bits = (0x1u << R_ID_AA64PFR0_EL0_SHIFT) |
8887                             (0x1u << R_ID_AA64PFR0_EL1_SHIFT) },
8888             { .name = "ID_AA64PFR1_EL1",
8889               .exported_bits = R_ID_AA64PFR1_BT_MASK |
8890                                R_ID_AA64PFR1_SSBS_MASK |
8891                                R_ID_AA64PFR1_MTE_MASK |
8892                                R_ID_AA64PFR1_SME_MASK },
8893             { .name = "ID_AA64PFR*_EL1_RESERVED",
8894               .is_glob = true },
8895             { .name = "ID_AA64ZFR0_EL1",
8896               .exported_bits = R_ID_AA64ZFR0_SVEVER_MASK |
8897                                R_ID_AA64ZFR0_AES_MASK |
8898                                R_ID_AA64ZFR0_BITPERM_MASK |
8899                                R_ID_AA64ZFR0_BFLOAT16_MASK |
8900                                R_ID_AA64ZFR0_SHA3_MASK |
8901                                R_ID_AA64ZFR0_SM4_MASK |
8902                                R_ID_AA64ZFR0_I8MM_MASK |
8903                                R_ID_AA64ZFR0_F32MM_MASK |
8904                                R_ID_AA64ZFR0_F64MM_MASK },
8905             { .name = "ID_AA64SMFR0_EL1",
8906               .exported_bits = R_ID_AA64SMFR0_F32F32_MASK |
8907                                R_ID_AA64SMFR0_BI32I32_MASK |
8908                                R_ID_AA64SMFR0_B16F32_MASK |
8909                                R_ID_AA64SMFR0_F16F32_MASK |
8910                                R_ID_AA64SMFR0_I8I32_MASK |
8911                                R_ID_AA64SMFR0_F16F16_MASK |
8912                                R_ID_AA64SMFR0_B16B16_MASK |
8913                                R_ID_AA64SMFR0_I16I32_MASK |
8914                                R_ID_AA64SMFR0_F64F64_MASK |
8915                                R_ID_AA64SMFR0_I16I64_MASK |
8916                                R_ID_AA64SMFR0_SMEVER_MASK |
8917                                R_ID_AA64SMFR0_FA64_MASK },
8918             { .name = "ID_AA64MMFR0_EL1",
8919               .exported_bits = R_ID_AA64MMFR0_ECV_MASK,
8920               .fixed_bits = (0xfu << R_ID_AA64MMFR0_TGRAN64_SHIFT) |
8921                             (0xfu << R_ID_AA64MMFR0_TGRAN4_SHIFT) },
8922             { .name = "ID_AA64MMFR1_EL1",
8923               .exported_bits = R_ID_AA64MMFR1_AFP_MASK },
8924             { .name = "ID_AA64MMFR2_EL1",
8925               .exported_bits = R_ID_AA64MMFR2_AT_MASK },
8926             { .name = "ID_AA64MMFR*_EL1_RESERVED",
8927               .is_glob = true },
8928             { .name = "ID_AA64DFR0_EL1",
8929               .fixed_bits = (0x6u << R_ID_AA64DFR0_DEBUGVER_SHIFT) },
8930             { .name = "ID_AA64DFR1_EL1" },
8931             { .name = "ID_AA64DFR*_EL1_RESERVED",
8932               .is_glob = true },
8933             { .name = "ID_AA64AFR*",
8934               .is_glob = true },
8935             { .name = "ID_AA64ISAR0_EL1",
8936               .exported_bits = R_ID_AA64ISAR0_AES_MASK |
8937                                R_ID_AA64ISAR0_SHA1_MASK |
8938                                R_ID_AA64ISAR0_SHA2_MASK |
8939                                R_ID_AA64ISAR0_CRC32_MASK |
8940                                R_ID_AA64ISAR0_ATOMIC_MASK |
8941                                R_ID_AA64ISAR0_RDM_MASK |
8942                                R_ID_AA64ISAR0_SHA3_MASK |
8943                                R_ID_AA64ISAR0_SM3_MASK |
8944                                R_ID_AA64ISAR0_SM4_MASK |
8945                                R_ID_AA64ISAR0_DP_MASK |
8946                                R_ID_AA64ISAR0_FHM_MASK |
8947                                R_ID_AA64ISAR0_TS_MASK |
8948                                R_ID_AA64ISAR0_RNDR_MASK },
8949             { .name = "ID_AA64ISAR1_EL1",
8950               .exported_bits = R_ID_AA64ISAR1_DPB_MASK |
8951                                R_ID_AA64ISAR1_APA_MASK |
8952                                R_ID_AA64ISAR1_API_MASK |
8953                                R_ID_AA64ISAR1_JSCVT_MASK |
8954                                R_ID_AA64ISAR1_FCMA_MASK |
8955                                R_ID_AA64ISAR1_LRCPC_MASK |
8956                                R_ID_AA64ISAR1_GPA_MASK |
8957                                R_ID_AA64ISAR1_GPI_MASK |
8958                                R_ID_AA64ISAR1_FRINTTS_MASK |
8959                                R_ID_AA64ISAR1_SB_MASK |
8960                                R_ID_AA64ISAR1_BF16_MASK |
8961                                R_ID_AA64ISAR1_DGH_MASK |
8962                                R_ID_AA64ISAR1_I8MM_MASK },
8963             { .name = "ID_AA64ISAR2_EL1",
8964               .exported_bits = R_ID_AA64ISAR2_WFXT_MASK |
8965                                R_ID_AA64ISAR2_RPRES_MASK |
8966                                R_ID_AA64ISAR2_GPA3_MASK |
8967                                R_ID_AA64ISAR2_APA3_MASK |
8968                                R_ID_AA64ISAR2_MOPS_MASK |
8969                                R_ID_AA64ISAR2_BC_MASK |
8970                                R_ID_AA64ISAR2_RPRFM_MASK |
8971                                R_ID_AA64ISAR2_CSSC_MASK },
8972             { .name = "ID_AA64ISAR*_EL1_RESERVED",
8973               .is_glob = true },
8974         };
8975         modify_arm_cp_regs(v8_idregs, v8_user_idregs);
8976 #endif
8977         /*
8978          * RVBAR_EL1 and RMR_EL1 only implemented if EL1 is the highest EL.
8979          * TODO: For RMR, a write with bit 1 set should do something with
8980          * cpu_reset(). In the meantime, "the bit is strictly a request",
8981          * so we are in spec just ignoring writes.
8982          */
8983         if (!arm_feature(env, ARM_FEATURE_EL3) &&
8984             !arm_feature(env, ARM_FEATURE_EL2)) {
8985             ARMCPRegInfo el1_reset_regs[] = {
8986                 { .name = "RVBAR_EL1", .state = ARM_CP_STATE_BOTH,
8987                   .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 0, .opc2 = 1,
8988                   .access = PL1_R,
8989                   .fieldoffset = offsetof(CPUARMState, cp15.rvbar) },
8990                 { .name = "RMR_EL1", .state = ARM_CP_STATE_BOTH,
8991                   .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 0, .opc2 = 2,
8992                   .access = PL1_RW, .type = ARM_CP_CONST,
8993                   .resetvalue = arm_feature(env, ARM_FEATURE_AARCH64) }
8994             };
8995             define_arm_cp_regs(cpu, el1_reset_regs);
8996         }
8997         define_arm_cp_regs(cpu, v8_idregs);
8998         define_arm_cp_regs(cpu, v8_cp_reginfo);
8999         if (cpu_isar_feature(aa64_aa32_el1, cpu)) {
9000             define_arm_cp_regs(cpu, v8_aa32_el1_reginfo);
9001         }
9002 
9003         for (i = 4; i < 16; i++) {
9004             /*
9005              * Encodings in "0, c0, {c4-c7}, {0-7}" are RAZ for AArch32.
9006              * For pre-v8 cores there are RAZ patterns for these in
9007              * id_pre_v8_midr_cp_reginfo[]; for v8 we do that here.
9008              * v8 extends the "must RAZ" part of the ID register space
9009              * to also cover c0, 0, c{8-15}, {0-7}.
9010              * These are STATE_AA32 because in the AArch64 sysreg space
9011              * c4-c7 is where the AArch64 ID registers live (and we've
9012              * already defined those in v8_idregs[]), and c8-c15 are not
9013              * "must RAZ" for AArch64.
9014              */
9015             g_autofree char *name = g_strdup_printf("RES_0_C0_C%d_X", i);
9016             ARMCPRegInfo v8_aa32_raz_idregs = {
9017                 .name = name,
9018                 .state = ARM_CP_STATE_AA32,
9019                 .cp = 15, .opc1 = 0, .crn = 0, .crm = i, .opc2 = CP_ANY,
9020                 .access = PL1_R, .type = ARM_CP_CONST,
9021                 .accessfn = access_aa64_tid3,
9022                 .resetvalue = 0 };
9023             define_one_arm_cp_reg(cpu, &v8_aa32_raz_idregs);
9024         }
9025     }
9026 
9027     /*
9028      * Register the base EL2 cpregs.
9029      * Pre v8, these registers are implemented only as part of the
9030      * Virtualization Extensions (EL2 present).  Beginning with v8,
9031      * if EL2 is missing but EL3 is enabled, mostly these become
9032      * RES0 from EL3, with some specific exceptions.
9033      */
9034     if (arm_feature(env, ARM_FEATURE_EL2)
9035         || (arm_feature(env, ARM_FEATURE_EL3)
9036             && arm_feature(env, ARM_FEATURE_V8))) {
9037         uint64_t vmpidr_def = mpidr_read_val(env);
9038         ARMCPRegInfo vpidr_regs[] = {
9039             { .name = "VPIDR", .state = ARM_CP_STATE_AA32,
9040               .cp = 15, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 0,
9041               .access = PL2_RW, .accessfn = access_el3_aa32ns,
9042               .resetvalue = cpu->midr,
9043               .type = ARM_CP_ALIAS | ARM_CP_EL3_NO_EL2_C_NZ,
9044               .fieldoffset = offsetoflow32(CPUARMState, cp15.vpidr_el2) },
9045             { .name = "VPIDR_EL2", .state = ARM_CP_STATE_AA64,
9046               .opc0 = 3, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 0,
9047               .access = PL2_RW, .resetvalue = cpu->midr,
9048               .type = ARM_CP_EL3_NO_EL2_C_NZ,
9049               .nv2_redirect_offset = 0x88,
9050               .fieldoffset = offsetof(CPUARMState, cp15.vpidr_el2) },
9051             { .name = "VMPIDR", .state = ARM_CP_STATE_AA32,
9052               .cp = 15, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 5,
9053               .access = PL2_RW, .accessfn = access_el3_aa32ns,
9054               .resetvalue = vmpidr_def,
9055               .type = ARM_CP_ALIAS | ARM_CP_EL3_NO_EL2_C_NZ,
9056               .fieldoffset = offsetoflow32(CPUARMState, cp15.vmpidr_el2) },
9057             { .name = "VMPIDR_EL2", .state = ARM_CP_STATE_AA64,
9058               .opc0 = 3, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 5,
9059               .access = PL2_RW, .resetvalue = vmpidr_def,
9060               .type = ARM_CP_EL3_NO_EL2_C_NZ,
9061               .nv2_redirect_offset = 0x50,
9062               .fieldoffset = offsetof(CPUARMState, cp15.vmpidr_el2) },
9063         };
9064         /*
9065          * The only field of MDCR_EL2 that has a defined architectural reset
9066          * value is MDCR_EL2.HPMN which should reset to the value of PMCR_EL0.N.
9067          */
9068         ARMCPRegInfo mdcr_el2 = {
9069             .name = "MDCR_EL2", .state = ARM_CP_STATE_BOTH, .type = ARM_CP_IO,
9070             .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 1,
9071             .writefn = mdcr_el2_write,
9072             .access = PL2_RW, .resetvalue = pmu_num_counters(env),
9073             .fieldoffset = offsetof(CPUARMState, cp15.mdcr_el2),
9074         };
9075         define_one_arm_cp_reg(cpu, &mdcr_el2);
9076         define_arm_cp_regs(cpu, vpidr_regs);
9077         define_arm_cp_regs(cpu, el2_cp_reginfo);
9078         if (arm_feature(env, ARM_FEATURE_V8)) {
9079             define_arm_cp_regs(cpu, el2_v8_cp_reginfo);
9080         }
9081         if (cpu_isar_feature(aa64_sel2, cpu)) {
9082             define_arm_cp_regs(cpu, el2_sec_cp_reginfo);
9083         }
9084         /*
9085          * RVBAR_EL2 and RMR_EL2 only implemented if EL2 is the highest EL.
9086          * See commentary near RMR_EL1.
9087          */
9088         if (!arm_feature(env, ARM_FEATURE_EL3)) {
9089             static const ARMCPRegInfo el2_reset_regs[] = {
9090                 { .name = "RVBAR_EL2", .state = ARM_CP_STATE_AA64,
9091                   .opc0 = 3, .opc1 = 4, .crn = 12, .crm = 0, .opc2 = 1,
9092                   .access = PL2_R,
9093                   .fieldoffset = offsetof(CPUARMState, cp15.rvbar) },
9094                 { .name = "RVBAR", .type = ARM_CP_ALIAS,
9095                   .cp = 15, .opc1 = 0, .crn = 12, .crm = 0, .opc2 = 1,
9096                   .access = PL2_R,
9097                   .fieldoffset = offsetof(CPUARMState, cp15.rvbar) },
9098                 { .name = "RMR_EL2", .state = ARM_CP_STATE_AA64,
9099                   .opc0 = 3, .opc1 = 4, .crn = 12, .crm = 0, .opc2 = 2,
9100                   .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 1 },
9101             };
9102             define_arm_cp_regs(cpu, el2_reset_regs);
9103         }
9104     }
9105 
9106     /* Register the base EL3 cpregs. */
9107     if (arm_feature(env, ARM_FEATURE_EL3)) {
9108         define_arm_cp_regs(cpu, el3_cp_reginfo);
9109         ARMCPRegInfo el3_regs[] = {
9110             { .name = "RVBAR_EL3", .state = ARM_CP_STATE_AA64,
9111               .opc0 = 3, .opc1 = 6, .crn = 12, .crm = 0, .opc2 = 1,
9112               .access = PL3_R,
9113               .fieldoffset = offsetof(CPUARMState, cp15.rvbar), },
9114             { .name = "RMR_EL3", .state = ARM_CP_STATE_AA64,
9115               .opc0 = 3, .opc1 = 6, .crn = 12, .crm = 0, .opc2 = 2,
9116               .access = PL3_RW, .type = ARM_CP_CONST, .resetvalue = 1 },
9117             { .name = "RMR", .state = ARM_CP_STATE_AA32,
9118               .cp = 15, .opc1 = 0, .crn = 12, .crm = 0, .opc2 = 2,
9119               .access = PL3_RW, .type = ARM_CP_CONST,
9120               .resetvalue = arm_feature(env, ARM_FEATURE_AARCH64) },
9121             { .name = "SCTLR_EL3", .state = ARM_CP_STATE_AA64,
9122               .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 0, .opc2 = 0,
9123               .access = PL3_RW,
9124               .raw_writefn = raw_write, .writefn = sctlr_write,
9125               .fieldoffset = offsetof(CPUARMState, cp15.sctlr_el[3]),
9126               .resetvalue = cpu->reset_sctlr },
9127         };
9128 
9129         define_arm_cp_regs(cpu, el3_regs);
9130     }
9131     /*
9132      * The behaviour of NSACR is sufficiently various that we don't
9133      * try to describe it in a single reginfo:
9134      *  if EL3 is 64 bit, then trap to EL3 from S EL1,
9135      *     reads as constant 0xc00 from NS EL1 and NS EL2
9136      *  if EL3 is 32 bit, then RW at EL3, RO at NS EL1 and NS EL2
9137      *  if v7 without EL3, register doesn't exist
9138      *  if v8 without EL3, reads as constant 0xc00 from NS EL1 and NS EL2
9139      */
9140     if (arm_feature(env, ARM_FEATURE_EL3)) {
9141         if (arm_feature(env, ARM_FEATURE_AARCH64)) {
9142             static const ARMCPRegInfo nsacr = {
9143                 .name = "NSACR", .type = ARM_CP_CONST,
9144                 .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 2,
9145                 .access = PL1_RW, .accessfn = nsacr_access,
9146                 .resetvalue = 0xc00
9147             };
9148             define_one_arm_cp_reg(cpu, &nsacr);
9149         } else {
9150             static const ARMCPRegInfo nsacr = {
9151                 .name = "NSACR",
9152                 .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 2,
9153                 .access = PL3_RW | PL1_R,
9154                 .resetvalue = 0,
9155                 .fieldoffset = offsetof(CPUARMState, cp15.nsacr)
9156             };
9157             define_one_arm_cp_reg(cpu, &nsacr);
9158         }
9159     } else {
9160         if (arm_feature(env, ARM_FEATURE_V8)) {
9161             static const ARMCPRegInfo nsacr = {
9162                 .name = "NSACR", .type = ARM_CP_CONST,
9163                 .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 2,
9164                 .access = PL1_R,
9165                 .resetvalue = 0xc00
9166             };
9167             define_one_arm_cp_reg(cpu, &nsacr);
9168         }
9169     }
9170 
9171     if (arm_feature(env, ARM_FEATURE_PMSA)) {
9172         if (arm_feature(env, ARM_FEATURE_V6)) {
9173             /* PMSAv6 not implemented */
9174             assert(arm_feature(env, ARM_FEATURE_V7));
9175             define_arm_cp_regs(cpu, vmsa_pmsa_cp_reginfo);
9176             define_arm_cp_regs(cpu, pmsav7_cp_reginfo);
9177         } else {
9178             define_arm_cp_regs(cpu, pmsav5_cp_reginfo);
9179         }
9180     } else {
9181         define_arm_cp_regs(cpu, vmsa_pmsa_cp_reginfo);
9182         define_arm_cp_regs(cpu, vmsa_cp_reginfo);
9183         /* TTCBR2 is introduced with ARMv8.2-AA32HPD.  */
9184         if (cpu_isar_feature(aa32_hpd, cpu)) {
9185             define_one_arm_cp_reg(cpu, &ttbcr2_reginfo);
9186         }
9187     }
9188     if (arm_feature(env, ARM_FEATURE_THUMB2EE)) {
9189         define_arm_cp_regs(cpu, t2ee_cp_reginfo);
9190     }
9191     if (arm_feature(env, ARM_FEATURE_GENERIC_TIMER)) {
9192         define_arm_cp_regs(cpu, generic_timer_cp_reginfo);
9193     }
9194     if (arm_feature(env, ARM_FEATURE_VAPA)) {
9195         ARMCPRegInfo vapa_cp_reginfo[] = {
9196             { .name = "PAR", .cp = 15, .crn = 7, .crm = 4, .opc1 = 0, .opc2 = 0,
9197               .access = PL1_RW, .resetvalue = 0,
9198               .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.par_s),
9199                                      offsetoflow32(CPUARMState, cp15.par_ns) },
9200               .writefn = par_write},
9201 #ifndef CONFIG_USER_ONLY
9202             /* This underdecoding is safe because the reginfo is NO_RAW. */
9203             { .name = "ATS", .cp = 15, .crn = 7, .crm = 8, .opc1 = 0, .opc2 = CP_ANY,
9204               .access = PL1_W, .accessfn = ats_access,
9205               .writefn = ats_write, .type = ARM_CP_NO_RAW | ARM_CP_RAISES_EXC },
9206 #endif
9207         };
9208 
9209         /*
9210          * When LPAE exists this 32-bit PAR register is an alias of the
9211          * 64-bit AArch32 PAR register defined in lpae_cp_reginfo[]
9212          */
9213         if (arm_feature(env, ARM_FEATURE_LPAE)) {
9214             vapa_cp_reginfo[0].type = ARM_CP_ALIAS | ARM_CP_NO_GDB;
9215         }
9216         define_arm_cp_regs(cpu, vapa_cp_reginfo);
9217     }
9218     if (arm_feature(env, ARM_FEATURE_CACHE_TEST_CLEAN)) {
9219         define_arm_cp_regs(cpu, cache_test_clean_cp_reginfo);
9220     }
9221     if (arm_feature(env, ARM_FEATURE_CACHE_DIRTY_REG)) {
9222         define_arm_cp_regs(cpu, cache_dirty_status_cp_reginfo);
9223     }
9224     if (arm_feature(env, ARM_FEATURE_CACHE_BLOCK_OPS)) {
9225         define_arm_cp_regs(cpu, cache_block_ops_cp_reginfo);
9226     }
9227     if (arm_feature(env, ARM_FEATURE_OMAPCP)) {
9228         define_arm_cp_regs(cpu, omap_cp_reginfo);
9229     }
9230     if (arm_feature(env, ARM_FEATURE_STRONGARM)) {
9231         define_arm_cp_regs(cpu, strongarm_cp_reginfo);
9232     }
9233     if (arm_feature(env, ARM_FEATURE_XSCALE)) {
9234         define_arm_cp_regs(cpu, xscale_cp_reginfo);
9235     }
9236     if (arm_feature(env, ARM_FEATURE_DUMMY_C15_REGS)) {
9237         define_arm_cp_regs(cpu, dummy_c15_cp_reginfo);
9238     }
9239     if (arm_feature(env, ARM_FEATURE_LPAE)) {
9240         define_arm_cp_regs(cpu, lpae_cp_reginfo);
9241     }
9242     if (cpu_isar_feature(aa32_jazelle, cpu)) {
9243         define_arm_cp_regs(cpu, jazelle_regs);
9244     }
9245     /*
9246      * Slightly awkwardly, the OMAP and StrongARM cores need all of
9247      * cp15 crn=0 to be writes-ignored, whereas for other cores they should
9248      * be read-only (ie write causes UNDEF exception).
9249      */
9250     {
9251         ARMCPRegInfo id_pre_v8_midr_cp_reginfo[] = {
9252             /*
9253              * Pre-v8 MIDR space.
9254              * Note that the MIDR isn't a simple constant register because
9255              * of the TI925 behaviour where writes to another register can
9256              * cause the MIDR value to change.
9257              *
9258              * Unimplemented registers in the c15 0 0 0 space default to
9259              * MIDR. Define MIDR first as this entire space, then CTR, TCMTR
9260              * and friends override accordingly.
9261              */
9262             { .name = "MIDR",
9263               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = CP_ANY,
9264               .access = PL1_R, .resetvalue = cpu->midr,
9265               .writefn = arm_cp_write_ignore, .raw_writefn = raw_write,
9266               .readfn = midr_read,
9267               .fieldoffset = offsetof(CPUARMState, cp15.c0_cpuid),
9268               .type = ARM_CP_OVERRIDE },
9269             /* crn = 0 op1 = 0 crm = 3..7 : currently unassigned; we RAZ. */
9270             { .name = "DUMMY",
9271               .cp = 15, .crn = 0, .crm = 3, .opc1 = 0, .opc2 = CP_ANY,
9272               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
9273             { .name = "DUMMY",
9274               .cp = 15, .crn = 0, .crm = 4, .opc1 = 0, .opc2 = CP_ANY,
9275               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
9276             { .name = "DUMMY",
9277               .cp = 15, .crn = 0, .crm = 5, .opc1 = 0, .opc2 = CP_ANY,
9278               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
9279             { .name = "DUMMY",
9280               .cp = 15, .crn = 0, .crm = 6, .opc1 = 0, .opc2 = CP_ANY,
9281               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
9282             { .name = "DUMMY",
9283               .cp = 15, .crn = 0, .crm = 7, .opc1 = 0, .opc2 = CP_ANY,
9284               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
9285         };
9286         ARMCPRegInfo id_v8_midr_cp_reginfo[] = {
9287             { .name = "MIDR_EL1", .state = ARM_CP_STATE_BOTH,
9288               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 0, .opc2 = 0,
9289               .access = PL1_R, .type = ARM_CP_NO_RAW, .resetvalue = cpu->midr,
9290               .fgt = FGT_MIDR_EL1,
9291               .fieldoffset = offsetof(CPUARMState, cp15.c0_cpuid),
9292               .readfn = midr_read },
9293             /* crn = 0 op1 = 0 crm = 0 op2 = 7 : AArch32 aliases of MIDR */
9294             { .name = "MIDR", .type = ARM_CP_ALIAS | ARM_CP_CONST,
9295               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 7,
9296               .access = PL1_R, .resetvalue = cpu->midr },
9297             { .name = "REVIDR_EL1", .state = ARM_CP_STATE_BOTH,
9298               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 0, .opc2 = 6,
9299               .access = PL1_R,
9300               .accessfn = access_aa64_tid1,
9301               .fgt = FGT_REVIDR_EL1,
9302               .type = ARM_CP_CONST, .resetvalue = cpu->revidr },
9303         };
9304         ARMCPRegInfo id_v8_midr_alias_cp_reginfo = {
9305             .name = "MIDR", .type = ARM_CP_ALIAS | ARM_CP_CONST | ARM_CP_NO_GDB,
9306             .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 4,
9307             .access = PL1_R, .resetvalue = cpu->midr
9308         };
9309         ARMCPRegInfo id_cp_reginfo[] = {
9310             /* These are common to v8 and pre-v8 */
9311             { .name = "CTR",
9312               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 1,
9313               .access = PL1_R, .accessfn = ctr_el0_access,
9314               .type = ARM_CP_CONST, .resetvalue = cpu->ctr },
9315             { .name = "CTR_EL0", .state = ARM_CP_STATE_AA64,
9316               .opc0 = 3, .opc1 = 3, .opc2 = 1, .crn = 0, .crm = 0,
9317               .access = PL0_R, .accessfn = ctr_el0_access,
9318               .fgt = FGT_CTR_EL0,
9319               .type = ARM_CP_CONST, .resetvalue = cpu->ctr },
9320             /* TCMTR and TLBTR exist in v8 but have no 64-bit versions */
9321             { .name = "TCMTR",
9322               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 2,
9323               .access = PL1_R,
9324               .accessfn = access_aa32_tid1,
9325               .type = ARM_CP_CONST, .resetvalue = 0 },
9326         };
9327         /* TLBTR is specific to VMSA */
9328         ARMCPRegInfo id_tlbtr_reginfo = {
9329               .name = "TLBTR",
9330               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 3,
9331               .access = PL1_R,
9332               .accessfn = access_aa32_tid1,
9333               .type = ARM_CP_CONST, .resetvalue = 0,
9334         };
9335         /* MPUIR is specific to PMSA V6+ */
9336         ARMCPRegInfo id_mpuir_reginfo = {
9337               .name = "MPUIR",
9338               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 4,
9339               .access = PL1_R, .type = ARM_CP_CONST,
9340               .resetvalue = cpu->pmsav7_dregion << 8
9341         };
9342         /* HMPUIR is specific to PMSA V8 */
9343         ARMCPRegInfo id_hmpuir_reginfo = {
9344             .name = "HMPUIR",
9345             .cp = 15, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 4,
9346             .access = PL2_R, .type = ARM_CP_CONST,
9347             .resetvalue = cpu->pmsav8r_hdregion
9348         };
9349         static const ARMCPRegInfo crn0_wi_reginfo = {
9350             .name = "CRN0_WI", .cp = 15, .crn = 0, .crm = CP_ANY,
9351             .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_W,
9352             .type = ARM_CP_NOP | ARM_CP_OVERRIDE
9353         };
9354 #ifdef CONFIG_USER_ONLY
9355         static const ARMCPRegUserSpaceInfo id_v8_user_midr_cp_reginfo[] = {
9356             { .name = "MIDR_EL1",
9357               .exported_bits = R_MIDR_EL1_REVISION_MASK |
9358                                R_MIDR_EL1_PARTNUM_MASK |
9359                                R_MIDR_EL1_ARCHITECTURE_MASK |
9360                                R_MIDR_EL1_VARIANT_MASK |
9361                                R_MIDR_EL1_IMPLEMENTER_MASK },
9362             { .name = "REVIDR_EL1" },
9363         };
9364         modify_arm_cp_regs(id_v8_midr_cp_reginfo, id_v8_user_midr_cp_reginfo);
9365 #endif
9366         if (arm_feature(env, ARM_FEATURE_OMAPCP) ||
9367             arm_feature(env, ARM_FEATURE_STRONGARM)) {
9368             size_t i;
9369             /*
9370              * Register the blanket "writes ignored" value first to cover the
9371              * whole space. Then update the specific ID registers to allow write
9372              * access, so that they ignore writes rather than causing them to
9373              * UNDEF.
9374              */
9375             define_one_arm_cp_reg(cpu, &crn0_wi_reginfo);
9376             for (i = 0; i < ARRAY_SIZE(id_pre_v8_midr_cp_reginfo); ++i) {
9377                 id_pre_v8_midr_cp_reginfo[i].access = PL1_RW;
9378             }
9379             for (i = 0; i < ARRAY_SIZE(id_cp_reginfo); ++i) {
9380                 id_cp_reginfo[i].access = PL1_RW;
9381             }
9382             id_mpuir_reginfo.access = PL1_RW;
9383             id_tlbtr_reginfo.access = PL1_RW;
9384         }
9385         if (arm_feature(env, ARM_FEATURE_V8)) {
9386             define_arm_cp_regs(cpu, id_v8_midr_cp_reginfo);
9387             if (!arm_feature(env, ARM_FEATURE_PMSA)) {
9388                 define_one_arm_cp_reg(cpu, &id_v8_midr_alias_cp_reginfo);
9389             }
9390         } else {
9391             define_arm_cp_regs(cpu, id_pre_v8_midr_cp_reginfo);
9392         }
9393         define_arm_cp_regs(cpu, id_cp_reginfo);
9394         if (!arm_feature(env, ARM_FEATURE_PMSA)) {
9395             define_one_arm_cp_reg(cpu, &id_tlbtr_reginfo);
9396         } else if (arm_feature(env, ARM_FEATURE_PMSA) &&
9397                    arm_feature(env, ARM_FEATURE_V8)) {
9398             uint32_t i = 0;
9399             char *tmp_string;
9400 
9401             define_one_arm_cp_reg(cpu, &id_mpuir_reginfo);
9402             define_one_arm_cp_reg(cpu, &id_hmpuir_reginfo);
9403             define_arm_cp_regs(cpu, pmsav8r_cp_reginfo);
9404 
9405             /* Register alias is only valid for first 32 indexes */
9406             for (i = 0; i < MIN(cpu->pmsav7_dregion, 32); ++i) {
9407                 uint8_t crm = 0b1000 | extract32(i, 1, 3);
9408                 uint8_t opc1 = extract32(i, 4, 1);
9409                 uint8_t opc2 = extract32(i, 0, 1) << 2;
9410 
9411                 tmp_string = g_strdup_printf("PRBAR%u", i);
9412                 ARMCPRegInfo tmp_prbarn_reginfo = {
9413                     .name = tmp_string, .type = ARM_CP_ALIAS | ARM_CP_NO_RAW,
9414                     .cp = 15, .opc1 = opc1, .crn = 6, .crm = crm, .opc2 = opc2,
9415                     .access = PL1_RW, .resetvalue = 0,
9416                     .accessfn = access_tvm_trvm,
9417                     .writefn = pmsav8r_regn_write, .readfn = pmsav8r_regn_read
9418                 };
9419                 define_one_arm_cp_reg(cpu, &tmp_prbarn_reginfo);
9420                 g_free(tmp_string);
9421 
9422                 opc2 = extract32(i, 0, 1) << 2 | 0x1;
9423                 tmp_string = g_strdup_printf("PRLAR%u", i);
9424                 ARMCPRegInfo tmp_prlarn_reginfo = {
9425                     .name = tmp_string, .type = ARM_CP_ALIAS | ARM_CP_NO_RAW,
9426                     .cp = 15, .opc1 = opc1, .crn = 6, .crm = crm, .opc2 = opc2,
9427                     .access = PL1_RW, .resetvalue = 0,
9428                     .accessfn = access_tvm_trvm,
9429                     .writefn = pmsav8r_regn_write, .readfn = pmsav8r_regn_read
9430                 };
9431                 define_one_arm_cp_reg(cpu, &tmp_prlarn_reginfo);
9432                 g_free(tmp_string);
9433             }
9434 
9435             /* Register alias is only valid for first 32 indexes */
9436             for (i = 0; i < MIN(cpu->pmsav8r_hdregion, 32); ++i) {
9437                 uint8_t crm = 0b1000 | extract32(i, 1, 3);
9438                 uint8_t opc1 = 0b100 | extract32(i, 4, 1);
9439                 uint8_t opc2 = extract32(i, 0, 1) << 2;
9440 
9441                 tmp_string = g_strdup_printf("HPRBAR%u", i);
9442                 ARMCPRegInfo tmp_hprbarn_reginfo = {
9443                     .name = tmp_string,
9444                     .type = ARM_CP_NO_RAW,
9445                     .cp = 15, .opc1 = opc1, .crn = 6, .crm = crm, .opc2 = opc2,
9446                     .access = PL2_RW, .resetvalue = 0,
9447                     .writefn = pmsav8r_regn_write, .readfn = pmsav8r_regn_read
9448                 };
9449                 define_one_arm_cp_reg(cpu, &tmp_hprbarn_reginfo);
9450                 g_free(tmp_string);
9451 
9452                 opc2 = extract32(i, 0, 1) << 2 | 0x1;
9453                 tmp_string = g_strdup_printf("HPRLAR%u", i);
9454                 ARMCPRegInfo tmp_hprlarn_reginfo = {
9455                     .name = tmp_string,
9456                     .type = ARM_CP_NO_RAW,
9457                     .cp = 15, .opc1 = opc1, .crn = 6, .crm = crm, .opc2 = opc2,
9458                     .access = PL2_RW, .resetvalue = 0,
9459                     .writefn = pmsav8r_regn_write, .readfn = pmsav8r_regn_read
9460                 };
9461                 define_one_arm_cp_reg(cpu, &tmp_hprlarn_reginfo);
9462                 g_free(tmp_string);
9463             }
9464         } else if (arm_feature(env, ARM_FEATURE_V7)) {
9465             define_one_arm_cp_reg(cpu, &id_mpuir_reginfo);
9466         }
9467     }
9468 
9469     if (arm_feature(env, ARM_FEATURE_MPIDR)) {
9470         ARMCPRegInfo mpidr_cp_reginfo[] = {
9471             { .name = "MPIDR_EL1", .state = ARM_CP_STATE_BOTH,
9472               .opc0 = 3, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 5,
9473               .fgt = FGT_MPIDR_EL1,
9474               .access = PL1_R, .readfn = mpidr_read, .type = ARM_CP_NO_RAW },
9475         };
9476 #ifdef CONFIG_USER_ONLY
9477         static const ARMCPRegUserSpaceInfo mpidr_user_cp_reginfo[] = {
9478             { .name = "MPIDR_EL1",
9479               .fixed_bits = 0x0000000080000000 },
9480         };
9481         modify_arm_cp_regs(mpidr_cp_reginfo, mpidr_user_cp_reginfo);
9482 #endif
9483         define_arm_cp_regs(cpu, mpidr_cp_reginfo);
9484     }
9485 
9486     if (arm_feature(env, ARM_FEATURE_AUXCR)) {
9487         ARMCPRegInfo auxcr_reginfo[] = {
9488             { .name = "ACTLR_EL1", .state = ARM_CP_STATE_BOTH,
9489               .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 1,
9490               .access = PL1_RW, .accessfn = access_tacr,
9491               .nv2_redirect_offset = 0x118,
9492               .type = ARM_CP_CONST, .resetvalue = cpu->reset_auxcr },
9493             { .name = "ACTLR_EL2", .state = ARM_CP_STATE_BOTH,
9494               .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 0, .opc2 = 1,
9495               .access = PL2_RW, .type = ARM_CP_CONST,
9496               .resetvalue = 0 },
9497             { .name = "ACTLR_EL3", .state = ARM_CP_STATE_AA64,
9498               .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 0, .opc2 = 1,
9499               .access = PL3_RW, .type = ARM_CP_CONST,
9500               .resetvalue = 0 },
9501         };
9502         define_arm_cp_regs(cpu, auxcr_reginfo);
9503         if (cpu_isar_feature(aa32_ac2, cpu)) {
9504             define_arm_cp_regs(cpu, actlr2_hactlr2_reginfo);
9505         }
9506     }
9507 
9508     if (arm_feature(env, ARM_FEATURE_CBAR)) {
9509         /*
9510          * CBAR is IMPDEF, but common on Arm Cortex-A implementations.
9511          * There are two flavours:
9512          *  (1) older 32-bit only cores have a simple 32-bit CBAR
9513          *  (2) 64-bit cores have a 64-bit CBAR visible to AArch64, plus a
9514          *      32-bit register visible to AArch32 at a different encoding
9515          *      to the "flavour 1" register and with the bits rearranged to
9516          *      be able to squash a 64-bit address into the 32-bit view.
9517          * We distinguish the two via the ARM_FEATURE_AARCH64 flag, but
9518          * in future if we support AArch32-only configs of some of the
9519          * AArch64 cores we might need to add a specific feature flag
9520          * to indicate cores with "flavour 2" CBAR.
9521          */
9522         if (arm_feature(env, ARM_FEATURE_AARCH64)) {
9523             /* 32 bit view is [31:18] 0...0 [43:32]. */
9524             uint32_t cbar32 = (extract64(cpu->reset_cbar, 18, 14) << 18)
9525                 | extract64(cpu->reset_cbar, 32, 12);
9526             ARMCPRegInfo cbar_reginfo[] = {
9527                 { .name = "CBAR",
9528                   .type = ARM_CP_CONST,
9529                   .cp = 15, .crn = 15, .crm = 3, .opc1 = 1, .opc2 = 0,
9530                   .access = PL1_R, .resetvalue = cbar32 },
9531                 { .name = "CBAR_EL1", .state = ARM_CP_STATE_AA64,
9532                   .type = ARM_CP_CONST,
9533                   .opc0 = 3, .opc1 = 1, .crn = 15, .crm = 3, .opc2 = 0,
9534                   .access = PL1_R, .resetvalue = cpu->reset_cbar },
9535             };
9536             /* We don't implement a r/w 64 bit CBAR currently */
9537             assert(arm_feature(env, ARM_FEATURE_CBAR_RO));
9538             define_arm_cp_regs(cpu, cbar_reginfo);
9539         } else {
9540             ARMCPRegInfo cbar = {
9541                 .name = "CBAR",
9542                 .cp = 15, .crn = 15, .crm = 0, .opc1 = 4, .opc2 = 0,
9543                 .access = PL1_R | PL3_W, .resetvalue = cpu->reset_cbar,
9544                 .fieldoffset = offsetof(CPUARMState,
9545                                         cp15.c15_config_base_address)
9546             };
9547             if (arm_feature(env, ARM_FEATURE_CBAR_RO)) {
9548                 cbar.access = PL1_R;
9549                 cbar.fieldoffset = 0;
9550                 cbar.type = ARM_CP_CONST;
9551             }
9552             define_one_arm_cp_reg(cpu, &cbar);
9553         }
9554     }
9555 
9556     if (arm_feature(env, ARM_FEATURE_VBAR)) {
9557         static const ARMCPRegInfo vbar_cp_reginfo[] = {
9558             { .name = "VBAR", .state = ARM_CP_STATE_BOTH,
9559               .opc0 = 3, .crn = 12, .crm = 0, .opc1 = 0, .opc2 = 0,
9560               .access = PL1_RW, .writefn = vbar_write,
9561               .accessfn = access_nv1,
9562               .fgt = FGT_VBAR_EL1,
9563               .nv2_redirect_offset = 0x250 | NV2_REDIR_NV1,
9564               .bank_fieldoffsets = { offsetof(CPUARMState, cp15.vbar_s),
9565                                      offsetof(CPUARMState, cp15.vbar_ns) },
9566               .resetvalue = 0 },
9567         };
9568         define_arm_cp_regs(cpu, vbar_cp_reginfo);
9569     }
9570 
9571     /* Generic registers whose values depend on the implementation */
9572     {
9573         ARMCPRegInfo sctlr = {
9574             .name = "SCTLR", .state = ARM_CP_STATE_BOTH,
9575             .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 0,
9576             .access = PL1_RW, .accessfn = access_tvm_trvm,
9577             .fgt = FGT_SCTLR_EL1,
9578             .nv2_redirect_offset = 0x110 | NV2_REDIR_NV1,
9579             .bank_fieldoffsets = { offsetof(CPUARMState, cp15.sctlr_s),
9580                                    offsetof(CPUARMState, cp15.sctlr_ns) },
9581             .writefn = sctlr_write, .resetvalue = cpu->reset_sctlr,
9582             .raw_writefn = raw_write,
9583         };
9584         if (arm_feature(env, ARM_FEATURE_XSCALE)) {
9585             /*
9586              * Normally we would always end the TB on an SCTLR write, but Linux
9587              * arch/arm/mach-pxa/sleep.S expects two instructions following
9588              * an MMU enable to execute from cache.  Imitate this behaviour.
9589              */
9590             sctlr.type |= ARM_CP_SUPPRESS_TB_END;
9591         }
9592         define_one_arm_cp_reg(cpu, &sctlr);
9593 
9594         if (arm_feature(env, ARM_FEATURE_PMSA) &&
9595             arm_feature(env, ARM_FEATURE_V8)) {
9596             ARMCPRegInfo vsctlr = {
9597                 .name = "VSCTLR", .state = ARM_CP_STATE_AA32,
9598                 .cp = 15, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 0,
9599                 .access = PL2_RW, .resetvalue = 0x0,
9600                 .fieldoffset = offsetoflow32(CPUARMState, cp15.vsctlr),
9601             };
9602             define_one_arm_cp_reg(cpu, &vsctlr);
9603         }
9604     }
9605 
9606     if (cpu_isar_feature(aa64_lor, cpu)) {
9607         define_arm_cp_regs(cpu, lor_reginfo);
9608     }
9609     if (cpu_isar_feature(aa64_pan, cpu)) {
9610         define_one_arm_cp_reg(cpu, &pan_reginfo);
9611     }
9612 #ifndef CONFIG_USER_ONLY
9613     if (cpu_isar_feature(aa64_ats1e1, cpu)) {
9614         define_arm_cp_regs(cpu, ats1e1_reginfo);
9615     }
9616     if (cpu_isar_feature(aa32_ats1e1, cpu)) {
9617         define_arm_cp_regs(cpu, ats1cp_reginfo);
9618     }
9619 #endif
9620     if (cpu_isar_feature(aa64_uao, cpu)) {
9621         define_one_arm_cp_reg(cpu, &uao_reginfo);
9622     }
9623 
9624     if (cpu_isar_feature(aa64_dit, cpu)) {
9625         define_one_arm_cp_reg(cpu, &dit_reginfo);
9626     }
9627     if (cpu_isar_feature(aa64_ssbs, cpu)) {
9628         define_one_arm_cp_reg(cpu, &ssbs_reginfo);
9629     }
9630     if (cpu_isar_feature(any_ras, cpu)) {
9631         define_arm_cp_regs(cpu, minimal_ras_reginfo);
9632     }
9633 
9634     if (cpu_isar_feature(aa64_vh, cpu) ||
9635         cpu_isar_feature(aa64_debugv8p2, cpu)) {
9636         define_one_arm_cp_reg(cpu, &contextidr_el2);
9637     }
9638     if (arm_feature(env, ARM_FEATURE_EL2) && cpu_isar_feature(aa64_vh, cpu)) {
9639         define_arm_cp_regs(cpu, vhe_reginfo);
9640     }
9641 
9642     if (cpu_isar_feature(aa64_sve, cpu)) {
9643         define_arm_cp_regs(cpu, zcr_reginfo);
9644     }
9645 
9646     if (cpu_isar_feature(aa64_hcx, cpu)) {
9647         define_one_arm_cp_reg(cpu, &hcrx_el2_reginfo);
9648     }
9649 
9650 #ifdef TARGET_AARCH64
9651     if (cpu_isar_feature(aa64_sme, cpu)) {
9652         define_arm_cp_regs(cpu, sme_reginfo);
9653     }
9654     if (cpu_isar_feature(aa64_pauth, cpu)) {
9655         define_arm_cp_regs(cpu, pauth_reginfo);
9656     }
9657     if (cpu_isar_feature(aa64_rndr, cpu)) {
9658         define_arm_cp_regs(cpu, rndr_reginfo);
9659     }
9660     if (cpu_isar_feature(aa64_tlbirange, cpu)) {
9661         define_arm_cp_regs(cpu, tlbirange_reginfo);
9662     }
9663     if (cpu_isar_feature(aa64_tlbios, cpu)) {
9664         define_arm_cp_regs(cpu, tlbios_reginfo);
9665     }
9666     /* Data Cache clean instructions up to PoP */
9667     if (cpu_isar_feature(aa64_dcpop, cpu)) {
9668         define_one_arm_cp_reg(cpu, dcpop_reg);
9669 
9670         if (cpu_isar_feature(aa64_dcpodp, cpu)) {
9671             define_one_arm_cp_reg(cpu, dcpodp_reg);
9672         }
9673     }
9674 
9675     /*
9676      * If full MTE is enabled, add all of the system registers.
9677      * If only "instructions available at EL0" are enabled,
9678      * then define only a RAZ/WI version of PSTATE.TCO.
9679      */
9680     if (cpu_isar_feature(aa64_mte, cpu)) {
9681         ARMCPRegInfo gmid_reginfo = {
9682             .name = "GMID_EL1", .state = ARM_CP_STATE_AA64,
9683             .opc0 = 3, .opc1 = 1, .crn = 0, .crm = 0, .opc2 = 4,
9684             .access = PL1_R, .accessfn = access_aa64_tid5,
9685             .type = ARM_CP_CONST, .resetvalue = cpu->gm_blocksize,
9686         };
9687         define_one_arm_cp_reg(cpu, &gmid_reginfo);
9688         define_arm_cp_regs(cpu, mte_reginfo);
9689         define_arm_cp_regs(cpu, mte_el0_cacheop_reginfo);
9690     } else if (cpu_isar_feature(aa64_mte_insn_reg, cpu)) {
9691         define_arm_cp_regs(cpu, mte_tco_ro_reginfo);
9692         define_arm_cp_regs(cpu, mte_el0_cacheop_reginfo);
9693     }
9694 
9695     if (cpu_isar_feature(aa64_scxtnum, cpu)) {
9696         define_arm_cp_regs(cpu, scxtnum_reginfo);
9697     }
9698 
9699     if (cpu_isar_feature(aa64_fgt, cpu)) {
9700         define_arm_cp_regs(cpu, fgt_reginfo);
9701     }
9702 
9703     if (cpu_isar_feature(aa64_rme, cpu)) {
9704         define_arm_cp_regs(cpu, rme_reginfo);
9705         if (cpu_isar_feature(aa64_mte, cpu)) {
9706             define_arm_cp_regs(cpu, rme_mte_reginfo);
9707         }
9708     }
9709 
9710     if (cpu_isar_feature(aa64_nv2, cpu)) {
9711         define_arm_cp_regs(cpu, nv2_reginfo);
9712     }
9713 #endif
9714 
9715     if (cpu_isar_feature(any_predinv, cpu)) {
9716         define_arm_cp_regs(cpu, predinv_reginfo);
9717     }
9718 
9719     if (cpu_isar_feature(any_ccidx, cpu)) {
9720         define_arm_cp_regs(cpu, ccsidr2_reginfo);
9721     }
9722 
9723 #ifndef CONFIG_USER_ONLY
9724     /*
9725      * Register redirections and aliases must be done last,
9726      * after the registers from the other extensions have been defined.
9727      */
9728     if (arm_feature(env, ARM_FEATURE_EL2) && cpu_isar_feature(aa64_vh, cpu)) {
9729         define_arm_vh_e2h_redirects_aliases(cpu);
9730     }
9731 #endif
9732 }
9733 
9734 /*
9735  * Private utility function for define_one_arm_cp_reg_with_opaque():
9736  * add a single reginfo struct to the hash table.
9737  */
9738 static void add_cpreg_to_hashtable(ARMCPU *cpu, const ARMCPRegInfo *r,
9739                                    void *opaque, CPState state,
9740                                    CPSecureState secstate,
9741                                    int crm, int opc1, int opc2,
9742                                    const char *name)
9743 {
9744     CPUARMState *env = &cpu->env;
9745     uint32_t key;
9746     ARMCPRegInfo *r2;
9747     bool is64 = r->type & ARM_CP_64BIT;
9748     bool ns = secstate & ARM_CP_SECSTATE_NS;
9749     int cp = r->cp;
9750     size_t name_len;
9751     bool make_const;
9752 
9753     switch (state) {
9754     case ARM_CP_STATE_AA32:
9755         /* We assume it is a cp15 register if the .cp field is left unset. */
9756         if (cp == 0 && r->state == ARM_CP_STATE_BOTH) {
9757             cp = 15;
9758         }
9759         key = ENCODE_CP_REG(cp, is64, ns, r->crn, crm, opc1, opc2);
9760         break;
9761     case ARM_CP_STATE_AA64:
9762         /*
9763          * To allow abbreviation of ARMCPRegInfo definitions, we treat
9764          * cp == 0 as equivalent to the value for "standard guest-visible
9765          * sysreg".  STATE_BOTH definitions are also always "standard sysreg"
9766          * in their AArch64 view (the .cp value may be non-zero for the
9767          * benefit of the AArch32 view).
9768          */
9769         if (cp == 0 || r->state == ARM_CP_STATE_BOTH) {
9770             cp = CP_REG_ARM64_SYSREG_CP;
9771         }
9772         key = ENCODE_AA64_CP_REG(cp, r->crn, crm, r->opc0, opc1, opc2);
9773         break;
9774     default:
9775         g_assert_not_reached();
9776     }
9777 
9778     /* Overriding of an existing definition must be explicitly requested. */
9779     if (!(r->type & ARM_CP_OVERRIDE)) {
9780         const ARMCPRegInfo *oldreg = get_arm_cp_reginfo(cpu->cp_regs, key);
9781         if (oldreg) {
9782             assert(oldreg->type & ARM_CP_OVERRIDE);
9783         }
9784     }
9785 
9786     /*
9787      * Eliminate registers that are not present because the EL is missing.
9788      * Doing this here makes it easier to put all registers for a given
9789      * feature into the same ARMCPRegInfo array and define them all at once.
9790      */
9791     make_const = false;
9792     if (arm_feature(env, ARM_FEATURE_EL3)) {
9793         /*
9794          * An EL2 register without EL2 but with EL3 is (usually) RES0.
9795          * See rule RJFFP in section D1.1.3 of DDI0487H.a.
9796          */
9797         int min_el = ctz32(r->access) / 2;
9798         if (min_el == 2 && !arm_feature(env, ARM_FEATURE_EL2)) {
9799             if (r->type & ARM_CP_EL3_NO_EL2_UNDEF) {
9800                 return;
9801             }
9802             make_const = !(r->type & ARM_CP_EL3_NO_EL2_KEEP);
9803         }
9804     } else {
9805         CPAccessRights max_el = (arm_feature(env, ARM_FEATURE_EL2)
9806                                  ? PL2_RW : PL1_RW);
9807         if ((r->access & max_el) == 0) {
9808             return;
9809         }
9810     }
9811 
9812     /* Combine cpreg and name into one allocation. */
9813     name_len = strlen(name) + 1;
9814     r2 = g_malloc(sizeof(*r2) + name_len);
9815     *r2 = *r;
9816     r2->name = memcpy(r2 + 1, name, name_len);
9817 
9818     /*
9819      * Update fields to match the instantiation, overwiting wildcards
9820      * such as CP_ANY, ARM_CP_STATE_BOTH, or ARM_CP_SECSTATE_BOTH.
9821      */
9822     r2->cp = cp;
9823     r2->crm = crm;
9824     r2->opc1 = opc1;
9825     r2->opc2 = opc2;
9826     r2->state = state;
9827     r2->secure = secstate;
9828     if (opaque) {
9829         r2->opaque = opaque;
9830     }
9831 
9832     if (make_const) {
9833         /* This should not have been a very special register to begin. */
9834         int old_special = r2->type & ARM_CP_SPECIAL_MASK;
9835         assert(old_special == 0 || old_special == ARM_CP_NOP);
9836         /*
9837          * Set the special function to CONST, retaining the other flags.
9838          * This is important for e.g. ARM_CP_SVE so that we still
9839          * take the SVE trap if CPTR_EL3.EZ == 0.
9840          */
9841         r2->type = (r2->type & ~ARM_CP_SPECIAL_MASK) | ARM_CP_CONST;
9842         /*
9843          * Usually, these registers become RES0, but there are a few
9844          * special cases like VPIDR_EL2 which have a constant non-zero
9845          * value with writes ignored.
9846          */
9847         if (!(r->type & ARM_CP_EL3_NO_EL2_C_NZ)) {
9848             r2->resetvalue = 0;
9849         }
9850         /*
9851          * ARM_CP_CONST has precedence, so removing the callbacks and
9852          * offsets are not strictly necessary, but it is potentially
9853          * less confusing to debug later.
9854          */
9855         r2->readfn = NULL;
9856         r2->writefn = NULL;
9857         r2->raw_readfn = NULL;
9858         r2->raw_writefn = NULL;
9859         r2->resetfn = NULL;
9860         r2->fieldoffset = 0;
9861         r2->bank_fieldoffsets[0] = 0;
9862         r2->bank_fieldoffsets[1] = 0;
9863     } else {
9864         bool isbanked = r->bank_fieldoffsets[0] && r->bank_fieldoffsets[1];
9865 
9866         if (isbanked) {
9867             /*
9868              * Register is banked (using both entries in array).
9869              * Overwriting fieldoffset as the array is only used to define
9870              * banked registers but later only fieldoffset is used.
9871              */
9872             r2->fieldoffset = r->bank_fieldoffsets[ns];
9873         }
9874         if (state == ARM_CP_STATE_AA32) {
9875             if (isbanked) {
9876                 /*
9877                  * If the register is banked then we don't need to migrate or
9878                  * reset the 32-bit instance in certain cases:
9879                  *
9880                  * 1) If the register has both 32-bit and 64-bit instances
9881                  *    then we can count on the 64-bit instance taking care
9882                  *    of the non-secure bank.
9883                  * 2) If ARMv8 is enabled then we can count on a 64-bit
9884                  *    version taking care of the secure bank.  This requires
9885                  *    that separate 32 and 64-bit definitions are provided.
9886                  */
9887                 if ((r->state == ARM_CP_STATE_BOTH && ns) ||
9888                     (arm_feature(env, ARM_FEATURE_V8) && !ns)) {
9889                     r2->type |= ARM_CP_ALIAS;
9890                 }
9891             } else if ((secstate != r->secure) && !ns) {
9892                 /*
9893                  * The register is not banked so we only want to allow
9894                  * migration of the non-secure instance.
9895                  */
9896                 r2->type |= ARM_CP_ALIAS;
9897             }
9898 
9899             if (HOST_BIG_ENDIAN &&
9900                 r->state == ARM_CP_STATE_BOTH && r2->fieldoffset) {
9901                 r2->fieldoffset += sizeof(uint32_t);
9902             }
9903         }
9904     }
9905 
9906     /*
9907      * By convention, for wildcarded registers only the first
9908      * entry is used for migration; the others are marked as
9909      * ALIAS so we don't try to transfer the register
9910      * multiple times. Special registers (ie NOP/WFI) are
9911      * never migratable and not even raw-accessible.
9912      */
9913     if (r2->type & ARM_CP_SPECIAL_MASK) {
9914         r2->type |= ARM_CP_NO_RAW;
9915     }
9916     if (((r->crm == CP_ANY) && crm != 0) ||
9917         ((r->opc1 == CP_ANY) && opc1 != 0) ||
9918         ((r->opc2 == CP_ANY) && opc2 != 0)) {
9919         r2->type |= ARM_CP_ALIAS | ARM_CP_NO_GDB;
9920     }
9921 
9922     /*
9923      * Check that raw accesses are either forbidden or handled. Note that
9924      * we can't assert this earlier because the setup of fieldoffset for
9925      * banked registers has to be done first.
9926      */
9927     if (!(r2->type & ARM_CP_NO_RAW)) {
9928         assert(!raw_accessors_invalid(r2));
9929     }
9930 
9931     g_hash_table_insert(cpu->cp_regs, (gpointer)(uintptr_t)key, r2);
9932 }
9933 
9934 
9935 void define_one_arm_cp_reg_with_opaque(ARMCPU *cpu,
9936                                        const ARMCPRegInfo *r, void *opaque)
9937 {
9938     /*
9939      * Define implementations of coprocessor registers.
9940      * We store these in a hashtable because typically
9941      * there are less than 150 registers in a space which
9942      * is 16*16*16*8*8 = 262144 in size.
9943      * Wildcarding is supported for the crm, opc1 and opc2 fields.
9944      * If a register is defined twice then the second definition is
9945      * used, so this can be used to define some generic registers and
9946      * then override them with implementation specific variations.
9947      * At least one of the original and the second definition should
9948      * include ARM_CP_OVERRIDE in its type bits -- this is just a guard
9949      * against accidental use.
9950      *
9951      * The state field defines whether the register is to be
9952      * visible in the AArch32 or AArch64 execution state. If the
9953      * state is set to ARM_CP_STATE_BOTH then we synthesise a
9954      * reginfo structure for the AArch32 view, which sees the lower
9955      * 32 bits of the 64 bit register.
9956      *
9957      * Only registers visible in AArch64 may set r->opc0; opc0 cannot
9958      * be wildcarded. AArch64 registers are always considered to be 64
9959      * bits; the ARM_CP_64BIT* flag applies only to the AArch32 view of
9960      * the register, if any.
9961      */
9962     int crm, opc1, opc2;
9963     int crmmin = (r->crm == CP_ANY) ? 0 : r->crm;
9964     int crmmax = (r->crm == CP_ANY) ? 15 : r->crm;
9965     int opc1min = (r->opc1 == CP_ANY) ? 0 : r->opc1;
9966     int opc1max = (r->opc1 == CP_ANY) ? 7 : r->opc1;
9967     int opc2min = (r->opc2 == CP_ANY) ? 0 : r->opc2;
9968     int opc2max = (r->opc2 == CP_ANY) ? 7 : r->opc2;
9969     CPState state;
9970 
9971     /* 64 bit registers have only CRm and Opc1 fields */
9972     assert(!((r->type & ARM_CP_64BIT) && (r->opc2 || r->crn)));
9973     /* op0 only exists in the AArch64 encodings */
9974     assert((r->state != ARM_CP_STATE_AA32) || (r->opc0 == 0));
9975     /* AArch64 regs are all 64 bit so ARM_CP_64BIT is meaningless */
9976     assert((r->state != ARM_CP_STATE_AA64) || !(r->type & ARM_CP_64BIT));
9977     /*
9978      * This API is only for Arm's system coprocessors (14 and 15) or
9979      * (M-profile or v7A-and-earlier only) for implementation defined
9980      * coprocessors in the range 0..7.  Our decode assumes this, since
9981      * 8..13 can be used for other insns including VFP and Neon. See
9982      * valid_cp() in translate.c.  Assert here that we haven't tried
9983      * to use an invalid coprocessor number.
9984      */
9985     switch (r->state) {
9986     case ARM_CP_STATE_BOTH:
9987         /* 0 has a special meaning, but otherwise the same rules as AA32. */
9988         if (r->cp == 0) {
9989             break;
9990         }
9991         /* fall through */
9992     case ARM_CP_STATE_AA32:
9993         if (arm_feature(&cpu->env, ARM_FEATURE_V8) &&
9994             !arm_feature(&cpu->env, ARM_FEATURE_M)) {
9995             assert(r->cp >= 14 && r->cp <= 15);
9996         } else {
9997             assert(r->cp < 8 || (r->cp >= 14 && r->cp <= 15));
9998         }
9999         break;
10000     case ARM_CP_STATE_AA64:
10001         assert(r->cp == 0 || r->cp == CP_REG_ARM64_SYSREG_CP);
10002         break;
10003     default:
10004         g_assert_not_reached();
10005     }
10006     /*
10007      * The AArch64 pseudocode CheckSystemAccess() specifies that op1
10008      * encodes a minimum access level for the register. We roll this
10009      * runtime check into our general permission check code, so check
10010      * here that the reginfo's specified permissions are strict enough
10011      * to encompass the generic architectural permission check.
10012      */
10013     if (r->state != ARM_CP_STATE_AA32) {
10014         CPAccessRights mask;
10015         switch (r->opc1) {
10016         case 0:
10017             /* min_EL EL1, but some accessible to EL0 via kernel ABI */
10018             mask = PL0U_R | PL1_RW;
10019             break;
10020         case 1: case 2:
10021             /* min_EL EL1 */
10022             mask = PL1_RW;
10023             break;
10024         case 3:
10025             /* min_EL EL0 */
10026             mask = PL0_RW;
10027             break;
10028         case 4:
10029         case 5:
10030             /* min_EL EL2 */
10031             mask = PL2_RW;
10032             break;
10033         case 6:
10034             /* min_EL EL3 */
10035             mask = PL3_RW;
10036             break;
10037         case 7:
10038             /* min_EL EL1, secure mode only (we don't check the latter) */
10039             mask = PL1_RW;
10040             break;
10041         default:
10042             /* broken reginfo with out-of-range opc1 */
10043             g_assert_not_reached();
10044         }
10045         /* assert our permissions are not too lax (stricter is fine) */
10046         assert((r->access & ~mask) == 0);
10047     }
10048 
10049     /*
10050      * Check that the register definition has enough info to handle
10051      * reads and writes if they are permitted.
10052      */
10053     if (!(r->type & (ARM_CP_SPECIAL_MASK | ARM_CP_CONST))) {
10054         if (r->access & PL3_R) {
10055             assert((r->fieldoffset ||
10056                    (r->bank_fieldoffsets[0] && r->bank_fieldoffsets[1])) ||
10057                    r->readfn);
10058         }
10059         if (r->access & PL3_W) {
10060             assert((r->fieldoffset ||
10061                    (r->bank_fieldoffsets[0] && r->bank_fieldoffsets[1])) ||
10062                    r->writefn);
10063         }
10064     }
10065 
10066     for (crm = crmmin; crm <= crmmax; crm++) {
10067         for (opc1 = opc1min; opc1 <= opc1max; opc1++) {
10068             for (opc2 = opc2min; opc2 <= opc2max; opc2++) {
10069                 for (state = ARM_CP_STATE_AA32;
10070                      state <= ARM_CP_STATE_AA64; state++) {
10071                     if (r->state != state && r->state != ARM_CP_STATE_BOTH) {
10072                         continue;
10073                     }
10074                     if (state == ARM_CP_STATE_AA32) {
10075                         /*
10076                          * Under AArch32 CP registers can be common
10077                          * (same for secure and non-secure world) or banked.
10078                          */
10079                         char *name;
10080 
10081                         switch (r->secure) {
10082                         case ARM_CP_SECSTATE_S:
10083                         case ARM_CP_SECSTATE_NS:
10084                             add_cpreg_to_hashtable(cpu, r, opaque, state,
10085                                                    r->secure, crm, opc1, opc2,
10086                                                    r->name);
10087                             break;
10088                         case ARM_CP_SECSTATE_BOTH:
10089                             name = g_strdup_printf("%s_S", r->name);
10090                             add_cpreg_to_hashtable(cpu, r, opaque, state,
10091                                                    ARM_CP_SECSTATE_S,
10092                                                    crm, opc1, opc2, name);
10093                             g_free(name);
10094                             add_cpreg_to_hashtable(cpu, r, opaque, state,
10095                                                    ARM_CP_SECSTATE_NS,
10096                                                    crm, opc1, opc2, r->name);
10097                             break;
10098                         default:
10099                             g_assert_not_reached();
10100                         }
10101                     } else {
10102                         /*
10103                          * AArch64 registers get mapped to non-secure instance
10104                          * of AArch32
10105                          */
10106                         add_cpreg_to_hashtable(cpu, r, opaque, state,
10107                                                ARM_CP_SECSTATE_NS,
10108                                                crm, opc1, opc2, r->name);
10109                     }
10110                 }
10111             }
10112         }
10113     }
10114 }
10115 
10116 /* Define a whole list of registers */
10117 void define_arm_cp_regs_with_opaque_len(ARMCPU *cpu, const ARMCPRegInfo *regs,
10118                                         void *opaque, size_t len)
10119 {
10120     size_t i;
10121     for (i = 0; i < len; ++i) {
10122         define_one_arm_cp_reg_with_opaque(cpu, regs + i, opaque);
10123     }
10124 }
10125 
10126 /*
10127  * Modify ARMCPRegInfo for access from userspace.
10128  *
10129  * This is a data driven modification directed by
10130  * ARMCPRegUserSpaceInfo. All registers become ARM_CP_CONST as
10131  * user-space cannot alter any values and dynamic values pertaining to
10132  * execution state are hidden from user space view anyway.
10133  */
10134 void modify_arm_cp_regs_with_len(ARMCPRegInfo *regs, size_t regs_len,
10135                                  const ARMCPRegUserSpaceInfo *mods,
10136                                  size_t mods_len)
10137 {
10138     for (size_t mi = 0; mi < mods_len; ++mi) {
10139         const ARMCPRegUserSpaceInfo *m = mods + mi;
10140         GPatternSpec *pat = NULL;
10141 
10142         if (m->is_glob) {
10143             pat = g_pattern_spec_new(m->name);
10144         }
10145         for (size_t ri = 0; ri < regs_len; ++ri) {
10146             ARMCPRegInfo *r = regs + ri;
10147 
10148             if (pat && g_pattern_match_string(pat, r->name)) {
10149                 r->type = ARM_CP_CONST;
10150                 r->access = PL0U_R;
10151                 r->resetvalue = 0;
10152                 /* continue */
10153             } else if (strcmp(r->name, m->name) == 0) {
10154                 r->type = ARM_CP_CONST;
10155                 r->access = PL0U_R;
10156                 r->resetvalue &= m->exported_bits;
10157                 r->resetvalue |= m->fixed_bits;
10158                 break;
10159             }
10160         }
10161         if (pat) {
10162             g_pattern_spec_free(pat);
10163         }
10164     }
10165 }
10166 
10167 const ARMCPRegInfo *get_arm_cp_reginfo(GHashTable *cpregs, uint32_t encoded_cp)
10168 {
10169     return g_hash_table_lookup(cpregs, (gpointer)(uintptr_t)encoded_cp);
10170 }
10171 
10172 void arm_cp_write_ignore(CPUARMState *env, const ARMCPRegInfo *ri,
10173                          uint64_t value)
10174 {
10175     /* Helper coprocessor write function for write-ignore registers */
10176 }
10177 
10178 uint64_t arm_cp_read_zero(CPUARMState *env, const ARMCPRegInfo *ri)
10179 {
10180     /* Helper coprocessor write function for read-as-zero registers */
10181     return 0;
10182 }
10183 
10184 void arm_cp_reset_ignore(CPUARMState *env, const ARMCPRegInfo *opaque)
10185 {
10186     /* Helper coprocessor reset function for do-nothing-on-reset registers */
10187 }
10188 
10189 static int bad_mode_switch(CPUARMState *env, int mode, CPSRWriteType write_type)
10190 {
10191     /*
10192      * Return true if it is not valid for us to switch to
10193      * this CPU mode (ie all the UNPREDICTABLE cases in
10194      * the ARM ARM CPSRWriteByInstr pseudocode).
10195      */
10196 
10197     /* Changes to or from Hyp via MSR and CPS are illegal. */
10198     if (write_type == CPSRWriteByInstr &&
10199         ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_HYP ||
10200          mode == ARM_CPU_MODE_HYP)) {
10201         return 1;
10202     }
10203 
10204     switch (mode) {
10205     case ARM_CPU_MODE_USR:
10206         return 0;
10207     case ARM_CPU_MODE_SYS:
10208     case ARM_CPU_MODE_SVC:
10209     case ARM_CPU_MODE_ABT:
10210     case ARM_CPU_MODE_UND:
10211     case ARM_CPU_MODE_IRQ:
10212     case ARM_CPU_MODE_FIQ:
10213         /*
10214          * Note that we don't implement the IMPDEF NSACR.RFR which in v7
10215          * allows FIQ mode to be Secure-only. (In v8 this doesn't exist.)
10216          */
10217         /*
10218          * If HCR.TGE is set then changes from Monitor to NS PL1 via MSR
10219          * and CPS are treated as illegal mode changes.
10220          */
10221         if (write_type == CPSRWriteByInstr &&
10222             (env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_MON &&
10223             (arm_hcr_el2_eff(env) & HCR_TGE)) {
10224             return 1;
10225         }
10226         return 0;
10227     case ARM_CPU_MODE_HYP:
10228         return !arm_is_el2_enabled(env) || arm_current_el(env) < 2;
10229     case ARM_CPU_MODE_MON:
10230         return arm_current_el(env) < 3;
10231     default:
10232         return 1;
10233     }
10234 }
10235 
10236 uint32_t cpsr_read(CPUARMState *env)
10237 {
10238     int ZF;
10239     ZF = (env->ZF == 0);
10240     return env->uncached_cpsr | (env->NF & 0x80000000) | (ZF << 30) |
10241         (env->CF << 29) | ((env->VF & 0x80000000) >> 3) | (env->QF << 27)
10242         | (env->thumb << 5) | ((env->condexec_bits & 3) << 25)
10243         | ((env->condexec_bits & 0xfc) << 8)
10244         | (env->GE << 16) | (env->daif & CPSR_AIF);
10245 }
10246 
10247 void cpsr_write(CPUARMState *env, uint32_t val, uint32_t mask,
10248                 CPSRWriteType write_type)
10249 {
10250     uint32_t changed_daif;
10251     bool rebuild_hflags = (write_type != CPSRWriteRaw) &&
10252         (mask & (CPSR_M | CPSR_E | CPSR_IL));
10253 
10254     if (mask & CPSR_NZCV) {
10255         env->ZF = (~val) & CPSR_Z;
10256         env->NF = val;
10257         env->CF = (val >> 29) & 1;
10258         env->VF = (val << 3) & 0x80000000;
10259     }
10260     if (mask & CPSR_Q) {
10261         env->QF = ((val & CPSR_Q) != 0);
10262     }
10263     if (mask & CPSR_T) {
10264         env->thumb = ((val & CPSR_T) != 0);
10265     }
10266     if (mask & CPSR_IT_0_1) {
10267         env->condexec_bits &= ~3;
10268         env->condexec_bits |= (val >> 25) & 3;
10269     }
10270     if (mask & CPSR_IT_2_7) {
10271         env->condexec_bits &= 3;
10272         env->condexec_bits |= (val >> 8) & 0xfc;
10273     }
10274     if (mask & CPSR_GE) {
10275         env->GE = (val >> 16) & 0xf;
10276     }
10277 
10278     /*
10279      * In a V7 implementation that includes the security extensions but does
10280      * not include Virtualization Extensions the SCR.FW and SCR.AW bits control
10281      * whether non-secure software is allowed to change the CPSR_F and CPSR_A
10282      * bits respectively.
10283      *
10284      * In a V8 implementation, it is permitted for privileged software to
10285      * change the CPSR A/F bits regardless of the SCR.AW/FW bits.
10286      */
10287     if (write_type != CPSRWriteRaw && !arm_feature(env, ARM_FEATURE_V8) &&
10288         arm_feature(env, ARM_FEATURE_EL3) &&
10289         !arm_feature(env, ARM_FEATURE_EL2) &&
10290         !arm_is_secure(env)) {
10291 
10292         changed_daif = (env->daif ^ val) & mask;
10293 
10294         if (changed_daif & CPSR_A) {
10295             /*
10296              * Check to see if we are allowed to change the masking of async
10297              * abort exceptions from a non-secure state.
10298              */
10299             if (!(env->cp15.scr_el3 & SCR_AW)) {
10300                 qemu_log_mask(LOG_GUEST_ERROR,
10301                               "Ignoring attempt to switch CPSR_A flag from "
10302                               "non-secure world with SCR.AW bit clear\n");
10303                 mask &= ~CPSR_A;
10304             }
10305         }
10306 
10307         if (changed_daif & CPSR_F) {
10308             /*
10309              * Check to see if we are allowed to change the masking of FIQ
10310              * exceptions from a non-secure state.
10311              */
10312             if (!(env->cp15.scr_el3 & SCR_FW)) {
10313                 qemu_log_mask(LOG_GUEST_ERROR,
10314                               "Ignoring attempt to switch CPSR_F flag from "
10315                               "non-secure world with SCR.FW bit clear\n");
10316                 mask &= ~CPSR_F;
10317             }
10318 
10319             /*
10320              * Check whether non-maskable FIQ (NMFI) support is enabled.
10321              * If this bit is set software is not allowed to mask
10322              * FIQs, but is allowed to set CPSR_F to 0.
10323              */
10324             if ((A32_BANKED_CURRENT_REG_GET(env, sctlr) & SCTLR_NMFI) &&
10325                 (val & CPSR_F)) {
10326                 qemu_log_mask(LOG_GUEST_ERROR,
10327                               "Ignoring attempt to enable CPSR_F flag "
10328                               "(non-maskable FIQ [NMFI] support enabled)\n");
10329                 mask &= ~CPSR_F;
10330             }
10331         }
10332     }
10333 
10334     env->daif &= ~(CPSR_AIF & mask);
10335     env->daif |= val & CPSR_AIF & mask;
10336 
10337     if (write_type != CPSRWriteRaw &&
10338         ((env->uncached_cpsr ^ val) & mask & CPSR_M)) {
10339         if ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_USR) {
10340             /*
10341              * Note that we can only get here in USR mode if this is a
10342              * gdb stub write; for this case we follow the architectural
10343              * behaviour for guest writes in USR mode of ignoring an attempt
10344              * to switch mode. (Those are caught by translate.c for writes
10345              * triggered by guest instructions.)
10346              */
10347             mask &= ~CPSR_M;
10348         } else if (bad_mode_switch(env, val & CPSR_M, write_type)) {
10349             /*
10350              * Attempt to switch to an invalid mode: this is UNPREDICTABLE in
10351              * v7, and has defined behaviour in v8:
10352              *  + leave CPSR.M untouched
10353              *  + allow changes to the other CPSR fields
10354              *  + set PSTATE.IL
10355              * For user changes via the GDB stub, we don't set PSTATE.IL,
10356              * as this would be unnecessarily harsh for a user error.
10357              */
10358             mask &= ~CPSR_M;
10359             if (write_type != CPSRWriteByGDBStub &&
10360                 arm_feature(env, ARM_FEATURE_V8)) {
10361                 mask |= CPSR_IL;
10362                 val |= CPSR_IL;
10363             }
10364             qemu_log_mask(LOG_GUEST_ERROR,
10365                           "Illegal AArch32 mode switch attempt from %s to %s\n",
10366                           aarch32_mode_name(env->uncached_cpsr),
10367                           aarch32_mode_name(val));
10368         } else {
10369             qemu_log_mask(CPU_LOG_INT, "%s %s to %s PC 0x%" PRIx32 "\n",
10370                           write_type == CPSRWriteExceptionReturn ?
10371                           "Exception return from AArch32" :
10372                           "AArch32 mode switch from",
10373                           aarch32_mode_name(env->uncached_cpsr),
10374                           aarch32_mode_name(val), env->regs[15]);
10375             switch_mode(env, val & CPSR_M);
10376         }
10377     }
10378     mask &= ~CACHED_CPSR_BITS;
10379     env->uncached_cpsr = (env->uncached_cpsr & ~mask) | (val & mask);
10380     if (tcg_enabled() && rebuild_hflags) {
10381         arm_rebuild_hflags(env);
10382     }
10383 }
10384 
10385 #ifdef CONFIG_USER_ONLY
10386 
10387 static void switch_mode(CPUARMState *env, int mode)
10388 {
10389     ARMCPU *cpu = env_archcpu(env);
10390 
10391     if (mode != ARM_CPU_MODE_USR) {
10392         cpu_abort(CPU(cpu), "Tried to switch out of user mode\n");
10393     }
10394 }
10395 
10396 uint32_t arm_phys_excp_target_el(CPUState *cs, uint32_t excp_idx,
10397                                  uint32_t cur_el, bool secure)
10398 {
10399     return 1;
10400 }
10401 
10402 void aarch64_sync_64_to_32(CPUARMState *env)
10403 {
10404     g_assert_not_reached();
10405 }
10406 
10407 #else
10408 
10409 static void switch_mode(CPUARMState *env, int mode)
10410 {
10411     int old_mode;
10412     int i;
10413 
10414     old_mode = env->uncached_cpsr & CPSR_M;
10415     if (mode == old_mode) {
10416         return;
10417     }
10418 
10419     if (old_mode == ARM_CPU_MODE_FIQ) {
10420         memcpy(env->fiq_regs, env->regs + 8, 5 * sizeof(uint32_t));
10421         memcpy(env->regs + 8, env->usr_regs, 5 * sizeof(uint32_t));
10422     } else if (mode == ARM_CPU_MODE_FIQ) {
10423         memcpy(env->usr_regs, env->regs + 8, 5 * sizeof(uint32_t));
10424         memcpy(env->regs + 8, env->fiq_regs, 5 * sizeof(uint32_t));
10425     }
10426 
10427     i = bank_number(old_mode);
10428     env->banked_r13[i] = env->regs[13];
10429     env->banked_spsr[i] = env->spsr;
10430 
10431     i = bank_number(mode);
10432     env->regs[13] = env->banked_r13[i];
10433     env->spsr = env->banked_spsr[i];
10434 
10435     env->banked_r14[r14_bank_number(old_mode)] = env->regs[14];
10436     env->regs[14] = env->banked_r14[r14_bank_number(mode)];
10437 }
10438 
10439 /*
10440  * Physical Interrupt Target EL Lookup Table
10441  *
10442  * [ From ARM ARM section G1.13.4 (Table G1-15) ]
10443  *
10444  * The below multi-dimensional table is used for looking up the target
10445  * exception level given numerous condition criteria.  Specifically, the
10446  * target EL is based on SCR and HCR routing controls as well as the
10447  * currently executing EL and secure state.
10448  *
10449  *    Dimensions:
10450  *    target_el_table[2][2][2][2][2][4]
10451  *                    |  |  |  |  |  +--- Current EL
10452  *                    |  |  |  |  +------ Non-secure(0)/Secure(1)
10453  *                    |  |  |  +--------- HCR mask override
10454  *                    |  |  +------------ SCR exec state control
10455  *                    |  +--------------- SCR mask override
10456  *                    +------------------ 32-bit(0)/64-bit(1) EL3
10457  *
10458  *    The table values are as such:
10459  *    0-3 = EL0-EL3
10460  *     -1 = Cannot occur
10461  *
10462  * The ARM ARM target EL table includes entries indicating that an "exception
10463  * is not taken".  The two cases where this is applicable are:
10464  *    1) An exception is taken from EL3 but the SCR does not have the exception
10465  *    routed to EL3.
10466  *    2) An exception is taken from EL2 but the HCR does not have the exception
10467  *    routed to EL2.
10468  * In these two cases, the below table contain a target of EL1.  This value is
10469  * returned as it is expected that the consumer of the table data will check
10470  * for "target EL >= current EL" to ensure the exception is not taken.
10471  *
10472  *            SCR     HCR
10473  *         64  EA     AMO                 From
10474  *        BIT IRQ     IMO      Non-secure         Secure
10475  *        EL3 FIQ  RW FMO   EL0 EL1 EL2 EL3   EL0 EL1 EL2 EL3
10476  */
10477 static const int8_t target_el_table[2][2][2][2][2][4] = {
10478     {{{{/* 0   0   0   0 */{ 1,  1,  2, -1 },{ 3, -1, -1,  3 },},
10479        {/* 0   0   0   1 */{ 2,  2,  2, -1 },{ 3, -1, -1,  3 },},},
10480       {{/* 0   0   1   0 */{ 1,  1,  2, -1 },{ 3, -1, -1,  3 },},
10481        {/* 0   0   1   1 */{ 2,  2,  2, -1 },{ 3, -1, -1,  3 },},},},
10482      {{{/* 0   1   0   0 */{ 3,  3,  3, -1 },{ 3, -1, -1,  3 },},
10483        {/* 0   1   0   1 */{ 3,  3,  3, -1 },{ 3, -1, -1,  3 },},},
10484       {{/* 0   1   1   0 */{ 3,  3,  3, -1 },{ 3, -1, -1,  3 },},
10485        {/* 0   1   1   1 */{ 3,  3,  3, -1 },{ 3, -1, -1,  3 },},},},},
10486     {{{{/* 1   0   0   0 */{ 1,  1,  2, -1 },{ 1,  1, -1,  1 },},
10487        {/* 1   0   0   1 */{ 2,  2,  2, -1 },{ 2,  2, -1,  1 },},},
10488       {{/* 1   0   1   0 */{ 1,  1,  1, -1 },{ 1,  1,  1,  1 },},
10489        {/* 1   0   1   1 */{ 2,  2,  2, -1 },{ 2,  2,  2,  1 },},},},
10490      {{{/* 1   1   0   0 */{ 3,  3,  3, -1 },{ 3,  3, -1,  3 },},
10491        {/* 1   1   0   1 */{ 3,  3,  3, -1 },{ 3,  3, -1,  3 },},},
10492       {{/* 1   1   1   0 */{ 3,  3,  3, -1 },{ 3,  3,  3,  3 },},
10493        {/* 1   1   1   1 */{ 3,  3,  3, -1 },{ 3,  3,  3,  3 },},},},},
10494 };
10495 
10496 /*
10497  * Determine the target EL for physical exceptions
10498  */
10499 uint32_t arm_phys_excp_target_el(CPUState *cs, uint32_t excp_idx,
10500                                  uint32_t cur_el, bool secure)
10501 {
10502     CPUARMState *env = cpu_env(cs);
10503     bool rw;
10504     bool scr;
10505     bool hcr;
10506     int target_el;
10507     /* Is the highest EL AArch64? */
10508     bool is64 = arm_feature(env, ARM_FEATURE_AARCH64);
10509     uint64_t hcr_el2;
10510 
10511     if (arm_feature(env, ARM_FEATURE_EL3)) {
10512         rw = ((env->cp15.scr_el3 & SCR_RW) == SCR_RW);
10513     } else {
10514         /*
10515          * Either EL2 is the highest EL (and so the EL2 register width
10516          * is given by is64); or there is no EL2 or EL3, in which case
10517          * the value of 'rw' does not affect the table lookup anyway.
10518          */
10519         rw = is64;
10520     }
10521 
10522     hcr_el2 = arm_hcr_el2_eff(env);
10523     switch (excp_idx) {
10524     case EXCP_IRQ:
10525         scr = ((env->cp15.scr_el3 & SCR_IRQ) == SCR_IRQ);
10526         hcr = hcr_el2 & HCR_IMO;
10527         break;
10528     case EXCP_FIQ:
10529         scr = ((env->cp15.scr_el3 & SCR_FIQ) == SCR_FIQ);
10530         hcr = hcr_el2 & HCR_FMO;
10531         break;
10532     default:
10533         scr = ((env->cp15.scr_el3 & SCR_EA) == SCR_EA);
10534         hcr = hcr_el2 & HCR_AMO;
10535         break;
10536     };
10537 
10538     /*
10539      * For these purposes, TGE and AMO/IMO/FMO both force the
10540      * interrupt to EL2.  Fold TGE into the bit extracted above.
10541      */
10542     hcr |= (hcr_el2 & HCR_TGE) != 0;
10543 
10544     /* Perform a table-lookup for the target EL given the current state */
10545     target_el = target_el_table[is64][scr][rw][hcr][secure][cur_el];
10546 
10547     assert(target_el > 0);
10548 
10549     return target_el;
10550 }
10551 
10552 void arm_log_exception(CPUState *cs)
10553 {
10554     int idx = cs->exception_index;
10555 
10556     if (qemu_loglevel_mask(CPU_LOG_INT)) {
10557         const char *exc = NULL;
10558         static const char * const excnames[] = {
10559             [EXCP_UDEF] = "Undefined Instruction",
10560             [EXCP_SWI] = "SVC",
10561             [EXCP_PREFETCH_ABORT] = "Prefetch Abort",
10562             [EXCP_DATA_ABORT] = "Data Abort",
10563             [EXCP_IRQ] = "IRQ",
10564             [EXCP_FIQ] = "FIQ",
10565             [EXCP_BKPT] = "Breakpoint",
10566             [EXCP_EXCEPTION_EXIT] = "QEMU v7M exception exit",
10567             [EXCP_KERNEL_TRAP] = "QEMU intercept of kernel commpage",
10568             [EXCP_HVC] = "Hypervisor Call",
10569             [EXCP_HYP_TRAP] = "Hypervisor Trap",
10570             [EXCP_SMC] = "Secure Monitor Call",
10571             [EXCP_VIRQ] = "Virtual IRQ",
10572             [EXCP_VFIQ] = "Virtual FIQ",
10573             [EXCP_SEMIHOST] = "Semihosting call",
10574             [EXCP_NOCP] = "v7M NOCP UsageFault",
10575             [EXCP_INVSTATE] = "v7M INVSTATE UsageFault",
10576             [EXCP_STKOF] = "v8M STKOF UsageFault",
10577             [EXCP_LAZYFP] = "v7M exception during lazy FP stacking",
10578             [EXCP_LSERR] = "v8M LSERR UsageFault",
10579             [EXCP_UNALIGNED] = "v7M UNALIGNED UsageFault",
10580             [EXCP_DIVBYZERO] = "v7M DIVBYZERO UsageFault",
10581             [EXCP_VSERR] = "Virtual SERR",
10582             [EXCP_GPC] = "Granule Protection Check",
10583         };
10584 
10585         if (idx >= 0 && idx < ARRAY_SIZE(excnames)) {
10586             exc = excnames[idx];
10587         }
10588         if (!exc) {
10589             exc = "unknown";
10590         }
10591         qemu_log_mask(CPU_LOG_INT, "Taking exception %d [%s] on CPU %d\n",
10592                       idx, exc, cs->cpu_index);
10593     }
10594 }
10595 
10596 /*
10597  * Function used to synchronize QEMU's AArch64 register set with AArch32
10598  * register set.  This is necessary when switching between AArch32 and AArch64
10599  * execution state.
10600  */
10601 void aarch64_sync_32_to_64(CPUARMState *env)
10602 {
10603     int i;
10604     uint32_t mode = env->uncached_cpsr & CPSR_M;
10605 
10606     /* We can blanket copy R[0:7] to X[0:7] */
10607     for (i = 0; i < 8; i++) {
10608         env->xregs[i] = env->regs[i];
10609     }
10610 
10611     /*
10612      * Unless we are in FIQ mode, x8-x12 come from the user registers r8-r12.
10613      * Otherwise, they come from the banked user regs.
10614      */
10615     if (mode == ARM_CPU_MODE_FIQ) {
10616         for (i = 8; i < 13; i++) {
10617             env->xregs[i] = env->usr_regs[i - 8];
10618         }
10619     } else {
10620         for (i = 8; i < 13; i++) {
10621             env->xregs[i] = env->regs[i];
10622         }
10623     }
10624 
10625     /*
10626      * Registers x13-x23 are the various mode SP and FP registers. Registers
10627      * r13 and r14 are only copied if we are in that mode, otherwise we copy
10628      * from the mode banked register.
10629      */
10630     if (mode == ARM_CPU_MODE_USR || mode == ARM_CPU_MODE_SYS) {
10631         env->xregs[13] = env->regs[13];
10632         env->xregs[14] = env->regs[14];
10633     } else {
10634         env->xregs[13] = env->banked_r13[bank_number(ARM_CPU_MODE_USR)];
10635         /* HYP is an exception in that it is copied from r14 */
10636         if (mode == ARM_CPU_MODE_HYP) {
10637             env->xregs[14] = env->regs[14];
10638         } else {
10639             env->xregs[14] = env->banked_r14[r14_bank_number(ARM_CPU_MODE_USR)];
10640         }
10641     }
10642 
10643     if (mode == ARM_CPU_MODE_HYP) {
10644         env->xregs[15] = env->regs[13];
10645     } else {
10646         env->xregs[15] = env->banked_r13[bank_number(ARM_CPU_MODE_HYP)];
10647     }
10648 
10649     if (mode == ARM_CPU_MODE_IRQ) {
10650         env->xregs[16] = env->regs[14];
10651         env->xregs[17] = env->regs[13];
10652     } else {
10653         env->xregs[16] = env->banked_r14[r14_bank_number(ARM_CPU_MODE_IRQ)];
10654         env->xregs[17] = env->banked_r13[bank_number(ARM_CPU_MODE_IRQ)];
10655     }
10656 
10657     if (mode == ARM_CPU_MODE_SVC) {
10658         env->xregs[18] = env->regs[14];
10659         env->xregs[19] = env->regs[13];
10660     } else {
10661         env->xregs[18] = env->banked_r14[r14_bank_number(ARM_CPU_MODE_SVC)];
10662         env->xregs[19] = env->banked_r13[bank_number(ARM_CPU_MODE_SVC)];
10663     }
10664 
10665     if (mode == ARM_CPU_MODE_ABT) {
10666         env->xregs[20] = env->regs[14];
10667         env->xregs[21] = env->regs[13];
10668     } else {
10669         env->xregs[20] = env->banked_r14[r14_bank_number(ARM_CPU_MODE_ABT)];
10670         env->xregs[21] = env->banked_r13[bank_number(ARM_CPU_MODE_ABT)];
10671     }
10672 
10673     if (mode == ARM_CPU_MODE_UND) {
10674         env->xregs[22] = env->regs[14];
10675         env->xregs[23] = env->regs[13];
10676     } else {
10677         env->xregs[22] = env->banked_r14[r14_bank_number(ARM_CPU_MODE_UND)];
10678         env->xregs[23] = env->banked_r13[bank_number(ARM_CPU_MODE_UND)];
10679     }
10680 
10681     /*
10682      * Registers x24-x30 are mapped to r8-r14 in FIQ mode.  If we are in FIQ
10683      * mode, then we can copy from r8-r14.  Otherwise, we copy from the
10684      * FIQ bank for r8-r14.
10685      */
10686     if (mode == ARM_CPU_MODE_FIQ) {
10687         for (i = 24; i < 31; i++) {
10688             env->xregs[i] = env->regs[i - 16];   /* X[24:30] <- R[8:14] */
10689         }
10690     } else {
10691         for (i = 24; i < 29; i++) {
10692             env->xregs[i] = env->fiq_regs[i - 24];
10693         }
10694         env->xregs[29] = env->banked_r13[bank_number(ARM_CPU_MODE_FIQ)];
10695         env->xregs[30] = env->banked_r14[r14_bank_number(ARM_CPU_MODE_FIQ)];
10696     }
10697 
10698     env->pc = env->regs[15];
10699 }
10700 
10701 /*
10702  * Function used to synchronize QEMU's AArch32 register set with AArch64
10703  * register set.  This is necessary when switching between AArch32 and AArch64
10704  * execution state.
10705  */
10706 void aarch64_sync_64_to_32(CPUARMState *env)
10707 {
10708     int i;
10709     uint32_t mode = env->uncached_cpsr & CPSR_M;
10710 
10711     /* We can blanket copy X[0:7] to R[0:7] */
10712     for (i = 0; i < 8; i++) {
10713         env->regs[i] = env->xregs[i];
10714     }
10715 
10716     /*
10717      * Unless we are in FIQ mode, r8-r12 come from the user registers x8-x12.
10718      * Otherwise, we copy x8-x12 into the banked user regs.
10719      */
10720     if (mode == ARM_CPU_MODE_FIQ) {
10721         for (i = 8; i < 13; i++) {
10722             env->usr_regs[i - 8] = env->xregs[i];
10723         }
10724     } else {
10725         for (i = 8; i < 13; i++) {
10726             env->regs[i] = env->xregs[i];
10727         }
10728     }
10729 
10730     /*
10731      * Registers r13 & r14 depend on the current mode.
10732      * If we are in a given mode, we copy the corresponding x registers to r13
10733      * and r14.  Otherwise, we copy the x register to the banked r13 and r14
10734      * for the mode.
10735      */
10736     if (mode == ARM_CPU_MODE_USR || mode == ARM_CPU_MODE_SYS) {
10737         env->regs[13] = env->xregs[13];
10738         env->regs[14] = env->xregs[14];
10739     } else {
10740         env->banked_r13[bank_number(ARM_CPU_MODE_USR)] = env->xregs[13];
10741 
10742         /*
10743          * HYP is an exception in that it does not have its own banked r14 but
10744          * shares the USR r14
10745          */
10746         if (mode == ARM_CPU_MODE_HYP) {
10747             env->regs[14] = env->xregs[14];
10748         } else {
10749             env->banked_r14[r14_bank_number(ARM_CPU_MODE_USR)] = env->xregs[14];
10750         }
10751     }
10752 
10753     if (mode == ARM_CPU_MODE_HYP) {
10754         env->regs[13] = env->xregs[15];
10755     } else {
10756         env->banked_r13[bank_number(ARM_CPU_MODE_HYP)] = env->xregs[15];
10757     }
10758 
10759     if (mode == ARM_CPU_MODE_IRQ) {
10760         env->regs[14] = env->xregs[16];
10761         env->regs[13] = env->xregs[17];
10762     } else {
10763         env->banked_r14[r14_bank_number(ARM_CPU_MODE_IRQ)] = env->xregs[16];
10764         env->banked_r13[bank_number(ARM_CPU_MODE_IRQ)] = env->xregs[17];
10765     }
10766 
10767     if (mode == ARM_CPU_MODE_SVC) {
10768         env->regs[14] = env->xregs[18];
10769         env->regs[13] = env->xregs[19];
10770     } else {
10771         env->banked_r14[r14_bank_number(ARM_CPU_MODE_SVC)] = env->xregs[18];
10772         env->banked_r13[bank_number(ARM_CPU_MODE_SVC)] = env->xregs[19];
10773     }
10774 
10775     if (mode == ARM_CPU_MODE_ABT) {
10776         env->regs[14] = env->xregs[20];
10777         env->regs[13] = env->xregs[21];
10778     } else {
10779         env->banked_r14[r14_bank_number(ARM_CPU_MODE_ABT)] = env->xregs[20];
10780         env->banked_r13[bank_number(ARM_CPU_MODE_ABT)] = env->xregs[21];
10781     }
10782 
10783     if (mode == ARM_CPU_MODE_UND) {
10784         env->regs[14] = env->xregs[22];
10785         env->regs[13] = env->xregs[23];
10786     } else {
10787         env->banked_r14[r14_bank_number(ARM_CPU_MODE_UND)] = env->xregs[22];
10788         env->banked_r13[bank_number(ARM_CPU_MODE_UND)] = env->xregs[23];
10789     }
10790 
10791     /*
10792      * Registers x24-x30 are mapped to r8-r14 in FIQ mode.  If we are in FIQ
10793      * mode, then we can copy to r8-r14.  Otherwise, we copy to the
10794      * FIQ bank for r8-r14.
10795      */
10796     if (mode == ARM_CPU_MODE_FIQ) {
10797         for (i = 24; i < 31; i++) {
10798             env->regs[i - 16] = env->xregs[i];   /* X[24:30] -> R[8:14] */
10799         }
10800     } else {
10801         for (i = 24; i < 29; i++) {
10802             env->fiq_regs[i - 24] = env->xregs[i];
10803         }
10804         env->banked_r13[bank_number(ARM_CPU_MODE_FIQ)] = env->xregs[29];
10805         env->banked_r14[r14_bank_number(ARM_CPU_MODE_FIQ)] = env->xregs[30];
10806     }
10807 
10808     env->regs[15] = env->pc;
10809 }
10810 
10811 static void take_aarch32_exception(CPUARMState *env, int new_mode,
10812                                    uint32_t mask, uint32_t offset,
10813                                    uint32_t newpc)
10814 {
10815     int new_el;
10816 
10817     /* Change the CPU state so as to actually take the exception. */
10818     switch_mode(env, new_mode);
10819 
10820     /*
10821      * For exceptions taken to AArch32 we must clear the SS bit in both
10822      * PSTATE and in the old-state value we save to SPSR_<mode>, so zero it now.
10823      */
10824     env->pstate &= ~PSTATE_SS;
10825     env->spsr = cpsr_read(env);
10826     /* Clear IT bits.  */
10827     env->condexec_bits = 0;
10828     /* Switch to the new mode, and to the correct instruction set.  */
10829     env->uncached_cpsr = (env->uncached_cpsr & ~CPSR_M) | new_mode;
10830 
10831     /* This must be after mode switching. */
10832     new_el = arm_current_el(env);
10833 
10834     /* Set new mode endianness */
10835     env->uncached_cpsr &= ~CPSR_E;
10836     if (env->cp15.sctlr_el[new_el] & SCTLR_EE) {
10837         env->uncached_cpsr |= CPSR_E;
10838     }
10839     /* J and IL must always be cleared for exception entry */
10840     env->uncached_cpsr &= ~(CPSR_IL | CPSR_J);
10841     env->daif |= mask;
10842 
10843     if (cpu_isar_feature(aa32_ssbs, env_archcpu(env))) {
10844         if (env->cp15.sctlr_el[new_el] & SCTLR_DSSBS_32) {
10845             env->uncached_cpsr |= CPSR_SSBS;
10846         } else {
10847             env->uncached_cpsr &= ~CPSR_SSBS;
10848         }
10849     }
10850 
10851     if (new_mode == ARM_CPU_MODE_HYP) {
10852         env->thumb = (env->cp15.sctlr_el[2] & SCTLR_TE) != 0;
10853         env->elr_el[2] = env->regs[15];
10854     } else {
10855         /* CPSR.PAN is normally preserved preserved unless...  */
10856         if (cpu_isar_feature(aa32_pan, env_archcpu(env))) {
10857             switch (new_el) {
10858             case 3:
10859                 if (!arm_is_secure_below_el3(env)) {
10860                     /* ... the target is EL3, from non-secure state.  */
10861                     env->uncached_cpsr &= ~CPSR_PAN;
10862                     break;
10863                 }
10864                 /* ... the target is EL3, from secure state ... */
10865                 /* fall through */
10866             case 1:
10867                 /* ... the target is EL1 and SCTLR.SPAN is 0.  */
10868                 if (!(env->cp15.sctlr_el[new_el] & SCTLR_SPAN)) {
10869                     env->uncached_cpsr |= CPSR_PAN;
10870                 }
10871                 break;
10872             }
10873         }
10874         /*
10875          * this is a lie, as there was no c1_sys on V4T/V5, but who cares
10876          * and we should just guard the thumb mode on V4
10877          */
10878         if (arm_feature(env, ARM_FEATURE_V4T)) {
10879             env->thumb =
10880                 (A32_BANKED_CURRENT_REG_GET(env, sctlr) & SCTLR_TE) != 0;
10881         }
10882         env->regs[14] = env->regs[15] + offset;
10883     }
10884     env->regs[15] = newpc;
10885 
10886     if (tcg_enabled()) {
10887         arm_rebuild_hflags(env);
10888     }
10889 }
10890 
10891 static void arm_cpu_do_interrupt_aarch32_hyp(CPUState *cs)
10892 {
10893     /*
10894      * Handle exception entry to Hyp mode; this is sufficiently
10895      * different to entry to other AArch32 modes that we handle it
10896      * separately here.
10897      *
10898      * The vector table entry used is always the 0x14 Hyp mode entry point,
10899      * unless this is an UNDEF/SVC/HVC/abort taken from Hyp to Hyp.
10900      * The offset applied to the preferred return address is always zero
10901      * (see DDI0487C.a section G1.12.3).
10902      * PSTATE A/I/F masks are set based only on the SCR.EA/IRQ/FIQ values.
10903      */
10904     uint32_t addr, mask;
10905     ARMCPU *cpu = ARM_CPU(cs);
10906     CPUARMState *env = &cpu->env;
10907 
10908     switch (cs->exception_index) {
10909     case EXCP_UDEF:
10910         addr = 0x04;
10911         break;
10912     case EXCP_SWI:
10913         addr = 0x08;
10914         break;
10915     case EXCP_BKPT:
10916         /* Fall through to prefetch abort.  */
10917     case EXCP_PREFETCH_ABORT:
10918         env->cp15.ifar_s = env->exception.vaddress;
10919         qemu_log_mask(CPU_LOG_INT, "...with HIFAR 0x%x\n",
10920                       (uint32_t)env->exception.vaddress);
10921         addr = 0x0c;
10922         break;
10923     case EXCP_DATA_ABORT:
10924         env->cp15.dfar_s = env->exception.vaddress;
10925         qemu_log_mask(CPU_LOG_INT, "...with HDFAR 0x%x\n",
10926                       (uint32_t)env->exception.vaddress);
10927         addr = 0x10;
10928         break;
10929     case EXCP_IRQ:
10930         addr = 0x18;
10931         break;
10932     case EXCP_FIQ:
10933         addr = 0x1c;
10934         break;
10935     case EXCP_HVC:
10936         addr = 0x08;
10937         break;
10938     case EXCP_HYP_TRAP:
10939         addr = 0x14;
10940         break;
10941     default:
10942         cpu_abort(cs, "Unhandled exception 0x%x\n", cs->exception_index);
10943     }
10944 
10945     if (cs->exception_index != EXCP_IRQ && cs->exception_index != EXCP_FIQ) {
10946         if (!arm_feature(env, ARM_FEATURE_V8)) {
10947             /*
10948              * QEMU syndrome values are v8-style. v7 has the IL bit
10949              * UNK/SBZP for "field not valid" cases, where v8 uses RES1.
10950              * If this is a v7 CPU, squash the IL bit in those cases.
10951              */
10952             if (cs->exception_index == EXCP_PREFETCH_ABORT ||
10953                 (cs->exception_index == EXCP_DATA_ABORT &&
10954                  !(env->exception.syndrome & ARM_EL_ISV)) ||
10955                 syn_get_ec(env->exception.syndrome) == EC_UNCATEGORIZED) {
10956                 env->exception.syndrome &= ~ARM_EL_IL;
10957             }
10958         }
10959         env->cp15.esr_el[2] = env->exception.syndrome;
10960     }
10961 
10962     if (arm_current_el(env) != 2 && addr < 0x14) {
10963         addr = 0x14;
10964     }
10965 
10966     mask = 0;
10967     if (!(env->cp15.scr_el3 & SCR_EA)) {
10968         mask |= CPSR_A;
10969     }
10970     if (!(env->cp15.scr_el3 & SCR_IRQ)) {
10971         mask |= CPSR_I;
10972     }
10973     if (!(env->cp15.scr_el3 & SCR_FIQ)) {
10974         mask |= CPSR_F;
10975     }
10976 
10977     addr += env->cp15.hvbar;
10978 
10979     take_aarch32_exception(env, ARM_CPU_MODE_HYP, mask, 0, addr);
10980 }
10981 
10982 static void arm_cpu_do_interrupt_aarch32(CPUState *cs)
10983 {
10984     ARMCPU *cpu = ARM_CPU(cs);
10985     CPUARMState *env = &cpu->env;
10986     uint32_t addr;
10987     uint32_t mask;
10988     int new_mode;
10989     uint32_t offset;
10990     uint32_t moe;
10991 
10992     /* If this is a debug exception we must update the DBGDSCR.MOE bits */
10993     switch (syn_get_ec(env->exception.syndrome)) {
10994     case EC_BREAKPOINT:
10995     case EC_BREAKPOINT_SAME_EL:
10996         moe = 1;
10997         break;
10998     case EC_WATCHPOINT:
10999     case EC_WATCHPOINT_SAME_EL:
11000         moe = 10;
11001         break;
11002     case EC_AA32_BKPT:
11003         moe = 3;
11004         break;
11005     case EC_VECTORCATCH:
11006         moe = 5;
11007         break;
11008     default:
11009         moe = 0;
11010         break;
11011     }
11012 
11013     if (moe) {
11014         env->cp15.mdscr_el1 = deposit64(env->cp15.mdscr_el1, 2, 4, moe);
11015     }
11016 
11017     if (env->exception.target_el == 2) {
11018         arm_cpu_do_interrupt_aarch32_hyp(cs);
11019         return;
11020     }
11021 
11022     switch (cs->exception_index) {
11023     case EXCP_UDEF:
11024         new_mode = ARM_CPU_MODE_UND;
11025         addr = 0x04;
11026         mask = CPSR_I;
11027         if (env->thumb) {
11028             offset = 2;
11029         } else {
11030             offset = 4;
11031         }
11032         break;
11033     case EXCP_SWI:
11034         new_mode = ARM_CPU_MODE_SVC;
11035         addr = 0x08;
11036         mask = CPSR_I;
11037         /* The PC already points to the next instruction.  */
11038         offset = 0;
11039         break;
11040     case EXCP_BKPT:
11041         /* Fall through to prefetch abort.  */
11042     case EXCP_PREFETCH_ABORT:
11043         A32_BANKED_CURRENT_REG_SET(env, ifsr, env->exception.fsr);
11044         A32_BANKED_CURRENT_REG_SET(env, ifar, env->exception.vaddress);
11045         qemu_log_mask(CPU_LOG_INT, "...with IFSR 0x%x IFAR 0x%x\n",
11046                       env->exception.fsr, (uint32_t)env->exception.vaddress);
11047         new_mode = ARM_CPU_MODE_ABT;
11048         addr = 0x0c;
11049         mask = CPSR_A | CPSR_I;
11050         offset = 4;
11051         break;
11052     case EXCP_DATA_ABORT:
11053         A32_BANKED_CURRENT_REG_SET(env, dfsr, env->exception.fsr);
11054         A32_BANKED_CURRENT_REG_SET(env, dfar, env->exception.vaddress);
11055         qemu_log_mask(CPU_LOG_INT, "...with DFSR 0x%x DFAR 0x%x\n",
11056                       env->exception.fsr,
11057                       (uint32_t)env->exception.vaddress);
11058         new_mode = ARM_CPU_MODE_ABT;
11059         addr = 0x10;
11060         mask = CPSR_A | CPSR_I;
11061         offset = 8;
11062         break;
11063     case EXCP_IRQ:
11064         new_mode = ARM_CPU_MODE_IRQ;
11065         addr = 0x18;
11066         /* Disable IRQ and imprecise data aborts.  */
11067         mask = CPSR_A | CPSR_I;
11068         offset = 4;
11069         if (env->cp15.scr_el3 & SCR_IRQ) {
11070             /* IRQ routed to monitor mode */
11071             new_mode = ARM_CPU_MODE_MON;
11072             mask |= CPSR_F;
11073         }
11074         break;
11075     case EXCP_FIQ:
11076         new_mode = ARM_CPU_MODE_FIQ;
11077         addr = 0x1c;
11078         /* Disable FIQ, IRQ and imprecise data aborts.  */
11079         mask = CPSR_A | CPSR_I | CPSR_F;
11080         if (env->cp15.scr_el3 & SCR_FIQ) {
11081             /* FIQ routed to monitor mode */
11082             new_mode = ARM_CPU_MODE_MON;
11083         }
11084         offset = 4;
11085         break;
11086     case EXCP_VIRQ:
11087         new_mode = ARM_CPU_MODE_IRQ;
11088         addr = 0x18;
11089         /* Disable IRQ and imprecise data aborts.  */
11090         mask = CPSR_A | CPSR_I;
11091         offset = 4;
11092         break;
11093     case EXCP_VFIQ:
11094         new_mode = ARM_CPU_MODE_FIQ;
11095         addr = 0x1c;
11096         /* Disable FIQ, IRQ and imprecise data aborts.  */
11097         mask = CPSR_A | CPSR_I | CPSR_F;
11098         offset = 4;
11099         break;
11100     case EXCP_VSERR:
11101         {
11102             /*
11103              * Note that this is reported as a data abort, but the DFAR
11104              * has an UNKNOWN value.  Construct the SError syndrome from
11105              * AET and ExT fields.
11106              */
11107             ARMMMUFaultInfo fi = { .type = ARMFault_AsyncExternal, };
11108 
11109             if (extended_addresses_enabled(env)) {
11110                 env->exception.fsr = arm_fi_to_lfsc(&fi);
11111             } else {
11112                 env->exception.fsr = arm_fi_to_sfsc(&fi);
11113             }
11114             env->exception.fsr |= env->cp15.vsesr_el2 & 0xd000;
11115             A32_BANKED_CURRENT_REG_SET(env, dfsr, env->exception.fsr);
11116             qemu_log_mask(CPU_LOG_INT, "...with IFSR 0x%x\n",
11117                           env->exception.fsr);
11118 
11119             new_mode = ARM_CPU_MODE_ABT;
11120             addr = 0x10;
11121             mask = CPSR_A | CPSR_I;
11122             offset = 8;
11123         }
11124         break;
11125     case EXCP_SMC:
11126         new_mode = ARM_CPU_MODE_MON;
11127         addr = 0x08;
11128         mask = CPSR_A | CPSR_I | CPSR_F;
11129         offset = 0;
11130         break;
11131     default:
11132         cpu_abort(cs, "Unhandled exception 0x%x\n", cs->exception_index);
11133         return; /* Never happens.  Keep compiler happy.  */
11134     }
11135 
11136     if (new_mode == ARM_CPU_MODE_MON) {
11137         addr += env->cp15.mvbar;
11138     } else if (A32_BANKED_CURRENT_REG_GET(env, sctlr) & SCTLR_V) {
11139         /* High vectors. When enabled, base address cannot be remapped. */
11140         addr += 0xffff0000;
11141     } else {
11142         /*
11143          * ARM v7 architectures provide a vector base address register to remap
11144          * the interrupt vector table.
11145          * This register is only followed in non-monitor mode, and is banked.
11146          * Note: only bits 31:5 are valid.
11147          */
11148         addr += A32_BANKED_CURRENT_REG_GET(env, vbar);
11149     }
11150 
11151     if ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_MON) {
11152         env->cp15.scr_el3 &= ~SCR_NS;
11153     }
11154 
11155     take_aarch32_exception(env, new_mode, mask, offset, addr);
11156 }
11157 
11158 static int aarch64_regnum(CPUARMState *env, int aarch32_reg)
11159 {
11160     /*
11161      * Return the register number of the AArch64 view of the AArch32
11162      * register @aarch32_reg. The CPUARMState CPSR is assumed to still
11163      * be that of the AArch32 mode the exception came from.
11164      */
11165     int mode = env->uncached_cpsr & CPSR_M;
11166 
11167     switch (aarch32_reg) {
11168     case 0 ... 7:
11169         return aarch32_reg;
11170     case 8 ... 12:
11171         return mode == ARM_CPU_MODE_FIQ ? aarch32_reg + 16 : aarch32_reg;
11172     case 13:
11173         switch (mode) {
11174         case ARM_CPU_MODE_USR:
11175         case ARM_CPU_MODE_SYS:
11176             return 13;
11177         case ARM_CPU_MODE_HYP:
11178             return 15;
11179         case ARM_CPU_MODE_IRQ:
11180             return 17;
11181         case ARM_CPU_MODE_SVC:
11182             return 19;
11183         case ARM_CPU_MODE_ABT:
11184             return 21;
11185         case ARM_CPU_MODE_UND:
11186             return 23;
11187         case ARM_CPU_MODE_FIQ:
11188             return 29;
11189         default:
11190             g_assert_not_reached();
11191         }
11192     case 14:
11193         switch (mode) {
11194         case ARM_CPU_MODE_USR:
11195         case ARM_CPU_MODE_SYS:
11196         case ARM_CPU_MODE_HYP:
11197             return 14;
11198         case ARM_CPU_MODE_IRQ:
11199             return 16;
11200         case ARM_CPU_MODE_SVC:
11201             return 18;
11202         case ARM_CPU_MODE_ABT:
11203             return 20;
11204         case ARM_CPU_MODE_UND:
11205             return 22;
11206         case ARM_CPU_MODE_FIQ:
11207             return 30;
11208         default:
11209             g_assert_not_reached();
11210         }
11211     case 15:
11212         return 31;
11213     default:
11214         g_assert_not_reached();
11215     }
11216 }
11217 
11218 static uint32_t cpsr_read_for_spsr_elx(CPUARMState *env)
11219 {
11220     uint32_t ret = cpsr_read(env);
11221 
11222     /* Move DIT to the correct location for SPSR_ELx */
11223     if (ret & CPSR_DIT) {
11224         ret &= ~CPSR_DIT;
11225         ret |= PSTATE_DIT;
11226     }
11227     /* Merge PSTATE.SS into SPSR_ELx */
11228     ret |= env->pstate & PSTATE_SS;
11229 
11230     return ret;
11231 }
11232 
11233 static bool syndrome_is_sync_extabt(uint32_t syndrome)
11234 {
11235     /* Return true if this syndrome value is a synchronous external abort */
11236     switch (syn_get_ec(syndrome)) {
11237     case EC_INSNABORT:
11238     case EC_INSNABORT_SAME_EL:
11239     case EC_DATAABORT:
11240     case EC_DATAABORT_SAME_EL:
11241         /* Look at fault status code for all the synchronous ext abort cases */
11242         switch (syndrome & 0x3f) {
11243         case 0x10:
11244         case 0x13:
11245         case 0x14:
11246         case 0x15:
11247         case 0x16:
11248         case 0x17:
11249             return true;
11250         default:
11251             return false;
11252         }
11253     default:
11254         return false;
11255     }
11256 }
11257 
11258 /* Handle exception entry to a target EL which is using AArch64 */
11259 static void arm_cpu_do_interrupt_aarch64(CPUState *cs)
11260 {
11261     ARMCPU *cpu = ARM_CPU(cs);
11262     CPUARMState *env = &cpu->env;
11263     unsigned int new_el = env->exception.target_el;
11264     target_ulong addr = env->cp15.vbar_el[new_el];
11265     unsigned int new_mode = aarch64_pstate_mode(new_el, true);
11266     unsigned int old_mode;
11267     unsigned int cur_el = arm_current_el(env);
11268     int rt;
11269 
11270     if (tcg_enabled()) {
11271         /*
11272          * Note that new_el can never be 0.  If cur_el is 0, then
11273          * el0_a64 is is_a64(), else el0_a64 is ignored.
11274          */
11275         aarch64_sve_change_el(env, cur_el, new_el, is_a64(env));
11276     }
11277 
11278     if (cur_el < new_el) {
11279         /*
11280          * Entry vector offset depends on whether the implemented EL
11281          * immediately lower than the target level is using AArch32 or AArch64
11282          */
11283         bool is_aa64;
11284         uint64_t hcr;
11285 
11286         switch (new_el) {
11287         case 3:
11288             is_aa64 = (env->cp15.scr_el3 & SCR_RW) != 0;
11289             break;
11290         case 2:
11291             hcr = arm_hcr_el2_eff(env);
11292             if ((hcr & (HCR_E2H | HCR_TGE)) != (HCR_E2H | HCR_TGE)) {
11293                 is_aa64 = (hcr & HCR_RW) != 0;
11294                 break;
11295             }
11296             /* fall through */
11297         case 1:
11298             is_aa64 = is_a64(env);
11299             break;
11300         default:
11301             g_assert_not_reached();
11302         }
11303 
11304         if (is_aa64) {
11305             addr += 0x400;
11306         } else {
11307             addr += 0x600;
11308         }
11309     } else if (pstate_read(env) & PSTATE_SP) {
11310         addr += 0x200;
11311     }
11312 
11313     switch (cs->exception_index) {
11314     case EXCP_GPC:
11315         qemu_log_mask(CPU_LOG_INT, "...with MFAR 0x%" PRIx64 "\n",
11316                       env->cp15.mfar_el3);
11317         /* fall through */
11318     case EXCP_PREFETCH_ABORT:
11319     case EXCP_DATA_ABORT:
11320         /*
11321          * FEAT_DoubleFault allows synchronous external aborts taken to EL3
11322          * to be taken to the SError vector entrypoint.
11323          */
11324         if (new_el == 3 && (env->cp15.scr_el3 & SCR_EASE) &&
11325             syndrome_is_sync_extabt(env->exception.syndrome)) {
11326             addr += 0x180;
11327         }
11328         env->cp15.far_el[new_el] = env->exception.vaddress;
11329         qemu_log_mask(CPU_LOG_INT, "...with FAR 0x%" PRIx64 "\n",
11330                       env->cp15.far_el[new_el]);
11331         /* fall through */
11332     case EXCP_BKPT:
11333     case EXCP_UDEF:
11334     case EXCP_SWI:
11335     case EXCP_HVC:
11336     case EXCP_HYP_TRAP:
11337     case EXCP_SMC:
11338         switch (syn_get_ec(env->exception.syndrome)) {
11339         case EC_ADVSIMDFPACCESSTRAP:
11340             /*
11341              * QEMU internal FP/SIMD syndromes from AArch32 include the
11342              * TA and coproc fields which are only exposed if the exception
11343              * is taken to AArch32 Hyp mode. Mask them out to get a valid
11344              * AArch64 format syndrome.
11345              */
11346             env->exception.syndrome &= ~MAKE_64BIT_MASK(0, 20);
11347             break;
11348         case EC_CP14RTTRAP:
11349         case EC_CP15RTTRAP:
11350         case EC_CP14DTTRAP:
11351             /*
11352              * For a trap on AArch32 MRC/MCR/LDC/STC the Rt field is currently
11353              * the raw register field from the insn; when taking this to
11354              * AArch64 we must convert it to the AArch64 view of the register
11355              * number. Notice that we read a 4-bit AArch32 register number and
11356              * write back a 5-bit AArch64 one.
11357              */
11358             rt = extract32(env->exception.syndrome, 5, 4);
11359             rt = aarch64_regnum(env, rt);
11360             env->exception.syndrome = deposit32(env->exception.syndrome,
11361                                                 5, 5, rt);
11362             break;
11363         case EC_CP15RRTTRAP:
11364         case EC_CP14RRTTRAP:
11365             /* Similarly for MRRC/MCRR traps for Rt and Rt2 fields */
11366             rt = extract32(env->exception.syndrome, 5, 4);
11367             rt = aarch64_regnum(env, rt);
11368             env->exception.syndrome = deposit32(env->exception.syndrome,
11369                                                 5, 5, rt);
11370             rt = extract32(env->exception.syndrome, 10, 4);
11371             rt = aarch64_regnum(env, rt);
11372             env->exception.syndrome = deposit32(env->exception.syndrome,
11373                                                 10, 5, rt);
11374             break;
11375         }
11376         env->cp15.esr_el[new_el] = env->exception.syndrome;
11377         break;
11378     case EXCP_IRQ:
11379     case EXCP_VIRQ:
11380         addr += 0x80;
11381         break;
11382     case EXCP_FIQ:
11383     case EXCP_VFIQ:
11384         addr += 0x100;
11385         break;
11386     case EXCP_VSERR:
11387         addr += 0x180;
11388         /* Construct the SError syndrome from IDS and ISS fields. */
11389         env->exception.syndrome = syn_serror(env->cp15.vsesr_el2 & 0x1ffffff);
11390         env->cp15.esr_el[new_el] = env->exception.syndrome;
11391         break;
11392     default:
11393         cpu_abort(cs, "Unhandled exception 0x%x\n", cs->exception_index);
11394     }
11395 
11396     if (is_a64(env)) {
11397         old_mode = pstate_read(env);
11398         aarch64_save_sp(env, arm_current_el(env));
11399         env->elr_el[new_el] = env->pc;
11400 
11401         if (cur_el == 1 && new_el == 1) {
11402             uint64_t hcr = arm_hcr_el2_eff(env);
11403             if ((hcr & (HCR_NV | HCR_NV1 | HCR_NV2)) == HCR_NV ||
11404                 (hcr & (HCR_NV | HCR_NV2)) == (HCR_NV | HCR_NV2)) {
11405                 /*
11406                  * FEAT_NV, FEAT_NV2 may need to report EL2 in the SPSR
11407                  * by setting M[3:2] to 0b10.
11408                  * If NV2 is disabled, change SPSR when NV,NV1 == 1,0 (I_ZJRNN)
11409                  * If NV2 is enabled, change SPSR when NV is 1 (I_DBTLM)
11410                  */
11411                 old_mode = deposit32(old_mode, 2, 2, 2);
11412             }
11413         }
11414     } else {
11415         old_mode = cpsr_read_for_spsr_elx(env);
11416         env->elr_el[new_el] = env->regs[15];
11417 
11418         aarch64_sync_32_to_64(env);
11419 
11420         env->condexec_bits = 0;
11421     }
11422     env->banked_spsr[aarch64_banked_spsr_index(new_el)] = old_mode;
11423 
11424     qemu_log_mask(CPU_LOG_INT, "...with SPSR 0x%x\n", old_mode);
11425     qemu_log_mask(CPU_LOG_INT, "...with ELR 0x%" PRIx64 "\n",
11426                   env->elr_el[new_el]);
11427 
11428     if (cpu_isar_feature(aa64_pan, cpu)) {
11429         /* The value of PSTATE.PAN is normally preserved, except when ... */
11430         new_mode |= old_mode & PSTATE_PAN;
11431         switch (new_el) {
11432         case 2:
11433             /* ... the target is EL2 with HCR_EL2.{E2H,TGE} == '11' ...  */
11434             if ((arm_hcr_el2_eff(env) & (HCR_E2H | HCR_TGE))
11435                 != (HCR_E2H | HCR_TGE)) {
11436                 break;
11437             }
11438             /* fall through */
11439         case 1:
11440             /* ... the target is EL1 ... */
11441             /* ... and SCTLR_ELx.SPAN == 0, then set to 1.  */
11442             if ((env->cp15.sctlr_el[new_el] & SCTLR_SPAN) == 0) {
11443                 new_mode |= PSTATE_PAN;
11444             }
11445             break;
11446         }
11447     }
11448     if (cpu_isar_feature(aa64_mte, cpu)) {
11449         new_mode |= PSTATE_TCO;
11450     }
11451 
11452     if (cpu_isar_feature(aa64_ssbs, cpu)) {
11453         if (env->cp15.sctlr_el[new_el] & SCTLR_DSSBS_64) {
11454             new_mode |= PSTATE_SSBS;
11455         } else {
11456             new_mode &= ~PSTATE_SSBS;
11457         }
11458     }
11459 
11460     pstate_write(env, PSTATE_DAIF | new_mode);
11461     env->aarch64 = true;
11462     aarch64_restore_sp(env, new_el);
11463 
11464     if (tcg_enabled()) {
11465         helper_rebuild_hflags_a64(env, new_el);
11466     }
11467 
11468     env->pc = addr;
11469 
11470     qemu_log_mask(CPU_LOG_INT, "...to EL%d PC 0x%" PRIx64 " PSTATE 0x%x\n",
11471                   new_el, env->pc, pstate_read(env));
11472 }
11473 
11474 /*
11475  * Do semihosting call and set the appropriate return value. All the
11476  * permission and validity checks have been done at translate time.
11477  *
11478  * We only see semihosting exceptions in TCG only as they are not
11479  * trapped to the hypervisor in KVM.
11480  */
11481 #ifdef CONFIG_TCG
11482 static void tcg_handle_semihosting(CPUState *cs)
11483 {
11484     ARMCPU *cpu = ARM_CPU(cs);
11485     CPUARMState *env = &cpu->env;
11486 
11487     if (is_a64(env)) {
11488         qemu_log_mask(CPU_LOG_INT,
11489                       "...handling as semihosting call 0x%" PRIx64 "\n",
11490                       env->xregs[0]);
11491         do_common_semihosting(cs);
11492         env->pc += 4;
11493     } else {
11494         qemu_log_mask(CPU_LOG_INT,
11495                       "...handling as semihosting call 0x%x\n",
11496                       env->regs[0]);
11497         do_common_semihosting(cs);
11498         env->regs[15] += env->thumb ? 2 : 4;
11499     }
11500 }
11501 #endif
11502 
11503 /*
11504  * Handle a CPU exception for A and R profile CPUs.
11505  * Do any appropriate logging, handle PSCI calls, and then hand off
11506  * to the AArch64-entry or AArch32-entry function depending on the
11507  * target exception level's register width.
11508  *
11509  * Note: this is used for both TCG (as the do_interrupt tcg op),
11510  *       and KVM to re-inject guest debug exceptions, and to
11511  *       inject a Synchronous-External-Abort.
11512  */
11513 void arm_cpu_do_interrupt(CPUState *cs)
11514 {
11515     ARMCPU *cpu = ARM_CPU(cs);
11516     CPUARMState *env = &cpu->env;
11517     unsigned int new_el = env->exception.target_el;
11518 
11519     assert(!arm_feature(env, ARM_FEATURE_M));
11520 
11521     arm_log_exception(cs);
11522     qemu_log_mask(CPU_LOG_INT, "...from EL%d to EL%d\n", arm_current_el(env),
11523                   new_el);
11524     if (qemu_loglevel_mask(CPU_LOG_INT)
11525         && !excp_is_internal(cs->exception_index)) {
11526         qemu_log_mask(CPU_LOG_INT, "...with ESR 0x%x/0x%" PRIx32 "\n",
11527                       syn_get_ec(env->exception.syndrome),
11528                       env->exception.syndrome);
11529     }
11530 
11531     if (tcg_enabled() && arm_is_psci_call(cpu, cs->exception_index)) {
11532         arm_handle_psci_call(cpu);
11533         qemu_log_mask(CPU_LOG_INT, "...handled as PSCI call\n");
11534         return;
11535     }
11536 
11537     /*
11538      * Semihosting semantics depend on the register width of the code
11539      * that caused the exception, not the target exception level, so
11540      * must be handled here.
11541      */
11542 #ifdef CONFIG_TCG
11543     if (cs->exception_index == EXCP_SEMIHOST) {
11544         tcg_handle_semihosting(cs);
11545         return;
11546     }
11547 #endif
11548 
11549     /*
11550      * Hooks may change global state so BQL should be held, also the
11551      * BQL needs to be held for any modification of
11552      * cs->interrupt_request.
11553      */
11554     g_assert(bql_locked());
11555 
11556     arm_call_pre_el_change_hook(cpu);
11557 
11558     assert(!excp_is_internal(cs->exception_index));
11559     if (arm_el_is_aa64(env, new_el)) {
11560         arm_cpu_do_interrupt_aarch64(cs);
11561     } else {
11562         arm_cpu_do_interrupt_aarch32(cs);
11563     }
11564 
11565     arm_call_el_change_hook(cpu);
11566 
11567     if (!kvm_enabled()) {
11568         cs->interrupt_request |= CPU_INTERRUPT_EXITTB;
11569     }
11570 }
11571 #endif /* !CONFIG_USER_ONLY */
11572 
11573 uint64_t arm_sctlr(CPUARMState *env, int el)
11574 {
11575     /* Only EL0 needs to be adjusted for EL1&0 or EL2&0. */
11576     if (el == 0) {
11577         ARMMMUIdx mmu_idx = arm_mmu_idx_el(env, 0);
11578         el = mmu_idx == ARMMMUIdx_E20_0 ? 2 : 1;
11579     }
11580     return env->cp15.sctlr_el[el];
11581 }
11582 
11583 int aa64_va_parameter_tbi(uint64_t tcr, ARMMMUIdx mmu_idx)
11584 {
11585     if (regime_has_2_ranges(mmu_idx)) {
11586         return extract64(tcr, 37, 2);
11587     } else if (regime_is_stage2(mmu_idx)) {
11588         return 0; /* VTCR_EL2 */
11589     } else {
11590         /* Replicate the single TBI bit so we always have 2 bits.  */
11591         return extract32(tcr, 20, 1) * 3;
11592     }
11593 }
11594 
11595 int aa64_va_parameter_tbid(uint64_t tcr, ARMMMUIdx mmu_idx)
11596 {
11597     if (regime_has_2_ranges(mmu_idx)) {
11598         return extract64(tcr, 51, 2);
11599     } else if (regime_is_stage2(mmu_idx)) {
11600         return 0; /* VTCR_EL2 */
11601     } else {
11602         /* Replicate the single TBID bit so we always have 2 bits.  */
11603         return extract32(tcr, 29, 1) * 3;
11604     }
11605 }
11606 
11607 int aa64_va_parameter_tcma(uint64_t tcr, ARMMMUIdx mmu_idx)
11608 {
11609     if (regime_has_2_ranges(mmu_idx)) {
11610         return extract64(tcr, 57, 2);
11611     } else {
11612         /* Replicate the single TCMA bit so we always have 2 bits.  */
11613         return extract32(tcr, 30, 1) * 3;
11614     }
11615 }
11616 
11617 static ARMGranuleSize tg0_to_gran_size(int tg)
11618 {
11619     switch (tg) {
11620     case 0:
11621         return Gran4K;
11622     case 1:
11623         return Gran64K;
11624     case 2:
11625         return Gran16K;
11626     default:
11627         return GranInvalid;
11628     }
11629 }
11630 
11631 static ARMGranuleSize tg1_to_gran_size(int tg)
11632 {
11633     switch (tg) {
11634     case 1:
11635         return Gran16K;
11636     case 2:
11637         return Gran4K;
11638     case 3:
11639         return Gran64K;
11640     default:
11641         return GranInvalid;
11642     }
11643 }
11644 
11645 static inline bool have4k(ARMCPU *cpu, bool stage2)
11646 {
11647     return stage2 ? cpu_isar_feature(aa64_tgran4_2, cpu)
11648         : cpu_isar_feature(aa64_tgran4, cpu);
11649 }
11650 
11651 static inline bool have16k(ARMCPU *cpu, bool stage2)
11652 {
11653     return stage2 ? cpu_isar_feature(aa64_tgran16_2, cpu)
11654         : cpu_isar_feature(aa64_tgran16, cpu);
11655 }
11656 
11657 static inline bool have64k(ARMCPU *cpu, bool stage2)
11658 {
11659     return stage2 ? cpu_isar_feature(aa64_tgran64_2, cpu)
11660         : cpu_isar_feature(aa64_tgran64, cpu);
11661 }
11662 
11663 static ARMGranuleSize sanitize_gran_size(ARMCPU *cpu, ARMGranuleSize gran,
11664                                          bool stage2)
11665 {
11666     switch (gran) {
11667     case Gran4K:
11668         if (have4k(cpu, stage2)) {
11669             return gran;
11670         }
11671         break;
11672     case Gran16K:
11673         if (have16k(cpu, stage2)) {
11674             return gran;
11675         }
11676         break;
11677     case Gran64K:
11678         if (have64k(cpu, stage2)) {
11679             return gran;
11680         }
11681         break;
11682     case GranInvalid:
11683         break;
11684     }
11685     /*
11686      * If the guest selects a granule size that isn't implemented,
11687      * the architecture requires that we behave as if it selected one
11688      * that is (with an IMPDEF choice of which one to pick). We choose
11689      * to implement the smallest supported granule size.
11690      */
11691     if (have4k(cpu, stage2)) {
11692         return Gran4K;
11693     }
11694     if (have16k(cpu, stage2)) {
11695         return Gran16K;
11696     }
11697     assert(have64k(cpu, stage2));
11698     return Gran64K;
11699 }
11700 
11701 ARMVAParameters aa64_va_parameters(CPUARMState *env, uint64_t va,
11702                                    ARMMMUIdx mmu_idx, bool data,
11703                                    bool el1_is_aa32)
11704 {
11705     uint64_t tcr = regime_tcr(env, mmu_idx);
11706     bool epd, hpd, tsz_oob, ds, ha, hd;
11707     int select, tsz, tbi, max_tsz, min_tsz, ps, sh;
11708     ARMGranuleSize gran;
11709     ARMCPU *cpu = env_archcpu(env);
11710     bool stage2 = regime_is_stage2(mmu_idx);
11711 
11712     if (!regime_has_2_ranges(mmu_idx)) {
11713         select = 0;
11714         tsz = extract32(tcr, 0, 6);
11715         gran = tg0_to_gran_size(extract32(tcr, 14, 2));
11716         if (stage2) {
11717             /* VTCR_EL2 */
11718             hpd = false;
11719         } else {
11720             hpd = extract32(tcr, 24, 1);
11721         }
11722         epd = false;
11723         sh = extract32(tcr, 12, 2);
11724         ps = extract32(tcr, 16, 3);
11725         ha = extract32(tcr, 21, 1) && cpu_isar_feature(aa64_hafs, cpu);
11726         hd = extract32(tcr, 22, 1) && cpu_isar_feature(aa64_hdbs, cpu);
11727         ds = extract64(tcr, 32, 1);
11728     } else {
11729         bool e0pd;
11730 
11731         /*
11732          * Bit 55 is always between the two regions, and is canonical for
11733          * determining if address tagging is enabled.
11734          */
11735         select = extract64(va, 55, 1);
11736         if (!select) {
11737             tsz = extract32(tcr, 0, 6);
11738             gran = tg0_to_gran_size(extract32(tcr, 14, 2));
11739             epd = extract32(tcr, 7, 1);
11740             sh = extract32(tcr, 12, 2);
11741             hpd = extract64(tcr, 41, 1);
11742             e0pd = extract64(tcr, 55, 1);
11743         } else {
11744             tsz = extract32(tcr, 16, 6);
11745             gran = tg1_to_gran_size(extract32(tcr, 30, 2));
11746             epd = extract32(tcr, 23, 1);
11747             sh = extract32(tcr, 28, 2);
11748             hpd = extract64(tcr, 42, 1);
11749             e0pd = extract64(tcr, 56, 1);
11750         }
11751         ps = extract64(tcr, 32, 3);
11752         ha = extract64(tcr, 39, 1) && cpu_isar_feature(aa64_hafs, cpu);
11753         hd = extract64(tcr, 40, 1) && cpu_isar_feature(aa64_hdbs, cpu);
11754         ds = extract64(tcr, 59, 1);
11755 
11756         if (e0pd && cpu_isar_feature(aa64_e0pd, cpu) &&
11757             regime_is_user(env, mmu_idx)) {
11758             epd = true;
11759         }
11760     }
11761 
11762     gran = sanitize_gran_size(cpu, gran, stage2);
11763 
11764     if (cpu_isar_feature(aa64_st, cpu)) {
11765         max_tsz = 48 - (gran == Gran64K);
11766     } else {
11767         max_tsz = 39;
11768     }
11769 
11770     /*
11771      * DS is RES0 unless FEAT_LPA2 is supported for the given page size;
11772      * adjust the effective value of DS, as documented.
11773      */
11774     min_tsz = 16;
11775     if (gran == Gran64K) {
11776         if (cpu_isar_feature(aa64_lva, cpu)) {
11777             min_tsz = 12;
11778         }
11779         ds = false;
11780     } else if (ds) {
11781         if (regime_is_stage2(mmu_idx)) {
11782             if (gran == Gran16K) {
11783                 ds = cpu_isar_feature(aa64_tgran16_2_lpa2, cpu);
11784             } else {
11785                 ds = cpu_isar_feature(aa64_tgran4_2_lpa2, cpu);
11786             }
11787         } else {
11788             if (gran == Gran16K) {
11789                 ds = cpu_isar_feature(aa64_tgran16_lpa2, cpu);
11790             } else {
11791                 ds = cpu_isar_feature(aa64_tgran4_lpa2, cpu);
11792             }
11793         }
11794         if (ds) {
11795             min_tsz = 12;
11796         }
11797     }
11798 
11799     if (stage2 && el1_is_aa32) {
11800         /*
11801          * For AArch32 EL1 the min txsz (and thus max IPA size) requirements
11802          * are loosened: a configured IPA of 40 bits is permitted even if
11803          * the implemented PA is less than that (and so a 40 bit IPA would
11804          * fault for an AArch64 EL1). See R_DTLMN.
11805          */
11806         min_tsz = MIN(min_tsz, 24);
11807     }
11808 
11809     if (tsz > max_tsz) {
11810         tsz = max_tsz;
11811         tsz_oob = true;
11812     } else if (tsz < min_tsz) {
11813         tsz = min_tsz;
11814         tsz_oob = true;
11815     } else {
11816         tsz_oob = false;
11817     }
11818 
11819     /* Present TBI as a composite with TBID.  */
11820     tbi = aa64_va_parameter_tbi(tcr, mmu_idx);
11821     if (!data) {
11822         tbi &= ~aa64_va_parameter_tbid(tcr, mmu_idx);
11823     }
11824     tbi = (tbi >> select) & 1;
11825 
11826     return (ARMVAParameters) {
11827         .tsz = tsz,
11828         .ps = ps,
11829         .sh = sh,
11830         .select = select,
11831         .tbi = tbi,
11832         .epd = epd,
11833         .hpd = hpd,
11834         .tsz_oob = tsz_oob,
11835         .ds = ds,
11836         .ha = ha,
11837         .hd = ha && hd,
11838         .gran = gran,
11839     };
11840 }
11841 
11842 /*
11843  * Note that signed overflow is undefined in C.  The following routines are
11844  * careful to use unsigned types where modulo arithmetic is required.
11845  * Failure to do so _will_ break on newer gcc.
11846  */
11847 
11848 /* Signed saturating arithmetic.  */
11849 
11850 /* Perform 16-bit signed saturating addition.  */
11851 static inline uint16_t add16_sat(uint16_t a, uint16_t b)
11852 {
11853     uint16_t res;
11854 
11855     res = a + b;
11856     if (((res ^ a) & 0x8000) && !((a ^ b) & 0x8000)) {
11857         if (a & 0x8000) {
11858             res = 0x8000;
11859         } else {
11860             res = 0x7fff;
11861         }
11862     }
11863     return res;
11864 }
11865 
11866 /* Perform 8-bit signed saturating addition.  */
11867 static inline uint8_t add8_sat(uint8_t a, uint8_t b)
11868 {
11869     uint8_t res;
11870 
11871     res = a + b;
11872     if (((res ^ a) & 0x80) && !((a ^ b) & 0x80)) {
11873         if (a & 0x80) {
11874             res = 0x80;
11875         } else {
11876             res = 0x7f;
11877         }
11878     }
11879     return res;
11880 }
11881 
11882 /* Perform 16-bit signed saturating subtraction.  */
11883 static inline uint16_t sub16_sat(uint16_t a, uint16_t b)
11884 {
11885     uint16_t res;
11886 
11887     res = a - b;
11888     if (((res ^ a) & 0x8000) && ((a ^ b) & 0x8000)) {
11889         if (a & 0x8000) {
11890             res = 0x8000;
11891         } else {
11892             res = 0x7fff;
11893         }
11894     }
11895     return res;
11896 }
11897 
11898 /* Perform 8-bit signed saturating subtraction.  */
11899 static inline uint8_t sub8_sat(uint8_t a, uint8_t b)
11900 {
11901     uint8_t res;
11902 
11903     res = a - b;
11904     if (((res ^ a) & 0x80) && ((a ^ b) & 0x80)) {
11905         if (a & 0x80) {
11906             res = 0x80;
11907         } else {
11908             res = 0x7f;
11909         }
11910     }
11911     return res;
11912 }
11913 
11914 #define ADD16(a, b, n) RESULT(add16_sat(a, b), n, 16);
11915 #define SUB16(a, b, n) RESULT(sub16_sat(a, b), n, 16);
11916 #define ADD8(a, b, n)  RESULT(add8_sat(a, b), n, 8);
11917 #define SUB8(a, b, n)  RESULT(sub8_sat(a, b), n, 8);
11918 #define PFX q
11919 
11920 #include "op_addsub.h"
11921 
11922 /* Unsigned saturating arithmetic.  */
11923 static inline uint16_t add16_usat(uint16_t a, uint16_t b)
11924 {
11925     uint16_t res;
11926     res = a + b;
11927     if (res < a) {
11928         res = 0xffff;
11929     }
11930     return res;
11931 }
11932 
11933 static inline uint16_t sub16_usat(uint16_t a, uint16_t b)
11934 {
11935     if (a > b) {
11936         return a - b;
11937     } else {
11938         return 0;
11939     }
11940 }
11941 
11942 static inline uint8_t add8_usat(uint8_t a, uint8_t b)
11943 {
11944     uint8_t res;
11945     res = a + b;
11946     if (res < a) {
11947         res = 0xff;
11948     }
11949     return res;
11950 }
11951 
11952 static inline uint8_t sub8_usat(uint8_t a, uint8_t b)
11953 {
11954     if (a > b) {
11955         return a - b;
11956     } else {
11957         return 0;
11958     }
11959 }
11960 
11961 #define ADD16(a, b, n) RESULT(add16_usat(a, b), n, 16);
11962 #define SUB16(a, b, n) RESULT(sub16_usat(a, b), n, 16);
11963 #define ADD8(a, b, n)  RESULT(add8_usat(a, b), n, 8);
11964 #define SUB8(a, b, n)  RESULT(sub8_usat(a, b), n, 8);
11965 #define PFX uq
11966 
11967 #include "op_addsub.h"
11968 
11969 /* Signed modulo arithmetic.  */
11970 #define SARITH16(a, b, n, op) do { \
11971     int32_t sum; \
11972     sum = (int32_t)(int16_t)(a) op (int32_t)(int16_t)(b); \
11973     RESULT(sum, n, 16); \
11974     if (sum >= 0) \
11975         ge |= 3 << (n * 2); \
11976     } while (0)
11977 
11978 #define SARITH8(a, b, n, op) do { \
11979     int32_t sum; \
11980     sum = (int32_t)(int8_t)(a) op (int32_t)(int8_t)(b); \
11981     RESULT(sum, n, 8); \
11982     if (sum >= 0) \
11983         ge |= 1 << n; \
11984     } while (0)
11985 
11986 
11987 #define ADD16(a, b, n) SARITH16(a, b, n, +)
11988 #define SUB16(a, b, n) SARITH16(a, b, n, -)
11989 #define ADD8(a, b, n)  SARITH8(a, b, n, +)
11990 #define SUB8(a, b, n)  SARITH8(a, b, n, -)
11991 #define PFX s
11992 #define ARITH_GE
11993 
11994 #include "op_addsub.h"
11995 
11996 /* Unsigned modulo arithmetic.  */
11997 #define ADD16(a, b, n) do { \
11998     uint32_t sum; \
11999     sum = (uint32_t)(uint16_t)(a) + (uint32_t)(uint16_t)(b); \
12000     RESULT(sum, n, 16); \
12001     if ((sum >> 16) == 1) \
12002         ge |= 3 << (n * 2); \
12003     } while (0)
12004 
12005 #define ADD8(a, b, n) do { \
12006     uint32_t sum; \
12007     sum = (uint32_t)(uint8_t)(a) + (uint32_t)(uint8_t)(b); \
12008     RESULT(sum, n, 8); \
12009     if ((sum >> 8) == 1) \
12010         ge |= 1 << n; \
12011     } while (0)
12012 
12013 #define SUB16(a, b, n) do { \
12014     uint32_t sum; \
12015     sum = (uint32_t)(uint16_t)(a) - (uint32_t)(uint16_t)(b); \
12016     RESULT(sum, n, 16); \
12017     if ((sum >> 16) == 0) \
12018         ge |= 3 << (n * 2); \
12019     } while (0)
12020 
12021 #define SUB8(a, b, n) do { \
12022     uint32_t sum; \
12023     sum = (uint32_t)(uint8_t)(a) - (uint32_t)(uint8_t)(b); \
12024     RESULT(sum, n, 8); \
12025     if ((sum >> 8) == 0) \
12026         ge |= 1 << n; \
12027     } while (0)
12028 
12029 #define PFX u
12030 #define ARITH_GE
12031 
12032 #include "op_addsub.h"
12033 
12034 /* Halved signed arithmetic.  */
12035 #define ADD16(a, b, n) \
12036   RESULT(((int32_t)(int16_t)(a) + (int32_t)(int16_t)(b)) >> 1, n, 16)
12037 #define SUB16(a, b, n) \
12038   RESULT(((int32_t)(int16_t)(a) - (int32_t)(int16_t)(b)) >> 1, n, 16)
12039 #define ADD8(a, b, n) \
12040   RESULT(((int32_t)(int8_t)(a) + (int32_t)(int8_t)(b)) >> 1, n, 8)
12041 #define SUB8(a, b, n) \
12042   RESULT(((int32_t)(int8_t)(a) - (int32_t)(int8_t)(b)) >> 1, n, 8)
12043 #define PFX sh
12044 
12045 #include "op_addsub.h"
12046 
12047 /* Halved unsigned arithmetic.  */
12048 #define ADD16(a, b, n) \
12049   RESULT(((uint32_t)(uint16_t)(a) + (uint32_t)(uint16_t)(b)) >> 1, n, 16)
12050 #define SUB16(a, b, n) \
12051   RESULT(((uint32_t)(uint16_t)(a) - (uint32_t)(uint16_t)(b)) >> 1, n, 16)
12052 #define ADD8(a, b, n) \
12053   RESULT(((uint32_t)(uint8_t)(a) + (uint32_t)(uint8_t)(b)) >> 1, n, 8)
12054 #define SUB8(a, b, n) \
12055   RESULT(((uint32_t)(uint8_t)(a) - (uint32_t)(uint8_t)(b)) >> 1, n, 8)
12056 #define PFX uh
12057 
12058 #include "op_addsub.h"
12059 
12060 static inline uint8_t do_usad(uint8_t a, uint8_t b)
12061 {
12062     if (a > b) {
12063         return a - b;
12064     } else {
12065         return b - a;
12066     }
12067 }
12068 
12069 /* Unsigned sum of absolute byte differences.  */
12070 uint32_t HELPER(usad8)(uint32_t a, uint32_t b)
12071 {
12072     uint32_t sum;
12073     sum = do_usad(a, b);
12074     sum += do_usad(a >> 8, b >> 8);
12075     sum += do_usad(a >> 16, b >> 16);
12076     sum += do_usad(a >> 24, b >> 24);
12077     return sum;
12078 }
12079 
12080 /* For ARMv6 SEL instruction.  */
12081 uint32_t HELPER(sel_flags)(uint32_t flags, uint32_t a, uint32_t b)
12082 {
12083     uint32_t mask;
12084 
12085     mask = 0;
12086     if (flags & 1) {
12087         mask |= 0xff;
12088     }
12089     if (flags & 2) {
12090         mask |= 0xff00;
12091     }
12092     if (flags & 4) {
12093         mask |= 0xff0000;
12094     }
12095     if (flags & 8) {
12096         mask |= 0xff000000;
12097     }
12098     return (a & mask) | (b & ~mask);
12099 }
12100 
12101 /*
12102  * CRC helpers.
12103  * The upper bytes of val (above the number specified by 'bytes') must have
12104  * been zeroed out by the caller.
12105  */
12106 uint32_t HELPER(crc32)(uint32_t acc, uint32_t val, uint32_t bytes)
12107 {
12108     uint8_t buf[4];
12109 
12110     stl_le_p(buf, val);
12111 
12112     /* zlib crc32 converts the accumulator and output to one's complement.  */
12113     return crc32(acc ^ 0xffffffff, buf, bytes) ^ 0xffffffff;
12114 }
12115 
12116 uint32_t HELPER(crc32c)(uint32_t acc, uint32_t val, uint32_t bytes)
12117 {
12118     uint8_t buf[4];
12119 
12120     stl_le_p(buf, val);
12121 
12122     /* Linux crc32c converts the output to one's complement.  */
12123     return crc32c(acc, buf, bytes) ^ 0xffffffff;
12124 }
12125 
12126 /*
12127  * Return the exception level to which FP-disabled exceptions should
12128  * be taken, or 0 if FP is enabled.
12129  */
12130 int fp_exception_el(CPUARMState *env, int cur_el)
12131 {
12132 #ifndef CONFIG_USER_ONLY
12133     uint64_t hcr_el2;
12134 
12135     /*
12136      * CPACR and the CPTR registers don't exist before v6, so FP is
12137      * always accessible
12138      */
12139     if (!arm_feature(env, ARM_FEATURE_V6)) {
12140         return 0;
12141     }
12142 
12143     if (arm_feature(env, ARM_FEATURE_M)) {
12144         /* CPACR can cause a NOCP UsageFault taken to current security state */
12145         if (!v7m_cpacr_pass(env, env->v7m.secure, cur_el != 0)) {
12146             return 1;
12147         }
12148 
12149         if (arm_feature(env, ARM_FEATURE_M_SECURITY) && !env->v7m.secure) {
12150             if (!extract32(env->v7m.nsacr, 10, 1)) {
12151                 /* FP insns cause a NOCP UsageFault taken to Secure */
12152                 return 3;
12153             }
12154         }
12155 
12156         return 0;
12157     }
12158 
12159     hcr_el2 = arm_hcr_el2_eff(env);
12160 
12161     /*
12162      * The CPACR controls traps to EL1, or PL1 if we're 32 bit:
12163      * 0, 2 : trap EL0 and EL1/PL1 accesses
12164      * 1    : trap only EL0 accesses
12165      * 3    : trap no accesses
12166      * This register is ignored if E2H+TGE are both set.
12167      */
12168     if ((hcr_el2 & (HCR_E2H | HCR_TGE)) != (HCR_E2H | HCR_TGE)) {
12169         int fpen = FIELD_EX64(env->cp15.cpacr_el1, CPACR_EL1, FPEN);
12170 
12171         switch (fpen) {
12172         case 1:
12173             if (cur_el != 0) {
12174                 break;
12175             }
12176             /* fall through */
12177         case 0:
12178         case 2:
12179             /* Trap from Secure PL0 or PL1 to Secure PL1. */
12180             if (!arm_el_is_aa64(env, 3)
12181                 && (cur_el == 3 || arm_is_secure_below_el3(env))) {
12182                 return 3;
12183             }
12184             if (cur_el <= 1) {
12185                 return 1;
12186             }
12187             break;
12188         }
12189     }
12190 
12191     /*
12192      * The NSACR allows A-profile AArch32 EL3 and M-profile secure mode
12193      * to control non-secure access to the FPU. It doesn't have any
12194      * effect if EL3 is AArch64 or if EL3 doesn't exist at all.
12195      */
12196     if ((arm_feature(env, ARM_FEATURE_EL3) && !arm_el_is_aa64(env, 3) &&
12197          cur_el <= 2 && !arm_is_secure_below_el3(env))) {
12198         if (!extract32(env->cp15.nsacr, 10, 1)) {
12199             /* FP insns act as UNDEF */
12200             return cur_el == 2 ? 2 : 1;
12201         }
12202     }
12203 
12204     /*
12205      * CPTR_EL2 is present in v7VE or v8, and changes format
12206      * with HCR_EL2.E2H (regardless of TGE).
12207      */
12208     if (cur_el <= 2) {
12209         if (hcr_el2 & HCR_E2H) {
12210             switch (FIELD_EX64(env->cp15.cptr_el[2], CPTR_EL2, FPEN)) {
12211             case 1:
12212                 if (cur_el != 0 || !(hcr_el2 & HCR_TGE)) {
12213                     break;
12214                 }
12215                 /* fall through */
12216             case 0:
12217             case 2:
12218                 return 2;
12219             }
12220         } else if (arm_is_el2_enabled(env)) {
12221             if (FIELD_EX64(env->cp15.cptr_el[2], CPTR_EL2, TFP)) {
12222                 return 2;
12223             }
12224         }
12225     }
12226 
12227     /* CPTR_EL3 : present in v8 */
12228     if (FIELD_EX64(env->cp15.cptr_el[3], CPTR_EL3, TFP)) {
12229         /* Trap all FP ops to EL3 */
12230         return 3;
12231     }
12232 #endif
12233     return 0;
12234 }
12235 
12236 /* Return the exception level we're running at if this is our mmu_idx */
12237 int arm_mmu_idx_to_el(ARMMMUIdx mmu_idx)
12238 {
12239     if (mmu_idx & ARM_MMU_IDX_M) {
12240         return mmu_idx & ARM_MMU_IDX_M_PRIV;
12241     }
12242 
12243     switch (mmu_idx) {
12244     case ARMMMUIdx_E10_0:
12245     case ARMMMUIdx_E20_0:
12246         return 0;
12247     case ARMMMUIdx_E10_1:
12248     case ARMMMUIdx_E10_1_PAN:
12249         return 1;
12250     case ARMMMUIdx_E2:
12251     case ARMMMUIdx_E20_2:
12252     case ARMMMUIdx_E20_2_PAN:
12253         return 2;
12254     case ARMMMUIdx_E3:
12255         return 3;
12256     default:
12257         g_assert_not_reached();
12258     }
12259 }
12260 
12261 #ifndef CONFIG_TCG
12262 ARMMMUIdx arm_v7m_mmu_idx_for_secstate(CPUARMState *env, bool secstate)
12263 {
12264     g_assert_not_reached();
12265 }
12266 #endif
12267 
12268 ARMMMUIdx arm_mmu_idx_el(CPUARMState *env, int el)
12269 {
12270     ARMMMUIdx idx;
12271     uint64_t hcr;
12272 
12273     if (arm_feature(env, ARM_FEATURE_M)) {
12274         return arm_v7m_mmu_idx_for_secstate(env, env->v7m.secure);
12275     }
12276 
12277     /* See ARM pseudo-function ELIsInHost.  */
12278     switch (el) {
12279     case 0:
12280         hcr = arm_hcr_el2_eff(env);
12281         if ((hcr & (HCR_E2H | HCR_TGE)) == (HCR_E2H | HCR_TGE)) {
12282             idx = ARMMMUIdx_E20_0;
12283         } else {
12284             idx = ARMMMUIdx_E10_0;
12285         }
12286         break;
12287     case 1:
12288         if (arm_pan_enabled(env)) {
12289             idx = ARMMMUIdx_E10_1_PAN;
12290         } else {
12291             idx = ARMMMUIdx_E10_1;
12292         }
12293         break;
12294     case 2:
12295         /* Note that TGE does not apply at EL2.  */
12296         if (arm_hcr_el2_eff(env) & HCR_E2H) {
12297             if (arm_pan_enabled(env)) {
12298                 idx = ARMMMUIdx_E20_2_PAN;
12299             } else {
12300                 idx = ARMMMUIdx_E20_2;
12301             }
12302         } else {
12303             idx = ARMMMUIdx_E2;
12304         }
12305         break;
12306     case 3:
12307         return ARMMMUIdx_E3;
12308     default:
12309         g_assert_not_reached();
12310     }
12311 
12312     return idx;
12313 }
12314 
12315 ARMMMUIdx arm_mmu_idx(CPUARMState *env)
12316 {
12317     return arm_mmu_idx_el(env, arm_current_el(env));
12318 }
12319 
12320 static bool mve_no_pred(CPUARMState *env)
12321 {
12322     /*
12323      * Return true if there is definitely no predication of MVE
12324      * instructions by VPR or LTPSIZE. (Returning false even if there
12325      * isn't any predication is OK; generated code will just be
12326      * a little worse.)
12327      * If the CPU does not implement MVE then this TB flag is always 0.
12328      *
12329      * NOTE: if you change this logic, the "recalculate s->mve_no_pred"
12330      * logic in gen_update_fp_context() needs to be updated to match.
12331      *
12332      * We do not include the effect of the ECI bits here -- they are
12333      * tracked in other TB flags. This simplifies the logic for
12334      * "when did we emit code that changes the MVE_NO_PRED TB flag
12335      * and thus need to end the TB?".
12336      */
12337     if (cpu_isar_feature(aa32_mve, env_archcpu(env))) {
12338         return false;
12339     }
12340     if (env->v7m.vpr) {
12341         return false;
12342     }
12343     if (env->v7m.ltpsize < 4) {
12344         return false;
12345     }
12346     return true;
12347 }
12348 
12349 void cpu_get_tb_cpu_state(CPUARMState *env, vaddr *pc,
12350                           uint64_t *cs_base, uint32_t *pflags)
12351 {
12352     CPUARMTBFlags flags;
12353 
12354     assert_hflags_rebuild_correctly(env);
12355     flags = env->hflags;
12356 
12357     if (EX_TBFLAG_ANY(flags, AARCH64_STATE)) {
12358         *pc = env->pc;
12359         if (cpu_isar_feature(aa64_bti, env_archcpu(env))) {
12360             DP_TBFLAG_A64(flags, BTYPE, env->btype);
12361         }
12362     } else {
12363         *pc = env->regs[15];
12364 
12365         if (arm_feature(env, ARM_FEATURE_M)) {
12366             if (arm_feature(env, ARM_FEATURE_M_SECURITY) &&
12367                 FIELD_EX32(env->v7m.fpccr[M_REG_S], V7M_FPCCR, S)
12368                 != env->v7m.secure) {
12369                 DP_TBFLAG_M32(flags, FPCCR_S_WRONG, 1);
12370             }
12371 
12372             if ((env->v7m.fpccr[env->v7m.secure] & R_V7M_FPCCR_ASPEN_MASK) &&
12373                 (!(env->v7m.control[M_REG_S] & R_V7M_CONTROL_FPCA_MASK) ||
12374                  (env->v7m.secure &&
12375                   !(env->v7m.control[M_REG_S] & R_V7M_CONTROL_SFPA_MASK)))) {
12376                 /*
12377                  * ASPEN is set, but FPCA/SFPA indicate that there is no
12378                  * active FP context; we must create a new FP context before
12379                  * executing any FP insn.
12380                  */
12381                 DP_TBFLAG_M32(flags, NEW_FP_CTXT_NEEDED, 1);
12382             }
12383 
12384             bool is_secure = env->v7m.fpccr[M_REG_S] & R_V7M_FPCCR_S_MASK;
12385             if (env->v7m.fpccr[is_secure] & R_V7M_FPCCR_LSPACT_MASK) {
12386                 DP_TBFLAG_M32(flags, LSPACT, 1);
12387             }
12388 
12389             if (mve_no_pred(env)) {
12390                 DP_TBFLAG_M32(flags, MVE_NO_PRED, 1);
12391             }
12392         } else {
12393             /*
12394              * Note that XSCALE_CPAR shares bits with VECSTRIDE.
12395              * Note that VECLEN+VECSTRIDE are RES0 for M-profile.
12396              */
12397             if (arm_feature(env, ARM_FEATURE_XSCALE)) {
12398                 DP_TBFLAG_A32(flags, XSCALE_CPAR, env->cp15.c15_cpar);
12399             } else {
12400                 DP_TBFLAG_A32(flags, VECLEN, env->vfp.vec_len);
12401                 DP_TBFLAG_A32(flags, VECSTRIDE, env->vfp.vec_stride);
12402             }
12403             if (env->vfp.xregs[ARM_VFP_FPEXC] & (1 << 30)) {
12404                 DP_TBFLAG_A32(flags, VFPEN, 1);
12405             }
12406         }
12407 
12408         DP_TBFLAG_AM32(flags, THUMB, env->thumb);
12409         DP_TBFLAG_AM32(flags, CONDEXEC, env->condexec_bits);
12410     }
12411 
12412     /*
12413      * The SS_ACTIVE and PSTATE_SS bits correspond to the state machine
12414      * states defined in the ARM ARM for software singlestep:
12415      *  SS_ACTIVE   PSTATE.SS   State
12416      *     0            x       Inactive (the TB flag for SS is always 0)
12417      *     1            0       Active-pending
12418      *     1            1       Active-not-pending
12419      * SS_ACTIVE is set in hflags; PSTATE__SS is computed every TB.
12420      */
12421     if (EX_TBFLAG_ANY(flags, SS_ACTIVE) && (env->pstate & PSTATE_SS)) {
12422         DP_TBFLAG_ANY(flags, PSTATE__SS, 1);
12423     }
12424 
12425     *pflags = flags.flags;
12426     *cs_base = flags.flags2;
12427 }
12428 
12429 #ifdef TARGET_AARCH64
12430 /*
12431  * The manual says that when SVE is enabled and VQ is widened the
12432  * implementation is allowed to zero the previously inaccessible
12433  * portion of the registers.  The corollary to that is that when
12434  * SVE is enabled and VQ is narrowed we are also allowed to zero
12435  * the now inaccessible portion of the registers.
12436  *
12437  * The intent of this is that no predicate bit beyond VQ is ever set.
12438  * Which means that some operations on predicate registers themselves
12439  * may operate on full uint64_t or even unrolled across the maximum
12440  * uint64_t[4].  Performing 4 bits of host arithmetic unconditionally
12441  * may well be cheaper than conditionals to restrict the operation
12442  * to the relevant portion of a uint16_t[16].
12443  */
12444 void aarch64_sve_narrow_vq(CPUARMState *env, unsigned vq)
12445 {
12446     int i, j;
12447     uint64_t pmask;
12448 
12449     assert(vq >= 1 && vq <= ARM_MAX_VQ);
12450     assert(vq <= env_archcpu(env)->sve_max_vq);
12451 
12452     /* Zap the high bits of the zregs.  */
12453     for (i = 0; i < 32; i++) {
12454         memset(&env->vfp.zregs[i].d[2 * vq], 0, 16 * (ARM_MAX_VQ - vq));
12455     }
12456 
12457     /* Zap the high bits of the pregs and ffr.  */
12458     pmask = 0;
12459     if (vq & 3) {
12460         pmask = ~(-1ULL << (16 * (vq & 3)));
12461     }
12462     for (j = vq / 4; j < ARM_MAX_VQ / 4; j++) {
12463         for (i = 0; i < 17; ++i) {
12464             env->vfp.pregs[i].p[j] &= pmask;
12465         }
12466         pmask = 0;
12467     }
12468 }
12469 
12470 static uint32_t sve_vqm1_for_el_sm_ena(CPUARMState *env, int el, bool sm)
12471 {
12472     int exc_el;
12473 
12474     if (sm) {
12475         exc_el = sme_exception_el(env, el);
12476     } else {
12477         exc_el = sve_exception_el(env, el);
12478     }
12479     if (exc_el) {
12480         return 0; /* disabled */
12481     }
12482     return sve_vqm1_for_el_sm(env, el, sm);
12483 }
12484 
12485 /*
12486  * Notice a change in SVE vector size when changing EL.
12487  */
12488 void aarch64_sve_change_el(CPUARMState *env, int old_el,
12489                            int new_el, bool el0_a64)
12490 {
12491     ARMCPU *cpu = env_archcpu(env);
12492     int old_len, new_len;
12493     bool old_a64, new_a64, sm;
12494 
12495     /* Nothing to do if no SVE.  */
12496     if (!cpu_isar_feature(aa64_sve, cpu)) {
12497         return;
12498     }
12499 
12500     /* Nothing to do if FP is disabled in either EL.  */
12501     if (fp_exception_el(env, old_el) || fp_exception_el(env, new_el)) {
12502         return;
12503     }
12504 
12505     old_a64 = old_el ? arm_el_is_aa64(env, old_el) : el0_a64;
12506     new_a64 = new_el ? arm_el_is_aa64(env, new_el) : el0_a64;
12507 
12508     /*
12509      * Both AArch64.TakeException and AArch64.ExceptionReturn
12510      * invoke ResetSVEState when taking an exception from, or
12511      * returning to, AArch32 state when PSTATE.SM is enabled.
12512      */
12513     sm = FIELD_EX64(env->svcr, SVCR, SM);
12514     if (old_a64 != new_a64 && sm) {
12515         arm_reset_sve_state(env);
12516         return;
12517     }
12518 
12519     /*
12520      * DDI0584A.d sec 3.2: "If SVE instructions are disabled or trapped
12521      * at ELx, or not available because the EL is in AArch32 state, then
12522      * for all purposes other than a direct read, the ZCR_ELx.LEN field
12523      * has an effective value of 0".
12524      *
12525      * Consider EL2 (aa64, vq=4) -> EL0 (aa32) -> EL1 (aa64, vq=0).
12526      * If we ignore aa32 state, we would fail to see the vq4->vq0 transition
12527      * from EL2->EL1.  Thus we go ahead and narrow when entering aa32 so that
12528      * we already have the correct register contents when encountering the
12529      * vq0->vq0 transition between EL0->EL1.
12530      */
12531     old_len = new_len = 0;
12532     if (old_a64) {
12533         old_len = sve_vqm1_for_el_sm_ena(env, old_el, sm);
12534     }
12535     if (new_a64) {
12536         new_len = sve_vqm1_for_el_sm_ena(env, new_el, sm);
12537     }
12538 
12539     /* When changing vector length, clear inaccessible state.  */
12540     if (new_len < old_len) {
12541         aarch64_sve_narrow_vq(env, new_len + 1);
12542     }
12543 }
12544 #endif
12545 
12546 #ifndef CONFIG_USER_ONLY
12547 ARMSecuritySpace arm_security_space(CPUARMState *env)
12548 {
12549     if (arm_feature(env, ARM_FEATURE_M)) {
12550         return arm_secure_to_space(env->v7m.secure);
12551     }
12552 
12553     /*
12554      * If EL3 is not supported then the secure state is implementation
12555      * defined, in which case QEMU defaults to non-secure.
12556      */
12557     if (!arm_feature(env, ARM_FEATURE_EL3)) {
12558         return ARMSS_NonSecure;
12559     }
12560 
12561     /* Check for AArch64 EL3 or AArch32 Mon. */
12562     if (is_a64(env)) {
12563         if (extract32(env->pstate, 2, 2) == 3) {
12564             if (cpu_isar_feature(aa64_rme, env_archcpu(env))) {
12565                 return ARMSS_Root;
12566             } else {
12567                 return ARMSS_Secure;
12568             }
12569         }
12570     } else {
12571         if ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_MON) {
12572             return ARMSS_Secure;
12573         }
12574     }
12575 
12576     return arm_security_space_below_el3(env);
12577 }
12578 
12579 ARMSecuritySpace arm_security_space_below_el3(CPUARMState *env)
12580 {
12581     assert(!arm_feature(env, ARM_FEATURE_M));
12582 
12583     /*
12584      * If EL3 is not supported then the secure state is implementation
12585      * defined, in which case QEMU defaults to non-secure.
12586      */
12587     if (!arm_feature(env, ARM_FEATURE_EL3)) {
12588         return ARMSS_NonSecure;
12589     }
12590 
12591     /*
12592      * Note NSE cannot be set without RME, and NSE & !NS is Reserved.
12593      * Ignoring NSE when !NS retains consistency without having to
12594      * modify other predicates.
12595      */
12596     if (!(env->cp15.scr_el3 & SCR_NS)) {
12597         return ARMSS_Secure;
12598     } else if (env->cp15.scr_el3 & SCR_NSE) {
12599         return ARMSS_Realm;
12600     } else {
12601         return ARMSS_NonSecure;
12602     }
12603 }
12604 #endif /* !CONFIG_USER_ONLY */
12605