xref: /openbmc/linux/arch/arm64/kvm/sys_regs.c (revision 151f4e2b)
1 /*
2  * Copyright (C) 2012,2013 - ARM Ltd
3  * Author: Marc Zyngier <marc.zyngier@arm.com>
4  *
5  * Derived from arch/arm/kvm/coproc.c:
6  * Copyright (C) 2012 - Virtual Open Systems and Columbia University
7  * Authors: Rusty Russell <rusty@rustcorp.com.au>
8  *          Christoffer Dall <c.dall@virtualopensystems.com>
9  *
10  * This program is free software; you can redistribute it and/or modify
11  * it under the terms of the GNU General Public License, version 2, as
12  * published by the Free Software Foundation.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17  * GNU General Public License for more details.
18  *
19  * You should have received a copy of the GNU General Public License
20  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
21  */
22 
23 #include <linux/bsearch.h>
24 #include <linux/kvm_host.h>
25 #include <linux/mm.h>
26 #include <linux/printk.h>
27 #include <linux/uaccess.h>
28 
29 #include <asm/cacheflush.h>
30 #include <asm/cputype.h>
31 #include <asm/debug-monitors.h>
32 #include <asm/esr.h>
33 #include <asm/kvm_arm.h>
34 #include <asm/kvm_coproc.h>
35 #include <asm/kvm_emulate.h>
36 #include <asm/kvm_host.h>
37 #include <asm/kvm_hyp.h>
38 #include <asm/kvm_mmu.h>
39 #include <asm/perf_event.h>
40 #include <asm/sysreg.h>
41 
42 #include <trace/events/kvm.h>
43 
44 #include "sys_regs.h"
45 
46 #include "trace.h"
47 
48 /*
49  * All of this file is extremly similar to the ARM coproc.c, but the
50  * types are different. My gut feeling is that it should be pretty
51  * easy to merge, but that would be an ABI breakage -- again. VFP
52  * would also need to be abstracted.
53  *
54  * For AArch32, we only take care of what is being trapped. Anything
55  * that has to do with init and userspace access has to go via the
56  * 64bit interface.
57  */
58 
59 static bool read_from_write_only(struct kvm_vcpu *vcpu,
60 				 struct sys_reg_params *params,
61 				 const struct sys_reg_desc *r)
62 {
63 	WARN_ONCE(1, "Unexpected sys_reg read to write-only register\n");
64 	print_sys_reg_instr(params);
65 	kvm_inject_undefined(vcpu);
66 	return false;
67 }
68 
69 static bool write_to_read_only(struct kvm_vcpu *vcpu,
70 			       struct sys_reg_params *params,
71 			       const struct sys_reg_desc *r)
72 {
73 	WARN_ONCE(1, "Unexpected sys_reg write to read-only register\n");
74 	print_sys_reg_instr(params);
75 	kvm_inject_undefined(vcpu);
76 	return false;
77 }
78 
79 u64 vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, int reg)
80 {
81 	if (!vcpu->arch.sysregs_loaded_on_cpu)
82 		goto immediate_read;
83 
84 	/*
85 	 * System registers listed in the switch are not saved on every
86 	 * exit from the guest but are only saved on vcpu_put.
87 	 *
88 	 * Note that MPIDR_EL1 for the guest is set by KVM via VMPIDR_EL2 but
89 	 * should never be listed below, because the guest cannot modify its
90 	 * own MPIDR_EL1 and MPIDR_EL1 is accessed for VCPU A from VCPU B's
91 	 * thread when emulating cross-VCPU communication.
92 	 */
93 	switch (reg) {
94 	case CSSELR_EL1:	return read_sysreg_s(SYS_CSSELR_EL1);
95 	case SCTLR_EL1:		return read_sysreg_s(sctlr_EL12);
96 	case ACTLR_EL1:		return read_sysreg_s(SYS_ACTLR_EL1);
97 	case CPACR_EL1:		return read_sysreg_s(cpacr_EL12);
98 	case TTBR0_EL1:		return read_sysreg_s(ttbr0_EL12);
99 	case TTBR1_EL1:		return read_sysreg_s(ttbr1_EL12);
100 	case TCR_EL1:		return read_sysreg_s(tcr_EL12);
101 	case ESR_EL1:		return read_sysreg_s(esr_EL12);
102 	case AFSR0_EL1:		return read_sysreg_s(afsr0_EL12);
103 	case AFSR1_EL1:		return read_sysreg_s(afsr1_EL12);
104 	case FAR_EL1:		return read_sysreg_s(far_EL12);
105 	case MAIR_EL1:		return read_sysreg_s(mair_EL12);
106 	case VBAR_EL1:		return read_sysreg_s(vbar_EL12);
107 	case CONTEXTIDR_EL1:	return read_sysreg_s(contextidr_EL12);
108 	case TPIDR_EL0:		return read_sysreg_s(SYS_TPIDR_EL0);
109 	case TPIDRRO_EL0:	return read_sysreg_s(SYS_TPIDRRO_EL0);
110 	case TPIDR_EL1:		return read_sysreg_s(SYS_TPIDR_EL1);
111 	case AMAIR_EL1:		return read_sysreg_s(amair_EL12);
112 	case CNTKCTL_EL1:	return read_sysreg_s(cntkctl_EL12);
113 	case PAR_EL1:		return read_sysreg_s(SYS_PAR_EL1);
114 	case DACR32_EL2:	return read_sysreg_s(SYS_DACR32_EL2);
115 	case IFSR32_EL2:	return read_sysreg_s(SYS_IFSR32_EL2);
116 	case DBGVCR32_EL2:	return read_sysreg_s(SYS_DBGVCR32_EL2);
117 	}
118 
119 immediate_read:
120 	return __vcpu_sys_reg(vcpu, reg);
121 }
122 
123 void vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg)
124 {
125 	if (!vcpu->arch.sysregs_loaded_on_cpu)
126 		goto immediate_write;
127 
128 	/*
129 	 * System registers listed in the switch are not restored on every
130 	 * entry to the guest but are only restored on vcpu_load.
131 	 *
132 	 * Note that MPIDR_EL1 for the guest is set by KVM via VMPIDR_EL2 but
133 	 * should never be listed below, because the the MPIDR should only be
134 	 * set once, before running the VCPU, and never changed later.
135 	 */
136 	switch (reg) {
137 	case CSSELR_EL1:	write_sysreg_s(val, SYS_CSSELR_EL1);	return;
138 	case SCTLR_EL1:		write_sysreg_s(val, sctlr_EL12);	return;
139 	case ACTLR_EL1:		write_sysreg_s(val, SYS_ACTLR_EL1);	return;
140 	case CPACR_EL1:		write_sysreg_s(val, cpacr_EL12);	return;
141 	case TTBR0_EL1:		write_sysreg_s(val, ttbr0_EL12);	return;
142 	case TTBR1_EL1:		write_sysreg_s(val, ttbr1_EL12);	return;
143 	case TCR_EL1:		write_sysreg_s(val, tcr_EL12);		return;
144 	case ESR_EL1:		write_sysreg_s(val, esr_EL12);		return;
145 	case AFSR0_EL1:		write_sysreg_s(val, afsr0_EL12);	return;
146 	case AFSR1_EL1:		write_sysreg_s(val, afsr1_EL12);	return;
147 	case FAR_EL1:		write_sysreg_s(val, far_EL12);		return;
148 	case MAIR_EL1:		write_sysreg_s(val, mair_EL12);		return;
149 	case VBAR_EL1:		write_sysreg_s(val, vbar_EL12);		return;
150 	case CONTEXTIDR_EL1:	write_sysreg_s(val, contextidr_EL12);	return;
151 	case TPIDR_EL0:		write_sysreg_s(val, SYS_TPIDR_EL0);	return;
152 	case TPIDRRO_EL0:	write_sysreg_s(val, SYS_TPIDRRO_EL0);	return;
153 	case TPIDR_EL1:		write_sysreg_s(val, SYS_TPIDR_EL1);	return;
154 	case AMAIR_EL1:		write_sysreg_s(val, amair_EL12);	return;
155 	case CNTKCTL_EL1:	write_sysreg_s(val, cntkctl_EL12);	return;
156 	case PAR_EL1:		write_sysreg_s(val, SYS_PAR_EL1);	return;
157 	case DACR32_EL2:	write_sysreg_s(val, SYS_DACR32_EL2);	return;
158 	case IFSR32_EL2:	write_sysreg_s(val, SYS_IFSR32_EL2);	return;
159 	case DBGVCR32_EL2:	write_sysreg_s(val, SYS_DBGVCR32_EL2);	return;
160 	}
161 
162 immediate_write:
163 	 __vcpu_sys_reg(vcpu, reg) = val;
164 }
165 
166 /* 3 bits per cache level, as per CLIDR, but non-existent caches always 0 */
167 static u32 cache_levels;
168 
169 /* CSSELR values; used to index KVM_REG_ARM_DEMUX_ID_CCSIDR */
170 #define CSSELR_MAX 12
171 
172 /* Which cache CCSIDR represents depends on CSSELR value. */
173 static u32 get_ccsidr(u32 csselr)
174 {
175 	u32 ccsidr;
176 
177 	/* Make sure noone else changes CSSELR during this! */
178 	local_irq_disable();
179 	write_sysreg(csselr, csselr_el1);
180 	isb();
181 	ccsidr = read_sysreg(ccsidr_el1);
182 	local_irq_enable();
183 
184 	return ccsidr;
185 }
186 
187 /*
188  * See note at ARMv7 ARM B1.14.4 (TL;DR: S/W ops are not easily virtualized).
189  */
190 static bool access_dcsw(struct kvm_vcpu *vcpu,
191 			struct sys_reg_params *p,
192 			const struct sys_reg_desc *r)
193 {
194 	if (!p->is_write)
195 		return read_from_write_only(vcpu, p, r);
196 
197 	/*
198 	 * Only track S/W ops if we don't have FWB. It still indicates
199 	 * that the guest is a bit broken (S/W operations should only
200 	 * be done by firmware, knowing that there is only a single
201 	 * CPU left in the system, and certainly not from non-secure
202 	 * software).
203 	 */
204 	if (!cpus_have_const_cap(ARM64_HAS_STAGE2_FWB))
205 		kvm_set_way_flush(vcpu);
206 
207 	return true;
208 }
209 
210 /*
211  * Generic accessor for VM registers. Only called as long as HCR_TVM
212  * is set. If the guest enables the MMU, we stop trapping the VM
213  * sys_regs and leave it in complete control of the caches.
214  */
215 static bool access_vm_reg(struct kvm_vcpu *vcpu,
216 			  struct sys_reg_params *p,
217 			  const struct sys_reg_desc *r)
218 {
219 	bool was_enabled = vcpu_has_cache_enabled(vcpu);
220 	u64 val;
221 	int reg = r->reg;
222 
223 	BUG_ON(!p->is_write);
224 
225 	/* See the 32bit mapping in kvm_host.h */
226 	if (p->is_aarch32)
227 		reg = r->reg / 2;
228 
229 	if (!p->is_aarch32 || !p->is_32bit) {
230 		val = p->regval;
231 	} else {
232 		val = vcpu_read_sys_reg(vcpu, reg);
233 		if (r->reg % 2)
234 			val = (p->regval << 32) | (u64)lower_32_bits(val);
235 		else
236 			val = ((u64)upper_32_bits(val) << 32) |
237 				lower_32_bits(p->regval);
238 	}
239 	vcpu_write_sys_reg(vcpu, val, reg);
240 
241 	kvm_toggle_cache(vcpu, was_enabled);
242 	return true;
243 }
244 
245 /*
246  * Trap handler for the GICv3 SGI generation system register.
247  * Forward the request to the VGIC emulation.
248  * The cp15_64 code makes sure this automatically works
249  * for both AArch64 and AArch32 accesses.
250  */
251 static bool access_gic_sgi(struct kvm_vcpu *vcpu,
252 			   struct sys_reg_params *p,
253 			   const struct sys_reg_desc *r)
254 {
255 	bool g1;
256 
257 	if (!p->is_write)
258 		return read_from_write_only(vcpu, p, r);
259 
260 	/*
261 	 * In a system where GICD_CTLR.DS=1, a ICC_SGI0R_EL1 access generates
262 	 * Group0 SGIs only, while ICC_SGI1R_EL1 can generate either group,
263 	 * depending on the SGI configuration. ICC_ASGI1R_EL1 is effectively
264 	 * equivalent to ICC_SGI0R_EL1, as there is no "alternative" secure
265 	 * group.
266 	 */
267 	if (p->is_aarch32) {
268 		switch (p->Op1) {
269 		default:		/* Keep GCC quiet */
270 		case 0:			/* ICC_SGI1R */
271 			g1 = true;
272 			break;
273 		case 1:			/* ICC_ASGI1R */
274 		case 2:			/* ICC_SGI0R */
275 			g1 = false;
276 			break;
277 		}
278 	} else {
279 		switch (p->Op2) {
280 		default:		/* Keep GCC quiet */
281 		case 5:			/* ICC_SGI1R_EL1 */
282 			g1 = true;
283 			break;
284 		case 6:			/* ICC_ASGI1R_EL1 */
285 		case 7:			/* ICC_SGI0R_EL1 */
286 			g1 = false;
287 			break;
288 		}
289 	}
290 
291 	vgic_v3_dispatch_sgi(vcpu, p->regval, g1);
292 
293 	return true;
294 }
295 
296 static bool access_gic_sre(struct kvm_vcpu *vcpu,
297 			   struct sys_reg_params *p,
298 			   const struct sys_reg_desc *r)
299 {
300 	if (p->is_write)
301 		return ignore_write(vcpu, p);
302 
303 	p->regval = vcpu->arch.vgic_cpu.vgic_v3.vgic_sre;
304 	return true;
305 }
306 
307 static bool trap_raz_wi(struct kvm_vcpu *vcpu,
308 			struct sys_reg_params *p,
309 			const struct sys_reg_desc *r)
310 {
311 	if (p->is_write)
312 		return ignore_write(vcpu, p);
313 	else
314 		return read_zero(vcpu, p);
315 }
316 
317 /*
318  * ARMv8.1 mandates at least a trivial LORegion implementation, where all the
319  * RW registers are RES0 (which we can implement as RAZ/WI). On an ARMv8.0
320  * system, these registers should UNDEF. LORID_EL1 being a RO register, we
321  * treat it separately.
322  */
323 static bool trap_loregion(struct kvm_vcpu *vcpu,
324 			  struct sys_reg_params *p,
325 			  const struct sys_reg_desc *r)
326 {
327 	u64 val = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
328 	u32 sr = sys_reg((u32)r->Op0, (u32)r->Op1,
329 			 (u32)r->CRn, (u32)r->CRm, (u32)r->Op2);
330 
331 	if (!(val & (0xfUL << ID_AA64MMFR1_LOR_SHIFT))) {
332 		kvm_inject_undefined(vcpu);
333 		return false;
334 	}
335 
336 	if (p->is_write && sr == SYS_LORID_EL1)
337 		return write_to_read_only(vcpu, p, r);
338 
339 	return trap_raz_wi(vcpu, p, r);
340 }
341 
342 static bool trap_oslsr_el1(struct kvm_vcpu *vcpu,
343 			   struct sys_reg_params *p,
344 			   const struct sys_reg_desc *r)
345 {
346 	if (p->is_write) {
347 		return ignore_write(vcpu, p);
348 	} else {
349 		p->regval = (1 << 3);
350 		return true;
351 	}
352 }
353 
354 static bool trap_dbgauthstatus_el1(struct kvm_vcpu *vcpu,
355 				   struct sys_reg_params *p,
356 				   const struct sys_reg_desc *r)
357 {
358 	if (p->is_write) {
359 		return ignore_write(vcpu, p);
360 	} else {
361 		p->regval = read_sysreg(dbgauthstatus_el1);
362 		return true;
363 	}
364 }
365 
366 /*
367  * We want to avoid world-switching all the DBG registers all the
368  * time:
369  *
370  * - If we've touched any debug register, it is likely that we're
371  *   going to touch more of them. It then makes sense to disable the
372  *   traps and start doing the save/restore dance
373  * - If debug is active (DBG_MDSCR_KDE or DBG_MDSCR_MDE set), it is
374  *   then mandatory to save/restore the registers, as the guest
375  *   depends on them.
376  *
377  * For this, we use a DIRTY bit, indicating the guest has modified the
378  * debug registers, used as follow:
379  *
380  * On guest entry:
381  * - If the dirty bit is set (because we're coming back from trapping),
382  *   disable the traps, save host registers, restore guest registers.
383  * - If debug is actively in use (DBG_MDSCR_KDE or DBG_MDSCR_MDE set),
384  *   set the dirty bit, disable the traps, save host registers,
385  *   restore guest registers.
386  * - Otherwise, enable the traps
387  *
388  * On guest exit:
389  * - If the dirty bit is set, save guest registers, restore host
390  *   registers and clear the dirty bit. This ensure that the host can
391  *   now use the debug registers.
392  */
393 static bool trap_debug_regs(struct kvm_vcpu *vcpu,
394 			    struct sys_reg_params *p,
395 			    const struct sys_reg_desc *r)
396 {
397 	if (p->is_write) {
398 		vcpu_write_sys_reg(vcpu, p->regval, r->reg);
399 		vcpu->arch.flags |= KVM_ARM64_DEBUG_DIRTY;
400 	} else {
401 		p->regval = vcpu_read_sys_reg(vcpu, r->reg);
402 	}
403 
404 	trace_trap_reg(__func__, r->reg, p->is_write, p->regval);
405 
406 	return true;
407 }
408 
409 /*
410  * reg_to_dbg/dbg_to_reg
411  *
412  * A 32 bit write to a debug register leave top bits alone
413  * A 32 bit read from a debug register only returns the bottom bits
414  *
415  * All writes will set the KVM_ARM64_DEBUG_DIRTY flag to ensure the
416  * hyp.S code switches between host and guest values in future.
417  */
418 static void reg_to_dbg(struct kvm_vcpu *vcpu,
419 		       struct sys_reg_params *p,
420 		       u64 *dbg_reg)
421 {
422 	u64 val = p->regval;
423 
424 	if (p->is_32bit) {
425 		val &= 0xffffffffUL;
426 		val |= ((*dbg_reg >> 32) << 32);
427 	}
428 
429 	*dbg_reg = val;
430 	vcpu->arch.flags |= KVM_ARM64_DEBUG_DIRTY;
431 }
432 
433 static void dbg_to_reg(struct kvm_vcpu *vcpu,
434 		       struct sys_reg_params *p,
435 		       u64 *dbg_reg)
436 {
437 	p->regval = *dbg_reg;
438 	if (p->is_32bit)
439 		p->regval &= 0xffffffffUL;
440 }
441 
442 static bool trap_bvr(struct kvm_vcpu *vcpu,
443 		     struct sys_reg_params *p,
444 		     const struct sys_reg_desc *rd)
445 {
446 	u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg];
447 
448 	if (p->is_write)
449 		reg_to_dbg(vcpu, p, dbg_reg);
450 	else
451 		dbg_to_reg(vcpu, p, dbg_reg);
452 
453 	trace_trap_reg(__func__, rd->reg, p->is_write, *dbg_reg);
454 
455 	return true;
456 }
457 
458 static int set_bvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
459 		const struct kvm_one_reg *reg, void __user *uaddr)
460 {
461 	__u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg];
462 
463 	if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
464 		return -EFAULT;
465 	return 0;
466 }
467 
468 static int get_bvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
469 	const struct kvm_one_reg *reg, void __user *uaddr)
470 {
471 	__u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg];
472 
473 	if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
474 		return -EFAULT;
475 	return 0;
476 }
477 
478 static void reset_bvr(struct kvm_vcpu *vcpu,
479 		      const struct sys_reg_desc *rd)
480 {
481 	vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg] = rd->val;
482 }
483 
484 static bool trap_bcr(struct kvm_vcpu *vcpu,
485 		     struct sys_reg_params *p,
486 		     const struct sys_reg_desc *rd)
487 {
488 	u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg];
489 
490 	if (p->is_write)
491 		reg_to_dbg(vcpu, p, dbg_reg);
492 	else
493 		dbg_to_reg(vcpu, p, dbg_reg);
494 
495 	trace_trap_reg(__func__, rd->reg, p->is_write, *dbg_reg);
496 
497 	return true;
498 }
499 
500 static int set_bcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
501 		const struct kvm_one_reg *reg, void __user *uaddr)
502 {
503 	__u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg];
504 
505 	if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
506 		return -EFAULT;
507 
508 	return 0;
509 }
510 
511 static int get_bcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
512 	const struct kvm_one_reg *reg, void __user *uaddr)
513 {
514 	__u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg];
515 
516 	if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
517 		return -EFAULT;
518 	return 0;
519 }
520 
521 static void reset_bcr(struct kvm_vcpu *vcpu,
522 		      const struct sys_reg_desc *rd)
523 {
524 	vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg] = rd->val;
525 }
526 
527 static bool trap_wvr(struct kvm_vcpu *vcpu,
528 		     struct sys_reg_params *p,
529 		     const struct sys_reg_desc *rd)
530 {
531 	u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg];
532 
533 	if (p->is_write)
534 		reg_to_dbg(vcpu, p, dbg_reg);
535 	else
536 		dbg_to_reg(vcpu, p, dbg_reg);
537 
538 	trace_trap_reg(__func__, rd->reg, p->is_write,
539 		vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg]);
540 
541 	return true;
542 }
543 
544 static int set_wvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
545 		const struct kvm_one_reg *reg, void __user *uaddr)
546 {
547 	__u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg];
548 
549 	if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
550 		return -EFAULT;
551 	return 0;
552 }
553 
554 static int get_wvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
555 	const struct kvm_one_reg *reg, void __user *uaddr)
556 {
557 	__u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg];
558 
559 	if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
560 		return -EFAULT;
561 	return 0;
562 }
563 
564 static void reset_wvr(struct kvm_vcpu *vcpu,
565 		      const struct sys_reg_desc *rd)
566 {
567 	vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg] = rd->val;
568 }
569 
570 static bool trap_wcr(struct kvm_vcpu *vcpu,
571 		     struct sys_reg_params *p,
572 		     const struct sys_reg_desc *rd)
573 {
574 	u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg];
575 
576 	if (p->is_write)
577 		reg_to_dbg(vcpu, p, dbg_reg);
578 	else
579 		dbg_to_reg(vcpu, p, dbg_reg);
580 
581 	trace_trap_reg(__func__, rd->reg, p->is_write, *dbg_reg);
582 
583 	return true;
584 }
585 
586 static int set_wcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
587 		const struct kvm_one_reg *reg, void __user *uaddr)
588 {
589 	__u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg];
590 
591 	if (copy_from_user(r, uaddr, KVM_REG_SIZE(reg->id)) != 0)
592 		return -EFAULT;
593 	return 0;
594 }
595 
596 static int get_wcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
597 	const struct kvm_one_reg *reg, void __user *uaddr)
598 {
599 	__u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg];
600 
601 	if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0)
602 		return -EFAULT;
603 	return 0;
604 }
605 
606 static void reset_wcr(struct kvm_vcpu *vcpu,
607 		      const struct sys_reg_desc *rd)
608 {
609 	vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg] = rd->val;
610 }
611 
612 static void reset_amair_el1(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
613 {
614 	u64 amair = read_sysreg(amair_el1);
615 	vcpu_write_sys_reg(vcpu, amair, AMAIR_EL1);
616 }
617 
618 static void reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
619 {
620 	u64 mpidr;
621 
622 	/*
623 	 * Map the vcpu_id into the first three affinity level fields of
624 	 * the MPIDR. We limit the number of VCPUs in level 0 due to a
625 	 * limitation to 16 CPUs in that level in the ICC_SGIxR registers
626 	 * of the GICv3 to be able to address each CPU directly when
627 	 * sending IPIs.
628 	 */
629 	mpidr = (vcpu->vcpu_id & 0x0f) << MPIDR_LEVEL_SHIFT(0);
630 	mpidr |= ((vcpu->vcpu_id >> 4) & 0xff) << MPIDR_LEVEL_SHIFT(1);
631 	mpidr |= ((vcpu->vcpu_id >> 12) & 0xff) << MPIDR_LEVEL_SHIFT(2);
632 	vcpu_write_sys_reg(vcpu, (1ULL << 31) | mpidr, MPIDR_EL1);
633 }
634 
635 static void reset_pmcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
636 {
637 	u64 pmcr, val;
638 
639 	pmcr = read_sysreg(pmcr_el0);
640 	/*
641 	 * Writable bits of PMCR_EL0 (ARMV8_PMU_PMCR_MASK) are reset to UNKNOWN
642 	 * except PMCR.E resetting to zero.
643 	 */
644 	val = ((pmcr & ~ARMV8_PMU_PMCR_MASK)
645 	       | (ARMV8_PMU_PMCR_MASK & 0xdecafbad)) & (~ARMV8_PMU_PMCR_E);
646 	__vcpu_sys_reg(vcpu, PMCR_EL0) = val;
647 }
648 
649 static bool check_pmu_access_disabled(struct kvm_vcpu *vcpu, u64 flags)
650 {
651 	u64 reg = __vcpu_sys_reg(vcpu, PMUSERENR_EL0);
652 	bool enabled = (reg & flags) || vcpu_mode_priv(vcpu);
653 
654 	if (!enabled)
655 		kvm_inject_undefined(vcpu);
656 
657 	return !enabled;
658 }
659 
660 static bool pmu_access_el0_disabled(struct kvm_vcpu *vcpu)
661 {
662 	return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_EN);
663 }
664 
665 static bool pmu_write_swinc_el0_disabled(struct kvm_vcpu *vcpu)
666 {
667 	return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_SW | ARMV8_PMU_USERENR_EN);
668 }
669 
670 static bool pmu_access_cycle_counter_el0_disabled(struct kvm_vcpu *vcpu)
671 {
672 	return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_CR | ARMV8_PMU_USERENR_EN);
673 }
674 
675 static bool pmu_access_event_counter_el0_disabled(struct kvm_vcpu *vcpu)
676 {
677 	return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_ER | ARMV8_PMU_USERENR_EN);
678 }
679 
680 static bool access_pmcr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
681 			const struct sys_reg_desc *r)
682 {
683 	u64 val;
684 
685 	if (!kvm_arm_pmu_v3_ready(vcpu))
686 		return trap_raz_wi(vcpu, p, r);
687 
688 	if (pmu_access_el0_disabled(vcpu))
689 		return false;
690 
691 	if (p->is_write) {
692 		/* Only update writeable bits of PMCR */
693 		val = __vcpu_sys_reg(vcpu, PMCR_EL0);
694 		val &= ~ARMV8_PMU_PMCR_MASK;
695 		val |= p->regval & ARMV8_PMU_PMCR_MASK;
696 		__vcpu_sys_reg(vcpu, PMCR_EL0) = val;
697 		kvm_pmu_handle_pmcr(vcpu, val);
698 		kvm_vcpu_pmu_restore_guest(vcpu);
699 	} else {
700 		/* PMCR.P & PMCR.C are RAZ */
701 		val = __vcpu_sys_reg(vcpu, PMCR_EL0)
702 		      & ~(ARMV8_PMU_PMCR_P | ARMV8_PMU_PMCR_C);
703 		p->regval = val;
704 	}
705 
706 	return true;
707 }
708 
709 static bool access_pmselr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
710 			  const struct sys_reg_desc *r)
711 {
712 	if (!kvm_arm_pmu_v3_ready(vcpu))
713 		return trap_raz_wi(vcpu, p, r);
714 
715 	if (pmu_access_event_counter_el0_disabled(vcpu))
716 		return false;
717 
718 	if (p->is_write)
719 		__vcpu_sys_reg(vcpu, PMSELR_EL0) = p->regval;
720 	else
721 		/* return PMSELR.SEL field */
722 		p->regval = __vcpu_sys_reg(vcpu, PMSELR_EL0)
723 			    & ARMV8_PMU_COUNTER_MASK;
724 
725 	return true;
726 }
727 
728 static bool access_pmceid(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
729 			  const struct sys_reg_desc *r)
730 {
731 	u64 pmceid;
732 
733 	if (!kvm_arm_pmu_v3_ready(vcpu))
734 		return trap_raz_wi(vcpu, p, r);
735 
736 	BUG_ON(p->is_write);
737 
738 	if (pmu_access_el0_disabled(vcpu))
739 		return false;
740 
741 	if (!(p->Op2 & 1))
742 		pmceid = read_sysreg(pmceid0_el0);
743 	else
744 		pmceid = read_sysreg(pmceid1_el0);
745 
746 	p->regval = pmceid;
747 
748 	return true;
749 }
750 
751 static bool pmu_counter_idx_valid(struct kvm_vcpu *vcpu, u64 idx)
752 {
753 	u64 pmcr, val;
754 
755 	pmcr = __vcpu_sys_reg(vcpu, PMCR_EL0);
756 	val = (pmcr >> ARMV8_PMU_PMCR_N_SHIFT) & ARMV8_PMU_PMCR_N_MASK;
757 	if (idx >= val && idx != ARMV8_PMU_CYCLE_IDX) {
758 		kvm_inject_undefined(vcpu);
759 		return false;
760 	}
761 
762 	return true;
763 }
764 
765 static bool access_pmu_evcntr(struct kvm_vcpu *vcpu,
766 			      struct sys_reg_params *p,
767 			      const struct sys_reg_desc *r)
768 {
769 	u64 idx;
770 
771 	if (!kvm_arm_pmu_v3_ready(vcpu))
772 		return trap_raz_wi(vcpu, p, r);
773 
774 	if (r->CRn == 9 && r->CRm == 13) {
775 		if (r->Op2 == 2) {
776 			/* PMXEVCNTR_EL0 */
777 			if (pmu_access_event_counter_el0_disabled(vcpu))
778 				return false;
779 
780 			idx = __vcpu_sys_reg(vcpu, PMSELR_EL0)
781 			      & ARMV8_PMU_COUNTER_MASK;
782 		} else if (r->Op2 == 0) {
783 			/* PMCCNTR_EL0 */
784 			if (pmu_access_cycle_counter_el0_disabled(vcpu))
785 				return false;
786 
787 			idx = ARMV8_PMU_CYCLE_IDX;
788 		} else {
789 			return false;
790 		}
791 	} else if (r->CRn == 0 && r->CRm == 9) {
792 		/* PMCCNTR */
793 		if (pmu_access_event_counter_el0_disabled(vcpu))
794 			return false;
795 
796 		idx = ARMV8_PMU_CYCLE_IDX;
797 	} else if (r->CRn == 14 && (r->CRm & 12) == 8) {
798 		/* PMEVCNTRn_EL0 */
799 		if (pmu_access_event_counter_el0_disabled(vcpu))
800 			return false;
801 
802 		idx = ((r->CRm & 3) << 3) | (r->Op2 & 7);
803 	} else {
804 		return false;
805 	}
806 
807 	if (!pmu_counter_idx_valid(vcpu, idx))
808 		return false;
809 
810 	if (p->is_write) {
811 		if (pmu_access_el0_disabled(vcpu))
812 			return false;
813 
814 		kvm_pmu_set_counter_value(vcpu, idx, p->regval);
815 	} else {
816 		p->regval = kvm_pmu_get_counter_value(vcpu, idx);
817 	}
818 
819 	return true;
820 }
821 
822 static bool access_pmu_evtyper(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
823 			       const struct sys_reg_desc *r)
824 {
825 	u64 idx, reg;
826 
827 	if (!kvm_arm_pmu_v3_ready(vcpu))
828 		return trap_raz_wi(vcpu, p, r);
829 
830 	if (pmu_access_el0_disabled(vcpu))
831 		return false;
832 
833 	if (r->CRn == 9 && r->CRm == 13 && r->Op2 == 1) {
834 		/* PMXEVTYPER_EL0 */
835 		idx = __vcpu_sys_reg(vcpu, PMSELR_EL0) & ARMV8_PMU_COUNTER_MASK;
836 		reg = PMEVTYPER0_EL0 + idx;
837 	} else if (r->CRn == 14 && (r->CRm & 12) == 12) {
838 		idx = ((r->CRm & 3) << 3) | (r->Op2 & 7);
839 		if (idx == ARMV8_PMU_CYCLE_IDX)
840 			reg = PMCCFILTR_EL0;
841 		else
842 			/* PMEVTYPERn_EL0 */
843 			reg = PMEVTYPER0_EL0 + idx;
844 	} else {
845 		BUG();
846 	}
847 
848 	if (!pmu_counter_idx_valid(vcpu, idx))
849 		return false;
850 
851 	if (p->is_write) {
852 		kvm_pmu_set_counter_event_type(vcpu, p->regval, idx);
853 		__vcpu_sys_reg(vcpu, reg) = p->regval & ARMV8_PMU_EVTYPE_MASK;
854 		kvm_vcpu_pmu_restore_guest(vcpu);
855 	} else {
856 		p->regval = __vcpu_sys_reg(vcpu, reg) & ARMV8_PMU_EVTYPE_MASK;
857 	}
858 
859 	return true;
860 }
861 
862 static bool access_pmcnten(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
863 			   const struct sys_reg_desc *r)
864 {
865 	u64 val, mask;
866 
867 	if (!kvm_arm_pmu_v3_ready(vcpu))
868 		return trap_raz_wi(vcpu, p, r);
869 
870 	if (pmu_access_el0_disabled(vcpu))
871 		return false;
872 
873 	mask = kvm_pmu_valid_counter_mask(vcpu);
874 	if (p->is_write) {
875 		val = p->regval & mask;
876 		if (r->Op2 & 0x1) {
877 			/* accessing PMCNTENSET_EL0 */
878 			__vcpu_sys_reg(vcpu, PMCNTENSET_EL0) |= val;
879 			kvm_pmu_enable_counter(vcpu, val);
880 			kvm_vcpu_pmu_restore_guest(vcpu);
881 		} else {
882 			/* accessing PMCNTENCLR_EL0 */
883 			__vcpu_sys_reg(vcpu, PMCNTENSET_EL0) &= ~val;
884 			kvm_pmu_disable_counter(vcpu, val);
885 		}
886 	} else {
887 		p->regval = __vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & mask;
888 	}
889 
890 	return true;
891 }
892 
893 static bool access_pminten(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
894 			   const struct sys_reg_desc *r)
895 {
896 	u64 mask = kvm_pmu_valid_counter_mask(vcpu);
897 
898 	if (!kvm_arm_pmu_v3_ready(vcpu))
899 		return trap_raz_wi(vcpu, p, r);
900 
901 	if (!vcpu_mode_priv(vcpu)) {
902 		kvm_inject_undefined(vcpu);
903 		return false;
904 	}
905 
906 	if (p->is_write) {
907 		u64 val = p->regval & mask;
908 
909 		if (r->Op2 & 0x1)
910 			/* accessing PMINTENSET_EL1 */
911 			__vcpu_sys_reg(vcpu, PMINTENSET_EL1) |= val;
912 		else
913 			/* accessing PMINTENCLR_EL1 */
914 			__vcpu_sys_reg(vcpu, PMINTENSET_EL1) &= ~val;
915 	} else {
916 		p->regval = __vcpu_sys_reg(vcpu, PMINTENSET_EL1) & mask;
917 	}
918 
919 	return true;
920 }
921 
922 static bool access_pmovs(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
923 			 const struct sys_reg_desc *r)
924 {
925 	u64 mask = kvm_pmu_valid_counter_mask(vcpu);
926 
927 	if (!kvm_arm_pmu_v3_ready(vcpu))
928 		return trap_raz_wi(vcpu, p, r);
929 
930 	if (pmu_access_el0_disabled(vcpu))
931 		return false;
932 
933 	if (p->is_write) {
934 		if (r->CRm & 0x2)
935 			/* accessing PMOVSSET_EL0 */
936 			__vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= (p->regval & mask);
937 		else
938 			/* accessing PMOVSCLR_EL0 */
939 			__vcpu_sys_reg(vcpu, PMOVSSET_EL0) &= ~(p->regval & mask);
940 	} else {
941 		p->regval = __vcpu_sys_reg(vcpu, PMOVSSET_EL0) & mask;
942 	}
943 
944 	return true;
945 }
946 
947 static bool access_pmswinc(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
948 			   const struct sys_reg_desc *r)
949 {
950 	u64 mask;
951 
952 	if (!kvm_arm_pmu_v3_ready(vcpu))
953 		return trap_raz_wi(vcpu, p, r);
954 
955 	if (!p->is_write)
956 		return read_from_write_only(vcpu, p, r);
957 
958 	if (pmu_write_swinc_el0_disabled(vcpu))
959 		return false;
960 
961 	mask = kvm_pmu_valid_counter_mask(vcpu);
962 	kvm_pmu_software_increment(vcpu, p->regval & mask);
963 	return true;
964 }
965 
966 static bool access_pmuserenr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
967 			     const struct sys_reg_desc *r)
968 {
969 	if (!kvm_arm_pmu_v3_ready(vcpu))
970 		return trap_raz_wi(vcpu, p, r);
971 
972 	if (p->is_write) {
973 		if (!vcpu_mode_priv(vcpu)) {
974 			kvm_inject_undefined(vcpu);
975 			return false;
976 		}
977 
978 		__vcpu_sys_reg(vcpu, PMUSERENR_EL0) =
979 			       p->regval & ARMV8_PMU_USERENR_MASK;
980 	} else {
981 		p->regval = __vcpu_sys_reg(vcpu, PMUSERENR_EL0)
982 			    & ARMV8_PMU_USERENR_MASK;
983 	}
984 
985 	return true;
986 }
987 
988 #define reg_to_encoding(x)						\
989 	sys_reg((u32)(x)->Op0, (u32)(x)->Op1,				\
990 		(u32)(x)->CRn, (u32)(x)->CRm, (u32)(x)->Op2);
991 
992 /* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
993 #define DBG_BCR_BVR_WCR_WVR_EL1(n)					\
994 	{ SYS_DESC(SYS_DBGBVRn_EL1(n)),					\
995 	  trap_bvr, reset_bvr, n, 0, get_bvr, set_bvr },		\
996 	{ SYS_DESC(SYS_DBGBCRn_EL1(n)),					\
997 	  trap_bcr, reset_bcr, n, 0, get_bcr, set_bcr },		\
998 	{ SYS_DESC(SYS_DBGWVRn_EL1(n)),					\
999 	  trap_wvr, reset_wvr, n, 0,  get_wvr, set_wvr },		\
1000 	{ SYS_DESC(SYS_DBGWCRn_EL1(n)),					\
1001 	  trap_wcr, reset_wcr, n, 0,  get_wcr, set_wcr }
1002 
1003 /* Macro to expand the PMEVCNTRn_EL0 register */
1004 #define PMU_PMEVCNTR_EL0(n)						\
1005 	{ SYS_DESC(SYS_PMEVCNTRn_EL0(n)),					\
1006 	  access_pmu_evcntr, reset_unknown, (PMEVCNTR0_EL0 + n), }
1007 
1008 /* Macro to expand the PMEVTYPERn_EL0 register */
1009 #define PMU_PMEVTYPER_EL0(n)						\
1010 	{ SYS_DESC(SYS_PMEVTYPERn_EL0(n)),					\
1011 	  access_pmu_evtyper, reset_unknown, (PMEVTYPER0_EL0 + n), }
1012 
1013 static bool trap_ptrauth(struct kvm_vcpu *vcpu,
1014 			 struct sys_reg_params *p,
1015 			 const struct sys_reg_desc *rd)
1016 {
1017 	kvm_arm_vcpu_ptrauth_trap(vcpu);
1018 
1019 	/*
1020 	 * Return false for both cases as we never skip the trapped
1021 	 * instruction:
1022 	 *
1023 	 * - Either we re-execute the same key register access instruction
1024 	 *   after enabling ptrauth.
1025 	 * - Or an UNDEF is injected as ptrauth is not supported/enabled.
1026 	 */
1027 	return false;
1028 }
1029 
1030 static unsigned int ptrauth_visibility(const struct kvm_vcpu *vcpu,
1031 			const struct sys_reg_desc *rd)
1032 {
1033 	return vcpu_has_ptrauth(vcpu) ? 0 : REG_HIDDEN_USER | REG_HIDDEN_GUEST;
1034 }
1035 
1036 #define __PTRAUTH_KEY(k)						\
1037 	{ SYS_DESC(SYS_## k), trap_ptrauth, reset_unknown, k,		\
1038 	.visibility = ptrauth_visibility}
1039 
1040 #define PTRAUTH_KEY(k)							\
1041 	__PTRAUTH_KEY(k ## KEYLO_EL1),					\
1042 	__PTRAUTH_KEY(k ## KEYHI_EL1)
1043 
1044 static bool access_arch_timer(struct kvm_vcpu *vcpu,
1045 			      struct sys_reg_params *p,
1046 			      const struct sys_reg_desc *r)
1047 {
1048 	enum kvm_arch_timers tmr;
1049 	enum kvm_arch_timer_regs treg;
1050 	u64 reg = reg_to_encoding(r);
1051 
1052 	switch (reg) {
1053 	case SYS_CNTP_TVAL_EL0:
1054 	case SYS_AARCH32_CNTP_TVAL:
1055 		tmr = TIMER_PTIMER;
1056 		treg = TIMER_REG_TVAL;
1057 		break;
1058 	case SYS_CNTP_CTL_EL0:
1059 	case SYS_AARCH32_CNTP_CTL:
1060 		tmr = TIMER_PTIMER;
1061 		treg = TIMER_REG_CTL;
1062 		break;
1063 	case SYS_CNTP_CVAL_EL0:
1064 	case SYS_AARCH32_CNTP_CVAL:
1065 		tmr = TIMER_PTIMER;
1066 		treg = TIMER_REG_CVAL;
1067 		break;
1068 	default:
1069 		BUG();
1070 	}
1071 
1072 	if (p->is_write)
1073 		kvm_arm_timer_write_sysreg(vcpu, tmr, treg, p->regval);
1074 	else
1075 		p->regval = kvm_arm_timer_read_sysreg(vcpu, tmr, treg);
1076 
1077 	return true;
1078 }
1079 
1080 /* Read a sanitised cpufeature ID register by sys_reg_desc */
1081 static u64 read_id_reg(const struct kvm_vcpu *vcpu,
1082 		struct sys_reg_desc const *r, bool raz)
1083 {
1084 	u32 id = sys_reg((u32)r->Op0, (u32)r->Op1,
1085 			 (u32)r->CRn, (u32)r->CRm, (u32)r->Op2);
1086 	u64 val = raz ? 0 : read_sanitised_ftr_reg(id);
1087 
1088 	if (id == SYS_ID_AA64PFR0_EL1 && !vcpu_has_sve(vcpu)) {
1089 		val &= ~(0xfUL << ID_AA64PFR0_SVE_SHIFT);
1090 	} else if (id == SYS_ID_AA64ISAR1_EL1 && !vcpu_has_ptrauth(vcpu)) {
1091 		val &= ~((0xfUL << ID_AA64ISAR1_APA_SHIFT) |
1092 			 (0xfUL << ID_AA64ISAR1_API_SHIFT) |
1093 			 (0xfUL << ID_AA64ISAR1_GPA_SHIFT) |
1094 			 (0xfUL << ID_AA64ISAR1_GPI_SHIFT));
1095 	}
1096 
1097 	return val;
1098 }
1099 
1100 /* cpufeature ID register access trap handlers */
1101 
1102 static bool __access_id_reg(struct kvm_vcpu *vcpu,
1103 			    struct sys_reg_params *p,
1104 			    const struct sys_reg_desc *r,
1105 			    bool raz)
1106 {
1107 	if (p->is_write)
1108 		return write_to_read_only(vcpu, p, r);
1109 
1110 	p->regval = read_id_reg(vcpu, r, raz);
1111 	return true;
1112 }
1113 
1114 static bool access_id_reg(struct kvm_vcpu *vcpu,
1115 			  struct sys_reg_params *p,
1116 			  const struct sys_reg_desc *r)
1117 {
1118 	return __access_id_reg(vcpu, p, r, false);
1119 }
1120 
1121 static bool access_raz_id_reg(struct kvm_vcpu *vcpu,
1122 			      struct sys_reg_params *p,
1123 			      const struct sys_reg_desc *r)
1124 {
1125 	return __access_id_reg(vcpu, p, r, true);
1126 }
1127 
1128 static int reg_from_user(u64 *val, const void __user *uaddr, u64 id);
1129 static int reg_to_user(void __user *uaddr, const u64 *val, u64 id);
1130 static u64 sys_reg_to_index(const struct sys_reg_desc *reg);
1131 
1132 /* Visibility overrides for SVE-specific control registers */
1133 static unsigned int sve_visibility(const struct kvm_vcpu *vcpu,
1134 				   const struct sys_reg_desc *rd)
1135 {
1136 	if (vcpu_has_sve(vcpu))
1137 		return 0;
1138 
1139 	return REG_HIDDEN_USER | REG_HIDDEN_GUEST;
1140 }
1141 
1142 /* Visibility overrides for SVE-specific ID registers */
1143 static unsigned int sve_id_visibility(const struct kvm_vcpu *vcpu,
1144 				      const struct sys_reg_desc *rd)
1145 {
1146 	if (vcpu_has_sve(vcpu))
1147 		return 0;
1148 
1149 	return REG_HIDDEN_USER;
1150 }
1151 
1152 /* Generate the emulated ID_AA64ZFR0_EL1 value exposed to the guest */
1153 static u64 guest_id_aa64zfr0_el1(const struct kvm_vcpu *vcpu)
1154 {
1155 	if (!vcpu_has_sve(vcpu))
1156 		return 0;
1157 
1158 	return read_sanitised_ftr_reg(SYS_ID_AA64ZFR0_EL1);
1159 }
1160 
1161 static bool access_id_aa64zfr0_el1(struct kvm_vcpu *vcpu,
1162 				   struct sys_reg_params *p,
1163 				   const struct sys_reg_desc *rd)
1164 {
1165 	if (p->is_write)
1166 		return write_to_read_only(vcpu, p, rd);
1167 
1168 	p->regval = guest_id_aa64zfr0_el1(vcpu);
1169 	return true;
1170 }
1171 
1172 static int get_id_aa64zfr0_el1(struct kvm_vcpu *vcpu,
1173 		const struct sys_reg_desc *rd,
1174 		const struct kvm_one_reg *reg, void __user *uaddr)
1175 {
1176 	u64 val;
1177 
1178 	if (WARN_ON(!vcpu_has_sve(vcpu)))
1179 		return -ENOENT;
1180 
1181 	val = guest_id_aa64zfr0_el1(vcpu);
1182 	return reg_to_user(uaddr, &val, reg->id);
1183 }
1184 
1185 static int set_id_aa64zfr0_el1(struct kvm_vcpu *vcpu,
1186 		const struct sys_reg_desc *rd,
1187 		const struct kvm_one_reg *reg, void __user *uaddr)
1188 {
1189 	const u64 id = sys_reg_to_index(rd);
1190 	int err;
1191 	u64 val;
1192 
1193 	if (WARN_ON(!vcpu_has_sve(vcpu)))
1194 		return -ENOENT;
1195 
1196 	err = reg_from_user(&val, uaddr, id);
1197 	if (err)
1198 		return err;
1199 
1200 	/* This is what we mean by invariant: you can't change it. */
1201 	if (val != guest_id_aa64zfr0_el1(vcpu))
1202 		return -EINVAL;
1203 
1204 	return 0;
1205 }
1206 
1207 /*
1208  * cpufeature ID register user accessors
1209  *
1210  * For now, these registers are immutable for userspace, so no values
1211  * are stored, and for set_id_reg() we don't allow the effective value
1212  * to be changed.
1213  */
1214 static int __get_id_reg(const struct kvm_vcpu *vcpu,
1215 			const struct sys_reg_desc *rd, void __user *uaddr,
1216 			bool raz)
1217 {
1218 	const u64 id = sys_reg_to_index(rd);
1219 	const u64 val = read_id_reg(vcpu, rd, raz);
1220 
1221 	return reg_to_user(uaddr, &val, id);
1222 }
1223 
1224 static int __set_id_reg(const struct kvm_vcpu *vcpu,
1225 			const struct sys_reg_desc *rd, void __user *uaddr,
1226 			bool raz)
1227 {
1228 	const u64 id = sys_reg_to_index(rd);
1229 	int err;
1230 	u64 val;
1231 
1232 	err = reg_from_user(&val, uaddr, id);
1233 	if (err)
1234 		return err;
1235 
1236 	/* This is what we mean by invariant: you can't change it. */
1237 	if (val != read_id_reg(vcpu, rd, raz))
1238 		return -EINVAL;
1239 
1240 	return 0;
1241 }
1242 
1243 static int get_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1244 		      const struct kvm_one_reg *reg, void __user *uaddr)
1245 {
1246 	return __get_id_reg(vcpu, rd, uaddr, false);
1247 }
1248 
1249 static int set_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1250 		      const struct kvm_one_reg *reg, void __user *uaddr)
1251 {
1252 	return __set_id_reg(vcpu, rd, uaddr, false);
1253 }
1254 
1255 static int get_raz_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1256 			  const struct kvm_one_reg *reg, void __user *uaddr)
1257 {
1258 	return __get_id_reg(vcpu, rd, uaddr, true);
1259 }
1260 
1261 static int set_raz_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1262 			  const struct kvm_one_reg *reg, void __user *uaddr)
1263 {
1264 	return __set_id_reg(vcpu, rd, uaddr, true);
1265 }
1266 
1267 static bool access_ctr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1268 		       const struct sys_reg_desc *r)
1269 {
1270 	if (p->is_write)
1271 		return write_to_read_only(vcpu, p, r);
1272 
1273 	p->regval = read_sanitised_ftr_reg(SYS_CTR_EL0);
1274 	return true;
1275 }
1276 
1277 static bool access_clidr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1278 			 const struct sys_reg_desc *r)
1279 {
1280 	if (p->is_write)
1281 		return write_to_read_only(vcpu, p, r);
1282 
1283 	p->regval = read_sysreg(clidr_el1);
1284 	return true;
1285 }
1286 
1287 static bool access_csselr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1288 			  const struct sys_reg_desc *r)
1289 {
1290 	if (p->is_write)
1291 		vcpu_write_sys_reg(vcpu, p->regval, r->reg);
1292 	else
1293 		p->regval = vcpu_read_sys_reg(vcpu, r->reg);
1294 	return true;
1295 }
1296 
1297 static bool access_ccsidr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1298 			  const struct sys_reg_desc *r)
1299 {
1300 	u32 csselr;
1301 
1302 	if (p->is_write)
1303 		return write_to_read_only(vcpu, p, r);
1304 
1305 	csselr = vcpu_read_sys_reg(vcpu, CSSELR_EL1);
1306 	p->regval = get_ccsidr(csselr);
1307 
1308 	/*
1309 	 * Guests should not be doing cache operations by set/way at all, and
1310 	 * for this reason, we trap them and attempt to infer the intent, so
1311 	 * that we can flush the entire guest's address space at the appropriate
1312 	 * time.
1313 	 * To prevent this trapping from causing performance problems, let's
1314 	 * expose the geometry of all data and unified caches (which are
1315 	 * guaranteed to be PIPT and thus non-aliasing) as 1 set and 1 way.
1316 	 * [If guests should attempt to infer aliasing properties from the
1317 	 * geometry (which is not permitted by the architecture), they would
1318 	 * only do so for virtually indexed caches.]
1319 	 */
1320 	if (!(csselr & 1)) // data or unified cache
1321 		p->regval &= ~GENMASK(27, 3);
1322 	return true;
1323 }
1324 
1325 /* sys_reg_desc initialiser for known cpufeature ID registers */
1326 #define ID_SANITISED(name) {			\
1327 	SYS_DESC(SYS_##name),			\
1328 	.access	= access_id_reg,		\
1329 	.get_user = get_id_reg,			\
1330 	.set_user = set_id_reg,			\
1331 }
1332 
1333 /*
1334  * sys_reg_desc initialiser for architecturally unallocated cpufeature ID
1335  * register with encoding Op0=3, Op1=0, CRn=0, CRm=crm, Op2=op2
1336  * (1 <= crm < 8, 0 <= Op2 < 8).
1337  */
1338 #define ID_UNALLOCATED(crm, op2) {			\
1339 	Op0(3), Op1(0), CRn(0), CRm(crm), Op2(op2),	\
1340 	.access = access_raz_id_reg,			\
1341 	.get_user = get_raz_id_reg,			\
1342 	.set_user = set_raz_id_reg,			\
1343 }
1344 
1345 /*
1346  * sys_reg_desc initialiser for known ID registers that we hide from guests.
1347  * For now, these are exposed just like unallocated ID regs: they appear
1348  * RAZ for the guest.
1349  */
1350 #define ID_HIDDEN(name) {			\
1351 	SYS_DESC(SYS_##name),			\
1352 	.access = access_raz_id_reg,		\
1353 	.get_user = get_raz_id_reg,		\
1354 	.set_user = set_raz_id_reg,		\
1355 }
1356 
1357 /*
1358  * Architected system registers.
1359  * Important: Must be sorted ascending by Op0, Op1, CRn, CRm, Op2
1360  *
1361  * Debug handling: We do trap most, if not all debug related system
1362  * registers. The implementation is good enough to ensure that a guest
1363  * can use these with minimal performance degradation. The drawback is
1364  * that we don't implement any of the external debug, none of the
1365  * OSlock protocol. This should be revisited if we ever encounter a
1366  * more demanding guest...
1367  */
1368 static const struct sys_reg_desc sys_reg_descs[] = {
1369 	{ SYS_DESC(SYS_DC_ISW), access_dcsw },
1370 	{ SYS_DESC(SYS_DC_CSW), access_dcsw },
1371 	{ SYS_DESC(SYS_DC_CISW), access_dcsw },
1372 
1373 	DBG_BCR_BVR_WCR_WVR_EL1(0),
1374 	DBG_BCR_BVR_WCR_WVR_EL1(1),
1375 	{ SYS_DESC(SYS_MDCCINT_EL1), trap_debug_regs, reset_val, MDCCINT_EL1, 0 },
1376 	{ SYS_DESC(SYS_MDSCR_EL1), trap_debug_regs, reset_val, MDSCR_EL1, 0 },
1377 	DBG_BCR_BVR_WCR_WVR_EL1(2),
1378 	DBG_BCR_BVR_WCR_WVR_EL1(3),
1379 	DBG_BCR_BVR_WCR_WVR_EL1(4),
1380 	DBG_BCR_BVR_WCR_WVR_EL1(5),
1381 	DBG_BCR_BVR_WCR_WVR_EL1(6),
1382 	DBG_BCR_BVR_WCR_WVR_EL1(7),
1383 	DBG_BCR_BVR_WCR_WVR_EL1(8),
1384 	DBG_BCR_BVR_WCR_WVR_EL1(9),
1385 	DBG_BCR_BVR_WCR_WVR_EL1(10),
1386 	DBG_BCR_BVR_WCR_WVR_EL1(11),
1387 	DBG_BCR_BVR_WCR_WVR_EL1(12),
1388 	DBG_BCR_BVR_WCR_WVR_EL1(13),
1389 	DBG_BCR_BVR_WCR_WVR_EL1(14),
1390 	DBG_BCR_BVR_WCR_WVR_EL1(15),
1391 
1392 	{ SYS_DESC(SYS_MDRAR_EL1), trap_raz_wi },
1393 	{ SYS_DESC(SYS_OSLAR_EL1), trap_raz_wi },
1394 	{ SYS_DESC(SYS_OSLSR_EL1), trap_oslsr_el1 },
1395 	{ SYS_DESC(SYS_OSDLR_EL1), trap_raz_wi },
1396 	{ SYS_DESC(SYS_DBGPRCR_EL1), trap_raz_wi },
1397 	{ SYS_DESC(SYS_DBGCLAIMSET_EL1), trap_raz_wi },
1398 	{ SYS_DESC(SYS_DBGCLAIMCLR_EL1), trap_raz_wi },
1399 	{ SYS_DESC(SYS_DBGAUTHSTATUS_EL1), trap_dbgauthstatus_el1 },
1400 
1401 	{ SYS_DESC(SYS_MDCCSR_EL0), trap_raz_wi },
1402 	{ SYS_DESC(SYS_DBGDTR_EL0), trap_raz_wi },
1403 	// DBGDTR[TR]X_EL0 share the same encoding
1404 	{ SYS_DESC(SYS_DBGDTRTX_EL0), trap_raz_wi },
1405 
1406 	{ SYS_DESC(SYS_DBGVCR32_EL2), NULL, reset_val, DBGVCR32_EL2, 0 },
1407 
1408 	{ SYS_DESC(SYS_MPIDR_EL1), NULL, reset_mpidr, MPIDR_EL1 },
1409 
1410 	/*
1411 	 * ID regs: all ID_SANITISED() entries here must have corresponding
1412 	 * entries in arm64_ftr_regs[].
1413 	 */
1414 
1415 	/* AArch64 mappings of the AArch32 ID registers */
1416 	/* CRm=1 */
1417 	ID_SANITISED(ID_PFR0_EL1),
1418 	ID_SANITISED(ID_PFR1_EL1),
1419 	ID_SANITISED(ID_DFR0_EL1),
1420 	ID_HIDDEN(ID_AFR0_EL1),
1421 	ID_SANITISED(ID_MMFR0_EL1),
1422 	ID_SANITISED(ID_MMFR1_EL1),
1423 	ID_SANITISED(ID_MMFR2_EL1),
1424 	ID_SANITISED(ID_MMFR3_EL1),
1425 
1426 	/* CRm=2 */
1427 	ID_SANITISED(ID_ISAR0_EL1),
1428 	ID_SANITISED(ID_ISAR1_EL1),
1429 	ID_SANITISED(ID_ISAR2_EL1),
1430 	ID_SANITISED(ID_ISAR3_EL1),
1431 	ID_SANITISED(ID_ISAR4_EL1),
1432 	ID_SANITISED(ID_ISAR5_EL1),
1433 	ID_SANITISED(ID_MMFR4_EL1),
1434 	ID_UNALLOCATED(2,7),
1435 
1436 	/* CRm=3 */
1437 	ID_SANITISED(MVFR0_EL1),
1438 	ID_SANITISED(MVFR1_EL1),
1439 	ID_SANITISED(MVFR2_EL1),
1440 	ID_UNALLOCATED(3,3),
1441 	ID_UNALLOCATED(3,4),
1442 	ID_UNALLOCATED(3,5),
1443 	ID_UNALLOCATED(3,6),
1444 	ID_UNALLOCATED(3,7),
1445 
1446 	/* AArch64 ID registers */
1447 	/* CRm=4 */
1448 	ID_SANITISED(ID_AA64PFR0_EL1),
1449 	ID_SANITISED(ID_AA64PFR1_EL1),
1450 	ID_UNALLOCATED(4,2),
1451 	ID_UNALLOCATED(4,3),
1452 	{ SYS_DESC(SYS_ID_AA64ZFR0_EL1), access_id_aa64zfr0_el1, .get_user = get_id_aa64zfr0_el1, .set_user = set_id_aa64zfr0_el1, .visibility = sve_id_visibility },
1453 	ID_UNALLOCATED(4,5),
1454 	ID_UNALLOCATED(4,6),
1455 	ID_UNALLOCATED(4,7),
1456 
1457 	/* CRm=5 */
1458 	ID_SANITISED(ID_AA64DFR0_EL1),
1459 	ID_SANITISED(ID_AA64DFR1_EL1),
1460 	ID_UNALLOCATED(5,2),
1461 	ID_UNALLOCATED(5,3),
1462 	ID_HIDDEN(ID_AA64AFR0_EL1),
1463 	ID_HIDDEN(ID_AA64AFR1_EL1),
1464 	ID_UNALLOCATED(5,6),
1465 	ID_UNALLOCATED(5,7),
1466 
1467 	/* CRm=6 */
1468 	ID_SANITISED(ID_AA64ISAR0_EL1),
1469 	ID_SANITISED(ID_AA64ISAR1_EL1),
1470 	ID_UNALLOCATED(6,2),
1471 	ID_UNALLOCATED(6,3),
1472 	ID_UNALLOCATED(6,4),
1473 	ID_UNALLOCATED(6,5),
1474 	ID_UNALLOCATED(6,6),
1475 	ID_UNALLOCATED(6,7),
1476 
1477 	/* CRm=7 */
1478 	ID_SANITISED(ID_AA64MMFR0_EL1),
1479 	ID_SANITISED(ID_AA64MMFR1_EL1),
1480 	ID_SANITISED(ID_AA64MMFR2_EL1),
1481 	ID_UNALLOCATED(7,3),
1482 	ID_UNALLOCATED(7,4),
1483 	ID_UNALLOCATED(7,5),
1484 	ID_UNALLOCATED(7,6),
1485 	ID_UNALLOCATED(7,7),
1486 
1487 	{ SYS_DESC(SYS_SCTLR_EL1), access_vm_reg, reset_val, SCTLR_EL1, 0x00C50078 },
1488 	{ SYS_DESC(SYS_CPACR_EL1), NULL, reset_val, CPACR_EL1, 0 },
1489 	{ SYS_DESC(SYS_ZCR_EL1), NULL, reset_val, ZCR_EL1, 0, .visibility = sve_visibility },
1490 	{ SYS_DESC(SYS_TTBR0_EL1), access_vm_reg, reset_unknown, TTBR0_EL1 },
1491 	{ SYS_DESC(SYS_TTBR1_EL1), access_vm_reg, reset_unknown, TTBR1_EL1 },
1492 	{ SYS_DESC(SYS_TCR_EL1), access_vm_reg, reset_val, TCR_EL1, 0 },
1493 
1494 	PTRAUTH_KEY(APIA),
1495 	PTRAUTH_KEY(APIB),
1496 	PTRAUTH_KEY(APDA),
1497 	PTRAUTH_KEY(APDB),
1498 	PTRAUTH_KEY(APGA),
1499 
1500 	{ SYS_DESC(SYS_AFSR0_EL1), access_vm_reg, reset_unknown, AFSR0_EL1 },
1501 	{ SYS_DESC(SYS_AFSR1_EL1), access_vm_reg, reset_unknown, AFSR1_EL1 },
1502 	{ SYS_DESC(SYS_ESR_EL1), access_vm_reg, reset_unknown, ESR_EL1 },
1503 
1504 	{ SYS_DESC(SYS_ERRIDR_EL1), trap_raz_wi },
1505 	{ SYS_DESC(SYS_ERRSELR_EL1), trap_raz_wi },
1506 	{ SYS_DESC(SYS_ERXFR_EL1), trap_raz_wi },
1507 	{ SYS_DESC(SYS_ERXCTLR_EL1), trap_raz_wi },
1508 	{ SYS_DESC(SYS_ERXSTATUS_EL1), trap_raz_wi },
1509 	{ SYS_DESC(SYS_ERXADDR_EL1), trap_raz_wi },
1510 	{ SYS_DESC(SYS_ERXMISC0_EL1), trap_raz_wi },
1511 	{ SYS_DESC(SYS_ERXMISC1_EL1), trap_raz_wi },
1512 
1513 	{ SYS_DESC(SYS_FAR_EL1), access_vm_reg, reset_unknown, FAR_EL1 },
1514 	{ SYS_DESC(SYS_PAR_EL1), NULL, reset_unknown, PAR_EL1 },
1515 
1516 	{ SYS_DESC(SYS_PMINTENSET_EL1), access_pminten, reset_unknown, PMINTENSET_EL1 },
1517 	{ SYS_DESC(SYS_PMINTENCLR_EL1), access_pminten, NULL, PMINTENSET_EL1 },
1518 
1519 	{ SYS_DESC(SYS_MAIR_EL1), access_vm_reg, reset_unknown, MAIR_EL1 },
1520 	{ SYS_DESC(SYS_AMAIR_EL1), access_vm_reg, reset_amair_el1, AMAIR_EL1 },
1521 
1522 	{ SYS_DESC(SYS_LORSA_EL1), trap_loregion },
1523 	{ SYS_DESC(SYS_LOREA_EL1), trap_loregion },
1524 	{ SYS_DESC(SYS_LORN_EL1), trap_loregion },
1525 	{ SYS_DESC(SYS_LORC_EL1), trap_loregion },
1526 	{ SYS_DESC(SYS_LORID_EL1), trap_loregion },
1527 
1528 	{ SYS_DESC(SYS_VBAR_EL1), NULL, reset_val, VBAR_EL1, 0 },
1529 	{ SYS_DESC(SYS_DISR_EL1), NULL, reset_val, DISR_EL1, 0 },
1530 
1531 	{ SYS_DESC(SYS_ICC_IAR0_EL1), write_to_read_only },
1532 	{ SYS_DESC(SYS_ICC_EOIR0_EL1), read_from_write_only },
1533 	{ SYS_DESC(SYS_ICC_HPPIR0_EL1), write_to_read_only },
1534 	{ SYS_DESC(SYS_ICC_DIR_EL1), read_from_write_only },
1535 	{ SYS_DESC(SYS_ICC_RPR_EL1), write_to_read_only },
1536 	{ SYS_DESC(SYS_ICC_SGI1R_EL1), access_gic_sgi },
1537 	{ SYS_DESC(SYS_ICC_ASGI1R_EL1), access_gic_sgi },
1538 	{ SYS_DESC(SYS_ICC_SGI0R_EL1), access_gic_sgi },
1539 	{ SYS_DESC(SYS_ICC_IAR1_EL1), write_to_read_only },
1540 	{ SYS_DESC(SYS_ICC_EOIR1_EL1), read_from_write_only },
1541 	{ SYS_DESC(SYS_ICC_HPPIR1_EL1), write_to_read_only },
1542 	{ SYS_DESC(SYS_ICC_SRE_EL1), access_gic_sre },
1543 
1544 	{ SYS_DESC(SYS_CONTEXTIDR_EL1), access_vm_reg, reset_val, CONTEXTIDR_EL1, 0 },
1545 	{ SYS_DESC(SYS_TPIDR_EL1), NULL, reset_unknown, TPIDR_EL1 },
1546 
1547 	{ SYS_DESC(SYS_CNTKCTL_EL1), NULL, reset_val, CNTKCTL_EL1, 0},
1548 
1549 	{ SYS_DESC(SYS_CCSIDR_EL1), access_ccsidr },
1550 	{ SYS_DESC(SYS_CLIDR_EL1), access_clidr },
1551 	{ SYS_DESC(SYS_CSSELR_EL1), access_csselr, reset_unknown, CSSELR_EL1 },
1552 	{ SYS_DESC(SYS_CTR_EL0), access_ctr },
1553 
1554 	{ SYS_DESC(SYS_PMCR_EL0), access_pmcr, reset_pmcr, },
1555 	{ SYS_DESC(SYS_PMCNTENSET_EL0), access_pmcnten, reset_unknown, PMCNTENSET_EL0 },
1556 	{ SYS_DESC(SYS_PMCNTENCLR_EL0), access_pmcnten, NULL, PMCNTENSET_EL0 },
1557 	{ SYS_DESC(SYS_PMOVSCLR_EL0), access_pmovs, NULL, PMOVSSET_EL0 },
1558 	{ SYS_DESC(SYS_PMSWINC_EL0), access_pmswinc, reset_unknown, PMSWINC_EL0 },
1559 	{ SYS_DESC(SYS_PMSELR_EL0), access_pmselr, reset_unknown, PMSELR_EL0 },
1560 	{ SYS_DESC(SYS_PMCEID0_EL0), access_pmceid },
1561 	{ SYS_DESC(SYS_PMCEID1_EL0), access_pmceid },
1562 	{ SYS_DESC(SYS_PMCCNTR_EL0), access_pmu_evcntr, reset_unknown, PMCCNTR_EL0 },
1563 	{ SYS_DESC(SYS_PMXEVTYPER_EL0), access_pmu_evtyper },
1564 	{ SYS_DESC(SYS_PMXEVCNTR_EL0), access_pmu_evcntr },
1565 	/*
1566 	 * PMUSERENR_EL0 resets as unknown in 64bit mode while it resets as zero
1567 	 * in 32bit mode. Here we choose to reset it as zero for consistency.
1568 	 */
1569 	{ SYS_DESC(SYS_PMUSERENR_EL0), access_pmuserenr, reset_val, PMUSERENR_EL0, 0 },
1570 	{ SYS_DESC(SYS_PMOVSSET_EL0), access_pmovs, reset_unknown, PMOVSSET_EL0 },
1571 
1572 	{ SYS_DESC(SYS_TPIDR_EL0), NULL, reset_unknown, TPIDR_EL0 },
1573 	{ SYS_DESC(SYS_TPIDRRO_EL0), NULL, reset_unknown, TPIDRRO_EL0 },
1574 
1575 	{ SYS_DESC(SYS_CNTP_TVAL_EL0), access_arch_timer },
1576 	{ SYS_DESC(SYS_CNTP_CTL_EL0), access_arch_timer },
1577 	{ SYS_DESC(SYS_CNTP_CVAL_EL0), access_arch_timer },
1578 
1579 	/* PMEVCNTRn_EL0 */
1580 	PMU_PMEVCNTR_EL0(0),
1581 	PMU_PMEVCNTR_EL0(1),
1582 	PMU_PMEVCNTR_EL0(2),
1583 	PMU_PMEVCNTR_EL0(3),
1584 	PMU_PMEVCNTR_EL0(4),
1585 	PMU_PMEVCNTR_EL0(5),
1586 	PMU_PMEVCNTR_EL0(6),
1587 	PMU_PMEVCNTR_EL0(7),
1588 	PMU_PMEVCNTR_EL0(8),
1589 	PMU_PMEVCNTR_EL0(9),
1590 	PMU_PMEVCNTR_EL0(10),
1591 	PMU_PMEVCNTR_EL0(11),
1592 	PMU_PMEVCNTR_EL0(12),
1593 	PMU_PMEVCNTR_EL0(13),
1594 	PMU_PMEVCNTR_EL0(14),
1595 	PMU_PMEVCNTR_EL0(15),
1596 	PMU_PMEVCNTR_EL0(16),
1597 	PMU_PMEVCNTR_EL0(17),
1598 	PMU_PMEVCNTR_EL0(18),
1599 	PMU_PMEVCNTR_EL0(19),
1600 	PMU_PMEVCNTR_EL0(20),
1601 	PMU_PMEVCNTR_EL0(21),
1602 	PMU_PMEVCNTR_EL0(22),
1603 	PMU_PMEVCNTR_EL0(23),
1604 	PMU_PMEVCNTR_EL0(24),
1605 	PMU_PMEVCNTR_EL0(25),
1606 	PMU_PMEVCNTR_EL0(26),
1607 	PMU_PMEVCNTR_EL0(27),
1608 	PMU_PMEVCNTR_EL0(28),
1609 	PMU_PMEVCNTR_EL0(29),
1610 	PMU_PMEVCNTR_EL0(30),
1611 	/* PMEVTYPERn_EL0 */
1612 	PMU_PMEVTYPER_EL0(0),
1613 	PMU_PMEVTYPER_EL0(1),
1614 	PMU_PMEVTYPER_EL0(2),
1615 	PMU_PMEVTYPER_EL0(3),
1616 	PMU_PMEVTYPER_EL0(4),
1617 	PMU_PMEVTYPER_EL0(5),
1618 	PMU_PMEVTYPER_EL0(6),
1619 	PMU_PMEVTYPER_EL0(7),
1620 	PMU_PMEVTYPER_EL0(8),
1621 	PMU_PMEVTYPER_EL0(9),
1622 	PMU_PMEVTYPER_EL0(10),
1623 	PMU_PMEVTYPER_EL0(11),
1624 	PMU_PMEVTYPER_EL0(12),
1625 	PMU_PMEVTYPER_EL0(13),
1626 	PMU_PMEVTYPER_EL0(14),
1627 	PMU_PMEVTYPER_EL0(15),
1628 	PMU_PMEVTYPER_EL0(16),
1629 	PMU_PMEVTYPER_EL0(17),
1630 	PMU_PMEVTYPER_EL0(18),
1631 	PMU_PMEVTYPER_EL0(19),
1632 	PMU_PMEVTYPER_EL0(20),
1633 	PMU_PMEVTYPER_EL0(21),
1634 	PMU_PMEVTYPER_EL0(22),
1635 	PMU_PMEVTYPER_EL0(23),
1636 	PMU_PMEVTYPER_EL0(24),
1637 	PMU_PMEVTYPER_EL0(25),
1638 	PMU_PMEVTYPER_EL0(26),
1639 	PMU_PMEVTYPER_EL0(27),
1640 	PMU_PMEVTYPER_EL0(28),
1641 	PMU_PMEVTYPER_EL0(29),
1642 	PMU_PMEVTYPER_EL0(30),
1643 	/*
1644 	 * PMCCFILTR_EL0 resets as unknown in 64bit mode while it resets as zero
1645 	 * in 32bit mode. Here we choose to reset it as zero for consistency.
1646 	 */
1647 	{ SYS_DESC(SYS_PMCCFILTR_EL0), access_pmu_evtyper, reset_val, PMCCFILTR_EL0, 0 },
1648 
1649 	{ SYS_DESC(SYS_DACR32_EL2), NULL, reset_unknown, DACR32_EL2 },
1650 	{ SYS_DESC(SYS_IFSR32_EL2), NULL, reset_unknown, IFSR32_EL2 },
1651 	{ SYS_DESC(SYS_FPEXC32_EL2), NULL, reset_val, FPEXC32_EL2, 0x700 },
1652 };
1653 
1654 static bool trap_dbgidr(struct kvm_vcpu *vcpu,
1655 			struct sys_reg_params *p,
1656 			const struct sys_reg_desc *r)
1657 {
1658 	if (p->is_write) {
1659 		return ignore_write(vcpu, p);
1660 	} else {
1661 		u64 dfr = read_sanitised_ftr_reg(SYS_ID_AA64DFR0_EL1);
1662 		u64 pfr = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
1663 		u32 el3 = !!cpuid_feature_extract_unsigned_field(pfr, ID_AA64PFR0_EL3_SHIFT);
1664 
1665 		p->regval = ((((dfr >> ID_AA64DFR0_WRPS_SHIFT) & 0xf) << 28) |
1666 			     (((dfr >> ID_AA64DFR0_BRPS_SHIFT) & 0xf) << 24) |
1667 			     (((dfr >> ID_AA64DFR0_CTX_CMPS_SHIFT) & 0xf) << 20)
1668 			     | (6 << 16) | (el3 << 14) | (el3 << 12));
1669 		return true;
1670 	}
1671 }
1672 
1673 static bool trap_debug32(struct kvm_vcpu *vcpu,
1674 			 struct sys_reg_params *p,
1675 			 const struct sys_reg_desc *r)
1676 {
1677 	if (p->is_write) {
1678 		vcpu_cp14(vcpu, r->reg) = p->regval;
1679 		vcpu->arch.flags |= KVM_ARM64_DEBUG_DIRTY;
1680 	} else {
1681 		p->regval = vcpu_cp14(vcpu, r->reg);
1682 	}
1683 
1684 	return true;
1685 }
1686 
1687 /* AArch32 debug register mappings
1688  *
1689  * AArch32 DBGBVRn is mapped to DBGBVRn_EL1[31:0]
1690  * AArch32 DBGBXVRn is mapped to DBGBVRn_EL1[63:32]
1691  *
1692  * All control registers and watchpoint value registers are mapped to
1693  * the lower 32 bits of their AArch64 equivalents. We share the trap
1694  * handlers with the above AArch64 code which checks what mode the
1695  * system is in.
1696  */
1697 
1698 static bool trap_xvr(struct kvm_vcpu *vcpu,
1699 		     struct sys_reg_params *p,
1700 		     const struct sys_reg_desc *rd)
1701 {
1702 	u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg];
1703 
1704 	if (p->is_write) {
1705 		u64 val = *dbg_reg;
1706 
1707 		val &= 0xffffffffUL;
1708 		val |= p->regval << 32;
1709 		*dbg_reg = val;
1710 
1711 		vcpu->arch.flags |= KVM_ARM64_DEBUG_DIRTY;
1712 	} else {
1713 		p->regval = *dbg_reg >> 32;
1714 	}
1715 
1716 	trace_trap_reg(__func__, rd->reg, p->is_write, *dbg_reg);
1717 
1718 	return true;
1719 }
1720 
1721 #define DBG_BCR_BVR_WCR_WVR(n)						\
1722 	/* DBGBVRn */							\
1723 	{ Op1( 0), CRn( 0), CRm((n)), Op2( 4), trap_bvr, NULL, n }, 	\
1724 	/* DBGBCRn */							\
1725 	{ Op1( 0), CRn( 0), CRm((n)), Op2( 5), trap_bcr, NULL, n },	\
1726 	/* DBGWVRn */							\
1727 	{ Op1( 0), CRn( 0), CRm((n)), Op2( 6), trap_wvr, NULL, n },	\
1728 	/* DBGWCRn */							\
1729 	{ Op1( 0), CRn( 0), CRm((n)), Op2( 7), trap_wcr, NULL, n }
1730 
1731 #define DBGBXVR(n)							\
1732 	{ Op1( 0), CRn( 1), CRm((n)), Op2( 1), trap_xvr, NULL, n }
1733 
1734 /*
1735  * Trapped cp14 registers. We generally ignore most of the external
1736  * debug, on the principle that they don't really make sense to a
1737  * guest. Revisit this one day, would this principle change.
1738  */
1739 static const struct sys_reg_desc cp14_regs[] = {
1740 	/* DBGIDR */
1741 	{ Op1( 0), CRn( 0), CRm( 0), Op2( 0), trap_dbgidr },
1742 	/* DBGDTRRXext */
1743 	{ Op1( 0), CRn( 0), CRm( 0), Op2( 2), trap_raz_wi },
1744 
1745 	DBG_BCR_BVR_WCR_WVR(0),
1746 	/* DBGDSCRint */
1747 	{ Op1( 0), CRn( 0), CRm( 1), Op2( 0), trap_raz_wi },
1748 	DBG_BCR_BVR_WCR_WVR(1),
1749 	/* DBGDCCINT */
1750 	{ Op1( 0), CRn( 0), CRm( 2), Op2( 0), trap_debug32 },
1751 	/* DBGDSCRext */
1752 	{ Op1( 0), CRn( 0), CRm( 2), Op2( 2), trap_debug32 },
1753 	DBG_BCR_BVR_WCR_WVR(2),
1754 	/* DBGDTR[RT]Xint */
1755 	{ Op1( 0), CRn( 0), CRm( 3), Op2( 0), trap_raz_wi },
1756 	/* DBGDTR[RT]Xext */
1757 	{ Op1( 0), CRn( 0), CRm( 3), Op2( 2), trap_raz_wi },
1758 	DBG_BCR_BVR_WCR_WVR(3),
1759 	DBG_BCR_BVR_WCR_WVR(4),
1760 	DBG_BCR_BVR_WCR_WVR(5),
1761 	/* DBGWFAR */
1762 	{ Op1( 0), CRn( 0), CRm( 6), Op2( 0), trap_raz_wi },
1763 	/* DBGOSECCR */
1764 	{ Op1( 0), CRn( 0), CRm( 6), Op2( 2), trap_raz_wi },
1765 	DBG_BCR_BVR_WCR_WVR(6),
1766 	/* DBGVCR */
1767 	{ Op1( 0), CRn( 0), CRm( 7), Op2( 0), trap_debug32 },
1768 	DBG_BCR_BVR_WCR_WVR(7),
1769 	DBG_BCR_BVR_WCR_WVR(8),
1770 	DBG_BCR_BVR_WCR_WVR(9),
1771 	DBG_BCR_BVR_WCR_WVR(10),
1772 	DBG_BCR_BVR_WCR_WVR(11),
1773 	DBG_BCR_BVR_WCR_WVR(12),
1774 	DBG_BCR_BVR_WCR_WVR(13),
1775 	DBG_BCR_BVR_WCR_WVR(14),
1776 	DBG_BCR_BVR_WCR_WVR(15),
1777 
1778 	/* DBGDRAR (32bit) */
1779 	{ Op1( 0), CRn( 1), CRm( 0), Op2( 0), trap_raz_wi },
1780 
1781 	DBGBXVR(0),
1782 	/* DBGOSLAR */
1783 	{ Op1( 0), CRn( 1), CRm( 0), Op2( 4), trap_raz_wi },
1784 	DBGBXVR(1),
1785 	/* DBGOSLSR */
1786 	{ Op1( 0), CRn( 1), CRm( 1), Op2( 4), trap_oslsr_el1 },
1787 	DBGBXVR(2),
1788 	DBGBXVR(3),
1789 	/* DBGOSDLR */
1790 	{ Op1( 0), CRn( 1), CRm( 3), Op2( 4), trap_raz_wi },
1791 	DBGBXVR(4),
1792 	/* DBGPRCR */
1793 	{ Op1( 0), CRn( 1), CRm( 4), Op2( 4), trap_raz_wi },
1794 	DBGBXVR(5),
1795 	DBGBXVR(6),
1796 	DBGBXVR(7),
1797 	DBGBXVR(8),
1798 	DBGBXVR(9),
1799 	DBGBXVR(10),
1800 	DBGBXVR(11),
1801 	DBGBXVR(12),
1802 	DBGBXVR(13),
1803 	DBGBXVR(14),
1804 	DBGBXVR(15),
1805 
1806 	/* DBGDSAR (32bit) */
1807 	{ Op1( 0), CRn( 2), CRm( 0), Op2( 0), trap_raz_wi },
1808 
1809 	/* DBGDEVID2 */
1810 	{ Op1( 0), CRn( 7), CRm( 0), Op2( 7), trap_raz_wi },
1811 	/* DBGDEVID1 */
1812 	{ Op1( 0), CRn( 7), CRm( 1), Op2( 7), trap_raz_wi },
1813 	/* DBGDEVID */
1814 	{ Op1( 0), CRn( 7), CRm( 2), Op2( 7), trap_raz_wi },
1815 	/* DBGCLAIMSET */
1816 	{ Op1( 0), CRn( 7), CRm( 8), Op2( 6), trap_raz_wi },
1817 	/* DBGCLAIMCLR */
1818 	{ Op1( 0), CRn( 7), CRm( 9), Op2( 6), trap_raz_wi },
1819 	/* DBGAUTHSTATUS */
1820 	{ Op1( 0), CRn( 7), CRm(14), Op2( 6), trap_dbgauthstatus_el1 },
1821 };
1822 
1823 /* Trapped cp14 64bit registers */
1824 static const struct sys_reg_desc cp14_64_regs[] = {
1825 	/* DBGDRAR (64bit) */
1826 	{ Op1( 0), CRm( 1), .access = trap_raz_wi },
1827 
1828 	/* DBGDSAR (64bit) */
1829 	{ Op1( 0), CRm( 2), .access = trap_raz_wi },
1830 };
1831 
1832 /* Macro to expand the PMEVCNTRn register */
1833 #define PMU_PMEVCNTR(n)							\
1834 	/* PMEVCNTRn */							\
1835 	{ Op1(0), CRn(0b1110),						\
1836 	  CRm((0b1000 | (((n) >> 3) & 0x3))), Op2(((n) & 0x7)),		\
1837 	  access_pmu_evcntr }
1838 
1839 /* Macro to expand the PMEVTYPERn register */
1840 #define PMU_PMEVTYPER(n)						\
1841 	/* PMEVTYPERn */						\
1842 	{ Op1(0), CRn(0b1110),						\
1843 	  CRm((0b1100 | (((n) >> 3) & 0x3))), Op2(((n) & 0x7)),		\
1844 	  access_pmu_evtyper }
1845 
1846 /*
1847  * Trapped cp15 registers. TTBR0/TTBR1 get a double encoding,
1848  * depending on the way they are accessed (as a 32bit or a 64bit
1849  * register).
1850  */
1851 static const struct sys_reg_desc cp15_regs[] = {
1852 	{ Op1( 0), CRn( 0), CRm( 0), Op2( 1), access_ctr },
1853 	{ Op1( 0), CRn( 1), CRm( 0), Op2( 0), access_vm_reg, NULL, c1_SCTLR },
1854 	{ Op1( 0), CRn( 2), CRm( 0), Op2( 0), access_vm_reg, NULL, c2_TTBR0 },
1855 	{ Op1( 0), CRn( 2), CRm( 0), Op2( 1), access_vm_reg, NULL, c2_TTBR1 },
1856 	{ Op1( 0), CRn( 2), CRm( 0), Op2( 2), access_vm_reg, NULL, c2_TTBCR },
1857 	{ Op1( 0), CRn( 3), CRm( 0), Op2( 0), access_vm_reg, NULL, c3_DACR },
1858 	{ Op1( 0), CRn( 5), CRm( 0), Op2( 0), access_vm_reg, NULL, c5_DFSR },
1859 	{ Op1( 0), CRn( 5), CRm( 0), Op2( 1), access_vm_reg, NULL, c5_IFSR },
1860 	{ Op1( 0), CRn( 5), CRm( 1), Op2( 0), access_vm_reg, NULL, c5_ADFSR },
1861 	{ Op1( 0), CRn( 5), CRm( 1), Op2( 1), access_vm_reg, NULL, c5_AIFSR },
1862 	{ Op1( 0), CRn( 6), CRm( 0), Op2( 0), access_vm_reg, NULL, c6_DFAR },
1863 	{ Op1( 0), CRn( 6), CRm( 0), Op2( 2), access_vm_reg, NULL, c6_IFAR },
1864 
1865 	/*
1866 	 * DC{C,I,CI}SW operations:
1867 	 */
1868 	{ Op1( 0), CRn( 7), CRm( 6), Op2( 2), access_dcsw },
1869 	{ Op1( 0), CRn( 7), CRm(10), Op2( 2), access_dcsw },
1870 	{ Op1( 0), CRn( 7), CRm(14), Op2( 2), access_dcsw },
1871 
1872 	/* PMU */
1873 	{ Op1( 0), CRn( 9), CRm(12), Op2( 0), access_pmcr },
1874 	{ Op1( 0), CRn( 9), CRm(12), Op2( 1), access_pmcnten },
1875 	{ Op1( 0), CRn( 9), CRm(12), Op2( 2), access_pmcnten },
1876 	{ Op1( 0), CRn( 9), CRm(12), Op2( 3), access_pmovs },
1877 	{ Op1( 0), CRn( 9), CRm(12), Op2( 4), access_pmswinc },
1878 	{ Op1( 0), CRn( 9), CRm(12), Op2( 5), access_pmselr },
1879 	{ Op1( 0), CRn( 9), CRm(12), Op2( 6), access_pmceid },
1880 	{ Op1( 0), CRn( 9), CRm(12), Op2( 7), access_pmceid },
1881 	{ Op1( 0), CRn( 9), CRm(13), Op2( 0), access_pmu_evcntr },
1882 	{ Op1( 0), CRn( 9), CRm(13), Op2( 1), access_pmu_evtyper },
1883 	{ Op1( 0), CRn( 9), CRm(13), Op2( 2), access_pmu_evcntr },
1884 	{ Op1( 0), CRn( 9), CRm(14), Op2( 0), access_pmuserenr },
1885 	{ Op1( 0), CRn( 9), CRm(14), Op2( 1), access_pminten },
1886 	{ Op1( 0), CRn( 9), CRm(14), Op2( 2), access_pminten },
1887 	{ Op1( 0), CRn( 9), CRm(14), Op2( 3), access_pmovs },
1888 
1889 	{ Op1( 0), CRn(10), CRm( 2), Op2( 0), access_vm_reg, NULL, c10_PRRR },
1890 	{ Op1( 0), CRn(10), CRm( 2), Op2( 1), access_vm_reg, NULL, c10_NMRR },
1891 	{ Op1( 0), CRn(10), CRm( 3), Op2( 0), access_vm_reg, NULL, c10_AMAIR0 },
1892 	{ Op1( 0), CRn(10), CRm( 3), Op2( 1), access_vm_reg, NULL, c10_AMAIR1 },
1893 
1894 	/* ICC_SRE */
1895 	{ Op1( 0), CRn(12), CRm(12), Op2( 5), access_gic_sre },
1896 
1897 	{ Op1( 0), CRn(13), CRm( 0), Op2( 1), access_vm_reg, NULL, c13_CID },
1898 
1899 	/* Arch Tmers */
1900 	{ SYS_DESC(SYS_AARCH32_CNTP_TVAL), access_arch_timer },
1901 	{ SYS_DESC(SYS_AARCH32_CNTP_CTL), access_arch_timer },
1902 
1903 	/* PMEVCNTRn */
1904 	PMU_PMEVCNTR(0),
1905 	PMU_PMEVCNTR(1),
1906 	PMU_PMEVCNTR(2),
1907 	PMU_PMEVCNTR(3),
1908 	PMU_PMEVCNTR(4),
1909 	PMU_PMEVCNTR(5),
1910 	PMU_PMEVCNTR(6),
1911 	PMU_PMEVCNTR(7),
1912 	PMU_PMEVCNTR(8),
1913 	PMU_PMEVCNTR(9),
1914 	PMU_PMEVCNTR(10),
1915 	PMU_PMEVCNTR(11),
1916 	PMU_PMEVCNTR(12),
1917 	PMU_PMEVCNTR(13),
1918 	PMU_PMEVCNTR(14),
1919 	PMU_PMEVCNTR(15),
1920 	PMU_PMEVCNTR(16),
1921 	PMU_PMEVCNTR(17),
1922 	PMU_PMEVCNTR(18),
1923 	PMU_PMEVCNTR(19),
1924 	PMU_PMEVCNTR(20),
1925 	PMU_PMEVCNTR(21),
1926 	PMU_PMEVCNTR(22),
1927 	PMU_PMEVCNTR(23),
1928 	PMU_PMEVCNTR(24),
1929 	PMU_PMEVCNTR(25),
1930 	PMU_PMEVCNTR(26),
1931 	PMU_PMEVCNTR(27),
1932 	PMU_PMEVCNTR(28),
1933 	PMU_PMEVCNTR(29),
1934 	PMU_PMEVCNTR(30),
1935 	/* PMEVTYPERn */
1936 	PMU_PMEVTYPER(0),
1937 	PMU_PMEVTYPER(1),
1938 	PMU_PMEVTYPER(2),
1939 	PMU_PMEVTYPER(3),
1940 	PMU_PMEVTYPER(4),
1941 	PMU_PMEVTYPER(5),
1942 	PMU_PMEVTYPER(6),
1943 	PMU_PMEVTYPER(7),
1944 	PMU_PMEVTYPER(8),
1945 	PMU_PMEVTYPER(9),
1946 	PMU_PMEVTYPER(10),
1947 	PMU_PMEVTYPER(11),
1948 	PMU_PMEVTYPER(12),
1949 	PMU_PMEVTYPER(13),
1950 	PMU_PMEVTYPER(14),
1951 	PMU_PMEVTYPER(15),
1952 	PMU_PMEVTYPER(16),
1953 	PMU_PMEVTYPER(17),
1954 	PMU_PMEVTYPER(18),
1955 	PMU_PMEVTYPER(19),
1956 	PMU_PMEVTYPER(20),
1957 	PMU_PMEVTYPER(21),
1958 	PMU_PMEVTYPER(22),
1959 	PMU_PMEVTYPER(23),
1960 	PMU_PMEVTYPER(24),
1961 	PMU_PMEVTYPER(25),
1962 	PMU_PMEVTYPER(26),
1963 	PMU_PMEVTYPER(27),
1964 	PMU_PMEVTYPER(28),
1965 	PMU_PMEVTYPER(29),
1966 	PMU_PMEVTYPER(30),
1967 	/* PMCCFILTR */
1968 	{ Op1(0), CRn(14), CRm(15), Op2(7), access_pmu_evtyper },
1969 
1970 	{ Op1(1), CRn( 0), CRm( 0), Op2(0), access_ccsidr },
1971 	{ Op1(1), CRn( 0), CRm( 0), Op2(1), access_clidr },
1972 	{ Op1(2), CRn( 0), CRm( 0), Op2(0), access_csselr, NULL, c0_CSSELR },
1973 };
1974 
1975 static const struct sys_reg_desc cp15_64_regs[] = {
1976 	{ Op1( 0), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, c2_TTBR0 },
1977 	{ Op1( 0), CRn( 0), CRm( 9), Op2( 0), access_pmu_evcntr },
1978 	{ Op1( 0), CRn( 0), CRm(12), Op2( 0), access_gic_sgi }, /* ICC_SGI1R */
1979 	{ Op1( 1), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, c2_TTBR1 },
1980 	{ Op1( 1), CRn( 0), CRm(12), Op2( 0), access_gic_sgi }, /* ICC_ASGI1R */
1981 	{ Op1( 2), CRn( 0), CRm(12), Op2( 0), access_gic_sgi }, /* ICC_SGI0R */
1982 	{ SYS_DESC(SYS_AARCH32_CNTP_CVAL),    access_arch_timer },
1983 };
1984 
1985 /* Target specific emulation tables */
1986 static struct kvm_sys_reg_target_table *target_tables[KVM_ARM_NUM_TARGETS];
1987 
1988 void kvm_register_target_sys_reg_table(unsigned int target,
1989 				       struct kvm_sys_reg_target_table *table)
1990 {
1991 	target_tables[target] = table;
1992 }
1993 
1994 /* Get specific register table for this target. */
1995 static const struct sys_reg_desc *get_target_table(unsigned target,
1996 						   bool mode_is_64,
1997 						   size_t *num)
1998 {
1999 	struct kvm_sys_reg_target_table *table;
2000 
2001 	table = target_tables[target];
2002 	if (mode_is_64) {
2003 		*num = table->table64.num;
2004 		return table->table64.table;
2005 	} else {
2006 		*num = table->table32.num;
2007 		return table->table32.table;
2008 	}
2009 }
2010 
2011 static int match_sys_reg(const void *key, const void *elt)
2012 {
2013 	const unsigned long pval = (unsigned long)key;
2014 	const struct sys_reg_desc *r = elt;
2015 
2016 	return pval - reg_to_encoding(r);
2017 }
2018 
2019 static const struct sys_reg_desc *find_reg(const struct sys_reg_params *params,
2020 					 const struct sys_reg_desc table[],
2021 					 unsigned int num)
2022 {
2023 	unsigned long pval = reg_to_encoding(params);
2024 
2025 	return bsearch((void *)pval, table, num, sizeof(table[0]), match_sys_reg);
2026 }
2027 
2028 int kvm_handle_cp14_load_store(struct kvm_vcpu *vcpu, struct kvm_run *run)
2029 {
2030 	kvm_inject_undefined(vcpu);
2031 	return 1;
2032 }
2033 
2034 static void perform_access(struct kvm_vcpu *vcpu,
2035 			   struct sys_reg_params *params,
2036 			   const struct sys_reg_desc *r)
2037 {
2038 	trace_kvm_sys_access(*vcpu_pc(vcpu), params, r);
2039 
2040 	/* Check for regs disabled by runtime config */
2041 	if (sysreg_hidden_from_guest(vcpu, r)) {
2042 		kvm_inject_undefined(vcpu);
2043 		return;
2044 	}
2045 
2046 	/*
2047 	 * Not having an accessor means that we have configured a trap
2048 	 * that we don't know how to handle. This certainly qualifies
2049 	 * as a gross bug that should be fixed right away.
2050 	 */
2051 	BUG_ON(!r->access);
2052 
2053 	/* Skip instruction if instructed so */
2054 	if (likely(r->access(vcpu, params, r)))
2055 		kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
2056 }
2057 
2058 /*
2059  * emulate_cp --  tries to match a sys_reg access in a handling table, and
2060  *                call the corresponding trap handler.
2061  *
2062  * @params: pointer to the descriptor of the access
2063  * @table: array of trap descriptors
2064  * @num: size of the trap descriptor array
2065  *
2066  * Return 0 if the access has been handled, and -1 if not.
2067  */
2068 static int emulate_cp(struct kvm_vcpu *vcpu,
2069 		      struct sys_reg_params *params,
2070 		      const struct sys_reg_desc *table,
2071 		      size_t num)
2072 {
2073 	const struct sys_reg_desc *r;
2074 
2075 	if (!table)
2076 		return -1;	/* Not handled */
2077 
2078 	r = find_reg(params, table, num);
2079 
2080 	if (r) {
2081 		perform_access(vcpu, params, r);
2082 		return 0;
2083 	}
2084 
2085 	/* Not handled */
2086 	return -1;
2087 }
2088 
2089 static void unhandled_cp_access(struct kvm_vcpu *vcpu,
2090 				struct sys_reg_params *params)
2091 {
2092 	u8 hsr_ec = kvm_vcpu_trap_get_class(vcpu);
2093 	int cp = -1;
2094 
2095 	switch(hsr_ec) {
2096 	case ESR_ELx_EC_CP15_32:
2097 	case ESR_ELx_EC_CP15_64:
2098 		cp = 15;
2099 		break;
2100 	case ESR_ELx_EC_CP14_MR:
2101 	case ESR_ELx_EC_CP14_64:
2102 		cp = 14;
2103 		break;
2104 	default:
2105 		WARN_ON(1);
2106 	}
2107 
2108 	kvm_err("Unsupported guest CP%d access at: %08lx [%08lx]\n",
2109 		cp, *vcpu_pc(vcpu), *vcpu_cpsr(vcpu));
2110 	print_sys_reg_instr(params);
2111 	kvm_inject_undefined(vcpu);
2112 }
2113 
2114 /**
2115  * kvm_handle_cp_64 -- handles a mrrc/mcrr trap on a guest CP14/CP15 access
2116  * @vcpu: The VCPU pointer
2117  * @run:  The kvm_run struct
2118  */
2119 static int kvm_handle_cp_64(struct kvm_vcpu *vcpu,
2120 			    const struct sys_reg_desc *global,
2121 			    size_t nr_global,
2122 			    const struct sys_reg_desc *target_specific,
2123 			    size_t nr_specific)
2124 {
2125 	struct sys_reg_params params;
2126 	u32 hsr = kvm_vcpu_get_hsr(vcpu);
2127 	int Rt = kvm_vcpu_sys_get_rt(vcpu);
2128 	int Rt2 = (hsr >> 10) & 0x1f;
2129 
2130 	params.is_aarch32 = true;
2131 	params.is_32bit = false;
2132 	params.CRm = (hsr >> 1) & 0xf;
2133 	params.is_write = ((hsr & 1) == 0);
2134 
2135 	params.Op0 = 0;
2136 	params.Op1 = (hsr >> 16) & 0xf;
2137 	params.Op2 = 0;
2138 	params.CRn = 0;
2139 
2140 	/*
2141 	 * Make a 64-bit value out of Rt and Rt2. As we use the same trap
2142 	 * backends between AArch32 and AArch64, we get away with it.
2143 	 */
2144 	if (params.is_write) {
2145 		params.regval = vcpu_get_reg(vcpu, Rt) & 0xffffffff;
2146 		params.regval |= vcpu_get_reg(vcpu, Rt2) << 32;
2147 	}
2148 
2149 	/*
2150 	 * Try to emulate the coprocessor access using the target
2151 	 * specific table first, and using the global table afterwards.
2152 	 * If either of the tables contains a handler, handle the
2153 	 * potential register operation in the case of a read and return
2154 	 * with success.
2155 	 */
2156 	if (!emulate_cp(vcpu, &params, target_specific, nr_specific) ||
2157 	    !emulate_cp(vcpu, &params, global, nr_global)) {
2158 		/* Split up the value between registers for the read side */
2159 		if (!params.is_write) {
2160 			vcpu_set_reg(vcpu, Rt, lower_32_bits(params.regval));
2161 			vcpu_set_reg(vcpu, Rt2, upper_32_bits(params.regval));
2162 		}
2163 
2164 		return 1;
2165 	}
2166 
2167 	unhandled_cp_access(vcpu, &params);
2168 	return 1;
2169 }
2170 
2171 /**
2172  * kvm_handle_cp_32 -- handles a mrc/mcr trap on a guest CP14/CP15 access
2173  * @vcpu: The VCPU pointer
2174  * @run:  The kvm_run struct
2175  */
2176 static int kvm_handle_cp_32(struct kvm_vcpu *vcpu,
2177 			    const struct sys_reg_desc *global,
2178 			    size_t nr_global,
2179 			    const struct sys_reg_desc *target_specific,
2180 			    size_t nr_specific)
2181 {
2182 	struct sys_reg_params params;
2183 	u32 hsr = kvm_vcpu_get_hsr(vcpu);
2184 	int Rt  = kvm_vcpu_sys_get_rt(vcpu);
2185 
2186 	params.is_aarch32 = true;
2187 	params.is_32bit = true;
2188 	params.CRm = (hsr >> 1) & 0xf;
2189 	params.regval = vcpu_get_reg(vcpu, Rt);
2190 	params.is_write = ((hsr & 1) == 0);
2191 	params.CRn = (hsr >> 10) & 0xf;
2192 	params.Op0 = 0;
2193 	params.Op1 = (hsr >> 14) & 0x7;
2194 	params.Op2 = (hsr >> 17) & 0x7;
2195 
2196 	if (!emulate_cp(vcpu, &params, target_specific, nr_specific) ||
2197 	    !emulate_cp(vcpu, &params, global, nr_global)) {
2198 		if (!params.is_write)
2199 			vcpu_set_reg(vcpu, Rt, params.regval);
2200 		return 1;
2201 	}
2202 
2203 	unhandled_cp_access(vcpu, &params);
2204 	return 1;
2205 }
2206 
2207 int kvm_handle_cp15_64(struct kvm_vcpu *vcpu, struct kvm_run *run)
2208 {
2209 	const struct sys_reg_desc *target_specific;
2210 	size_t num;
2211 
2212 	target_specific = get_target_table(vcpu->arch.target, false, &num);
2213 	return kvm_handle_cp_64(vcpu,
2214 				cp15_64_regs, ARRAY_SIZE(cp15_64_regs),
2215 				target_specific, num);
2216 }
2217 
2218 int kvm_handle_cp15_32(struct kvm_vcpu *vcpu, struct kvm_run *run)
2219 {
2220 	const struct sys_reg_desc *target_specific;
2221 	size_t num;
2222 
2223 	target_specific = get_target_table(vcpu->arch.target, false, &num);
2224 	return kvm_handle_cp_32(vcpu,
2225 				cp15_regs, ARRAY_SIZE(cp15_regs),
2226 				target_specific, num);
2227 }
2228 
2229 int kvm_handle_cp14_64(struct kvm_vcpu *vcpu, struct kvm_run *run)
2230 {
2231 	return kvm_handle_cp_64(vcpu,
2232 				cp14_64_regs, ARRAY_SIZE(cp14_64_regs),
2233 				NULL, 0);
2234 }
2235 
2236 int kvm_handle_cp14_32(struct kvm_vcpu *vcpu, struct kvm_run *run)
2237 {
2238 	return kvm_handle_cp_32(vcpu,
2239 				cp14_regs, ARRAY_SIZE(cp14_regs),
2240 				NULL, 0);
2241 }
2242 
2243 static int emulate_sys_reg(struct kvm_vcpu *vcpu,
2244 			   struct sys_reg_params *params)
2245 {
2246 	size_t num;
2247 	const struct sys_reg_desc *table, *r;
2248 
2249 	table = get_target_table(vcpu->arch.target, true, &num);
2250 
2251 	/* Search target-specific then generic table. */
2252 	r = find_reg(params, table, num);
2253 	if (!r)
2254 		r = find_reg(params, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
2255 
2256 	if (likely(r)) {
2257 		perform_access(vcpu, params, r);
2258 	} else {
2259 		kvm_err("Unsupported guest sys_reg access at: %lx [%08lx]\n",
2260 			*vcpu_pc(vcpu), *vcpu_cpsr(vcpu));
2261 		print_sys_reg_instr(params);
2262 		kvm_inject_undefined(vcpu);
2263 	}
2264 	return 1;
2265 }
2266 
2267 static void reset_sys_reg_descs(struct kvm_vcpu *vcpu,
2268 			      const struct sys_reg_desc *table, size_t num)
2269 {
2270 	unsigned long i;
2271 
2272 	for (i = 0; i < num; i++)
2273 		if (table[i].reset)
2274 			table[i].reset(vcpu, &table[i]);
2275 }
2276 
2277 /**
2278  * kvm_handle_sys_reg -- handles a mrs/msr trap on a guest sys_reg access
2279  * @vcpu: The VCPU pointer
2280  * @run:  The kvm_run struct
2281  */
2282 int kvm_handle_sys_reg(struct kvm_vcpu *vcpu, struct kvm_run *run)
2283 {
2284 	struct sys_reg_params params;
2285 	unsigned long esr = kvm_vcpu_get_hsr(vcpu);
2286 	int Rt = kvm_vcpu_sys_get_rt(vcpu);
2287 	int ret;
2288 
2289 	trace_kvm_handle_sys_reg(esr);
2290 
2291 	params.is_aarch32 = false;
2292 	params.is_32bit = false;
2293 	params.Op0 = (esr >> 20) & 3;
2294 	params.Op1 = (esr >> 14) & 0x7;
2295 	params.CRn = (esr >> 10) & 0xf;
2296 	params.CRm = (esr >> 1) & 0xf;
2297 	params.Op2 = (esr >> 17) & 0x7;
2298 	params.regval = vcpu_get_reg(vcpu, Rt);
2299 	params.is_write = !(esr & 1);
2300 
2301 	ret = emulate_sys_reg(vcpu, &params);
2302 
2303 	if (!params.is_write)
2304 		vcpu_set_reg(vcpu, Rt, params.regval);
2305 	return ret;
2306 }
2307 
2308 /******************************************************************************
2309  * Userspace API
2310  *****************************************************************************/
2311 
2312 static bool index_to_params(u64 id, struct sys_reg_params *params)
2313 {
2314 	switch (id & KVM_REG_SIZE_MASK) {
2315 	case KVM_REG_SIZE_U64:
2316 		/* Any unused index bits means it's not valid. */
2317 		if (id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK
2318 			      | KVM_REG_ARM_COPROC_MASK
2319 			      | KVM_REG_ARM64_SYSREG_OP0_MASK
2320 			      | KVM_REG_ARM64_SYSREG_OP1_MASK
2321 			      | KVM_REG_ARM64_SYSREG_CRN_MASK
2322 			      | KVM_REG_ARM64_SYSREG_CRM_MASK
2323 			      | KVM_REG_ARM64_SYSREG_OP2_MASK))
2324 			return false;
2325 		params->Op0 = ((id & KVM_REG_ARM64_SYSREG_OP0_MASK)
2326 			       >> KVM_REG_ARM64_SYSREG_OP0_SHIFT);
2327 		params->Op1 = ((id & KVM_REG_ARM64_SYSREG_OP1_MASK)
2328 			       >> KVM_REG_ARM64_SYSREG_OP1_SHIFT);
2329 		params->CRn = ((id & KVM_REG_ARM64_SYSREG_CRN_MASK)
2330 			       >> KVM_REG_ARM64_SYSREG_CRN_SHIFT);
2331 		params->CRm = ((id & KVM_REG_ARM64_SYSREG_CRM_MASK)
2332 			       >> KVM_REG_ARM64_SYSREG_CRM_SHIFT);
2333 		params->Op2 = ((id & KVM_REG_ARM64_SYSREG_OP2_MASK)
2334 			       >> KVM_REG_ARM64_SYSREG_OP2_SHIFT);
2335 		return true;
2336 	default:
2337 		return false;
2338 	}
2339 }
2340 
2341 const struct sys_reg_desc *find_reg_by_id(u64 id,
2342 					  struct sys_reg_params *params,
2343 					  const struct sys_reg_desc table[],
2344 					  unsigned int num)
2345 {
2346 	if (!index_to_params(id, params))
2347 		return NULL;
2348 
2349 	return find_reg(params, table, num);
2350 }
2351 
2352 /* Decode an index value, and find the sys_reg_desc entry. */
2353 static const struct sys_reg_desc *index_to_sys_reg_desc(struct kvm_vcpu *vcpu,
2354 						    u64 id)
2355 {
2356 	size_t num;
2357 	const struct sys_reg_desc *table, *r;
2358 	struct sys_reg_params params;
2359 
2360 	/* We only do sys_reg for now. */
2361 	if ((id & KVM_REG_ARM_COPROC_MASK) != KVM_REG_ARM64_SYSREG)
2362 		return NULL;
2363 
2364 	table = get_target_table(vcpu->arch.target, true, &num);
2365 	r = find_reg_by_id(id, &params, table, num);
2366 	if (!r)
2367 		r = find_reg(&params, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
2368 
2369 	/* Not saved in the sys_reg array and not otherwise accessible? */
2370 	if (r && !(r->reg || r->get_user))
2371 		r = NULL;
2372 
2373 	return r;
2374 }
2375 
2376 /*
2377  * These are the invariant sys_reg registers: we let the guest see the
2378  * host versions of these, so they're part of the guest state.
2379  *
2380  * A future CPU may provide a mechanism to present different values to
2381  * the guest, or a future kvm may trap them.
2382  */
2383 
2384 #define FUNCTION_INVARIANT(reg)						\
2385 	static void get_##reg(struct kvm_vcpu *v,			\
2386 			      const struct sys_reg_desc *r)		\
2387 	{								\
2388 		((struct sys_reg_desc *)r)->val = read_sysreg(reg);	\
2389 	}
2390 
2391 FUNCTION_INVARIANT(midr_el1)
2392 FUNCTION_INVARIANT(revidr_el1)
2393 FUNCTION_INVARIANT(clidr_el1)
2394 FUNCTION_INVARIANT(aidr_el1)
2395 
2396 static void get_ctr_el0(struct kvm_vcpu *v, const struct sys_reg_desc *r)
2397 {
2398 	((struct sys_reg_desc *)r)->val = read_sanitised_ftr_reg(SYS_CTR_EL0);
2399 }
2400 
2401 /* ->val is filled in by kvm_sys_reg_table_init() */
2402 static struct sys_reg_desc invariant_sys_regs[] = {
2403 	{ SYS_DESC(SYS_MIDR_EL1), NULL, get_midr_el1 },
2404 	{ SYS_DESC(SYS_REVIDR_EL1), NULL, get_revidr_el1 },
2405 	{ SYS_DESC(SYS_CLIDR_EL1), NULL, get_clidr_el1 },
2406 	{ SYS_DESC(SYS_AIDR_EL1), NULL, get_aidr_el1 },
2407 	{ SYS_DESC(SYS_CTR_EL0), NULL, get_ctr_el0 },
2408 };
2409 
2410 static int reg_from_user(u64 *val, const void __user *uaddr, u64 id)
2411 {
2412 	if (copy_from_user(val, uaddr, KVM_REG_SIZE(id)) != 0)
2413 		return -EFAULT;
2414 	return 0;
2415 }
2416 
2417 static int reg_to_user(void __user *uaddr, const u64 *val, u64 id)
2418 {
2419 	if (copy_to_user(uaddr, val, KVM_REG_SIZE(id)) != 0)
2420 		return -EFAULT;
2421 	return 0;
2422 }
2423 
2424 static int get_invariant_sys_reg(u64 id, void __user *uaddr)
2425 {
2426 	struct sys_reg_params params;
2427 	const struct sys_reg_desc *r;
2428 
2429 	r = find_reg_by_id(id, &params, invariant_sys_regs,
2430 			   ARRAY_SIZE(invariant_sys_regs));
2431 	if (!r)
2432 		return -ENOENT;
2433 
2434 	return reg_to_user(uaddr, &r->val, id);
2435 }
2436 
2437 static int set_invariant_sys_reg(u64 id, void __user *uaddr)
2438 {
2439 	struct sys_reg_params params;
2440 	const struct sys_reg_desc *r;
2441 	int err;
2442 	u64 val = 0; /* Make sure high bits are 0 for 32-bit regs */
2443 
2444 	r = find_reg_by_id(id, &params, invariant_sys_regs,
2445 			   ARRAY_SIZE(invariant_sys_regs));
2446 	if (!r)
2447 		return -ENOENT;
2448 
2449 	err = reg_from_user(&val, uaddr, id);
2450 	if (err)
2451 		return err;
2452 
2453 	/* This is what we mean by invariant: you can't change it. */
2454 	if (r->val != val)
2455 		return -EINVAL;
2456 
2457 	return 0;
2458 }
2459 
2460 static bool is_valid_cache(u32 val)
2461 {
2462 	u32 level, ctype;
2463 
2464 	if (val >= CSSELR_MAX)
2465 		return false;
2466 
2467 	/* Bottom bit is Instruction or Data bit.  Next 3 bits are level. */
2468 	level = (val >> 1);
2469 	ctype = (cache_levels >> (level * 3)) & 7;
2470 
2471 	switch (ctype) {
2472 	case 0: /* No cache */
2473 		return false;
2474 	case 1: /* Instruction cache only */
2475 		return (val & 1);
2476 	case 2: /* Data cache only */
2477 	case 4: /* Unified cache */
2478 		return !(val & 1);
2479 	case 3: /* Separate instruction and data caches */
2480 		return true;
2481 	default: /* Reserved: we can't know instruction or data. */
2482 		return false;
2483 	}
2484 }
2485 
2486 static int demux_c15_get(u64 id, void __user *uaddr)
2487 {
2488 	u32 val;
2489 	u32 __user *uval = uaddr;
2490 
2491 	/* Fail if we have unknown bits set. */
2492 	if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
2493 		   | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
2494 		return -ENOENT;
2495 
2496 	switch (id & KVM_REG_ARM_DEMUX_ID_MASK) {
2497 	case KVM_REG_ARM_DEMUX_ID_CCSIDR:
2498 		if (KVM_REG_SIZE(id) != 4)
2499 			return -ENOENT;
2500 		val = (id & KVM_REG_ARM_DEMUX_VAL_MASK)
2501 			>> KVM_REG_ARM_DEMUX_VAL_SHIFT;
2502 		if (!is_valid_cache(val))
2503 			return -ENOENT;
2504 
2505 		return put_user(get_ccsidr(val), uval);
2506 	default:
2507 		return -ENOENT;
2508 	}
2509 }
2510 
2511 static int demux_c15_set(u64 id, void __user *uaddr)
2512 {
2513 	u32 val, newval;
2514 	u32 __user *uval = uaddr;
2515 
2516 	/* Fail if we have unknown bits set. */
2517 	if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
2518 		   | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
2519 		return -ENOENT;
2520 
2521 	switch (id & KVM_REG_ARM_DEMUX_ID_MASK) {
2522 	case KVM_REG_ARM_DEMUX_ID_CCSIDR:
2523 		if (KVM_REG_SIZE(id) != 4)
2524 			return -ENOENT;
2525 		val = (id & KVM_REG_ARM_DEMUX_VAL_MASK)
2526 			>> KVM_REG_ARM_DEMUX_VAL_SHIFT;
2527 		if (!is_valid_cache(val))
2528 			return -ENOENT;
2529 
2530 		if (get_user(newval, uval))
2531 			return -EFAULT;
2532 
2533 		/* This is also invariant: you can't change it. */
2534 		if (newval != get_ccsidr(val))
2535 			return -EINVAL;
2536 		return 0;
2537 	default:
2538 		return -ENOENT;
2539 	}
2540 }
2541 
2542 int kvm_arm_sys_reg_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
2543 {
2544 	const struct sys_reg_desc *r;
2545 	void __user *uaddr = (void __user *)(unsigned long)reg->addr;
2546 
2547 	if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
2548 		return demux_c15_get(reg->id, uaddr);
2549 
2550 	if (KVM_REG_SIZE(reg->id) != sizeof(__u64))
2551 		return -ENOENT;
2552 
2553 	r = index_to_sys_reg_desc(vcpu, reg->id);
2554 	if (!r)
2555 		return get_invariant_sys_reg(reg->id, uaddr);
2556 
2557 	/* Check for regs disabled by runtime config */
2558 	if (sysreg_hidden_from_user(vcpu, r))
2559 		return -ENOENT;
2560 
2561 	if (r->get_user)
2562 		return (r->get_user)(vcpu, r, reg, uaddr);
2563 
2564 	return reg_to_user(uaddr, &__vcpu_sys_reg(vcpu, r->reg), reg->id);
2565 }
2566 
2567 int kvm_arm_sys_reg_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
2568 {
2569 	const struct sys_reg_desc *r;
2570 	void __user *uaddr = (void __user *)(unsigned long)reg->addr;
2571 
2572 	if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
2573 		return demux_c15_set(reg->id, uaddr);
2574 
2575 	if (KVM_REG_SIZE(reg->id) != sizeof(__u64))
2576 		return -ENOENT;
2577 
2578 	r = index_to_sys_reg_desc(vcpu, reg->id);
2579 	if (!r)
2580 		return set_invariant_sys_reg(reg->id, uaddr);
2581 
2582 	/* Check for regs disabled by runtime config */
2583 	if (sysreg_hidden_from_user(vcpu, r))
2584 		return -ENOENT;
2585 
2586 	if (r->set_user)
2587 		return (r->set_user)(vcpu, r, reg, uaddr);
2588 
2589 	return reg_from_user(&__vcpu_sys_reg(vcpu, r->reg), uaddr, reg->id);
2590 }
2591 
2592 static unsigned int num_demux_regs(void)
2593 {
2594 	unsigned int i, count = 0;
2595 
2596 	for (i = 0; i < CSSELR_MAX; i++)
2597 		if (is_valid_cache(i))
2598 			count++;
2599 
2600 	return count;
2601 }
2602 
2603 static int write_demux_regids(u64 __user *uindices)
2604 {
2605 	u64 val = KVM_REG_ARM64 | KVM_REG_SIZE_U32 | KVM_REG_ARM_DEMUX;
2606 	unsigned int i;
2607 
2608 	val |= KVM_REG_ARM_DEMUX_ID_CCSIDR;
2609 	for (i = 0; i < CSSELR_MAX; i++) {
2610 		if (!is_valid_cache(i))
2611 			continue;
2612 		if (put_user(val | i, uindices))
2613 			return -EFAULT;
2614 		uindices++;
2615 	}
2616 	return 0;
2617 }
2618 
2619 static u64 sys_reg_to_index(const struct sys_reg_desc *reg)
2620 {
2621 	return (KVM_REG_ARM64 | KVM_REG_SIZE_U64 |
2622 		KVM_REG_ARM64_SYSREG |
2623 		(reg->Op0 << KVM_REG_ARM64_SYSREG_OP0_SHIFT) |
2624 		(reg->Op1 << KVM_REG_ARM64_SYSREG_OP1_SHIFT) |
2625 		(reg->CRn << KVM_REG_ARM64_SYSREG_CRN_SHIFT) |
2626 		(reg->CRm << KVM_REG_ARM64_SYSREG_CRM_SHIFT) |
2627 		(reg->Op2 << KVM_REG_ARM64_SYSREG_OP2_SHIFT));
2628 }
2629 
2630 static bool copy_reg_to_user(const struct sys_reg_desc *reg, u64 __user **uind)
2631 {
2632 	if (!*uind)
2633 		return true;
2634 
2635 	if (put_user(sys_reg_to_index(reg), *uind))
2636 		return false;
2637 
2638 	(*uind)++;
2639 	return true;
2640 }
2641 
2642 static int walk_one_sys_reg(const struct kvm_vcpu *vcpu,
2643 			    const struct sys_reg_desc *rd,
2644 			    u64 __user **uind,
2645 			    unsigned int *total)
2646 {
2647 	/*
2648 	 * Ignore registers we trap but don't save,
2649 	 * and for which no custom user accessor is provided.
2650 	 */
2651 	if (!(rd->reg || rd->get_user))
2652 		return 0;
2653 
2654 	if (sysreg_hidden_from_user(vcpu, rd))
2655 		return 0;
2656 
2657 	if (!copy_reg_to_user(rd, uind))
2658 		return -EFAULT;
2659 
2660 	(*total)++;
2661 	return 0;
2662 }
2663 
2664 /* Assumed ordered tables, see kvm_sys_reg_table_init. */
2665 static int walk_sys_regs(struct kvm_vcpu *vcpu, u64 __user *uind)
2666 {
2667 	const struct sys_reg_desc *i1, *i2, *end1, *end2;
2668 	unsigned int total = 0;
2669 	size_t num;
2670 	int err;
2671 
2672 	/* We check for duplicates here, to allow arch-specific overrides. */
2673 	i1 = get_target_table(vcpu->arch.target, true, &num);
2674 	end1 = i1 + num;
2675 	i2 = sys_reg_descs;
2676 	end2 = sys_reg_descs + ARRAY_SIZE(sys_reg_descs);
2677 
2678 	BUG_ON(i1 == end1 || i2 == end2);
2679 
2680 	/* Walk carefully, as both tables may refer to the same register. */
2681 	while (i1 || i2) {
2682 		int cmp = cmp_sys_reg(i1, i2);
2683 		/* target-specific overrides generic entry. */
2684 		if (cmp <= 0)
2685 			err = walk_one_sys_reg(vcpu, i1, &uind, &total);
2686 		else
2687 			err = walk_one_sys_reg(vcpu, i2, &uind, &total);
2688 
2689 		if (err)
2690 			return err;
2691 
2692 		if (cmp <= 0 && ++i1 == end1)
2693 			i1 = NULL;
2694 		if (cmp >= 0 && ++i2 == end2)
2695 			i2 = NULL;
2696 	}
2697 	return total;
2698 }
2699 
2700 unsigned long kvm_arm_num_sys_reg_descs(struct kvm_vcpu *vcpu)
2701 {
2702 	return ARRAY_SIZE(invariant_sys_regs)
2703 		+ num_demux_regs()
2704 		+ walk_sys_regs(vcpu, (u64 __user *)NULL);
2705 }
2706 
2707 int kvm_arm_copy_sys_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
2708 {
2709 	unsigned int i;
2710 	int err;
2711 
2712 	/* Then give them all the invariant registers' indices. */
2713 	for (i = 0; i < ARRAY_SIZE(invariant_sys_regs); i++) {
2714 		if (put_user(sys_reg_to_index(&invariant_sys_regs[i]), uindices))
2715 			return -EFAULT;
2716 		uindices++;
2717 	}
2718 
2719 	err = walk_sys_regs(vcpu, uindices);
2720 	if (err < 0)
2721 		return err;
2722 	uindices += err;
2723 
2724 	return write_demux_regids(uindices);
2725 }
2726 
2727 static int check_sysreg_table(const struct sys_reg_desc *table, unsigned int n)
2728 {
2729 	unsigned int i;
2730 
2731 	for (i = 1; i < n; i++) {
2732 		if (cmp_sys_reg(&table[i-1], &table[i]) >= 0) {
2733 			kvm_err("sys_reg table %p out of order (%d)\n", table, i - 1);
2734 			return 1;
2735 		}
2736 	}
2737 
2738 	return 0;
2739 }
2740 
2741 void kvm_sys_reg_table_init(void)
2742 {
2743 	unsigned int i;
2744 	struct sys_reg_desc clidr;
2745 
2746 	/* Make sure tables are unique and in order. */
2747 	BUG_ON(check_sysreg_table(sys_reg_descs, ARRAY_SIZE(sys_reg_descs)));
2748 	BUG_ON(check_sysreg_table(cp14_regs, ARRAY_SIZE(cp14_regs)));
2749 	BUG_ON(check_sysreg_table(cp14_64_regs, ARRAY_SIZE(cp14_64_regs)));
2750 	BUG_ON(check_sysreg_table(cp15_regs, ARRAY_SIZE(cp15_regs)));
2751 	BUG_ON(check_sysreg_table(cp15_64_regs, ARRAY_SIZE(cp15_64_regs)));
2752 	BUG_ON(check_sysreg_table(invariant_sys_regs, ARRAY_SIZE(invariant_sys_regs)));
2753 
2754 	/* We abuse the reset function to overwrite the table itself. */
2755 	for (i = 0; i < ARRAY_SIZE(invariant_sys_regs); i++)
2756 		invariant_sys_regs[i].reset(NULL, &invariant_sys_regs[i]);
2757 
2758 	/*
2759 	 * CLIDR format is awkward, so clean it up.  See ARM B4.1.20:
2760 	 *
2761 	 *   If software reads the Cache Type fields from Ctype1
2762 	 *   upwards, once it has seen a value of 0b000, no caches
2763 	 *   exist at further-out levels of the hierarchy. So, for
2764 	 *   example, if Ctype3 is the first Cache Type field with a
2765 	 *   value of 0b000, the values of Ctype4 to Ctype7 must be
2766 	 *   ignored.
2767 	 */
2768 	get_clidr_el1(NULL, &clidr); /* Ugly... */
2769 	cache_levels = clidr.val;
2770 	for (i = 0; i < 7; i++)
2771 		if (((cache_levels >> (i*3)) & 7) == 0)
2772 			break;
2773 	/* Clear all higher bits. */
2774 	cache_levels &= (1 << (i*3))-1;
2775 }
2776 
2777 /**
2778  * kvm_reset_sys_regs - sets system registers to reset value
2779  * @vcpu: The VCPU pointer
2780  *
2781  * This function finds the right table above and sets the registers on the
2782  * virtual CPU struct to their architecturally defined reset values.
2783  */
2784 void kvm_reset_sys_regs(struct kvm_vcpu *vcpu)
2785 {
2786 	size_t num;
2787 	const struct sys_reg_desc *table;
2788 
2789 	/* Catch someone adding a register without putting in reset entry. */
2790 	memset(&vcpu->arch.ctxt.sys_regs, 0x42, sizeof(vcpu->arch.ctxt.sys_regs));
2791 
2792 	/* Generic chip reset first (so target could override). */
2793 	reset_sys_reg_descs(vcpu, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
2794 
2795 	table = get_target_table(vcpu->arch.target, true, &num);
2796 	reset_sys_reg_descs(vcpu, table, num);
2797 
2798 	for (num = 1; num < NR_SYS_REGS; num++) {
2799 		if (WARN(__vcpu_sys_reg(vcpu, num) == 0x4242424242424242,
2800 			 "Didn't reset __vcpu_sys_reg(%zi)\n", num))
2801 			break;
2802 	}
2803 }
2804