1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Check for KVM_GET_REG_LIST regressions.
4  *
5  * Copyright (C) 2020, Red Hat, Inc.
6  *
7  * While the blessed list should be created from the oldest possible
8  * kernel, we can't go older than v5.2, though, because that's the first
9  * release which includes df205b5c6328 ("KVM: arm64: Filter out invalid
10  * core register IDs in KVM_GET_REG_LIST"). Without that commit the core
11  * registers won't match expectations.
12  */
13 #include <stdio.h>
14 #include "kvm_util.h"
15 #include "test_util.h"
16 #include "processor.h"
17 
18 struct feature_id_reg {
19 	__u64 reg;
20 	__u64 id_reg;
21 	__u64 feat_shift;
22 	__u64 feat_min;
23 };
24 
25 static struct feature_id_reg feat_id_regs[] = {
26 	{
27 		ARM64_SYS_REG(3, 0, 2, 0, 3),	/* TCR2_EL1 */
28 		ARM64_SYS_REG(3, 0, 0, 7, 3),	/* ID_AA64MMFR3_EL1 */
29 		0,
30 		1
31 	},
32 	{
33 		ARM64_SYS_REG(3, 0, 10, 2, 2),	/* PIRE0_EL1 */
34 		ARM64_SYS_REG(3, 0, 0, 7, 3),	/* ID_AA64MMFR3_EL1 */
35 		4,
36 		1
37 	},
38 	{
39 		ARM64_SYS_REG(3, 0, 10, 2, 3),	/* PIR_EL1 */
40 		ARM64_SYS_REG(3, 0, 0, 7, 3),	/* ID_AA64MMFR3_EL1 */
41 		4,
42 		1
43 	}
44 };
45 
filter_reg(__u64 reg)46 bool filter_reg(__u64 reg)
47 {
48 	/*
49 	 * DEMUX register presence depends on the host's CLIDR_EL1.
50 	 * This means there's no set of them that we can bless.
51 	 */
52 	if ((reg & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
53 		return true;
54 
55 	return false;
56 }
57 
check_supported_feat_reg(struct kvm_vcpu * vcpu,__u64 reg)58 static bool check_supported_feat_reg(struct kvm_vcpu *vcpu, __u64 reg)
59 {
60 	int i, ret;
61 	__u64 data, feat_val;
62 
63 	for (i = 0; i < ARRAY_SIZE(feat_id_regs); i++) {
64 		if (feat_id_regs[i].reg == reg) {
65 			ret = __vcpu_get_reg(vcpu, feat_id_regs[i].id_reg, &data);
66 			if (ret < 0)
67 				return false;
68 
69 			feat_val = ((data >> feat_id_regs[i].feat_shift) & 0xf);
70 			return feat_val >= feat_id_regs[i].feat_min;
71 		}
72 	}
73 
74 	return true;
75 }
76 
check_supported_reg(struct kvm_vcpu * vcpu,__u64 reg)77 bool check_supported_reg(struct kvm_vcpu *vcpu, __u64 reg)
78 {
79 	return check_supported_feat_reg(vcpu, reg);
80 }
81 
check_reject_set(int err)82 bool check_reject_set(int err)
83 {
84 	return err == EPERM;
85 }
86 
finalize_vcpu(struct kvm_vcpu * vcpu,struct vcpu_reg_list * c)87 void finalize_vcpu(struct kvm_vcpu *vcpu, struct vcpu_reg_list *c)
88 {
89 	struct vcpu_reg_sublist *s;
90 	int feature;
91 
92 	for_each_sublist(c, s) {
93 		if (s->finalize) {
94 			feature = s->feature;
95 			vcpu_ioctl(vcpu, KVM_ARM_VCPU_FINALIZE, &feature);
96 		}
97 	}
98 }
99 
100 #define REG_MASK (KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK | KVM_REG_ARM_COPROC_MASK)
101 
102 #define CORE_REGS_XX_NR_WORDS	2
103 #define CORE_SPSR_XX_NR_WORDS	2
104 #define CORE_FPREGS_XX_NR_WORDS	4
105 
core_id_to_str(const char * prefix,__u64 id)106 static const char *core_id_to_str(const char *prefix, __u64 id)
107 {
108 	__u64 core_off = id & ~REG_MASK, idx;
109 
110 	/*
111 	 * core_off is the offset into struct kvm_regs
112 	 */
113 	switch (core_off) {
114 	case KVM_REG_ARM_CORE_REG(regs.regs[0]) ...
115 	     KVM_REG_ARM_CORE_REG(regs.regs[30]):
116 		idx = (core_off - KVM_REG_ARM_CORE_REG(regs.regs[0])) / CORE_REGS_XX_NR_WORDS;
117 		TEST_ASSERT(idx < 31, "%s: Unexpected regs.regs index: %lld", prefix, idx);
118 		return strdup_printf("KVM_REG_ARM_CORE_REG(regs.regs[%lld])", idx);
119 	case KVM_REG_ARM_CORE_REG(regs.sp):
120 		return "KVM_REG_ARM_CORE_REG(regs.sp)";
121 	case KVM_REG_ARM_CORE_REG(regs.pc):
122 		return "KVM_REG_ARM_CORE_REG(regs.pc)";
123 	case KVM_REG_ARM_CORE_REG(regs.pstate):
124 		return "KVM_REG_ARM_CORE_REG(regs.pstate)";
125 	case KVM_REG_ARM_CORE_REG(sp_el1):
126 		return "KVM_REG_ARM_CORE_REG(sp_el1)";
127 	case KVM_REG_ARM_CORE_REG(elr_el1):
128 		return "KVM_REG_ARM_CORE_REG(elr_el1)";
129 	case KVM_REG_ARM_CORE_REG(spsr[0]) ...
130 	     KVM_REG_ARM_CORE_REG(spsr[KVM_NR_SPSR - 1]):
131 		idx = (core_off - KVM_REG_ARM_CORE_REG(spsr[0])) / CORE_SPSR_XX_NR_WORDS;
132 		TEST_ASSERT(idx < KVM_NR_SPSR, "%s: Unexpected spsr index: %lld", prefix, idx);
133 		return strdup_printf("KVM_REG_ARM_CORE_REG(spsr[%lld])", idx);
134 	case KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]) ...
135 	     KVM_REG_ARM_CORE_REG(fp_regs.vregs[31]):
136 		idx = (core_off - KVM_REG_ARM_CORE_REG(fp_regs.vregs[0])) / CORE_FPREGS_XX_NR_WORDS;
137 		TEST_ASSERT(idx < 32, "%s: Unexpected fp_regs.vregs index: %lld", prefix, idx);
138 		return strdup_printf("KVM_REG_ARM_CORE_REG(fp_regs.vregs[%lld])", idx);
139 	case KVM_REG_ARM_CORE_REG(fp_regs.fpsr):
140 		return "KVM_REG_ARM_CORE_REG(fp_regs.fpsr)";
141 	case KVM_REG_ARM_CORE_REG(fp_regs.fpcr):
142 		return "KVM_REG_ARM_CORE_REG(fp_regs.fpcr)";
143 	}
144 
145 	TEST_FAIL("%s: Unknown core reg id: 0x%llx", prefix, id);
146 	return NULL;
147 }
148 
sve_id_to_str(const char * prefix,__u64 id)149 static const char *sve_id_to_str(const char *prefix, __u64 id)
150 {
151 	__u64 sve_off, n, i;
152 
153 	if (id == KVM_REG_ARM64_SVE_VLS)
154 		return "KVM_REG_ARM64_SVE_VLS";
155 
156 	sve_off = id & ~(REG_MASK | ((1ULL << 5) - 1));
157 	i = id & (KVM_ARM64_SVE_MAX_SLICES - 1);
158 
159 	TEST_ASSERT(i == 0, "%s: Currently we don't expect slice > 0, reg id 0x%llx", prefix, id);
160 
161 	switch (sve_off) {
162 	case KVM_REG_ARM64_SVE_ZREG_BASE ...
163 	     KVM_REG_ARM64_SVE_ZREG_BASE + (1ULL << 5) * KVM_ARM64_SVE_NUM_ZREGS - 1:
164 		n = (id >> 5) & (KVM_ARM64_SVE_NUM_ZREGS - 1);
165 		TEST_ASSERT(id == KVM_REG_ARM64_SVE_ZREG(n, 0),
166 			    "%s: Unexpected bits set in SVE ZREG id: 0x%llx", prefix, id);
167 		return strdup_printf("KVM_REG_ARM64_SVE_ZREG(%lld, 0)", n);
168 	case KVM_REG_ARM64_SVE_PREG_BASE ...
169 	     KVM_REG_ARM64_SVE_PREG_BASE + (1ULL << 5) * KVM_ARM64_SVE_NUM_PREGS - 1:
170 		n = (id >> 5) & (KVM_ARM64_SVE_NUM_PREGS - 1);
171 		TEST_ASSERT(id == KVM_REG_ARM64_SVE_PREG(n, 0),
172 			    "%s: Unexpected bits set in SVE PREG id: 0x%llx", prefix, id);
173 		return strdup_printf("KVM_REG_ARM64_SVE_PREG(%lld, 0)", n);
174 	case KVM_REG_ARM64_SVE_FFR_BASE:
175 		TEST_ASSERT(id == KVM_REG_ARM64_SVE_FFR(0),
176 			    "%s: Unexpected bits set in SVE FFR id: 0x%llx", prefix, id);
177 		return "KVM_REG_ARM64_SVE_FFR(0)";
178 	}
179 
180 	return NULL;
181 }
182 
print_reg(const char * prefix,__u64 id)183 void print_reg(const char *prefix, __u64 id)
184 {
185 	unsigned op0, op1, crn, crm, op2;
186 	const char *reg_size = NULL;
187 
188 	TEST_ASSERT((id & KVM_REG_ARCH_MASK) == KVM_REG_ARM64,
189 		    "%s: KVM_REG_ARM64 missing in reg id: 0x%llx", prefix, id);
190 
191 	switch (id & KVM_REG_SIZE_MASK) {
192 	case KVM_REG_SIZE_U8:
193 		reg_size = "KVM_REG_SIZE_U8";
194 		break;
195 	case KVM_REG_SIZE_U16:
196 		reg_size = "KVM_REG_SIZE_U16";
197 		break;
198 	case KVM_REG_SIZE_U32:
199 		reg_size = "KVM_REG_SIZE_U32";
200 		break;
201 	case KVM_REG_SIZE_U64:
202 		reg_size = "KVM_REG_SIZE_U64";
203 		break;
204 	case KVM_REG_SIZE_U128:
205 		reg_size = "KVM_REG_SIZE_U128";
206 		break;
207 	case KVM_REG_SIZE_U256:
208 		reg_size = "KVM_REG_SIZE_U256";
209 		break;
210 	case KVM_REG_SIZE_U512:
211 		reg_size = "KVM_REG_SIZE_U512";
212 		break;
213 	case KVM_REG_SIZE_U1024:
214 		reg_size = "KVM_REG_SIZE_U1024";
215 		break;
216 	case KVM_REG_SIZE_U2048:
217 		reg_size = "KVM_REG_SIZE_U2048";
218 		break;
219 	default:
220 		TEST_FAIL("%s: Unexpected reg size: 0x%llx in reg id: 0x%llx",
221 			  prefix, (id & KVM_REG_SIZE_MASK) >> KVM_REG_SIZE_SHIFT, id);
222 	}
223 
224 	switch (id & KVM_REG_ARM_COPROC_MASK) {
225 	case KVM_REG_ARM_CORE:
226 		printf("\tKVM_REG_ARM64 | %s | KVM_REG_ARM_CORE | %s,\n", reg_size, core_id_to_str(prefix, id));
227 		break;
228 	case KVM_REG_ARM_DEMUX:
229 		TEST_ASSERT(!(id & ~(REG_MASK | KVM_REG_ARM_DEMUX_ID_MASK | KVM_REG_ARM_DEMUX_VAL_MASK)),
230 			    "%s: Unexpected bits set in DEMUX reg id: 0x%llx", prefix, id);
231 		printf("\tKVM_REG_ARM64 | %s | KVM_REG_ARM_DEMUX | KVM_REG_ARM_DEMUX_ID_CCSIDR | %lld,\n",
232 		       reg_size, id & KVM_REG_ARM_DEMUX_VAL_MASK);
233 		break;
234 	case KVM_REG_ARM64_SYSREG:
235 		op0 = (id & KVM_REG_ARM64_SYSREG_OP0_MASK) >> KVM_REG_ARM64_SYSREG_OP0_SHIFT;
236 		op1 = (id & KVM_REG_ARM64_SYSREG_OP1_MASK) >> KVM_REG_ARM64_SYSREG_OP1_SHIFT;
237 		crn = (id & KVM_REG_ARM64_SYSREG_CRN_MASK) >> KVM_REG_ARM64_SYSREG_CRN_SHIFT;
238 		crm = (id & KVM_REG_ARM64_SYSREG_CRM_MASK) >> KVM_REG_ARM64_SYSREG_CRM_SHIFT;
239 		op2 = (id & KVM_REG_ARM64_SYSREG_OP2_MASK) >> KVM_REG_ARM64_SYSREG_OP2_SHIFT;
240 		TEST_ASSERT(id == ARM64_SYS_REG(op0, op1, crn, crm, op2),
241 			    "%s: Unexpected bits set in SYSREG reg id: 0x%llx", prefix, id);
242 		printf("\tARM64_SYS_REG(%d, %d, %d, %d, %d),\n", op0, op1, crn, crm, op2);
243 		break;
244 	case KVM_REG_ARM_FW:
245 		TEST_ASSERT(id == KVM_REG_ARM_FW_REG(id & 0xffff),
246 			    "%s: Unexpected bits set in FW reg id: 0x%llx", prefix, id);
247 		printf("\tKVM_REG_ARM_FW_REG(%lld),\n", id & 0xffff);
248 		break;
249 	case KVM_REG_ARM_FW_FEAT_BMAP:
250 		TEST_ASSERT(id == KVM_REG_ARM_FW_FEAT_BMAP_REG(id & 0xffff),
251 			    "%s: Unexpected bits set in the bitmap feature FW reg id: 0x%llx", prefix, id);
252 		printf("\tKVM_REG_ARM_FW_FEAT_BMAP_REG(%lld),\n", id & 0xffff);
253 		break;
254 	case KVM_REG_ARM64_SVE:
255 		printf("\t%s,\n", sve_id_to_str(prefix, id));
256 		break;
257 	default:
258 		TEST_FAIL("%s: Unexpected coproc type: 0x%llx in reg id: 0x%llx",
259 			  prefix, (id & KVM_REG_ARM_COPROC_MASK) >> KVM_REG_ARM_COPROC_SHIFT, id);
260 	}
261 }
262 
263 /*
264  * The original blessed list was primed with the output of kernel version
265  * v4.15 with --core-reg-fixup and then later updated with new registers.
266  * (The --core-reg-fixup option and it's fixup function have been removed
267  * from the test, as it's unlikely to use this type of test on a kernel
268  * older than v5.2.)
269  *
270  * The blessed list is up to date with kernel version v6.4 (or so we hope)
271  */
272 static __u64 base_regs[] = {
273 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[0]),
274 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[1]),
275 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[2]),
276 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[3]),
277 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[4]),
278 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[5]),
279 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[6]),
280 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[7]),
281 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[8]),
282 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[9]),
283 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[10]),
284 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[11]),
285 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[12]),
286 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[13]),
287 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[14]),
288 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[15]),
289 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[16]),
290 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[17]),
291 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[18]),
292 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[19]),
293 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[20]),
294 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[21]),
295 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[22]),
296 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[23]),
297 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[24]),
298 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[25]),
299 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[26]),
300 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[27]),
301 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[28]),
302 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[29]),
303 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[30]),
304 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.sp),
305 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.pc),
306 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.pstate),
307 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(sp_el1),
308 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(elr_el1),
309 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(spsr[0]),
310 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(spsr[1]),
311 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(spsr[2]),
312 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(spsr[3]),
313 	KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(spsr[4]),
314 	KVM_REG_ARM64 | KVM_REG_SIZE_U32 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.fpsr),
315 	KVM_REG_ARM64 | KVM_REG_SIZE_U32 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.fpcr),
316 	KVM_REG_ARM_FW_REG(0),		/* KVM_REG_ARM_PSCI_VERSION */
317 	KVM_REG_ARM_FW_REG(1),		/* KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1 */
318 	KVM_REG_ARM_FW_REG(2),		/* KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2 */
319 	KVM_REG_ARM_FW_REG(3),		/* KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3 */
320 	KVM_REG_ARM_FW_FEAT_BMAP_REG(0),	/* KVM_REG_ARM_STD_BMAP */
321 	KVM_REG_ARM_FW_FEAT_BMAP_REG(1),	/* KVM_REG_ARM_STD_HYP_BMAP */
322 	KVM_REG_ARM_FW_FEAT_BMAP_REG(2),	/* KVM_REG_ARM_VENDOR_HYP_BMAP */
323 	ARM64_SYS_REG(3, 3, 14, 3, 1),	/* CNTV_CTL_EL0 */
324 	ARM64_SYS_REG(3, 3, 14, 3, 2),	/* CNTV_CVAL_EL0 */
325 	ARM64_SYS_REG(3, 3, 14, 0, 2),
326 	ARM64_SYS_REG(3, 0, 0, 0, 0),	/* MIDR_EL1 */
327 	ARM64_SYS_REG(3, 0, 0, 0, 6),	/* REVIDR_EL1 */
328 	ARM64_SYS_REG(3, 1, 0, 0, 1),	/* CLIDR_EL1 */
329 	ARM64_SYS_REG(3, 1, 0, 0, 7),	/* AIDR_EL1 */
330 	ARM64_SYS_REG(3, 3, 0, 0, 1),	/* CTR_EL0 */
331 	ARM64_SYS_REG(2, 0, 0, 0, 4),
332 	ARM64_SYS_REG(2, 0, 0, 0, 5),
333 	ARM64_SYS_REG(2, 0, 0, 0, 6),
334 	ARM64_SYS_REG(2, 0, 0, 0, 7),
335 	ARM64_SYS_REG(2, 0, 0, 1, 4),
336 	ARM64_SYS_REG(2, 0, 0, 1, 5),
337 	ARM64_SYS_REG(2, 0, 0, 1, 6),
338 	ARM64_SYS_REG(2, 0, 0, 1, 7),
339 	ARM64_SYS_REG(2, 0, 0, 2, 0),	/* MDCCINT_EL1 */
340 	ARM64_SYS_REG(2, 0, 0, 2, 2),	/* MDSCR_EL1 */
341 	ARM64_SYS_REG(2, 0, 0, 2, 4),
342 	ARM64_SYS_REG(2, 0, 0, 2, 5),
343 	ARM64_SYS_REG(2, 0, 0, 2, 6),
344 	ARM64_SYS_REG(2, 0, 0, 2, 7),
345 	ARM64_SYS_REG(2, 0, 0, 3, 4),
346 	ARM64_SYS_REG(2, 0, 0, 3, 5),
347 	ARM64_SYS_REG(2, 0, 0, 3, 6),
348 	ARM64_SYS_REG(2, 0, 0, 3, 7),
349 	ARM64_SYS_REG(2, 0, 0, 4, 4),
350 	ARM64_SYS_REG(2, 0, 0, 4, 5),
351 	ARM64_SYS_REG(2, 0, 0, 4, 6),
352 	ARM64_SYS_REG(2, 0, 0, 4, 7),
353 	ARM64_SYS_REG(2, 0, 0, 5, 4),
354 	ARM64_SYS_REG(2, 0, 0, 5, 5),
355 	ARM64_SYS_REG(2, 0, 0, 5, 6),
356 	ARM64_SYS_REG(2, 0, 0, 5, 7),
357 	ARM64_SYS_REG(2, 0, 0, 6, 4),
358 	ARM64_SYS_REG(2, 0, 0, 6, 5),
359 	ARM64_SYS_REG(2, 0, 0, 6, 6),
360 	ARM64_SYS_REG(2, 0, 0, 6, 7),
361 	ARM64_SYS_REG(2, 0, 0, 7, 4),
362 	ARM64_SYS_REG(2, 0, 0, 7, 5),
363 	ARM64_SYS_REG(2, 0, 0, 7, 6),
364 	ARM64_SYS_REG(2, 0, 0, 7, 7),
365 	ARM64_SYS_REG(2, 0, 0, 8, 4),
366 	ARM64_SYS_REG(2, 0, 0, 8, 5),
367 	ARM64_SYS_REG(2, 0, 0, 8, 6),
368 	ARM64_SYS_REG(2, 0, 0, 8, 7),
369 	ARM64_SYS_REG(2, 0, 0, 9, 4),
370 	ARM64_SYS_REG(2, 0, 0, 9, 5),
371 	ARM64_SYS_REG(2, 0, 0, 9, 6),
372 	ARM64_SYS_REG(2, 0, 0, 9, 7),
373 	ARM64_SYS_REG(2, 0, 0, 10, 4),
374 	ARM64_SYS_REG(2, 0, 0, 10, 5),
375 	ARM64_SYS_REG(2, 0, 0, 10, 6),
376 	ARM64_SYS_REG(2, 0, 0, 10, 7),
377 	ARM64_SYS_REG(2, 0, 0, 11, 4),
378 	ARM64_SYS_REG(2, 0, 0, 11, 5),
379 	ARM64_SYS_REG(2, 0, 0, 11, 6),
380 	ARM64_SYS_REG(2, 0, 0, 11, 7),
381 	ARM64_SYS_REG(2, 0, 0, 12, 4),
382 	ARM64_SYS_REG(2, 0, 0, 12, 5),
383 	ARM64_SYS_REG(2, 0, 0, 12, 6),
384 	ARM64_SYS_REG(2, 0, 0, 12, 7),
385 	ARM64_SYS_REG(2, 0, 0, 13, 4),
386 	ARM64_SYS_REG(2, 0, 0, 13, 5),
387 	ARM64_SYS_REG(2, 0, 0, 13, 6),
388 	ARM64_SYS_REG(2, 0, 0, 13, 7),
389 	ARM64_SYS_REG(2, 0, 0, 14, 4),
390 	ARM64_SYS_REG(2, 0, 0, 14, 5),
391 	ARM64_SYS_REG(2, 0, 0, 14, 6),
392 	ARM64_SYS_REG(2, 0, 0, 14, 7),
393 	ARM64_SYS_REG(2, 0, 0, 15, 4),
394 	ARM64_SYS_REG(2, 0, 0, 15, 5),
395 	ARM64_SYS_REG(2, 0, 0, 15, 6),
396 	ARM64_SYS_REG(2, 0, 0, 15, 7),
397 	ARM64_SYS_REG(2, 0, 1, 1, 4),	/* OSLSR_EL1 */
398 	ARM64_SYS_REG(2, 4, 0, 7, 0),	/* DBGVCR32_EL2 */
399 	ARM64_SYS_REG(3, 0, 0, 0, 5),	/* MPIDR_EL1 */
400 	ARM64_SYS_REG(3, 0, 0, 1, 0),	/* ID_PFR0_EL1 */
401 	ARM64_SYS_REG(3, 0, 0, 1, 1),	/* ID_PFR1_EL1 */
402 	ARM64_SYS_REG(3, 0, 0, 1, 2),	/* ID_DFR0_EL1 */
403 	ARM64_SYS_REG(3, 0, 0, 1, 3),	/* ID_AFR0_EL1 */
404 	ARM64_SYS_REG(3, 0, 0, 1, 4),	/* ID_MMFR0_EL1 */
405 	ARM64_SYS_REG(3, 0, 0, 1, 5),	/* ID_MMFR1_EL1 */
406 	ARM64_SYS_REG(3, 0, 0, 1, 6),	/* ID_MMFR2_EL1 */
407 	ARM64_SYS_REG(3, 0, 0, 1, 7),	/* ID_MMFR3_EL1 */
408 	ARM64_SYS_REG(3, 0, 0, 2, 0),	/* ID_ISAR0_EL1 */
409 	ARM64_SYS_REG(3, 0, 0, 2, 1),	/* ID_ISAR1_EL1 */
410 	ARM64_SYS_REG(3, 0, 0, 2, 2),	/* ID_ISAR2_EL1 */
411 	ARM64_SYS_REG(3, 0, 0, 2, 3),	/* ID_ISAR3_EL1 */
412 	ARM64_SYS_REG(3, 0, 0, 2, 4),	/* ID_ISAR4_EL1 */
413 	ARM64_SYS_REG(3, 0, 0, 2, 5),	/* ID_ISAR5_EL1 */
414 	ARM64_SYS_REG(3, 0, 0, 2, 6),	/* ID_MMFR4_EL1 */
415 	ARM64_SYS_REG(3, 0, 0, 2, 7),	/* ID_ISAR6_EL1 */
416 	ARM64_SYS_REG(3, 0, 0, 3, 0),	/* MVFR0_EL1 */
417 	ARM64_SYS_REG(3, 0, 0, 3, 1),	/* MVFR1_EL1 */
418 	ARM64_SYS_REG(3, 0, 0, 3, 2),	/* MVFR2_EL1 */
419 	ARM64_SYS_REG(3, 0, 0, 3, 3),
420 	ARM64_SYS_REG(3, 0, 0, 3, 4),	/* ID_PFR2_EL1 */
421 	ARM64_SYS_REG(3, 0, 0, 3, 5),	/* ID_DFR1_EL1 */
422 	ARM64_SYS_REG(3, 0, 0, 3, 6),	/* ID_MMFR5_EL1 */
423 	ARM64_SYS_REG(3, 0, 0, 3, 7),
424 	ARM64_SYS_REG(3, 0, 0, 4, 0),	/* ID_AA64PFR0_EL1 */
425 	ARM64_SYS_REG(3, 0, 0, 4, 1),	/* ID_AA64PFR1_EL1 */
426 	ARM64_SYS_REG(3, 0, 0, 4, 2),	/* ID_AA64PFR2_EL1 */
427 	ARM64_SYS_REG(3, 0, 0, 4, 3),
428 	ARM64_SYS_REG(3, 0, 0, 4, 4),	/* ID_AA64ZFR0_EL1 */
429 	ARM64_SYS_REG(3, 0, 0, 4, 5),	/* ID_AA64SMFR0_EL1 */
430 	ARM64_SYS_REG(3, 0, 0, 4, 6),
431 	ARM64_SYS_REG(3, 0, 0, 4, 7),
432 	ARM64_SYS_REG(3, 0, 0, 5, 0),	/* ID_AA64DFR0_EL1 */
433 	ARM64_SYS_REG(3, 0, 0, 5, 1),	/* ID_AA64DFR1_EL1 */
434 	ARM64_SYS_REG(3, 0, 0, 5, 2),
435 	ARM64_SYS_REG(3, 0, 0, 5, 3),
436 	ARM64_SYS_REG(3, 0, 0, 5, 4),	/* ID_AA64AFR0_EL1 */
437 	ARM64_SYS_REG(3, 0, 0, 5, 5),	/* ID_AA64AFR1_EL1 */
438 	ARM64_SYS_REG(3, 0, 0, 5, 6),
439 	ARM64_SYS_REG(3, 0, 0, 5, 7),
440 	ARM64_SYS_REG(3, 0, 0, 6, 0),	/* ID_AA64ISAR0_EL1 */
441 	ARM64_SYS_REG(3, 0, 0, 6, 1),	/* ID_AA64ISAR1_EL1 */
442 	ARM64_SYS_REG(3, 0, 0, 6, 2),	/* ID_AA64ISAR2_EL1 */
443 	ARM64_SYS_REG(3, 0, 0, 6, 3),
444 	ARM64_SYS_REG(3, 0, 0, 6, 4),
445 	ARM64_SYS_REG(3, 0, 0, 6, 5),
446 	ARM64_SYS_REG(3, 0, 0, 6, 6),
447 	ARM64_SYS_REG(3, 0, 0, 6, 7),
448 	ARM64_SYS_REG(3, 0, 0, 7, 0),	/* ID_AA64MMFR0_EL1 */
449 	ARM64_SYS_REG(3, 0, 0, 7, 1),	/* ID_AA64MMFR1_EL1 */
450 	ARM64_SYS_REG(3, 0, 0, 7, 2),	/* ID_AA64MMFR2_EL1 */
451 	ARM64_SYS_REG(3, 0, 0, 7, 3),	/* ID_AA64MMFR3_EL1 */
452 	ARM64_SYS_REG(3, 0, 0, 7, 4),	/* ID_AA64MMFR4_EL1 */
453 	ARM64_SYS_REG(3, 0, 0, 7, 5),
454 	ARM64_SYS_REG(3, 0, 0, 7, 6),
455 	ARM64_SYS_REG(3, 0, 0, 7, 7),
456 	ARM64_SYS_REG(3, 0, 1, 0, 0),	/* SCTLR_EL1 */
457 	ARM64_SYS_REG(3, 0, 1, 0, 1),	/* ACTLR_EL1 */
458 	ARM64_SYS_REG(3, 0, 1, 0, 2),	/* CPACR_EL1 */
459 	ARM64_SYS_REG(3, 0, 2, 0, 0),	/* TTBR0_EL1 */
460 	ARM64_SYS_REG(3, 0, 2, 0, 1),	/* TTBR1_EL1 */
461 	ARM64_SYS_REG(3, 0, 2, 0, 2),	/* TCR_EL1 */
462 	ARM64_SYS_REG(3, 0, 2, 0, 3),	/* TCR2_EL1 */
463 	ARM64_SYS_REG(3, 0, 5, 1, 0),	/* AFSR0_EL1 */
464 	ARM64_SYS_REG(3, 0, 5, 1, 1),	/* AFSR1_EL1 */
465 	ARM64_SYS_REG(3, 0, 5, 2, 0),	/* ESR_EL1 */
466 	ARM64_SYS_REG(3, 0, 6, 0, 0),	/* FAR_EL1 */
467 	ARM64_SYS_REG(3, 0, 7, 4, 0),	/* PAR_EL1 */
468 	ARM64_SYS_REG(3, 0, 10, 2, 0),	/* MAIR_EL1 */
469 	ARM64_SYS_REG(3, 0, 10, 2, 2),	/* PIRE0_EL1 */
470 	ARM64_SYS_REG(3, 0, 10, 2, 3),	/* PIR_EL1 */
471 	ARM64_SYS_REG(3, 0, 10, 3, 0),	/* AMAIR_EL1 */
472 	ARM64_SYS_REG(3, 0, 12, 0, 0),	/* VBAR_EL1 */
473 	ARM64_SYS_REG(3, 0, 12, 1, 1),	/* DISR_EL1 */
474 	ARM64_SYS_REG(3, 0, 13, 0, 1),	/* CONTEXTIDR_EL1 */
475 	ARM64_SYS_REG(3, 0, 13, 0, 4),	/* TPIDR_EL1 */
476 	ARM64_SYS_REG(3, 0, 14, 1, 0),	/* CNTKCTL_EL1 */
477 	ARM64_SYS_REG(3, 2, 0, 0, 0),	/* CSSELR_EL1 */
478 	ARM64_SYS_REG(3, 3, 13, 0, 2),	/* TPIDR_EL0 */
479 	ARM64_SYS_REG(3, 3, 13, 0, 3),	/* TPIDRRO_EL0 */
480 	ARM64_SYS_REG(3, 3, 14, 0, 1),	/* CNTPCT_EL0 */
481 	ARM64_SYS_REG(3, 3, 14, 2, 1),	/* CNTP_CTL_EL0 */
482 	ARM64_SYS_REG(3, 3, 14, 2, 2),	/* CNTP_CVAL_EL0 */
483 	ARM64_SYS_REG(3, 4, 3, 0, 0),	/* DACR32_EL2 */
484 	ARM64_SYS_REG(3, 4, 5, 0, 1),	/* IFSR32_EL2 */
485 	ARM64_SYS_REG(3, 4, 5, 3, 0),	/* FPEXC32_EL2 */
486 };
487 
488 static __u64 pmu_regs[] = {
489 	ARM64_SYS_REG(3, 0, 9, 14, 1),	/* PMINTENSET_EL1 */
490 	ARM64_SYS_REG(3, 0, 9, 14, 2),	/* PMINTENCLR_EL1 */
491 	ARM64_SYS_REG(3, 3, 9, 12, 0),	/* PMCR_EL0 */
492 	ARM64_SYS_REG(3, 3, 9, 12, 1),	/* PMCNTENSET_EL0 */
493 	ARM64_SYS_REG(3, 3, 9, 12, 2),	/* PMCNTENCLR_EL0 */
494 	ARM64_SYS_REG(3, 3, 9, 12, 3),	/* PMOVSCLR_EL0 */
495 	ARM64_SYS_REG(3, 3, 9, 12, 4),	/* PMSWINC_EL0 */
496 	ARM64_SYS_REG(3, 3, 9, 12, 5),	/* PMSELR_EL0 */
497 	ARM64_SYS_REG(3, 3, 9, 13, 0),	/* PMCCNTR_EL0 */
498 	ARM64_SYS_REG(3, 3, 9, 14, 0),	/* PMUSERENR_EL0 */
499 	ARM64_SYS_REG(3, 3, 9, 14, 3),	/* PMOVSSET_EL0 */
500 	ARM64_SYS_REG(3, 3, 14, 8, 0),
501 	ARM64_SYS_REG(3, 3, 14, 8, 1),
502 	ARM64_SYS_REG(3, 3, 14, 8, 2),
503 	ARM64_SYS_REG(3, 3, 14, 8, 3),
504 	ARM64_SYS_REG(3, 3, 14, 8, 4),
505 	ARM64_SYS_REG(3, 3, 14, 8, 5),
506 	ARM64_SYS_REG(3, 3, 14, 8, 6),
507 	ARM64_SYS_REG(3, 3, 14, 8, 7),
508 	ARM64_SYS_REG(3, 3, 14, 9, 0),
509 	ARM64_SYS_REG(3, 3, 14, 9, 1),
510 	ARM64_SYS_REG(3, 3, 14, 9, 2),
511 	ARM64_SYS_REG(3, 3, 14, 9, 3),
512 	ARM64_SYS_REG(3, 3, 14, 9, 4),
513 	ARM64_SYS_REG(3, 3, 14, 9, 5),
514 	ARM64_SYS_REG(3, 3, 14, 9, 6),
515 	ARM64_SYS_REG(3, 3, 14, 9, 7),
516 	ARM64_SYS_REG(3, 3, 14, 10, 0),
517 	ARM64_SYS_REG(3, 3, 14, 10, 1),
518 	ARM64_SYS_REG(3, 3, 14, 10, 2),
519 	ARM64_SYS_REG(3, 3, 14, 10, 3),
520 	ARM64_SYS_REG(3, 3, 14, 10, 4),
521 	ARM64_SYS_REG(3, 3, 14, 10, 5),
522 	ARM64_SYS_REG(3, 3, 14, 10, 6),
523 	ARM64_SYS_REG(3, 3, 14, 10, 7),
524 	ARM64_SYS_REG(3, 3, 14, 11, 0),
525 	ARM64_SYS_REG(3, 3, 14, 11, 1),
526 	ARM64_SYS_REG(3, 3, 14, 11, 2),
527 	ARM64_SYS_REG(3, 3, 14, 11, 3),
528 	ARM64_SYS_REG(3, 3, 14, 11, 4),
529 	ARM64_SYS_REG(3, 3, 14, 11, 5),
530 	ARM64_SYS_REG(3, 3, 14, 11, 6),
531 	ARM64_SYS_REG(3, 3, 14, 12, 0),
532 	ARM64_SYS_REG(3, 3, 14, 12, 1),
533 	ARM64_SYS_REG(3, 3, 14, 12, 2),
534 	ARM64_SYS_REG(3, 3, 14, 12, 3),
535 	ARM64_SYS_REG(3, 3, 14, 12, 4),
536 	ARM64_SYS_REG(3, 3, 14, 12, 5),
537 	ARM64_SYS_REG(3, 3, 14, 12, 6),
538 	ARM64_SYS_REG(3, 3, 14, 12, 7),
539 	ARM64_SYS_REG(3, 3, 14, 13, 0),
540 	ARM64_SYS_REG(3, 3, 14, 13, 1),
541 	ARM64_SYS_REG(3, 3, 14, 13, 2),
542 	ARM64_SYS_REG(3, 3, 14, 13, 3),
543 	ARM64_SYS_REG(3, 3, 14, 13, 4),
544 	ARM64_SYS_REG(3, 3, 14, 13, 5),
545 	ARM64_SYS_REG(3, 3, 14, 13, 6),
546 	ARM64_SYS_REG(3, 3, 14, 13, 7),
547 	ARM64_SYS_REG(3, 3, 14, 14, 0),
548 	ARM64_SYS_REG(3, 3, 14, 14, 1),
549 	ARM64_SYS_REG(3, 3, 14, 14, 2),
550 	ARM64_SYS_REG(3, 3, 14, 14, 3),
551 	ARM64_SYS_REG(3, 3, 14, 14, 4),
552 	ARM64_SYS_REG(3, 3, 14, 14, 5),
553 	ARM64_SYS_REG(3, 3, 14, 14, 6),
554 	ARM64_SYS_REG(3, 3, 14, 14, 7),
555 	ARM64_SYS_REG(3, 3, 14, 15, 0),
556 	ARM64_SYS_REG(3, 3, 14, 15, 1),
557 	ARM64_SYS_REG(3, 3, 14, 15, 2),
558 	ARM64_SYS_REG(3, 3, 14, 15, 3),
559 	ARM64_SYS_REG(3, 3, 14, 15, 4),
560 	ARM64_SYS_REG(3, 3, 14, 15, 5),
561 	ARM64_SYS_REG(3, 3, 14, 15, 6),
562 	ARM64_SYS_REG(3, 3, 14, 15, 7),	/* PMCCFILTR_EL0 */
563 };
564 
565 static __u64 vregs[] = {
566 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]),
567 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[1]),
568 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[2]),
569 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[3]),
570 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[4]),
571 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[5]),
572 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[6]),
573 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[7]),
574 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[8]),
575 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[9]),
576 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[10]),
577 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[11]),
578 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[12]),
579 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[13]),
580 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[14]),
581 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[15]),
582 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[16]),
583 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[17]),
584 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[18]),
585 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[19]),
586 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[20]),
587 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[21]),
588 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[22]),
589 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[23]),
590 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[24]),
591 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[25]),
592 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[26]),
593 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[27]),
594 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[28]),
595 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[29]),
596 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[30]),
597 	KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[31]),
598 };
599 
600 static __u64 sve_regs[] = {
601 	KVM_REG_ARM64_SVE_VLS,
602 	KVM_REG_ARM64_SVE_ZREG(0, 0),
603 	KVM_REG_ARM64_SVE_ZREG(1, 0),
604 	KVM_REG_ARM64_SVE_ZREG(2, 0),
605 	KVM_REG_ARM64_SVE_ZREG(3, 0),
606 	KVM_REG_ARM64_SVE_ZREG(4, 0),
607 	KVM_REG_ARM64_SVE_ZREG(5, 0),
608 	KVM_REG_ARM64_SVE_ZREG(6, 0),
609 	KVM_REG_ARM64_SVE_ZREG(7, 0),
610 	KVM_REG_ARM64_SVE_ZREG(8, 0),
611 	KVM_REG_ARM64_SVE_ZREG(9, 0),
612 	KVM_REG_ARM64_SVE_ZREG(10, 0),
613 	KVM_REG_ARM64_SVE_ZREG(11, 0),
614 	KVM_REG_ARM64_SVE_ZREG(12, 0),
615 	KVM_REG_ARM64_SVE_ZREG(13, 0),
616 	KVM_REG_ARM64_SVE_ZREG(14, 0),
617 	KVM_REG_ARM64_SVE_ZREG(15, 0),
618 	KVM_REG_ARM64_SVE_ZREG(16, 0),
619 	KVM_REG_ARM64_SVE_ZREG(17, 0),
620 	KVM_REG_ARM64_SVE_ZREG(18, 0),
621 	KVM_REG_ARM64_SVE_ZREG(19, 0),
622 	KVM_REG_ARM64_SVE_ZREG(20, 0),
623 	KVM_REG_ARM64_SVE_ZREG(21, 0),
624 	KVM_REG_ARM64_SVE_ZREG(22, 0),
625 	KVM_REG_ARM64_SVE_ZREG(23, 0),
626 	KVM_REG_ARM64_SVE_ZREG(24, 0),
627 	KVM_REG_ARM64_SVE_ZREG(25, 0),
628 	KVM_REG_ARM64_SVE_ZREG(26, 0),
629 	KVM_REG_ARM64_SVE_ZREG(27, 0),
630 	KVM_REG_ARM64_SVE_ZREG(28, 0),
631 	KVM_REG_ARM64_SVE_ZREG(29, 0),
632 	KVM_REG_ARM64_SVE_ZREG(30, 0),
633 	KVM_REG_ARM64_SVE_ZREG(31, 0),
634 	KVM_REG_ARM64_SVE_PREG(0, 0),
635 	KVM_REG_ARM64_SVE_PREG(1, 0),
636 	KVM_REG_ARM64_SVE_PREG(2, 0),
637 	KVM_REG_ARM64_SVE_PREG(3, 0),
638 	KVM_REG_ARM64_SVE_PREG(4, 0),
639 	KVM_REG_ARM64_SVE_PREG(5, 0),
640 	KVM_REG_ARM64_SVE_PREG(6, 0),
641 	KVM_REG_ARM64_SVE_PREG(7, 0),
642 	KVM_REG_ARM64_SVE_PREG(8, 0),
643 	KVM_REG_ARM64_SVE_PREG(9, 0),
644 	KVM_REG_ARM64_SVE_PREG(10, 0),
645 	KVM_REG_ARM64_SVE_PREG(11, 0),
646 	KVM_REG_ARM64_SVE_PREG(12, 0),
647 	KVM_REG_ARM64_SVE_PREG(13, 0),
648 	KVM_REG_ARM64_SVE_PREG(14, 0),
649 	KVM_REG_ARM64_SVE_PREG(15, 0),
650 	KVM_REG_ARM64_SVE_FFR(0),
651 	ARM64_SYS_REG(3, 0, 1, 2, 0),   /* ZCR_EL1 */
652 };
653 
654 static __u64 sve_rejects_set[] = {
655 	KVM_REG_ARM64_SVE_VLS,
656 };
657 
658 static __u64 pauth_addr_regs[] = {
659 	ARM64_SYS_REG(3, 0, 2, 1, 0),	/* APIAKEYLO_EL1 */
660 	ARM64_SYS_REG(3, 0, 2, 1, 1),	/* APIAKEYHI_EL1 */
661 	ARM64_SYS_REG(3, 0, 2, 1, 2),	/* APIBKEYLO_EL1 */
662 	ARM64_SYS_REG(3, 0, 2, 1, 3),	/* APIBKEYHI_EL1 */
663 	ARM64_SYS_REG(3, 0, 2, 2, 0),	/* APDAKEYLO_EL1 */
664 	ARM64_SYS_REG(3, 0, 2, 2, 1),	/* APDAKEYHI_EL1 */
665 	ARM64_SYS_REG(3, 0, 2, 2, 2),	/* APDBKEYLO_EL1 */
666 	ARM64_SYS_REG(3, 0, 2, 2, 3)	/* APDBKEYHI_EL1 */
667 };
668 
669 static __u64 pauth_generic_regs[] = {
670 	ARM64_SYS_REG(3, 0, 2, 3, 0),	/* APGAKEYLO_EL1 */
671 	ARM64_SYS_REG(3, 0, 2, 3, 1),	/* APGAKEYHI_EL1 */
672 };
673 
674 #define BASE_SUBLIST \
675 	{ "base", .regs = base_regs, .regs_n = ARRAY_SIZE(base_regs), }
676 #define VREGS_SUBLIST \
677 	{ "vregs", .regs = vregs, .regs_n = ARRAY_SIZE(vregs), }
678 #define PMU_SUBLIST \
679 	{ "pmu", .capability = KVM_CAP_ARM_PMU_V3, .feature = KVM_ARM_VCPU_PMU_V3, \
680 	  .regs = pmu_regs, .regs_n = ARRAY_SIZE(pmu_regs), }
681 #define SVE_SUBLIST \
682 	{ "sve", .capability = KVM_CAP_ARM_SVE, .feature = KVM_ARM_VCPU_SVE, .finalize = true, \
683 	  .regs = sve_regs, .regs_n = ARRAY_SIZE(sve_regs), \
684 	  .rejects_set = sve_rejects_set, .rejects_set_n = ARRAY_SIZE(sve_rejects_set), }
685 #define PAUTH_SUBLIST							\
686 	{								\
687 		.name 		= "pauth_address",			\
688 		.capability	= KVM_CAP_ARM_PTRAUTH_ADDRESS,		\
689 		.feature	= KVM_ARM_VCPU_PTRAUTH_ADDRESS,		\
690 		.regs		= pauth_addr_regs,			\
691 		.regs_n		= ARRAY_SIZE(pauth_addr_regs),		\
692 	},								\
693 	{								\
694 		.name 		= "pauth_generic",			\
695 		.capability	= KVM_CAP_ARM_PTRAUTH_GENERIC,		\
696 		.feature	= KVM_ARM_VCPU_PTRAUTH_GENERIC,		\
697 		.regs		= pauth_generic_regs,			\
698 		.regs_n		= ARRAY_SIZE(pauth_generic_regs),	\
699 	}
700 
701 static struct vcpu_reg_list vregs_config = {
702 	.sublists = {
703 	BASE_SUBLIST,
704 	VREGS_SUBLIST,
705 	{0},
706 	},
707 };
708 static struct vcpu_reg_list vregs_pmu_config = {
709 	.sublists = {
710 	BASE_SUBLIST,
711 	VREGS_SUBLIST,
712 	PMU_SUBLIST,
713 	{0},
714 	},
715 };
716 static struct vcpu_reg_list sve_config = {
717 	.sublists = {
718 	BASE_SUBLIST,
719 	SVE_SUBLIST,
720 	{0},
721 	},
722 };
723 static struct vcpu_reg_list sve_pmu_config = {
724 	.sublists = {
725 	BASE_SUBLIST,
726 	SVE_SUBLIST,
727 	PMU_SUBLIST,
728 	{0},
729 	},
730 };
731 static struct vcpu_reg_list pauth_config = {
732 	.sublists = {
733 	BASE_SUBLIST,
734 	VREGS_SUBLIST,
735 	PAUTH_SUBLIST,
736 	{0},
737 	},
738 };
739 static struct vcpu_reg_list pauth_pmu_config = {
740 	.sublists = {
741 	BASE_SUBLIST,
742 	VREGS_SUBLIST,
743 	PAUTH_SUBLIST,
744 	PMU_SUBLIST,
745 	{0},
746 	},
747 };
748 
749 struct vcpu_reg_list *vcpu_configs[] = {
750 	&vregs_config,
751 	&vregs_pmu_config,
752 	&sve_config,
753 	&sve_pmu_config,
754 	&pauth_config,
755 	&pauth_pmu_config,
756 };
757 int vcpu_configs_n = ARRAY_SIZE(vcpu_configs);
758