xref: /openbmc/linux/arch/arm64/kernel/cpufeature.c (revision 479965a2)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Contains CPU feature definitions
4  *
5  * Copyright (C) 2015 ARM Ltd.
6  *
7  * A note for the weary kernel hacker: the code here is confusing and hard to
8  * follow! That's partly because it's solving a nasty problem, but also because
9  * there's a little bit of over-abstraction that tends to obscure what's going
10  * on behind a maze of helper functions and macros.
11  *
12  * The basic problem is that hardware folks have started gluing together CPUs
13  * with distinct architectural features; in some cases even creating SoCs where
14  * user-visible instructions are available only on a subset of the available
15  * cores. We try to address this by snapshotting the feature registers of the
16  * boot CPU and comparing these with the feature registers of each secondary
17  * CPU when bringing them up. If there is a mismatch, then we update the
18  * snapshot state to indicate the lowest-common denominator of the feature,
19  * known as the "safe" value. This snapshot state can be queried to view the
20  * "sanitised" value of a feature register.
21  *
22  * The sanitised register values are used to decide which capabilities we
23  * have in the system. These may be in the form of traditional "hwcaps"
24  * advertised to userspace or internal "cpucaps" which are used to configure
25  * things like alternative patching and static keys. While a feature mismatch
26  * may result in a TAINT_CPU_OUT_OF_SPEC kernel taint, a capability mismatch
27  * may prevent a CPU from being onlined at all.
28  *
29  * Some implementation details worth remembering:
30  *
31  * - Mismatched features are *always* sanitised to a "safe" value, which
32  *   usually indicates that the feature is not supported.
33  *
34  * - A mismatched feature marked with FTR_STRICT will cause a "SANITY CHECK"
35  *   warning when onlining an offending CPU and the kernel will be tainted
36  *   with TAINT_CPU_OUT_OF_SPEC.
37  *
38  * - Features marked as FTR_VISIBLE have their sanitised value visible to
39  *   userspace. FTR_VISIBLE features in registers that are only visible
40  *   to EL0 by trapping *must* have a corresponding HWCAP so that late
41  *   onlining of CPUs cannot lead to features disappearing at runtime.
42  *
43  * - A "feature" is typically a 4-bit register field. A "capability" is the
44  *   high-level description derived from the sanitised field value.
45  *
46  * - Read the Arm ARM (DDI 0487F.a) section D13.1.3 ("Principles of the ID
47  *   scheme for fields in ID registers") to understand when feature fields
48  *   may be signed or unsigned (FTR_SIGNED and FTR_UNSIGNED accordingly).
49  *
50  * - KVM exposes its own view of the feature registers to guest operating
51  *   systems regardless of FTR_VISIBLE. This is typically driven from the
52  *   sanitised register values to allow virtual CPUs to be migrated between
53  *   arbitrary physical CPUs, but some features not present on the host are
54  *   also advertised and emulated. Look at sys_reg_descs[] for the gory
55  *   details.
56  *
57  * - If the arm64_ftr_bits[] for a register has a missing field, then this
58  *   field is treated as STRICT RES0, including for read_sanitised_ftr_reg().
59  *   This is stronger than FTR_HIDDEN and can be used to hide features from
60  *   KVM guests.
61  */
62 
63 #define pr_fmt(fmt) "CPU features: " fmt
64 
65 #include <linux/bsearch.h>
66 #include <linux/cpumask.h>
67 #include <linux/crash_dump.h>
68 #include <linux/kstrtox.h>
69 #include <linux/sort.h>
70 #include <linux/stop_machine.h>
71 #include <linux/sysfs.h>
72 #include <linux/types.h>
73 #include <linux/minmax.h>
74 #include <linux/mm.h>
75 #include <linux/cpu.h>
76 #include <linux/kasan.h>
77 #include <linux/percpu.h>
78 
79 #include <asm/cpu.h>
80 #include <asm/cpufeature.h>
81 #include <asm/cpu_ops.h>
82 #include <asm/fpsimd.h>
83 #include <asm/hwcap.h>
84 #include <asm/insn.h>
85 #include <asm/kvm_host.h>
86 #include <asm/mmu_context.h>
87 #include <asm/mte.h>
88 #include <asm/processor.h>
89 #include <asm/smp.h>
90 #include <asm/sysreg.h>
91 #include <asm/traps.h>
92 #include <asm/vectors.h>
93 #include <asm/virt.h>
94 
95 /* Kernel representation of AT_HWCAP and AT_HWCAP2 */
96 static DECLARE_BITMAP(elf_hwcap, MAX_CPU_FEATURES) __read_mostly;
97 
98 #ifdef CONFIG_COMPAT
99 #define COMPAT_ELF_HWCAP_DEFAULT	\
100 				(COMPAT_HWCAP_HALF|COMPAT_HWCAP_THUMB|\
101 				 COMPAT_HWCAP_FAST_MULT|COMPAT_HWCAP_EDSP|\
102 				 COMPAT_HWCAP_TLS|COMPAT_HWCAP_IDIV|\
103 				 COMPAT_HWCAP_LPAE)
104 unsigned int compat_elf_hwcap __read_mostly = COMPAT_ELF_HWCAP_DEFAULT;
105 unsigned int compat_elf_hwcap2 __read_mostly;
106 #endif
107 
108 DECLARE_BITMAP(system_cpucaps, ARM64_NCAPS);
109 EXPORT_SYMBOL(system_cpucaps);
110 static struct arm64_cpu_capabilities const __ro_after_init *cpucap_ptrs[ARM64_NCAPS];
111 
112 DECLARE_BITMAP(boot_cpucaps, ARM64_NCAPS);
113 
114 bool arm64_use_ng_mappings = false;
115 EXPORT_SYMBOL(arm64_use_ng_mappings);
116 
117 DEFINE_PER_CPU_READ_MOSTLY(const char *, this_cpu_vector) = vectors;
118 
119 /*
120  * Permit PER_LINUX32 and execve() of 32-bit binaries even if not all CPUs
121  * support it?
122  */
123 static bool __read_mostly allow_mismatched_32bit_el0;
124 
125 /*
126  * Static branch enabled only if allow_mismatched_32bit_el0 is set and we have
127  * seen at least one CPU capable of 32-bit EL0.
128  */
129 DEFINE_STATIC_KEY_FALSE(arm64_mismatched_32bit_el0);
130 
131 /*
132  * Mask of CPUs supporting 32-bit EL0.
133  * Only valid if arm64_mismatched_32bit_el0 is enabled.
134  */
135 static cpumask_var_t cpu_32bit_el0_mask __cpumask_var_read_mostly;
136 
137 void dump_cpu_features(void)
138 {
139 	/* file-wide pr_fmt adds "CPU features: " prefix */
140 	pr_emerg("0x%*pb\n", ARM64_NCAPS, &system_cpucaps);
141 }
142 
143 #define ARM64_CPUID_FIELDS(reg, field, min_value)			\
144 		.sys_reg = SYS_##reg,							\
145 		.field_pos = reg##_##field##_SHIFT,						\
146 		.field_width = reg##_##field##_WIDTH,						\
147 		.sign = reg##_##field##_SIGNED,							\
148 		.min_field_value = reg##_##field##_##min_value,
149 
150 #define __ARM64_FTR_BITS(SIGNED, VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \
151 	{						\
152 		.sign = SIGNED,				\
153 		.visible = VISIBLE,			\
154 		.strict = STRICT,			\
155 		.type = TYPE,				\
156 		.shift = SHIFT,				\
157 		.width = WIDTH,				\
158 		.safe_val = SAFE_VAL,			\
159 	}
160 
161 /* Define a feature with unsigned values */
162 #define ARM64_FTR_BITS(VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \
163 	__ARM64_FTR_BITS(FTR_UNSIGNED, VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL)
164 
165 /* Define a feature with a signed value */
166 #define S_ARM64_FTR_BITS(VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \
167 	__ARM64_FTR_BITS(FTR_SIGNED, VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL)
168 
169 #define ARM64_FTR_END					\
170 	{						\
171 		.width = 0,				\
172 	}
173 
174 static void cpu_enable_cnp(struct arm64_cpu_capabilities const *cap);
175 
176 static bool __system_matches_cap(unsigned int n);
177 
178 /*
179  * NOTE: Any changes to the visibility of features should be kept in
180  * sync with the documentation of the CPU feature register ABI.
181  */
182 static const struct arm64_ftr_bits ftr_id_aa64isar0[] = {
183 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_RNDR_SHIFT, 4, 0),
184 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_TLB_SHIFT, 4, 0),
185 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_TS_SHIFT, 4, 0),
186 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_FHM_SHIFT, 4, 0),
187 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_DP_SHIFT, 4, 0),
188 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_SM4_SHIFT, 4, 0),
189 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_SM3_SHIFT, 4, 0),
190 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_SHA3_SHIFT, 4, 0),
191 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_RDM_SHIFT, 4, 0),
192 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_ATOMIC_SHIFT, 4, 0),
193 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_CRC32_SHIFT, 4, 0),
194 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_SHA2_SHIFT, 4, 0),
195 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_SHA1_SHIFT, 4, 0),
196 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_AES_SHIFT, 4, 0),
197 	ARM64_FTR_END,
198 };
199 
200 static const struct arm64_ftr_bits ftr_id_aa64isar1[] = {
201 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_I8MM_SHIFT, 4, 0),
202 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_DGH_SHIFT, 4, 0),
203 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_BF16_SHIFT, 4, 0),
204 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_SPECRES_SHIFT, 4, 0),
205 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_SB_SHIFT, 4, 0),
206 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_FRINTTS_SHIFT, 4, 0),
207 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH),
208 		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_GPI_SHIFT, 4, 0),
209 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH),
210 		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_GPA_SHIFT, 4, 0),
211 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_LRCPC_SHIFT, 4, 0),
212 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_FCMA_SHIFT, 4, 0),
213 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_JSCVT_SHIFT, 4, 0),
214 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH),
215 		       FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_EL1_API_SHIFT, 4, 0),
216 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH),
217 		       FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_EL1_APA_SHIFT, 4, 0),
218 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_DPB_SHIFT, 4, 0),
219 	ARM64_FTR_END,
220 };
221 
222 static const struct arm64_ftr_bits ftr_id_aa64isar2[] = {
223 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_CSSC_SHIFT, 4, 0),
224 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_RPRFM_SHIFT, 4, 0),
225 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_CLRBHB_SHIFT, 4, 0),
226 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_BC_SHIFT, 4, 0),
227 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_MOPS_SHIFT, 4, 0),
228 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH),
229 		       FTR_STRICT, FTR_EXACT, ID_AA64ISAR2_EL1_APA3_SHIFT, 4, 0),
230 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH),
231 		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_GPA3_SHIFT, 4, 0),
232 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_RPRES_SHIFT, 4, 0),
233 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_WFxT_SHIFT, 4, 0),
234 	ARM64_FTR_END,
235 };
236 
237 static const struct arm64_ftr_bits ftr_id_aa64pfr0[] = {
238 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_CSV3_SHIFT, 4, 0),
239 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_CSV2_SHIFT, 4, 0),
240 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_DIT_SHIFT, 4, 0),
241 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_AMU_SHIFT, 4, 0),
242 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_MPAM_SHIFT, 4, 0),
243 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_SEL2_SHIFT, 4, 0),
244 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
245 				   FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_SVE_SHIFT, 4, 0),
246 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_RAS_SHIFT, 4, 0),
247 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_GIC_SHIFT, 4, 0),
248 	S_ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_AdvSIMD_SHIFT, 4, ID_AA64PFR0_EL1_AdvSIMD_NI),
249 	S_ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_FP_SHIFT, 4, ID_AA64PFR0_EL1_FP_NI),
250 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_EL3_SHIFT, 4, 0),
251 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_EL2_SHIFT, 4, 0),
252 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_EL1_SHIFT, 4, ID_AA64PFR0_EL1_ELx_64BIT_ONLY),
253 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_EL0_SHIFT, 4, ID_AA64PFR0_EL1_ELx_64BIT_ONLY),
254 	ARM64_FTR_END,
255 };
256 
257 static const struct arm64_ftr_bits ftr_id_aa64pfr1[] = {
258 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
259 		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_SME_SHIFT, 4, 0),
260 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_MPAM_frac_SHIFT, 4, 0),
261 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_RAS_frac_SHIFT, 4, 0),
262 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_MTE),
263 		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_MTE_SHIFT, 4, ID_AA64PFR1_EL1_MTE_NI),
264 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_SSBS_SHIFT, 4, ID_AA64PFR1_EL1_SSBS_NI),
265 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_BTI),
266 				    FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_BT_SHIFT, 4, 0),
267 	ARM64_FTR_END,
268 };
269 
270 static const struct arm64_ftr_bits ftr_id_aa64zfr0[] = {
271 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
272 		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_F64MM_SHIFT, 4, 0),
273 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
274 		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_F32MM_SHIFT, 4, 0),
275 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
276 		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_I8MM_SHIFT, 4, 0),
277 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
278 		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_SM4_SHIFT, 4, 0),
279 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
280 		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_SHA3_SHIFT, 4, 0),
281 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
282 		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_BF16_SHIFT, 4, 0),
283 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
284 		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_BitPerm_SHIFT, 4, 0),
285 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
286 		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_AES_SHIFT, 4, 0),
287 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
288 		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_SVEver_SHIFT, 4, 0),
289 	ARM64_FTR_END,
290 };
291 
292 static const struct arm64_ftr_bits ftr_id_aa64smfr0[] = {
293 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
294 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_FA64_SHIFT, 1, 0),
295 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
296 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_SMEver_SHIFT, 4, 0),
297 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
298 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_I16I64_SHIFT, 4, 0),
299 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
300 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_F64F64_SHIFT, 1, 0),
301 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
302 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_I16I32_SHIFT, 4, 0),
303 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
304 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_B16B16_SHIFT, 1, 0),
305 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
306 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_F16F16_SHIFT, 1, 0),
307 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
308 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_I8I32_SHIFT, 4, 0),
309 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
310 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_F16F32_SHIFT, 1, 0),
311 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
312 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_B16F32_SHIFT, 1, 0),
313 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
314 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_BI32I32_SHIFT, 1, 0),
315 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
316 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_F32F32_SHIFT, 1, 0),
317 	ARM64_FTR_END,
318 };
319 
320 static const struct arm64_ftr_bits ftr_id_aa64mmfr0[] = {
321 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_ECV_SHIFT, 4, 0),
322 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_FGT_SHIFT, 4, 0),
323 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_EXS_SHIFT, 4, 0),
324 	/*
325 	 * Page size not being supported at Stage-2 is not fatal. You
326 	 * just give up KVM if PAGE_SIZE isn't supported there. Go fix
327 	 * your favourite nesting hypervisor.
328 	 *
329 	 * There is a small corner case where the hypervisor explicitly
330 	 * advertises a given granule size at Stage-2 (value 2) on some
331 	 * vCPUs, and uses the fallback to Stage-1 (value 0) for other
332 	 * vCPUs. Although this is not forbidden by the architecture, it
333 	 * indicates that the hypervisor is being silly (or buggy).
334 	 *
335 	 * We make no effort to cope with this and pretend that if these
336 	 * fields are inconsistent across vCPUs, then it isn't worth
337 	 * trying to bring KVM up.
338 	 */
339 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64MMFR0_EL1_TGRAN4_2_SHIFT, 4, 1),
340 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64MMFR0_EL1_TGRAN64_2_SHIFT, 4, 1),
341 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64MMFR0_EL1_TGRAN16_2_SHIFT, 4, 1),
342 	/*
343 	 * We already refuse to boot CPUs that don't support our configured
344 	 * page size, so we can only detect mismatches for a page size other
345 	 * than the one we're currently using. Unfortunately, SoCs like this
346 	 * exist in the wild so, even though we don't like it, we'll have to go
347 	 * along with it and treat them as non-strict.
348 	 */
349 	S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_TGRAN4_SHIFT, 4, ID_AA64MMFR0_EL1_TGRAN4_NI),
350 	S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_TGRAN64_SHIFT, 4, ID_AA64MMFR0_EL1_TGRAN64_NI),
351 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_TGRAN16_SHIFT, 4, ID_AA64MMFR0_EL1_TGRAN16_NI),
352 
353 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_BIGENDEL0_SHIFT, 4, 0),
354 	/* Linux shouldn't care about secure memory */
355 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_SNSMEM_SHIFT, 4, 0),
356 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_BIGEND_SHIFT, 4, 0),
357 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_ASIDBITS_SHIFT, 4, 0),
358 	/*
359 	 * Differing PARange is fine as long as all peripherals and memory are mapped
360 	 * within the minimum PARange of all CPUs
361 	 */
362 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_PARANGE_SHIFT, 4, 0),
363 	ARM64_FTR_END,
364 };
365 
366 static const struct arm64_ftr_bits ftr_id_aa64mmfr1[] = {
367 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_TIDCP1_SHIFT, 4, 0),
368 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_AFP_SHIFT, 4, 0),
369 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_HCX_SHIFT, 4, 0),
370 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_ETS_SHIFT, 4, 0),
371 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_TWED_SHIFT, 4, 0),
372 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_XNX_SHIFT, 4, 0),
373 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_AA64MMFR1_EL1_SpecSEI_SHIFT, 4, 0),
374 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_PAN_SHIFT, 4, 0),
375 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_LO_SHIFT, 4, 0),
376 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_HPDS_SHIFT, 4, 0),
377 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_VH_SHIFT, 4, 0),
378 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_VMIDBits_SHIFT, 4, 0),
379 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_HAFDBS_SHIFT, 4, 0),
380 	ARM64_FTR_END,
381 };
382 
383 static const struct arm64_ftr_bits ftr_id_aa64mmfr2[] = {
384 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_E0PD_SHIFT, 4, 0),
385 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_EVT_SHIFT, 4, 0),
386 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_BBM_SHIFT, 4, 0),
387 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_TTL_SHIFT, 4, 0),
388 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_FWB_SHIFT, 4, 0),
389 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_IDS_SHIFT, 4, 0),
390 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_AT_SHIFT, 4, 0),
391 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_ST_SHIFT, 4, 0),
392 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_NV_SHIFT, 4, 0),
393 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_CCIDX_SHIFT, 4, 0),
394 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_VARange_SHIFT, 4, 0),
395 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_IESB_SHIFT, 4, 0),
396 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_LSM_SHIFT, 4, 0),
397 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_UAO_SHIFT, 4, 0),
398 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_CnP_SHIFT, 4, 0),
399 	ARM64_FTR_END,
400 };
401 
402 static const struct arm64_ftr_bits ftr_id_aa64mmfr3[] = {
403 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR3_EL1_S1PIE_SHIFT, 4, 0),
404 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR3_EL1_TCRX_SHIFT, 4, 0),
405 	ARM64_FTR_END,
406 };
407 
408 static const struct arm64_ftr_bits ftr_ctr[] = {
409 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, 31, 1, 1), /* RES1 */
410 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, CTR_EL0_DIC_SHIFT, 1, 1),
411 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, CTR_EL0_IDC_SHIFT, 1, 1),
412 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_HIGHER_OR_ZERO_SAFE, CTR_EL0_CWG_SHIFT, 4, 0),
413 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_HIGHER_OR_ZERO_SAFE, CTR_EL0_ERG_SHIFT, 4, 0),
414 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, CTR_EL0_DminLine_SHIFT, 4, 1),
415 	/*
416 	 * Linux can handle differing I-cache policies. Userspace JITs will
417 	 * make use of *minLine.
418 	 * If we have differing I-cache policies, report it as the weakest - VIPT.
419 	 */
420 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_EXACT, CTR_EL0_L1Ip_SHIFT, 2, CTR_EL0_L1Ip_VIPT),	/* L1Ip */
421 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, CTR_EL0_IminLine_SHIFT, 4, 0),
422 	ARM64_FTR_END,
423 };
424 
425 static struct arm64_ftr_override __ro_after_init no_override = { };
426 
427 struct arm64_ftr_reg arm64_ftr_reg_ctrel0 = {
428 	.name		= "SYS_CTR_EL0",
429 	.ftr_bits	= ftr_ctr,
430 	.override	= &no_override,
431 };
432 
433 static const struct arm64_ftr_bits ftr_id_mmfr0[] = {
434 	S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_InnerShr_SHIFT, 4, 0xf),
435 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_FCSE_SHIFT, 4, 0),
436 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_AuxReg_SHIFT, 4, 0),
437 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_TCM_SHIFT, 4, 0),
438 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_ShareLvl_SHIFT, 4, 0),
439 	S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_OuterShr_SHIFT, 4, 0xf),
440 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_PMSA_SHIFT, 4, 0),
441 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_VMSA_SHIFT, 4, 0),
442 	ARM64_FTR_END,
443 };
444 
445 static const struct arm64_ftr_bits ftr_id_aa64dfr0[] = {
446 	S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_EL1_DoubleLock_SHIFT, 4, 0),
447 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64DFR0_EL1_PMSVer_SHIFT, 4, 0),
448 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_EL1_CTX_CMPs_SHIFT, 4, 0),
449 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_EL1_WRPs_SHIFT, 4, 0),
450 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_EL1_BRPs_SHIFT, 4, 0),
451 	/*
452 	 * We can instantiate multiple PMU instances with different levels
453 	 * of support.
454 	 */
455 	S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64DFR0_EL1_PMUVer_SHIFT, 4, 0),
456 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64DFR0_EL1_DebugVer_SHIFT, 4, 0x6),
457 	ARM64_FTR_END,
458 };
459 
460 static const struct arm64_ftr_bits ftr_mvfr0[] = {
461 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPRound_SHIFT, 4, 0),
462 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPShVec_SHIFT, 4, 0),
463 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPSqrt_SHIFT, 4, 0),
464 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPDivide_SHIFT, 4, 0),
465 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPTrap_SHIFT, 4, 0),
466 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPDP_SHIFT, 4, 0),
467 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPSP_SHIFT, 4, 0),
468 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_SIMDReg_SHIFT, 4, 0),
469 	ARM64_FTR_END,
470 };
471 
472 static const struct arm64_ftr_bits ftr_mvfr1[] = {
473 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_SIMDFMAC_SHIFT, 4, 0),
474 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_FPHP_SHIFT, 4, 0),
475 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_SIMDHP_SHIFT, 4, 0),
476 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_SIMDSP_SHIFT, 4, 0),
477 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_SIMDInt_SHIFT, 4, 0),
478 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_SIMDLS_SHIFT, 4, 0),
479 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_FPDNaN_SHIFT, 4, 0),
480 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_FPFtZ_SHIFT, 4, 0),
481 	ARM64_FTR_END,
482 };
483 
484 static const struct arm64_ftr_bits ftr_mvfr2[] = {
485 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR2_EL1_FPMisc_SHIFT, 4, 0),
486 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR2_EL1_SIMDMisc_SHIFT, 4, 0),
487 	ARM64_FTR_END,
488 };
489 
490 static const struct arm64_ftr_bits ftr_dczid[] = {
491 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, DCZID_EL0_DZP_SHIFT, 1, 1),
492 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, DCZID_EL0_BS_SHIFT, 4, 0),
493 	ARM64_FTR_END,
494 };
495 
496 static const struct arm64_ftr_bits ftr_gmid[] = {
497 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, GMID_EL1_BS_SHIFT, 4, 0),
498 	ARM64_FTR_END,
499 };
500 
501 static const struct arm64_ftr_bits ftr_id_isar0[] = {
502 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_Divide_SHIFT, 4, 0),
503 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_Debug_SHIFT, 4, 0),
504 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_Coproc_SHIFT, 4, 0),
505 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_CmpBranch_SHIFT, 4, 0),
506 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_BitField_SHIFT, 4, 0),
507 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_BitCount_SHIFT, 4, 0),
508 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_Swap_SHIFT, 4, 0),
509 	ARM64_FTR_END,
510 };
511 
512 static const struct arm64_ftr_bits ftr_id_isar5[] = {
513 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_RDM_SHIFT, 4, 0),
514 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_CRC32_SHIFT, 4, 0),
515 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_SHA2_SHIFT, 4, 0),
516 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_SHA1_SHIFT, 4, 0),
517 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_AES_SHIFT, 4, 0),
518 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_SEVL_SHIFT, 4, 0),
519 	ARM64_FTR_END,
520 };
521 
522 static const struct arm64_ftr_bits ftr_id_mmfr4[] = {
523 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_EVT_SHIFT, 4, 0),
524 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_CCIDX_SHIFT, 4, 0),
525 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_LSM_SHIFT, 4, 0),
526 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_HPDS_SHIFT, 4, 0),
527 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_CnP_SHIFT, 4, 0),
528 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_XNX_SHIFT, 4, 0),
529 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_AC2_SHIFT, 4, 0),
530 
531 	/*
532 	 * SpecSEI = 1 indicates that the PE might generate an SError on an
533 	 * external abort on speculative read. It is safe to assume that an
534 	 * SError might be generated than it will not be. Hence it has been
535 	 * classified as FTR_HIGHER_SAFE.
536 	 */
537 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_MMFR4_EL1_SpecSEI_SHIFT, 4, 0),
538 	ARM64_FTR_END,
539 };
540 
541 static const struct arm64_ftr_bits ftr_id_isar4[] = {
542 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_SWP_frac_SHIFT, 4, 0),
543 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_PSR_M_SHIFT, 4, 0),
544 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_SynchPrim_frac_SHIFT, 4, 0),
545 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_Barrier_SHIFT, 4, 0),
546 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_SMC_SHIFT, 4, 0),
547 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_Writeback_SHIFT, 4, 0),
548 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_WithShifts_SHIFT, 4, 0),
549 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_Unpriv_SHIFT, 4, 0),
550 	ARM64_FTR_END,
551 };
552 
553 static const struct arm64_ftr_bits ftr_id_mmfr5[] = {
554 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR5_EL1_ETS_SHIFT, 4, 0),
555 	ARM64_FTR_END,
556 };
557 
558 static const struct arm64_ftr_bits ftr_id_isar6[] = {
559 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_I8MM_SHIFT, 4, 0),
560 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_BF16_SHIFT, 4, 0),
561 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_SPECRES_SHIFT, 4, 0),
562 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_SB_SHIFT, 4, 0),
563 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_FHM_SHIFT, 4, 0),
564 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_DP_SHIFT, 4, 0),
565 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_JSCVT_SHIFT, 4, 0),
566 	ARM64_FTR_END,
567 };
568 
569 static const struct arm64_ftr_bits ftr_id_pfr0[] = {
570 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_DIT_SHIFT, 4, 0),
571 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_CSV2_SHIFT, 4, 0),
572 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_State3_SHIFT, 4, 0),
573 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_State2_SHIFT, 4, 0),
574 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_State1_SHIFT, 4, 0),
575 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_State0_SHIFT, 4, 0),
576 	ARM64_FTR_END,
577 };
578 
579 static const struct arm64_ftr_bits ftr_id_pfr1[] = {
580 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_GIC_SHIFT, 4, 0),
581 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_Virt_frac_SHIFT, 4, 0),
582 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_Sec_frac_SHIFT, 4, 0),
583 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_GenTimer_SHIFT, 4, 0),
584 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_Virtualization_SHIFT, 4, 0),
585 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_MProgMod_SHIFT, 4, 0),
586 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_Security_SHIFT, 4, 0),
587 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_ProgMod_SHIFT, 4, 0),
588 	ARM64_FTR_END,
589 };
590 
591 static const struct arm64_ftr_bits ftr_id_pfr2[] = {
592 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR2_EL1_SSBS_SHIFT, 4, 0),
593 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR2_EL1_CSV3_SHIFT, 4, 0),
594 	ARM64_FTR_END,
595 };
596 
597 static const struct arm64_ftr_bits ftr_id_dfr0[] = {
598 	/* [31:28] TraceFilt */
599 	S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_DFR0_EL1_PerfMon_SHIFT, 4, 0),
600 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_MProfDbg_SHIFT, 4, 0),
601 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_MMapTrc_SHIFT, 4, 0),
602 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_CopTrc_SHIFT, 4, 0),
603 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_MMapDbg_SHIFT, 4, 0),
604 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_CopSDbg_SHIFT, 4, 0),
605 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_CopDbg_SHIFT, 4, 0),
606 	ARM64_FTR_END,
607 };
608 
609 static const struct arm64_ftr_bits ftr_id_dfr1[] = {
610 	S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR1_EL1_MTPMU_SHIFT, 4, 0),
611 	ARM64_FTR_END,
612 };
613 
614 static const struct arm64_ftr_bits ftr_zcr[] = {
615 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE,
616 		ZCR_ELx_LEN_SHIFT, ZCR_ELx_LEN_WIDTH, 0),	/* LEN */
617 	ARM64_FTR_END,
618 };
619 
620 static const struct arm64_ftr_bits ftr_smcr[] = {
621 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE,
622 		SMCR_ELx_LEN_SHIFT, SMCR_ELx_LEN_WIDTH, 0),	/* LEN */
623 	ARM64_FTR_END,
624 };
625 
626 /*
627  * Common ftr bits for a 32bit register with all hidden, strict
628  * attributes, with 4bit feature fields and a default safe value of
629  * 0. Covers the following 32bit registers:
630  * id_isar[1-3], id_mmfr[1-3]
631  */
632 static const struct arm64_ftr_bits ftr_generic_32bits[] = {
633 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 28, 4, 0),
634 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 24, 4, 0),
635 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 20, 4, 0),
636 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 16, 4, 0),
637 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 12, 4, 0),
638 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 8, 4, 0),
639 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0),
640 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0),
641 	ARM64_FTR_END,
642 };
643 
644 /* Table for a single 32bit feature value */
645 static const struct arm64_ftr_bits ftr_single32[] = {
646 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 0, 32, 0),
647 	ARM64_FTR_END,
648 };
649 
650 static const struct arm64_ftr_bits ftr_raz[] = {
651 	ARM64_FTR_END,
652 };
653 
654 #define __ARM64_FTR_REG_OVERRIDE(id_str, id, table, ovr) {	\
655 		.sys_id = id,					\
656 		.reg = 	&(struct arm64_ftr_reg){		\
657 			.name = id_str,				\
658 			.override = (ovr),			\
659 			.ftr_bits = &((table)[0]),		\
660 	}}
661 
662 #define ARM64_FTR_REG_OVERRIDE(id, table, ovr)	\
663 	__ARM64_FTR_REG_OVERRIDE(#id, id, table, ovr)
664 
665 #define ARM64_FTR_REG(id, table)		\
666 	__ARM64_FTR_REG_OVERRIDE(#id, id, table, &no_override)
667 
668 struct arm64_ftr_override __ro_after_init id_aa64mmfr1_override;
669 struct arm64_ftr_override __ro_after_init id_aa64pfr0_override;
670 struct arm64_ftr_override __ro_after_init id_aa64pfr1_override;
671 struct arm64_ftr_override __ro_after_init id_aa64zfr0_override;
672 struct arm64_ftr_override __ro_after_init id_aa64smfr0_override;
673 struct arm64_ftr_override __ro_after_init id_aa64isar1_override;
674 struct arm64_ftr_override __ro_after_init id_aa64isar2_override;
675 
676 struct arm64_ftr_override arm64_sw_feature_override;
677 
678 static const struct __ftr_reg_entry {
679 	u32			sys_id;
680 	struct arm64_ftr_reg 	*reg;
681 } arm64_ftr_regs[] = {
682 
683 	/* Op1 = 0, CRn = 0, CRm = 1 */
684 	ARM64_FTR_REG(SYS_ID_PFR0_EL1, ftr_id_pfr0),
685 	ARM64_FTR_REG(SYS_ID_PFR1_EL1, ftr_id_pfr1),
686 	ARM64_FTR_REG(SYS_ID_DFR0_EL1, ftr_id_dfr0),
687 	ARM64_FTR_REG(SYS_ID_MMFR0_EL1, ftr_id_mmfr0),
688 	ARM64_FTR_REG(SYS_ID_MMFR1_EL1, ftr_generic_32bits),
689 	ARM64_FTR_REG(SYS_ID_MMFR2_EL1, ftr_generic_32bits),
690 	ARM64_FTR_REG(SYS_ID_MMFR3_EL1, ftr_generic_32bits),
691 
692 	/* Op1 = 0, CRn = 0, CRm = 2 */
693 	ARM64_FTR_REG(SYS_ID_ISAR0_EL1, ftr_id_isar0),
694 	ARM64_FTR_REG(SYS_ID_ISAR1_EL1, ftr_generic_32bits),
695 	ARM64_FTR_REG(SYS_ID_ISAR2_EL1, ftr_generic_32bits),
696 	ARM64_FTR_REG(SYS_ID_ISAR3_EL1, ftr_generic_32bits),
697 	ARM64_FTR_REG(SYS_ID_ISAR4_EL1, ftr_id_isar4),
698 	ARM64_FTR_REG(SYS_ID_ISAR5_EL1, ftr_id_isar5),
699 	ARM64_FTR_REG(SYS_ID_MMFR4_EL1, ftr_id_mmfr4),
700 	ARM64_FTR_REG(SYS_ID_ISAR6_EL1, ftr_id_isar6),
701 
702 	/* Op1 = 0, CRn = 0, CRm = 3 */
703 	ARM64_FTR_REG(SYS_MVFR0_EL1, ftr_mvfr0),
704 	ARM64_FTR_REG(SYS_MVFR1_EL1, ftr_mvfr1),
705 	ARM64_FTR_REG(SYS_MVFR2_EL1, ftr_mvfr2),
706 	ARM64_FTR_REG(SYS_ID_PFR2_EL1, ftr_id_pfr2),
707 	ARM64_FTR_REG(SYS_ID_DFR1_EL1, ftr_id_dfr1),
708 	ARM64_FTR_REG(SYS_ID_MMFR5_EL1, ftr_id_mmfr5),
709 
710 	/* Op1 = 0, CRn = 0, CRm = 4 */
711 	ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64PFR0_EL1, ftr_id_aa64pfr0,
712 			       &id_aa64pfr0_override),
713 	ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64PFR1_EL1, ftr_id_aa64pfr1,
714 			       &id_aa64pfr1_override),
715 	ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64ZFR0_EL1, ftr_id_aa64zfr0,
716 			       &id_aa64zfr0_override),
717 	ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64SMFR0_EL1, ftr_id_aa64smfr0,
718 			       &id_aa64smfr0_override),
719 
720 	/* Op1 = 0, CRn = 0, CRm = 5 */
721 	ARM64_FTR_REG(SYS_ID_AA64DFR0_EL1, ftr_id_aa64dfr0),
722 	ARM64_FTR_REG(SYS_ID_AA64DFR1_EL1, ftr_raz),
723 
724 	/* Op1 = 0, CRn = 0, CRm = 6 */
725 	ARM64_FTR_REG(SYS_ID_AA64ISAR0_EL1, ftr_id_aa64isar0),
726 	ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64ISAR1_EL1, ftr_id_aa64isar1,
727 			       &id_aa64isar1_override),
728 	ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64ISAR2_EL1, ftr_id_aa64isar2,
729 			       &id_aa64isar2_override),
730 
731 	/* Op1 = 0, CRn = 0, CRm = 7 */
732 	ARM64_FTR_REG(SYS_ID_AA64MMFR0_EL1, ftr_id_aa64mmfr0),
733 	ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64MMFR1_EL1, ftr_id_aa64mmfr1,
734 			       &id_aa64mmfr1_override),
735 	ARM64_FTR_REG(SYS_ID_AA64MMFR2_EL1, ftr_id_aa64mmfr2),
736 	ARM64_FTR_REG(SYS_ID_AA64MMFR3_EL1, ftr_id_aa64mmfr3),
737 
738 	/* Op1 = 0, CRn = 1, CRm = 2 */
739 	ARM64_FTR_REG(SYS_ZCR_EL1, ftr_zcr),
740 	ARM64_FTR_REG(SYS_SMCR_EL1, ftr_smcr),
741 
742 	/* Op1 = 1, CRn = 0, CRm = 0 */
743 	ARM64_FTR_REG(SYS_GMID_EL1, ftr_gmid),
744 
745 	/* Op1 = 3, CRn = 0, CRm = 0 */
746 	{ SYS_CTR_EL0, &arm64_ftr_reg_ctrel0 },
747 	ARM64_FTR_REG(SYS_DCZID_EL0, ftr_dczid),
748 
749 	/* Op1 = 3, CRn = 14, CRm = 0 */
750 	ARM64_FTR_REG(SYS_CNTFRQ_EL0, ftr_single32),
751 };
752 
753 static int search_cmp_ftr_reg(const void *id, const void *regp)
754 {
755 	return (int)(unsigned long)id - (int)((const struct __ftr_reg_entry *)regp)->sys_id;
756 }
757 
758 /*
759  * get_arm64_ftr_reg_nowarn - Looks up a feature register entry using
760  * its sys_reg() encoding. With the array arm64_ftr_regs sorted in the
761  * ascending order of sys_id, we use binary search to find a matching
762  * entry.
763  *
764  * returns - Upon success,  matching ftr_reg entry for id.
765  *         - NULL on failure. It is upto the caller to decide
766  *	     the impact of a failure.
767  */
768 static struct arm64_ftr_reg *get_arm64_ftr_reg_nowarn(u32 sys_id)
769 {
770 	const struct __ftr_reg_entry *ret;
771 
772 	ret = bsearch((const void *)(unsigned long)sys_id,
773 			arm64_ftr_regs,
774 			ARRAY_SIZE(arm64_ftr_regs),
775 			sizeof(arm64_ftr_regs[0]),
776 			search_cmp_ftr_reg);
777 	if (ret)
778 		return ret->reg;
779 	return NULL;
780 }
781 
782 /*
783  * get_arm64_ftr_reg - Looks up a feature register entry using
784  * its sys_reg() encoding. This calls get_arm64_ftr_reg_nowarn().
785  *
786  * returns - Upon success,  matching ftr_reg entry for id.
787  *         - NULL on failure but with an WARN_ON().
788  */
789 struct arm64_ftr_reg *get_arm64_ftr_reg(u32 sys_id)
790 {
791 	struct arm64_ftr_reg *reg;
792 
793 	reg = get_arm64_ftr_reg_nowarn(sys_id);
794 
795 	/*
796 	 * Requesting a non-existent register search is an error. Warn
797 	 * and let the caller handle it.
798 	 */
799 	WARN_ON(!reg);
800 	return reg;
801 }
802 
803 static u64 arm64_ftr_set_value(const struct arm64_ftr_bits *ftrp, s64 reg,
804 			       s64 ftr_val)
805 {
806 	u64 mask = arm64_ftr_mask(ftrp);
807 
808 	reg &= ~mask;
809 	reg |= (ftr_val << ftrp->shift) & mask;
810 	return reg;
811 }
812 
813 s64 arm64_ftr_safe_value(const struct arm64_ftr_bits *ftrp, s64 new,
814 				s64 cur)
815 {
816 	s64 ret = 0;
817 
818 	switch (ftrp->type) {
819 	case FTR_EXACT:
820 		ret = ftrp->safe_val;
821 		break;
822 	case FTR_LOWER_SAFE:
823 		ret = min(new, cur);
824 		break;
825 	case FTR_HIGHER_OR_ZERO_SAFE:
826 		if (!cur || !new)
827 			break;
828 		fallthrough;
829 	case FTR_HIGHER_SAFE:
830 		ret = max(new, cur);
831 		break;
832 	default:
833 		BUG();
834 	}
835 
836 	return ret;
837 }
838 
839 static void __init sort_ftr_regs(void)
840 {
841 	unsigned int i;
842 
843 	for (i = 0; i < ARRAY_SIZE(arm64_ftr_regs); i++) {
844 		const struct arm64_ftr_reg *ftr_reg = arm64_ftr_regs[i].reg;
845 		const struct arm64_ftr_bits *ftr_bits = ftr_reg->ftr_bits;
846 		unsigned int j = 0;
847 
848 		/*
849 		 * Features here must be sorted in descending order with respect
850 		 * to their shift values and should not overlap with each other.
851 		 */
852 		for (; ftr_bits->width != 0; ftr_bits++, j++) {
853 			unsigned int width = ftr_reg->ftr_bits[j].width;
854 			unsigned int shift = ftr_reg->ftr_bits[j].shift;
855 			unsigned int prev_shift;
856 
857 			WARN((shift  + width) > 64,
858 				"%s has invalid feature at shift %d\n",
859 				ftr_reg->name, shift);
860 
861 			/*
862 			 * Skip the first feature. There is nothing to
863 			 * compare against for now.
864 			 */
865 			if (j == 0)
866 				continue;
867 
868 			prev_shift = ftr_reg->ftr_bits[j - 1].shift;
869 			WARN((shift + width) > prev_shift,
870 				"%s has feature overlap at shift %d\n",
871 				ftr_reg->name, shift);
872 		}
873 
874 		/*
875 		 * Skip the first register. There is nothing to
876 		 * compare against for now.
877 		 */
878 		if (i == 0)
879 			continue;
880 		/*
881 		 * Registers here must be sorted in ascending order with respect
882 		 * to sys_id for subsequent binary search in get_arm64_ftr_reg()
883 		 * to work correctly.
884 		 */
885 		BUG_ON(arm64_ftr_regs[i].sys_id <= arm64_ftr_regs[i - 1].sys_id);
886 	}
887 }
888 
889 /*
890  * Initialise the CPU feature register from Boot CPU values.
891  * Also initiliases the strict_mask for the register.
892  * Any bits that are not covered by an arm64_ftr_bits entry are considered
893  * RES0 for the system-wide value, and must strictly match.
894  */
895 static void init_cpu_ftr_reg(u32 sys_reg, u64 new)
896 {
897 	u64 val = 0;
898 	u64 strict_mask = ~0x0ULL;
899 	u64 user_mask = 0;
900 	u64 valid_mask = 0;
901 
902 	const struct arm64_ftr_bits *ftrp;
903 	struct arm64_ftr_reg *reg = get_arm64_ftr_reg(sys_reg);
904 
905 	if (!reg)
906 		return;
907 
908 	for (ftrp = reg->ftr_bits; ftrp->width; ftrp++) {
909 		u64 ftr_mask = arm64_ftr_mask(ftrp);
910 		s64 ftr_new = arm64_ftr_value(ftrp, new);
911 		s64 ftr_ovr = arm64_ftr_value(ftrp, reg->override->val);
912 
913 		if ((ftr_mask & reg->override->mask) == ftr_mask) {
914 			s64 tmp = arm64_ftr_safe_value(ftrp, ftr_ovr, ftr_new);
915 			char *str = NULL;
916 
917 			if (ftr_ovr != tmp) {
918 				/* Unsafe, remove the override */
919 				reg->override->mask &= ~ftr_mask;
920 				reg->override->val &= ~ftr_mask;
921 				tmp = ftr_ovr;
922 				str = "ignoring override";
923 			} else if (ftr_new != tmp) {
924 				/* Override was valid */
925 				ftr_new = tmp;
926 				str = "forced";
927 			} else if (ftr_ovr == tmp) {
928 				/* Override was the safe value */
929 				str = "already set";
930 			}
931 
932 			if (str)
933 				pr_warn("%s[%d:%d]: %s to %llx\n",
934 					reg->name,
935 					ftrp->shift + ftrp->width - 1,
936 					ftrp->shift, str, tmp);
937 		} else if ((ftr_mask & reg->override->val) == ftr_mask) {
938 			reg->override->val &= ~ftr_mask;
939 			pr_warn("%s[%d:%d]: impossible override, ignored\n",
940 				reg->name,
941 				ftrp->shift + ftrp->width - 1,
942 				ftrp->shift);
943 		}
944 
945 		val = arm64_ftr_set_value(ftrp, val, ftr_new);
946 
947 		valid_mask |= ftr_mask;
948 		if (!ftrp->strict)
949 			strict_mask &= ~ftr_mask;
950 		if (ftrp->visible)
951 			user_mask |= ftr_mask;
952 		else
953 			reg->user_val = arm64_ftr_set_value(ftrp,
954 							    reg->user_val,
955 							    ftrp->safe_val);
956 	}
957 
958 	val &= valid_mask;
959 
960 	reg->sys_val = val;
961 	reg->strict_mask = strict_mask;
962 	reg->user_mask = user_mask;
963 }
964 
965 extern const struct arm64_cpu_capabilities arm64_errata[];
966 static const struct arm64_cpu_capabilities arm64_features[];
967 
968 static void __init
969 init_cpucap_indirect_list_from_array(const struct arm64_cpu_capabilities *caps)
970 {
971 	for (; caps->matches; caps++) {
972 		if (WARN(caps->capability >= ARM64_NCAPS,
973 			"Invalid capability %d\n", caps->capability))
974 			continue;
975 		if (WARN(cpucap_ptrs[caps->capability],
976 			"Duplicate entry for capability %d\n",
977 			caps->capability))
978 			continue;
979 		cpucap_ptrs[caps->capability] = caps;
980 	}
981 }
982 
983 static void __init init_cpucap_indirect_list(void)
984 {
985 	init_cpucap_indirect_list_from_array(arm64_features);
986 	init_cpucap_indirect_list_from_array(arm64_errata);
987 }
988 
989 static void __init setup_boot_cpu_capabilities(void);
990 
991 static void init_32bit_cpu_features(struct cpuinfo_32bit *info)
992 {
993 	init_cpu_ftr_reg(SYS_ID_DFR0_EL1, info->reg_id_dfr0);
994 	init_cpu_ftr_reg(SYS_ID_DFR1_EL1, info->reg_id_dfr1);
995 	init_cpu_ftr_reg(SYS_ID_ISAR0_EL1, info->reg_id_isar0);
996 	init_cpu_ftr_reg(SYS_ID_ISAR1_EL1, info->reg_id_isar1);
997 	init_cpu_ftr_reg(SYS_ID_ISAR2_EL1, info->reg_id_isar2);
998 	init_cpu_ftr_reg(SYS_ID_ISAR3_EL1, info->reg_id_isar3);
999 	init_cpu_ftr_reg(SYS_ID_ISAR4_EL1, info->reg_id_isar4);
1000 	init_cpu_ftr_reg(SYS_ID_ISAR5_EL1, info->reg_id_isar5);
1001 	init_cpu_ftr_reg(SYS_ID_ISAR6_EL1, info->reg_id_isar6);
1002 	init_cpu_ftr_reg(SYS_ID_MMFR0_EL1, info->reg_id_mmfr0);
1003 	init_cpu_ftr_reg(SYS_ID_MMFR1_EL1, info->reg_id_mmfr1);
1004 	init_cpu_ftr_reg(SYS_ID_MMFR2_EL1, info->reg_id_mmfr2);
1005 	init_cpu_ftr_reg(SYS_ID_MMFR3_EL1, info->reg_id_mmfr3);
1006 	init_cpu_ftr_reg(SYS_ID_MMFR4_EL1, info->reg_id_mmfr4);
1007 	init_cpu_ftr_reg(SYS_ID_MMFR5_EL1, info->reg_id_mmfr5);
1008 	init_cpu_ftr_reg(SYS_ID_PFR0_EL1, info->reg_id_pfr0);
1009 	init_cpu_ftr_reg(SYS_ID_PFR1_EL1, info->reg_id_pfr1);
1010 	init_cpu_ftr_reg(SYS_ID_PFR2_EL1, info->reg_id_pfr2);
1011 	init_cpu_ftr_reg(SYS_MVFR0_EL1, info->reg_mvfr0);
1012 	init_cpu_ftr_reg(SYS_MVFR1_EL1, info->reg_mvfr1);
1013 	init_cpu_ftr_reg(SYS_MVFR2_EL1, info->reg_mvfr2);
1014 }
1015 
1016 void __init init_cpu_features(struct cpuinfo_arm64 *info)
1017 {
1018 	/* Before we start using the tables, make sure it is sorted */
1019 	sort_ftr_regs();
1020 
1021 	init_cpu_ftr_reg(SYS_CTR_EL0, info->reg_ctr);
1022 	init_cpu_ftr_reg(SYS_DCZID_EL0, info->reg_dczid);
1023 	init_cpu_ftr_reg(SYS_CNTFRQ_EL0, info->reg_cntfrq);
1024 	init_cpu_ftr_reg(SYS_ID_AA64DFR0_EL1, info->reg_id_aa64dfr0);
1025 	init_cpu_ftr_reg(SYS_ID_AA64DFR1_EL1, info->reg_id_aa64dfr1);
1026 	init_cpu_ftr_reg(SYS_ID_AA64ISAR0_EL1, info->reg_id_aa64isar0);
1027 	init_cpu_ftr_reg(SYS_ID_AA64ISAR1_EL1, info->reg_id_aa64isar1);
1028 	init_cpu_ftr_reg(SYS_ID_AA64ISAR2_EL1, info->reg_id_aa64isar2);
1029 	init_cpu_ftr_reg(SYS_ID_AA64MMFR0_EL1, info->reg_id_aa64mmfr0);
1030 	init_cpu_ftr_reg(SYS_ID_AA64MMFR1_EL1, info->reg_id_aa64mmfr1);
1031 	init_cpu_ftr_reg(SYS_ID_AA64MMFR2_EL1, info->reg_id_aa64mmfr2);
1032 	init_cpu_ftr_reg(SYS_ID_AA64MMFR3_EL1, info->reg_id_aa64mmfr3);
1033 	init_cpu_ftr_reg(SYS_ID_AA64PFR0_EL1, info->reg_id_aa64pfr0);
1034 	init_cpu_ftr_reg(SYS_ID_AA64PFR1_EL1, info->reg_id_aa64pfr1);
1035 	init_cpu_ftr_reg(SYS_ID_AA64ZFR0_EL1, info->reg_id_aa64zfr0);
1036 	init_cpu_ftr_reg(SYS_ID_AA64SMFR0_EL1, info->reg_id_aa64smfr0);
1037 
1038 	if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0))
1039 		init_32bit_cpu_features(&info->aarch32);
1040 
1041 	if (IS_ENABLED(CONFIG_ARM64_SVE) &&
1042 	    id_aa64pfr0_sve(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1))) {
1043 		info->reg_zcr = read_zcr_features();
1044 		init_cpu_ftr_reg(SYS_ZCR_EL1, info->reg_zcr);
1045 		vec_init_vq_map(ARM64_VEC_SVE);
1046 	}
1047 
1048 	if (IS_ENABLED(CONFIG_ARM64_SME) &&
1049 	    id_aa64pfr1_sme(read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1))) {
1050 		info->reg_smcr = read_smcr_features();
1051 		/*
1052 		 * We mask out SMPS since even if the hardware
1053 		 * supports priorities the kernel does not at present
1054 		 * and we block access to them.
1055 		 */
1056 		info->reg_smidr = read_cpuid(SMIDR_EL1) & ~SMIDR_EL1_SMPS;
1057 		init_cpu_ftr_reg(SYS_SMCR_EL1, info->reg_smcr);
1058 		vec_init_vq_map(ARM64_VEC_SME);
1059 	}
1060 
1061 	if (id_aa64pfr1_mte(info->reg_id_aa64pfr1))
1062 		init_cpu_ftr_reg(SYS_GMID_EL1, info->reg_gmid);
1063 
1064 	/*
1065 	 * Initialize the indirect array of CPU capabilities pointers before we
1066 	 * handle the boot CPU below.
1067 	 */
1068 	init_cpucap_indirect_list();
1069 
1070 	/*
1071 	 * Detect and enable early CPU capabilities based on the boot CPU,
1072 	 * after we have initialised the CPU feature infrastructure.
1073 	 */
1074 	setup_boot_cpu_capabilities();
1075 }
1076 
1077 static void update_cpu_ftr_reg(struct arm64_ftr_reg *reg, u64 new)
1078 {
1079 	const struct arm64_ftr_bits *ftrp;
1080 
1081 	for (ftrp = reg->ftr_bits; ftrp->width; ftrp++) {
1082 		s64 ftr_cur = arm64_ftr_value(ftrp, reg->sys_val);
1083 		s64 ftr_new = arm64_ftr_value(ftrp, new);
1084 
1085 		if (ftr_cur == ftr_new)
1086 			continue;
1087 		/* Find a safe value */
1088 		ftr_new = arm64_ftr_safe_value(ftrp, ftr_new, ftr_cur);
1089 		reg->sys_val = arm64_ftr_set_value(ftrp, reg->sys_val, ftr_new);
1090 	}
1091 
1092 }
1093 
1094 static int check_update_ftr_reg(u32 sys_id, int cpu, u64 val, u64 boot)
1095 {
1096 	struct arm64_ftr_reg *regp = get_arm64_ftr_reg(sys_id);
1097 
1098 	if (!regp)
1099 		return 0;
1100 
1101 	update_cpu_ftr_reg(regp, val);
1102 	if ((boot & regp->strict_mask) == (val & regp->strict_mask))
1103 		return 0;
1104 	pr_warn("SANITY CHECK: Unexpected variation in %s. Boot CPU: %#016llx, CPU%d: %#016llx\n",
1105 			regp->name, boot, cpu, val);
1106 	return 1;
1107 }
1108 
1109 static void relax_cpu_ftr_reg(u32 sys_id, int field)
1110 {
1111 	const struct arm64_ftr_bits *ftrp;
1112 	struct arm64_ftr_reg *regp = get_arm64_ftr_reg(sys_id);
1113 
1114 	if (!regp)
1115 		return;
1116 
1117 	for (ftrp = regp->ftr_bits; ftrp->width; ftrp++) {
1118 		if (ftrp->shift == field) {
1119 			regp->strict_mask &= ~arm64_ftr_mask(ftrp);
1120 			break;
1121 		}
1122 	}
1123 
1124 	/* Bogus field? */
1125 	WARN_ON(!ftrp->width);
1126 }
1127 
1128 static void lazy_init_32bit_cpu_features(struct cpuinfo_arm64 *info,
1129 					 struct cpuinfo_arm64 *boot)
1130 {
1131 	static bool boot_cpu_32bit_regs_overridden = false;
1132 
1133 	if (!allow_mismatched_32bit_el0 || boot_cpu_32bit_regs_overridden)
1134 		return;
1135 
1136 	if (id_aa64pfr0_32bit_el0(boot->reg_id_aa64pfr0))
1137 		return;
1138 
1139 	boot->aarch32 = info->aarch32;
1140 	init_32bit_cpu_features(&boot->aarch32);
1141 	boot_cpu_32bit_regs_overridden = true;
1142 }
1143 
1144 static int update_32bit_cpu_features(int cpu, struct cpuinfo_32bit *info,
1145 				     struct cpuinfo_32bit *boot)
1146 {
1147 	int taint = 0;
1148 	u64 pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
1149 
1150 	/*
1151 	 * If we don't have AArch32 at EL1, then relax the strictness of
1152 	 * EL1-dependent register fields to avoid spurious sanity check fails.
1153 	 */
1154 	if (!id_aa64pfr0_32bit_el1(pfr0)) {
1155 		relax_cpu_ftr_reg(SYS_ID_ISAR4_EL1, ID_ISAR4_EL1_SMC_SHIFT);
1156 		relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_EL1_Virt_frac_SHIFT);
1157 		relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_EL1_Sec_frac_SHIFT);
1158 		relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_EL1_Virtualization_SHIFT);
1159 		relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_EL1_Security_SHIFT);
1160 		relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_EL1_ProgMod_SHIFT);
1161 	}
1162 
1163 	taint |= check_update_ftr_reg(SYS_ID_DFR0_EL1, cpu,
1164 				      info->reg_id_dfr0, boot->reg_id_dfr0);
1165 	taint |= check_update_ftr_reg(SYS_ID_DFR1_EL1, cpu,
1166 				      info->reg_id_dfr1, boot->reg_id_dfr1);
1167 	taint |= check_update_ftr_reg(SYS_ID_ISAR0_EL1, cpu,
1168 				      info->reg_id_isar0, boot->reg_id_isar0);
1169 	taint |= check_update_ftr_reg(SYS_ID_ISAR1_EL1, cpu,
1170 				      info->reg_id_isar1, boot->reg_id_isar1);
1171 	taint |= check_update_ftr_reg(SYS_ID_ISAR2_EL1, cpu,
1172 				      info->reg_id_isar2, boot->reg_id_isar2);
1173 	taint |= check_update_ftr_reg(SYS_ID_ISAR3_EL1, cpu,
1174 				      info->reg_id_isar3, boot->reg_id_isar3);
1175 	taint |= check_update_ftr_reg(SYS_ID_ISAR4_EL1, cpu,
1176 				      info->reg_id_isar4, boot->reg_id_isar4);
1177 	taint |= check_update_ftr_reg(SYS_ID_ISAR5_EL1, cpu,
1178 				      info->reg_id_isar5, boot->reg_id_isar5);
1179 	taint |= check_update_ftr_reg(SYS_ID_ISAR6_EL1, cpu,
1180 				      info->reg_id_isar6, boot->reg_id_isar6);
1181 
1182 	/*
1183 	 * Regardless of the value of the AuxReg field, the AIFSR, ADFSR, and
1184 	 * ACTLR formats could differ across CPUs and therefore would have to
1185 	 * be trapped for virtualization anyway.
1186 	 */
1187 	taint |= check_update_ftr_reg(SYS_ID_MMFR0_EL1, cpu,
1188 				      info->reg_id_mmfr0, boot->reg_id_mmfr0);
1189 	taint |= check_update_ftr_reg(SYS_ID_MMFR1_EL1, cpu,
1190 				      info->reg_id_mmfr1, boot->reg_id_mmfr1);
1191 	taint |= check_update_ftr_reg(SYS_ID_MMFR2_EL1, cpu,
1192 				      info->reg_id_mmfr2, boot->reg_id_mmfr2);
1193 	taint |= check_update_ftr_reg(SYS_ID_MMFR3_EL1, cpu,
1194 				      info->reg_id_mmfr3, boot->reg_id_mmfr3);
1195 	taint |= check_update_ftr_reg(SYS_ID_MMFR4_EL1, cpu,
1196 				      info->reg_id_mmfr4, boot->reg_id_mmfr4);
1197 	taint |= check_update_ftr_reg(SYS_ID_MMFR5_EL1, cpu,
1198 				      info->reg_id_mmfr5, boot->reg_id_mmfr5);
1199 	taint |= check_update_ftr_reg(SYS_ID_PFR0_EL1, cpu,
1200 				      info->reg_id_pfr0, boot->reg_id_pfr0);
1201 	taint |= check_update_ftr_reg(SYS_ID_PFR1_EL1, cpu,
1202 				      info->reg_id_pfr1, boot->reg_id_pfr1);
1203 	taint |= check_update_ftr_reg(SYS_ID_PFR2_EL1, cpu,
1204 				      info->reg_id_pfr2, boot->reg_id_pfr2);
1205 	taint |= check_update_ftr_reg(SYS_MVFR0_EL1, cpu,
1206 				      info->reg_mvfr0, boot->reg_mvfr0);
1207 	taint |= check_update_ftr_reg(SYS_MVFR1_EL1, cpu,
1208 				      info->reg_mvfr1, boot->reg_mvfr1);
1209 	taint |= check_update_ftr_reg(SYS_MVFR2_EL1, cpu,
1210 				      info->reg_mvfr2, boot->reg_mvfr2);
1211 
1212 	return taint;
1213 }
1214 
1215 /*
1216  * Update system wide CPU feature registers with the values from a
1217  * non-boot CPU. Also performs SANITY checks to make sure that there
1218  * aren't any insane variations from that of the boot CPU.
1219  */
1220 void update_cpu_features(int cpu,
1221 			 struct cpuinfo_arm64 *info,
1222 			 struct cpuinfo_arm64 *boot)
1223 {
1224 	int taint = 0;
1225 
1226 	/*
1227 	 * The kernel can handle differing I-cache policies, but otherwise
1228 	 * caches should look identical. Userspace JITs will make use of
1229 	 * *minLine.
1230 	 */
1231 	taint |= check_update_ftr_reg(SYS_CTR_EL0, cpu,
1232 				      info->reg_ctr, boot->reg_ctr);
1233 
1234 	/*
1235 	 * Userspace may perform DC ZVA instructions. Mismatched block sizes
1236 	 * could result in too much or too little memory being zeroed if a
1237 	 * process is preempted and migrated between CPUs.
1238 	 */
1239 	taint |= check_update_ftr_reg(SYS_DCZID_EL0, cpu,
1240 				      info->reg_dczid, boot->reg_dczid);
1241 
1242 	/* If different, timekeeping will be broken (especially with KVM) */
1243 	taint |= check_update_ftr_reg(SYS_CNTFRQ_EL0, cpu,
1244 				      info->reg_cntfrq, boot->reg_cntfrq);
1245 
1246 	/*
1247 	 * The kernel uses self-hosted debug features and expects CPUs to
1248 	 * support identical debug features. We presently need CTX_CMPs, WRPs,
1249 	 * and BRPs to be identical.
1250 	 * ID_AA64DFR1 is currently RES0.
1251 	 */
1252 	taint |= check_update_ftr_reg(SYS_ID_AA64DFR0_EL1, cpu,
1253 				      info->reg_id_aa64dfr0, boot->reg_id_aa64dfr0);
1254 	taint |= check_update_ftr_reg(SYS_ID_AA64DFR1_EL1, cpu,
1255 				      info->reg_id_aa64dfr1, boot->reg_id_aa64dfr1);
1256 	/*
1257 	 * Even in big.LITTLE, processors should be identical instruction-set
1258 	 * wise.
1259 	 */
1260 	taint |= check_update_ftr_reg(SYS_ID_AA64ISAR0_EL1, cpu,
1261 				      info->reg_id_aa64isar0, boot->reg_id_aa64isar0);
1262 	taint |= check_update_ftr_reg(SYS_ID_AA64ISAR1_EL1, cpu,
1263 				      info->reg_id_aa64isar1, boot->reg_id_aa64isar1);
1264 	taint |= check_update_ftr_reg(SYS_ID_AA64ISAR2_EL1, cpu,
1265 				      info->reg_id_aa64isar2, boot->reg_id_aa64isar2);
1266 
1267 	/*
1268 	 * Differing PARange support is fine as long as all peripherals and
1269 	 * memory are mapped within the minimum PARange of all CPUs.
1270 	 * Linux should not care about secure memory.
1271 	 */
1272 	taint |= check_update_ftr_reg(SYS_ID_AA64MMFR0_EL1, cpu,
1273 				      info->reg_id_aa64mmfr0, boot->reg_id_aa64mmfr0);
1274 	taint |= check_update_ftr_reg(SYS_ID_AA64MMFR1_EL1, cpu,
1275 				      info->reg_id_aa64mmfr1, boot->reg_id_aa64mmfr1);
1276 	taint |= check_update_ftr_reg(SYS_ID_AA64MMFR2_EL1, cpu,
1277 				      info->reg_id_aa64mmfr2, boot->reg_id_aa64mmfr2);
1278 	taint |= check_update_ftr_reg(SYS_ID_AA64MMFR3_EL1, cpu,
1279 				      info->reg_id_aa64mmfr3, boot->reg_id_aa64mmfr3);
1280 
1281 	taint |= check_update_ftr_reg(SYS_ID_AA64PFR0_EL1, cpu,
1282 				      info->reg_id_aa64pfr0, boot->reg_id_aa64pfr0);
1283 	taint |= check_update_ftr_reg(SYS_ID_AA64PFR1_EL1, cpu,
1284 				      info->reg_id_aa64pfr1, boot->reg_id_aa64pfr1);
1285 
1286 	taint |= check_update_ftr_reg(SYS_ID_AA64ZFR0_EL1, cpu,
1287 				      info->reg_id_aa64zfr0, boot->reg_id_aa64zfr0);
1288 
1289 	taint |= check_update_ftr_reg(SYS_ID_AA64SMFR0_EL1, cpu,
1290 				      info->reg_id_aa64smfr0, boot->reg_id_aa64smfr0);
1291 
1292 	if (IS_ENABLED(CONFIG_ARM64_SVE) &&
1293 	    id_aa64pfr0_sve(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1))) {
1294 		info->reg_zcr = read_zcr_features();
1295 		taint |= check_update_ftr_reg(SYS_ZCR_EL1, cpu,
1296 					info->reg_zcr, boot->reg_zcr);
1297 
1298 		/* Probe vector lengths */
1299 		if (!system_capabilities_finalized())
1300 			vec_update_vq_map(ARM64_VEC_SVE);
1301 	}
1302 
1303 	if (IS_ENABLED(CONFIG_ARM64_SME) &&
1304 	    id_aa64pfr1_sme(read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1))) {
1305 		info->reg_smcr = read_smcr_features();
1306 		/*
1307 		 * We mask out SMPS since even if the hardware
1308 		 * supports priorities the kernel does not at present
1309 		 * and we block access to them.
1310 		 */
1311 		info->reg_smidr = read_cpuid(SMIDR_EL1) & ~SMIDR_EL1_SMPS;
1312 		taint |= check_update_ftr_reg(SYS_SMCR_EL1, cpu,
1313 					info->reg_smcr, boot->reg_smcr);
1314 
1315 		/* Probe vector lengths */
1316 		if (!system_capabilities_finalized())
1317 			vec_update_vq_map(ARM64_VEC_SME);
1318 	}
1319 
1320 	/*
1321 	 * The kernel uses the LDGM/STGM instructions and the number of tags
1322 	 * they read/write depends on the GMID_EL1.BS field. Check that the
1323 	 * value is the same on all CPUs.
1324 	 */
1325 	if (IS_ENABLED(CONFIG_ARM64_MTE) &&
1326 	    id_aa64pfr1_mte(info->reg_id_aa64pfr1)) {
1327 		taint |= check_update_ftr_reg(SYS_GMID_EL1, cpu,
1328 					      info->reg_gmid, boot->reg_gmid);
1329 	}
1330 
1331 	/*
1332 	 * If we don't have AArch32 at all then skip the checks entirely
1333 	 * as the register values may be UNKNOWN and we're not going to be
1334 	 * using them for anything.
1335 	 *
1336 	 * This relies on a sanitised view of the AArch64 ID registers
1337 	 * (e.g. SYS_ID_AA64PFR0_EL1), so we call it last.
1338 	 */
1339 	if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) {
1340 		lazy_init_32bit_cpu_features(info, boot);
1341 		taint |= update_32bit_cpu_features(cpu, &info->aarch32,
1342 						   &boot->aarch32);
1343 	}
1344 
1345 	/*
1346 	 * Mismatched CPU features are a recipe for disaster. Don't even
1347 	 * pretend to support them.
1348 	 */
1349 	if (taint) {
1350 		pr_warn_once("Unsupported CPU feature variation detected.\n");
1351 		add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK);
1352 	}
1353 }
1354 
1355 u64 read_sanitised_ftr_reg(u32 id)
1356 {
1357 	struct arm64_ftr_reg *regp = get_arm64_ftr_reg(id);
1358 
1359 	if (!regp)
1360 		return 0;
1361 	return regp->sys_val;
1362 }
1363 EXPORT_SYMBOL_GPL(read_sanitised_ftr_reg);
1364 
1365 #define read_sysreg_case(r)	\
1366 	case r:		val = read_sysreg_s(r); break;
1367 
1368 /*
1369  * __read_sysreg_by_encoding() - Used by a STARTING cpu before cpuinfo is populated.
1370  * Read the system register on the current CPU
1371  */
1372 u64 __read_sysreg_by_encoding(u32 sys_id)
1373 {
1374 	struct arm64_ftr_reg *regp;
1375 	u64 val;
1376 
1377 	switch (sys_id) {
1378 	read_sysreg_case(SYS_ID_PFR0_EL1);
1379 	read_sysreg_case(SYS_ID_PFR1_EL1);
1380 	read_sysreg_case(SYS_ID_PFR2_EL1);
1381 	read_sysreg_case(SYS_ID_DFR0_EL1);
1382 	read_sysreg_case(SYS_ID_DFR1_EL1);
1383 	read_sysreg_case(SYS_ID_MMFR0_EL1);
1384 	read_sysreg_case(SYS_ID_MMFR1_EL1);
1385 	read_sysreg_case(SYS_ID_MMFR2_EL1);
1386 	read_sysreg_case(SYS_ID_MMFR3_EL1);
1387 	read_sysreg_case(SYS_ID_MMFR4_EL1);
1388 	read_sysreg_case(SYS_ID_MMFR5_EL1);
1389 	read_sysreg_case(SYS_ID_ISAR0_EL1);
1390 	read_sysreg_case(SYS_ID_ISAR1_EL1);
1391 	read_sysreg_case(SYS_ID_ISAR2_EL1);
1392 	read_sysreg_case(SYS_ID_ISAR3_EL1);
1393 	read_sysreg_case(SYS_ID_ISAR4_EL1);
1394 	read_sysreg_case(SYS_ID_ISAR5_EL1);
1395 	read_sysreg_case(SYS_ID_ISAR6_EL1);
1396 	read_sysreg_case(SYS_MVFR0_EL1);
1397 	read_sysreg_case(SYS_MVFR1_EL1);
1398 	read_sysreg_case(SYS_MVFR2_EL1);
1399 
1400 	read_sysreg_case(SYS_ID_AA64PFR0_EL1);
1401 	read_sysreg_case(SYS_ID_AA64PFR1_EL1);
1402 	read_sysreg_case(SYS_ID_AA64ZFR0_EL1);
1403 	read_sysreg_case(SYS_ID_AA64SMFR0_EL1);
1404 	read_sysreg_case(SYS_ID_AA64DFR0_EL1);
1405 	read_sysreg_case(SYS_ID_AA64DFR1_EL1);
1406 	read_sysreg_case(SYS_ID_AA64MMFR0_EL1);
1407 	read_sysreg_case(SYS_ID_AA64MMFR1_EL1);
1408 	read_sysreg_case(SYS_ID_AA64MMFR2_EL1);
1409 	read_sysreg_case(SYS_ID_AA64MMFR3_EL1);
1410 	read_sysreg_case(SYS_ID_AA64ISAR0_EL1);
1411 	read_sysreg_case(SYS_ID_AA64ISAR1_EL1);
1412 	read_sysreg_case(SYS_ID_AA64ISAR2_EL1);
1413 
1414 	read_sysreg_case(SYS_CNTFRQ_EL0);
1415 	read_sysreg_case(SYS_CTR_EL0);
1416 	read_sysreg_case(SYS_DCZID_EL0);
1417 
1418 	default:
1419 		BUG();
1420 		return 0;
1421 	}
1422 
1423 	regp  = get_arm64_ftr_reg(sys_id);
1424 	if (regp) {
1425 		val &= ~regp->override->mask;
1426 		val |= (regp->override->val & regp->override->mask);
1427 	}
1428 
1429 	return val;
1430 }
1431 
1432 #include <linux/irqchip/arm-gic-v3.h>
1433 
1434 static bool
1435 has_always(const struct arm64_cpu_capabilities *entry, int scope)
1436 {
1437 	return true;
1438 }
1439 
1440 static bool
1441 feature_matches(u64 reg, const struct arm64_cpu_capabilities *entry)
1442 {
1443 	int val = cpuid_feature_extract_field_width(reg, entry->field_pos,
1444 						    entry->field_width,
1445 						    entry->sign);
1446 
1447 	return val >= entry->min_field_value;
1448 }
1449 
1450 static u64
1451 read_scoped_sysreg(const struct arm64_cpu_capabilities *entry, int scope)
1452 {
1453 	WARN_ON(scope == SCOPE_LOCAL_CPU && preemptible());
1454 	if (scope == SCOPE_SYSTEM)
1455 		return read_sanitised_ftr_reg(entry->sys_reg);
1456 	else
1457 		return __read_sysreg_by_encoding(entry->sys_reg);
1458 }
1459 
1460 static bool
1461 has_user_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope)
1462 {
1463 	int mask;
1464 	struct arm64_ftr_reg *regp;
1465 	u64 val = read_scoped_sysreg(entry, scope);
1466 
1467 	regp = get_arm64_ftr_reg(entry->sys_reg);
1468 	if (!regp)
1469 		return false;
1470 
1471 	mask = cpuid_feature_extract_unsigned_field_width(regp->user_mask,
1472 							  entry->field_pos,
1473 							  entry->field_width);
1474 	if (!mask)
1475 		return false;
1476 
1477 	return feature_matches(val, entry);
1478 }
1479 
1480 static bool
1481 has_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope)
1482 {
1483 	u64 val = read_scoped_sysreg(entry, scope);
1484 	return feature_matches(val, entry);
1485 }
1486 
1487 const struct cpumask *system_32bit_el0_cpumask(void)
1488 {
1489 	if (!system_supports_32bit_el0())
1490 		return cpu_none_mask;
1491 
1492 	if (static_branch_unlikely(&arm64_mismatched_32bit_el0))
1493 		return cpu_32bit_el0_mask;
1494 
1495 	return cpu_possible_mask;
1496 }
1497 
1498 static int __init parse_32bit_el0_param(char *str)
1499 {
1500 	allow_mismatched_32bit_el0 = true;
1501 	return 0;
1502 }
1503 early_param("allow_mismatched_32bit_el0", parse_32bit_el0_param);
1504 
1505 static ssize_t aarch32_el0_show(struct device *dev,
1506 				struct device_attribute *attr, char *buf)
1507 {
1508 	const struct cpumask *mask = system_32bit_el0_cpumask();
1509 
1510 	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(mask));
1511 }
1512 static const DEVICE_ATTR_RO(aarch32_el0);
1513 
1514 static int __init aarch32_el0_sysfs_init(void)
1515 {
1516 	struct device *dev_root;
1517 	int ret = 0;
1518 
1519 	if (!allow_mismatched_32bit_el0)
1520 		return 0;
1521 
1522 	dev_root = bus_get_dev_root(&cpu_subsys);
1523 	if (dev_root) {
1524 		ret = device_create_file(dev_root, &dev_attr_aarch32_el0);
1525 		put_device(dev_root);
1526 	}
1527 	return ret;
1528 }
1529 device_initcall(aarch32_el0_sysfs_init);
1530 
1531 static bool has_32bit_el0(const struct arm64_cpu_capabilities *entry, int scope)
1532 {
1533 	if (!has_cpuid_feature(entry, scope))
1534 		return allow_mismatched_32bit_el0;
1535 
1536 	if (scope == SCOPE_SYSTEM)
1537 		pr_info("detected: 32-bit EL0 Support\n");
1538 
1539 	return true;
1540 }
1541 
1542 static bool has_useable_gicv3_cpuif(const struct arm64_cpu_capabilities *entry, int scope)
1543 {
1544 	bool has_sre;
1545 
1546 	if (!has_cpuid_feature(entry, scope))
1547 		return false;
1548 
1549 	has_sre = gic_enable_sre();
1550 	if (!has_sre)
1551 		pr_warn_once("%s present but disabled by higher exception level\n",
1552 			     entry->desc);
1553 
1554 	return has_sre;
1555 }
1556 
1557 static bool has_no_hw_prefetch(const struct arm64_cpu_capabilities *entry, int __unused)
1558 {
1559 	u32 midr = read_cpuid_id();
1560 
1561 	/* Cavium ThunderX pass 1.x and 2.x */
1562 	return midr_is_cpu_model_range(midr, MIDR_THUNDERX,
1563 		MIDR_CPU_VAR_REV(0, 0),
1564 		MIDR_CPU_VAR_REV(1, MIDR_REVISION_MASK));
1565 }
1566 
1567 static bool has_no_fpsimd(const struct arm64_cpu_capabilities *entry, int __unused)
1568 {
1569 	u64 pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
1570 
1571 	return cpuid_feature_extract_signed_field(pfr0,
1572 					ID_AA64PFR0_EL1_FP_SHIFT) < 0;
1573 }
1574 
1575 static bool has_cache_idc(const struct arm64_cpu_capabilities *entry,
1576 			  int scope)
1577 {
1578 	u64 ctr;
1579 
1580 	if (scope == SCOPE_SYSTEM)
1581 		ctr = arm64_ftr_reg_ctrel0.sys_val;
1582 	else
1583 		ctr = read_cpuid_effective_cachetype();
1584 
1585 	return ctr & BIT(CTR_EL0_IDC_SHIFT);
1586 }
1587 
1588 static void cpu_emulate_effective_ctr(const struct arm64_cpu_capabilities *__unused)
1589 {
1590 	/*
1591 	 * If the CPU exposes raw CTR_EL0.IDC = 0, while effectively
1592 	 * CTR_EL0.IDC = 1 (from CLIDR values), we need to trap accesses
1593 	 * to the CTR_EL0 on this CPU and emulate it with the real/safe
1594 	 * value.
1595 	 */
1596 	if (!(read_cpuid_cachetype() & BIT(CTR_EL0_IDC_SHIFT)))
1597 		sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCT, 0);
1598 }
1599 
1600 static bool has_cache_dic(const struct arm64_cpu_capabilities *entry,
1601 			  int scope)
1602 {
1603 	u64 ctr;
1604 
1605 	if (scope == SCOPE_SYSTEM)
1606 		ctr = arm64_ftr_reg_ctrel0.sys_val;
1607 	else
1608 		ctr = read_cpuid_cachetype();
1609 
1610 	return ctr & BIT(CTR_EL0_DIC_SHIFT);
1611 }
1612 
1613 static bool __maybe_unused
1614 has_useable_cnp(const struct arm64_cpu_capabilities *entry, int scope)
1615 {
1616 	/*
1617 	 * Kdump isn't guaranteed to power-off all secondary CPUs, CNP
1618 	 * may share TLB entries with a CPU stuck in the crashed
1619 	 * kernel.
1620 	 */
1621 	if (is_kdump_kernel())
1622 		return false;
1623 
1624 	if (cpus_have_const_cap(ARM64_WORKAROUND_NVIDIA_CARMEL_CNP))
1625 		return false;
1626 
1627 	return has_cpuid_feature(entry, scope);
1628 }
1629 
1630 /*
1631  * This check is triggered during the early boot before the cpufeature
1632  * is initialised. Checking the status on the local CPU allows the boot
1633  * CPU to detect the need for non-global mappings and thus avoiding a
1634  * pagetable re-write after all the CPUs are booted. This check will be
1635  * anyway run on individual CPUs, allowing us to get the consistent
1636  * state once the SMP CPUs are up and thus make the switch to non-global
1637  * mappings if required.
1638  */
1639 bool kaslr_requires_kpti(void)
1640 {
1641 	if (!IS_ENABLED(CONFIG_RANDOMIZE_BASE))
1642 		return false;
1643 
1644 	/*
1645 	 * E0PD does a similar job to KPTI so can be used instead
1646 	 * where available.
1647 	 */
1648 	if (IS_ENABLED(CONFIG_ARM64_E0PD)) {
1649 		u64 mmfr2 = read_sysreg_s(SYS_ID_AA64MMFR2_EL1);
1650 		if (cpuid_feature_extract_unsigned_field(mmfr2,
1651 						ID_AA64MMFR2_EL1_E0PD_SHIFT))
1652 			return false;
1653 	}
1654 
1655 	/*
1656 	 * Systems affected by Cavium erratum 24756 are incompatible
1657 	 * with KPTI.
1658 	 */
1659 	if (IS_ENABLED(CONFIG_CAVIUM_ERRATUM_27456)) {
1660 		extern const struct midr_range cavium_erratum_27456_cpus[];
1661 
1662 		if (is_midr_in_range_list(read_cpuid_id(),
1663 					  cavium_erratum_27456_cpus))
1664 			return false;
1665 	}
1666 
1667 	return kaslr_enabled();
1668 }
1669 
1670 static bool __meltdown_safe = true;
1671 static int __kpti_forced; /* 0: not forced, >0: forced on, <0: forced off */
1672 
1673 static bool unmap_kernel_at_el0(const struct arm64_cpu_capabilities *entry,
1674 				int scope)
1675 {
1676 	/* List of CPUs that are not vulnerable and don't need KPTI */
1677 	static const struct midr_range kpti_safe_list[] = {
1678 		MIDR_ALL_VERSIONS(MIDR_CAVIUM_THUNDERX2),
1679 		MIDR_ALL_VERSIONS(MIDR_BRCM_VULCAN),
1680 		MIDR_ALL_VERSIONS(MIDR_BRAHMA_B53),
1681 		MIDR_ALL_VERSIONS(MIDR_CORTEX_A35),
1682 		MIDR_ALL_VERSIONS(MIDR_CORTEX_A53),
1683 		MIDR_ALL_VERSIONS(MIDR_CORTEX_A55),
1684 		MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
1685 		MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
1686 		MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
1687 		MIDR_ALL_VERSIONS(MIDR_HISI_TSV110),
1688 		MIDR_ALL_VERSIONS(MIDR_NVIDIA_CARMEL),
1689 		MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_2XX_GOLD),
1690 		MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_2XX_SILVER),
1691 		MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_3XX_SILVER),
1692 		MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_4XX_SILVER),
1693 		{ /* sentinel */ }
1694 	};
1695 	char const *str = "kpti command line option";
1696 	bool meltdown_safe;
1697 
1698 	meltdown_safe = is_midr_in_range_list(read_cpuid_id(), kpti_safe_list);
1699 
1700 	/* Defer to CPU feature registers */
1701 	if (has_cpuid_feature(entry, scope))
1702 		meltdown_safe = true;
1703 
1704 	if (!meltdown_safe)
1705 		__meltdown_safe = false;
1706 
1707 	/*
1708 	 * For reasons that aren't entirely clear, enabling KPTI on Cavium
1709 	 * ThunderX leads to apparent I-cache corruption of kernel text, which
1710 	 * ends as well as you might imagine. Don't even try. We cannot rely
1711 	 * on the cpus_have_*cap() helpers here to detect the CPU erratum
1712 	 * because cpucap detection order may change. However, since we know
1713 	 * affected CPUs are always in a homogeneous configuration, it is
1714 	 * safe to rely on this_cpu_has_cap() here.
1715 	 */
1716 	if (this_cpu_has_cap(ARM64_WORKAROUND_CAVIUM_27456)) {
1717 		str = "ARM64_WORKAROUND_CAVIUM_27456";
1718 		__kpti_forced = -1;
1719 	}
1720 
1721 	/* Useful for KASLR robustness */
1722 	if (kaslr_requires_kpti()) {
1723 		if (!__kpti_forced) {
1724 			str = "KASLR";
1725 			__kpti_forced = 1;
1726 		}
1727 	}
1728 
1729 	if (cpu_mitigations_off() && !__kpti_forced) {
1730 		str = "mitigations=off";
1731 		__kpti_forced = -1;
1732 	}
1733 
1734 	if (!IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0)) {
1735 		pr_info_once("kernel page table isolation disabled by kernel configuration\n");
1736 		return false;
1737 	}
1738 
1739 	/* Forced? */
1740 	if (__kpti_forced) {
1741 		pr_info_once("kernel page table isolation forced %s by %s\n",
1742 			     __kpti_forced > 0 ? "ON" : "OFF", str);
1743 		return __kpti_forced > 0;
1744 	}
1745 
1746 	return !meltdown_safe;
1747 }
1748 
1749 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
1750 #define KPTI_NG_TEMP_VA		(-(1UL << PMD_SHIFT))
1751 
1752 extern
1753 void create_kpti_ng_temp_pgd(pgd_t *pgdir, phys_addr_t phys, unsigned long virt,
1754 			     phys_addr_t size, pgprot_t prot,
1755 			     phys_addr_t (*pgtable_alloc)(int), int flags);
1756 
1757 static phys_addr_t kpti_ng_temp_alloc;
1758 
1759 static phys_addr_t kpti_ng_pgd_alloc(int shift)
1760 {
1761 	kpti_ng_temp_alloc -= PAGE_SIZE;
1762 	return kpti_ng_temp_alloc;
1763 }
1764 
1765 static void
1766 kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused)
1767 {
1768 	typedef void (kpti_remap_fn)(int, int, phys_addr_t, unsigned long);
1769 	extern kpti_remap_fn idmap_kpti_install_ng_mappings;
1770 	kpti_remap_fn *remap_fn;
1771 
1772 	int cpu = smp_processor_id();
1773 	int levels = CONFIG_PGTABLE_LEVELS;
1774 	int order = order_base_2(levels);
1775 	u64 kpti_ng_temp_pgd_pa = 0;
1776 	pgd_t *kpti_ng_temp_pgd;
1777 	u64 alloc = 0;
1778 
1779 	if (__this_cpu_read(this_cpu_vector) == vectors) {
1780 		const char *v = arm64_get_bp_hardening_vector(EL1_VECTOR_KPTI);
1781 
1782 		__this_cpu_write(this_cpu_vector, v);
1783 	}
1784 
1785 	/*
1786 	 * We don't need to rewrite the page-tables if either we've done
1787 	 * it already or we have KASLR enabled and therefore have not
1788 	 * created any global mappings at all.
1789 	 */
1790 	if (arm64_use_ng_mappings)
1791 		return;
1792 
1793 	remap_fn = (void *)__pa_symbol(idmap_kpti_install_ng_mappings);
1794 
1795 	if (!cpu) {
1796 		alloc = __get_free_pages(GFP_ATOMIC | __GFP_ZERO, order);
1797 		kpti_ng_temp_pgd = (pgd_t *)(alloc + (levels - 1) * PAGE_SIZE);
1798 		kpti_ng_temp_alloc = kpti_ng_temp_pgd_pa = __pa(kpti_ng_temp_pgd);
1799 
1800 		//
1801 		// Create a minimal page table hierarchy that permits us to map
1802 		// the swapper page tables temporarily as we traverse them.
1803 		//
1804 		// The physical pages are laid out as follows:
1805 		//
1806 		// +--------+-/-------+-/------ +-\\--------+
1807 		// :  PTE[] : | PMD[] : | PUD[] : || PGD[]  :
1808 		// +--------+-\-------+-\------ +-//--------+
1809 		//      ^
1810 		// The first page is mapped into this hierarchy at a PMD_SHIFT
1811 		// aligned virtual address, so that we can manipulate the PTE
1812 		// level entries while the mapping is active. The first entry
1813 		// covers the PTE[] page itself, the remaining entries are free
1814 		// to be used as a ad-hoc fixmap.
1815 		//
1816 		create_kpti_ng_temp_pgd(kpti_ng_temp_pgd, __pa(alloc),
1817 					KPTI_NG_TEMP_VA, PAGE_SIZE, PAGE_KERNEL,
1818 					kpti_ng_pgd_alloc, 0);
1819 	}
1820 
1821 	cpu_install_idmap();
1822 	remap_fn(cpu, num_online_cpus(), kpti_ng_temp_pgd_pa, KPTI_NG_TEMP_VA);
1823 	cpu_uninstall_idmap();
1824 
1825 	if (!cpu) {
1826 		free_pages(alloc, order);
1827 		arm64_use_ng_mappings = true;
1828 	}
1829 }
1830 #else
1831 static void
1832 kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused)
1833 {
1834 }
1835 #endif	/* CONFIG_UNMAP_KERNEL_AT_EL0 */
1836 
1837 static int __init parse_kpti(char *str)
1838 {
1839 	bool enabled;
1840 	int ret = kstrtobool(str, &enabled);
1841 
1842 	if (ret)
1843 		return ret;
1844 
1845 	__kpti_forced = enabled ? 1 : -1;
1846 	return 0;
1847 }
1848 early_param("kpti", parse_kpti);
1849 
1850 #ifdef CONFIG_ARM64_HW_AFDBM
1851 static inline void __cpu_enable_hw_dbm(void)
1852 {
1853 	u64 tcr = read_sysreg(tcr_el1) | TCR_HD;
1854 
1855 	write_sysreg(tcr, tcr_el1);
1856 	isb();
1857 	local_flush_tlb_all();
1858 }
1859 
1860 static bool cpu_has_broken_dbm(void)
1861 {
1862 	/* List of CPUs which have broken DBM support. */
1863 	static const struct midr_range cpus[] = {
1864 #ifdef CONFIG_ARM64_ERRATUM_1024718
1865 		MIDR_ALL_VERSIONS(MIDR_CORTEX_A55),
1866 		/* Kryo4xx Silver (rdpe => r1p0) */
1867 		MIDR_REV(MIDR_QCOM_KRYO_4XX_SILVER, 0xd, 0xe),
1868 #endif
1869 #ifdef CONFIG_ARM64_ERRATUM_2051678
1870 		MIDR_REV_RANGE(MIDR_CORTEX_A510, 0, 0, 2),
1871 #endif
1872 		{},
1873 	};
1874 
1875 	return is_midr_in_range_list(read_cpuid_id(), cpus);
1876 }
1877 
1878 static bool cpu_can_use_dbm(const struct arm64_cpu_capabilities *cap)
1879 {
1880 	return has_cpuid_feature(cap, SCOPE_LOCAL_CPU) &&
1881 	       !cpu_has_broken_dbm();
1882 }
1883 
1884 static void cpu_enable_hw_dbm(struct arm64_cpu_capabilities const *cap)
1885 {
1886 	if (cpu_can_use_dbm(cap))
1887 		__cpu_enable_hw_dbm();
1888 }
1889 
1890 static bool has_hw_dbm(const struct arm64_cpu_capabilities *cap,
1891 		       int __unused)
1892 {
1893 	static bool detected = false;
1894 	/*
1895 	 * DBM is a non-conflicting feature. i.e, the kernel can safely
1896 	 * run a mix of CPUs with and without the feature. So, we
1897 	 * unconditionally enable the capability to allow any late CPU
1898 	 * to use the feature. We only enable the control bits on the
1899 	 * CPU, if it actually supports.
1900 	 *
1901 	 * We have to make sure we print the "feature" detection only
1902 	 * when at least one CPU actually uses it. So check if this CPU
1903 	 * can actually use it and print the message exactly once.
1904 	 *
1905 	 * This is safe as all CPUs (including secondary CPUs - due to the
1906 	 * LOCAL_CPU scope - and the hotplugged CPUs - via verification)
1907 	 * goes through the "matches" check exactly once. Also if a CPU
1908 	 * matches the criteria, it is guaranteed that the CPU will turn
1909 	 * the DBM on, as the capability is unconditionally enabled.
1910 	 */
1911 	if (!detected && cpu_can_use_dbm(cap)) {
1912 		detected = true;
1913 		pr_info("detected: Hardware dirty bit management\n");
1914 	}
1915 
1916 	return true;
1917 }
1918 
1919 #endif
1920 
1921 #ifdef CONFIG_ARM64_AMU_EXTN
1922 
1923 /*
1924  * The "amu_cpus" cpumask only signals that the CPU implementation for the
1925  * flagged CPUs supports the Activity Monitors Unit (AMU) but does not provide
1926  * information regarding all the events that it supports. When a CPU bit is
1927  * set in the cpumask, the user of this feature can only rely on the presence
1928  * of the 4 fixed counters for that CPU. But this does not guarantee that the
1929  * counters are enabled or access to these counters is enabled by code
1930  * executed at higher exception levels (firmware).
1931  */
1932 static struct cpumask amu_cpus __read_mostly;
1933 
1934 bool cpu_has_amu_feat(int cpu)
1935 {
1936 	return cpumask_test_cpu(cpu, &amu_cpus);
1937 }
1938 
1939 int get_cpu_with_amu_feat(void)
1940 {
1941 	return cpumask_any(&amu_cpus);
1942 }
1943 
1944 static void cpu_amu_enable(struct arm64_cpu_capabilities const *cap)
1945 {
1946 	if (has_cpuid_feature(cap, SCOPE_LOCAL_CPU)) {
1947 		pr_info("detected CPU%d: Activity Monitors Unit (AMU)\n",
1948 			smp_processor_id());
1949 		cpumask_set_cpu(smp_processor_id(), &amu_cpus);
1950 
1951 		/* 0 reference values signal broken/disabled counters */
1952 		if (!this_cpu_has_cap(ARM64_WORKAROUND_2457168))
1953 			update_freq_counters_refs();
1954 	}
1955 }
1956 
1957 static bool has_amu(const struct arm64_cpu_capabilities *cap,
1958 		    int __unused)
1959 {
1960 	/*
1961 	 * The AMU extension is a non-conflicting feature: the kernel can
1962 	 * safely run a mix of CPUs with and without support for the
1963 	 * activity monitors extension. Therefore, unconditionally enable
1964 	 * the capability to allow any late CPU to use the feature.
1965 	 *
1966 	 * With this feature unconditionally enabled, the cpu_enable
1967 	 * function will be called for all CPUs that match the criteria,
1968 	 * including secondary and hotplugged, marking this feature as
1969 	 * present on that respective CPU. The enable function will also
1970 	 * print a detection message.
1971 	 */
1972 
1973 	return true;
1974 }
1975 #else
1976 int get_cpu_with_amu_feat(void)
1977 {
1978 	return nr_cpu_ids;
1979 }
1980 #endif
1981 
1982 static bool runs_at_el2(const struct arm64_cpu_capabilities *entry, int __unused)
1983 {
1984 	return is_kernel_in_hyp_mode();
1985 }
1986 
1987 static void cpu_copy_el2regs(const struct arm64_cpu_capabilities *__unused)
1988 {
1989 	/*
1990 	 * Copy register values that aren't redirected by hardware.
1991 	 *
1992 	 * Before code patching, we only set tpidr_el1, all CPUs need to copy
1993 	 * this value to tpidr_el2 before we patch the code. Once we've done
1994 	 * that, freshly-onlined CPUs will set tpidr_el2, so we don't need to
1995 	 * do anything here.
1996 	 */
1997 	if (!alternative_is_applied(ARM64_HAS_VIRT_HOST_EXTN))
1998 		write_sysreg(read_sysreg(tpidr_el1), tpidr_el2);
1999 }
2000 
2001 static bool has_nested_virt_support(const struct arm64_cpu_capabilities *cap,
2002 				    int scope)
2003 {
2004 	if (kvm_get_mode() != KVM_MODE_NV)
2005 		return false;
2006 
2007 	if (!has_cpuid_feature(cap, scope)) {
2008 		pr_warn("unavailable: %s\n", cap->desc);
2009 		return false;
2010 	}
2011 
2012 	return true;
2013 }
2014 
2015 static bool hvhe_possible(const struct arm64_cpu_capabilities *entry,
2016 			  int __unused)
2017 {
2018 	u64 val;
2019 
2020 	val = read_sysreg(id_aa64mmfr1_el1);
2021 	if (!cpuid_feature_extract_unsigned_field(val, ID_AA64MMFR1_EL1_VH_SHIFT))
2022 		return false;
2023 
2024 	val = arm64_sw_feature_override.val & arm64_sw_feature_override.mask;
2025 	return cpuid_feature_extract_unsigned_field(val, ARM64_SW_FEATURE_OVERRIDE_HVHE);
2026 }
2027 
2028 #ifdef CONFIG_ARM64_PAN
2029 static void cpu_enable_pan(const struct arm64_cpu_capabilities *__unused)
2030 {
2031 	/*
2032 	 * We modify PSTATE. This won't work from irq context as the PSTATE
2033 	 * is discarded once we return from the exception.
2034 	 */
2035 	WARN_ON_ONCE(in_interrupt());
2036 
2037 	sysreg_clear_set(sctlr_el1, SCTLR_EL1_SPAN, 0);
2038 	set_pstate_pan(1);
2039 }
2040 #endif /* CONFIG_ARM64_PAN */
2041 
2042 #ifdef CONFIG_ARM64_RAS_EXTN
2043 static void cpu_clear_disr(const struct arm64_cpu_capabilities *__unused)
2044 {
2045 	/* Firmware may have left a deferred SError in this register. */
2046 	write_sysreg_s(0, SYS_DISR_EL1);
2047 }
2048 #endif /* CONFIG_ARM64_RAS_EXTN */
2049 
2050 #ifdef CONFIG_ARM64_PTR_AUTH
2051 static bool has_address_auth_cpucap(const struct arm64_cpu_capabilities *entry, int scope)
2052 {
2053 	int boot_val, sec_val;
2054 
2055 	/* We don't expect to be called with SCOPE_SYSTEM */
2056 	WARN_ON(scope == SCOPE_SYSTEM);
2057 	/*
2058 	 * The ptr-auth feature levels are not intercompatible with lower
2059 	 * levels. Hence we must match ptr-auth feature level of the secondary
2060 	 * CPUs with that of the boot CPU. The level of boot cpu is fetched
2061 	 * from the sanitised register whereas direct register read is done for
2062 	 * the secondary CPUs.
2063 	 * The sanitised feature state is guaranteed to match that of the
2064 	 * boot CPU as a mismatched secondary CPU is parked before it gets
2065 	 * a chance to update the state, with the capability.
2066 	 */
2067 	boot_val = cpuid_feature_extract_field(read_sanitised_ftr_reg(entry->sys_reg),
2068 					       entry->field_pos, entry->sign);
2069 	if (scope & SCOPE_BOOT_CPU)
2070 		return boot_val >= entry->min_field_value;
2071 	/* Now check for the secondary CPUs with SCOPE_LOCAL_CPU scope */
2072 	sec_val = cpuid_feature_extract_field(__read_sysreg_by_encoding(entry->sys_reg),
2073 					      entry->field_pos, entry->sign);
2074 	return (sec_val >= entry->min_field_value) && (sec_val == boot_val);
2075 }
2076 
2077 static bool has_address_auth_metacap(const struct arm64_cpu_capabilities *entry,
2078 				     int scope)
2079 {
2080 	bool api = has_address_auth_cpucap(cpucap_ptrs[ARM64_HAS_ADDRESS_AUTH_IMP_DEF], scope);
2081 	bool apa = has_address_auth_cpucap(cpucap_ptrs[ARM64_HAS_ADDRESS_AUTH_ARCH_QARMA5], scope);
2082 	bool apa3 = has_address_auth_cpucap(cpucap_ptrs[ARM64_HAS_ADDRESS_AUTH_ARCH_QARMA3], scope);
2083 
2084 	return apa || apa3 || api;
2085 }
2086 
2087 static bool has_generic_auth(const struct arm64_cpu_capabilities *entry,
2088 			     int __unused)
2089 {
2090 	bool gpi = __system_matches_cap(ARM64_HAS_GENERIC_AUTH_IMP_DEF);
2091 	bool gpa = __system_matches_cap(ARM64_HAS_GENERIC_AUTH_ARCH_QARMA5);
2092 	bool gpa3 = __system_matches_cap(ARM64_HAS_GENERIC_AUTH_ARCH_QARMA3);
2093 
2094 	return gpa || gpa3 || gpi;
2095 }
2096 #endif /* CONFIG_ARM64_PTR_AUTH */
2097 
2098 #ifdef CONFIG_ARM64_E0PD
2099 static void cpu_enable_e0pd(struct arm64_cpu_capabilities const *cap)
2100 {
2101 	if (this_cpu_has_cap(ARM64_HAS_E0PD))
2102 		sysreg_clear_set(tcr_el1, 0, TCR_E0PD1);
2103 }
2104 #endif /* CONFIG_ARM64_E0PD */
2105 
2106 #ifdef CONFIG_ARM64_PSEUDO_NMI
2107 static bool enable_pseudo_nmi;
2108 
2109 static int __init early_enable_pseudo_nmi(char *p)
2110 {
2111 	return kstrtobool(p, &enable_pseudo_nmi);
2112 }
2113 early_param("irqchip.gicv3_pseudo_nmi", early_enable_pseudo_nmi);
2114 
2115 static bool can_use_gic_priorities(const struct arm64_cpu_capabilities *entry,
2116 				   int scope)
2117 {
2118 	/*
2119 	 * ARM64_HAS_GIC_CPUIF_SYSREGS has a lower index, and is a boot CPU
2120 	 * feature, so will be detected earlier.
2121 	 */
2122 	BUILD_BUG_ON(ARM64_HAS_GIC_PRIO_MASKING <= ARM64_HAS_GIC_CPUIF_SYSREGS);
2123 	if (!cpus_have_cap(ARM64_HAS_GIC_CPUIF_SYSREGS))
2124 		return false;
2125 
2126 	return enable_pseudo_nmi;
2127 }
2128 
2129 static bool has_gic_prio_relaxed_sync(const struct arm64_cpu_capabilities *entry,
2130 				      int scope)
2131 {
2132 	/*
2133 	 * If we're not using priority masking then we won't be poking PMR_EL1,
2134 	 * and there's no need to relax synchronization of writes to it, and
2135 	 * ICC_CTLR_EL1 might not be accessible and we must avoid reads from
2136 	 * that.
2137 	 *
2138 	 * ARM64_HAS_GIC_PRIO_MASKING has a lower index, and is a boot CPU
2139 	 * feature, so will be detected earlier.
2140 	 */
2141 	BUILD_BUG_ON(ARM64_HAS_GIC_PRIO_RELAXED_SYNC <= ARM64_HAS_GIC_PRIO_MASKING);
2142 	if (!cpus_have_cap(ARM64_HAS_GIC_PRIO_MASKING))
2143 		return false;
2144 
2145 	/*
2146 	 * When Priority Mask Hint Enable (PMHE) == 0b0, PMR is not used as a
2147 	 * hint for interrupt distribution, a DSB is not necessary when
2148 	 * unmasking IRQs via PMR, and we can relax the barrier to a NOP.
2149 	 *
2150 	 * Linux itself doesn't use 1:N distribution, so has no need to
2151 	 * set PMHE. The only reason to have it set is if EL3 requires it
2152 	 * (and we can't change it).
2153 	 */
2154 	return (gic_read_ctlr() & ICC_CTLR_EL1_PMHE_MASK) == 0;
2155 }
2156 #endif
2157 
2158 #ifdef CONFIG_ARM64_BTI
2159 static void bti_enable(const struct arm64_cpu_capabilities *__unused)
2160 {
2161 	/*
2162 	 * Use of X16/X17 for tail-calls and trampolines that jump to
2163 	 * function entry points using BR is a requirement for
2164 	 * marking binaries with GNU_PROPERTY_AARCH64_FEATURE_1_BTI.
2165 	 * So, be strict and forbid other BRs using other registers to
2166 	 * jump onto a PACIxSP instruction:
2167 	 */
2168 	sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_BT0 | SCTLR_EL1_BT1);
2169 	isb();
2170 }
2171 #endif /* CONFIG_ARM64_BTI */
2172 
2173 #ifdef CONFIG_ARM64_MTE
2174 static void cpu_enable_mte(struct arm64_cpu_capabilities const *cap)
2175 {
2176 	sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_ATA | SCTLR_EL1_ATA0);
2177 
2178 	mte_cpu_setup();
2179 
2180 	/*
2181 	 * Clear the tags in the zero page. This needs to be done via the
2182 	 * linear map which has the Tagged attribute.
2183 	 */
2184 	if (try_page_mte_tagging(ZERO_PAGE(0))) {
2185 		mte_clear_page_tags(lm_alias(empty_zero_page));
2186 		set_page_mte_tagged(ZERO_PAGE(0));
2187 	}
2188 
2189 	kasan_init_hw_tags_cpu();
2190 }
2191 #endif /* CONFIG_ARM64_MTE */
2192 
2193 static void elf_hwcap_fixup(void)
2194 {
2195 #ifdef CONFIG_ARM64_ERRATUM_1742098
2196 	if (cpus_have_const_cap(ARM64_WORKAROUND_1742098))
2197 		compat_elf_hwcap2 &= ~COMPAT_HWCAP2_AES;
2198 #endif /* ARM64_ERRATUM_1742098 */
2199 }
2200 
2201 #ifdef CONFIG_KVM
2202 static bool is_kvm_protected_mode(const struct arm64_cpu_capabilities *entry, int __unused)
2203 {
2204 	return kvm_get_mode() == KVM_MODE_PROTECTED;
2205 }
2206 #endif /* CONFIG_KVM */
2207 
2208 static void cpu_trap_el0_impdef(const struct arm64_cpu_capabilities *__unused)
2209 {
2210 	sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_TIDCP);
2211 }
2212 
2213 static void cpu_enable_dit(const struct arm64_cpu_capabilities *__unused)
2214 {
2215 	set_pstate_dit(1);
2216 }
2217 
2218 static void cpu_enable_mops(const struct arm64_cpu_capabilities *__unused)
2219 {
2220 	sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_MSCEn);
2221 }
2222 
2223 /* Internal helper functions to match cpu capability type */
2224 static bool
2225 cpucap_late_cpu_optional(const struct arm64_cpu_capabilities *cap)
2226 {
2227 	return !!(cap->type & ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU);
2228 }
2229 
2230 static bool
2231 cpucap_late_cpu_permitted(const struct arm64_cpu_capabilities *cap)
2232 {
2233 	return !!(cap->type & ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU);
2234 }
2235 
2236 static bool
2237 cpucap_panic_on_conflict(const struct arm64_cpu_capabilities *cap)
2238 {
2239 	return !!(cap->type & ARM64_CPUCAP_PANIC_ON_CONFLICT);
2240 }
2241 
2242 static const struct arm64_cpu_capabilities arm64_features[] = {
2243 	{
2244 		.capability = ARM64_ALWAYS_BOOT,
2245 		.type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
2246 		.matches = has_always,
2247 	},
2248 	{
2249 		.capability = ARM64_ALWAYS_SYSTEM,
2250 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2251 		.matches = has_always,
2252 	},
2253 	{
2254 		.desc = "GIC system register CPU interface",
2255 		.capability = ARM64_HAS_GIC_CPUIF_SYSREGS,
2256 		.type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
2257 		.matches = has_useable_gicv3_cpuif,
2258 		ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, GIC, IMP)
2259 	},
2260 	{
2261 		.desc = "Enhanced Counter Virtualization",
2262 		.capability = ARM64_HAS_ECV,
2263 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2264 		.matches = has_cpuid_feature,
2265 		ARM64_CPUID_FIELDS(ID_AA64MMFR0_EL1, ECV, IMP)
2266 	},
2267 	{
2268 		.desc = "Enhanced Counter Virtualization (CNTPOFF)",
2269 		.capability = ARM64_HAS_ECV_CNTPOFF,
2270 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2271 		.matches = has_cpuid_feature,
2272 		ARM64_CPUID_FIELDS(ID_AA64MMFR0_EL1, ECV, CNTPOFF)
2273 	},
2274 #ifdef CONFIG_ARM64_PAN
2275 	{
2276 		.desc = "Privileged Access Never",
2277 		.capability = ARM64_HAS_PAN,
2278 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2279 		.matches = has_cpuid_feature,
2280 		.cpu_enable = cpu_enable_pan,
2281 		ARM64_CPUID_FIELDS(ID_AA64MMFR1_EL1, PAN, IMP)
2282 	},
2283 #endif /* CONFIG_ARM64_PAN */
2284 #ifdef CONFIG_ARM64_EPAN
2285 	{
2286 		.desc = "Enhanced Privileged Access Never",
2287 		.capability = ARM64_HAS_EPAN,
2288 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2289 		.matches = has_cpuid_feature,
2290 		ARM64_CPUID_FIELDS(ID_AA64MMFR1_EL1, PAN, PAN3)
2291 	},
2292 #endif /* CONFIG_ARM64_EPAN */
2293 #ifdef CONFIG_ARM64_LSE_ATOMICS
2294 	{
2295 		.desc = "LSE atomic instructions",
2296 		.capability = ARM64_HAS_LSE_ATOMICS,
2297 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2298 		.matches = has_cpuid_feature,
2299 		ARM64_CPUID_FIELDS(ID_AA64ISAR0_EL1, ATOMIC, IMP)
2300 	},
2301 #endif /* CONFIG_ARM64_LSE_ATOMICS */
2302 	{
2303 		.desc = "Software prefetching using PRFM",
2304 		.capability = ARM64_HAS_NO_HW_PREFETCH,
2305 		.type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE,
2306 		.matches = has_no_hw_prefetch,
2307 	},
2308 	{
2309 		.desc = "Virtualization Host Extensions",
2310 		.capability = ARM64_HAS_VIRT_HOST_EXTN,
2311 		.type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
2312 		.matches = runs_at_el2,
2313 		.cpu_enable = cpu_copy_el2regs,
2314 	},
2315 	{
2316 		.desc = "Nested Virtualization Support",
2317 		.capability = ARM64_HAS_NESTED_VIRT,
2318 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2319 		.matches = has_nested_virt_support,
2320 		ARM64_CPUID_FIELDS(ID_AA64MMFR2_EL1, NV, IMP)
2321 	},
2322 	{
2323 		.capability = ARM64_HAS_32BIT_EL0_DO_NOT_USE,
2324 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2325 		.matches = has_32bit_el0,
2326 		ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, EL0, AARCH32)
2327 	},
2328 #ifdef CONFIG_KVM
2329 	{
2330 		.desc = "32-bit EL1 Support",
2331 		.capability = ARM64_HAS_32BIT_EL1,
2332 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2333 		.matches = has_cpuid_feature,
2334 		ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, EL1, AARCH32)
2335 	},
2336 	{
2337 		.desc = "Protected KVM",
2338 		.capability = ARM64_KVM_PROTECTED_MODE,
2339 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2340 		.matches = is_kvm_protected_mode,
2341 	},
2342 	{
2343 		.desc = "HCRX_EL2 register",
2344 		.capability = ARM64_HAS_HCX,
2345 		.type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
2346 		.matches = has_cpuid_feature,
2347 		ARM64_CPUID_FIELDS(ID_AA64MMFR1_EL1, HCX, IMP)
2348 	},
2349 #endif
2350 	{
2351 		.desc = "Kernel page table isolation (KPTI)",
2352 		.capability = ARM64_UNMAP_KERNEL_AT_EL0,
2353 		.type = ARM64_CPUCAP_BOOT_RESTRICTED_CPU_LOCAL_FEATURE,
2354 		.cpu_enable = kpti_install_ng_mappings,
2355 		.matches = unmap_kernel_at_el0,
2356 		/*
2357 		 * The ID feature fields below are used to indicate that
2358 		 * the CPU doesn't need KPTI. See unmap_kernel_at_el0 for
2359 		 * more details.
2360 		 */
2361 		ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, CSV3, IMP)
2362 	},
2363 	{
2364 		/* FP/SIMD is not implemented */
2365 		.capability = ARM64_HAS_NO_FPSIMD,
2366 		.type = ARM64_CPUCAP_BOOT_RESTRICTED_CPU_LOCAL_FEATURE,
2367 		.min_field_value = 0,
2368 		.matches = has_no_fpsimd,
2369 	},
2370 #ifdef CONFIG_ARM64_PMEM
2371 	{
2372 		.desc = "Data cache clean to Point of Persistence",
2373 		.capability = ARM64_HAS_DCPOP,
2374 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2375 		.matches = has_cpuid_feature,
2376 		ARM64_CPUID_FIELDS(ID_AA64ISAR1_EL1, DPB, IMP)
2377 	},
2378 	{
2379 		.desc = "Data cache clean to Point of Deep Persistence",
2380 		.capability = ARM64_HAS_DCPODP,
2381 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2382 		.matches = has_cpuid_feature,
2383 		ARM64_CPUID_FIELDS(ID_AA64ISAR1_EL1, DPB, DPB2)
2384 	},
2385 #endif
2386 #ifdef CONFIG_ARM64_SVE
2387 	{
2388 		.desc = "Scalable Vector Extension",
2389 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2390 		.capability = ARM64_SVE,
2391 		.cpu_enable = sve_kernel_enable,
2392 		.matches = has_cpuid_feature,
2393 		ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, SVE, IMP)
2394 	},
2395 #endif /* CONFIG_ARM64_SVE */
2396 #ifdef CONFIG_ARM64_RAS_EXTN
2397 	{
2398 		.desc = "RAS Extension Support",
2399 		.capability = ARM64_HAS_RAS_EXTN,
2400 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2401 		.matches = has_cpuid_feature,
2402 		.cpu_enable = cpu_clear_disr,
2403 		ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, RAS, IMP)
2404 	},
2405 #endif /* CONFIG_ARM64_RAS_EXTN */
2406 #ifdef CONFIG_ARM64_AMU_EXTN
2407 	{
2408 		/*
2409 		 * The feature is enabled by default if CONFIG_ARM64_AMU_EXTN=y.
2410 		 * Therefore, don't provide .desc as we don't want the detection
2411 		 * message to be shown until at least one CPU is detected to
2412 		 * support the feature.
2413 		 */
2414 		.capability = ARM64_HAS_AMU_EXTN,
2415 		.type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE,
2416 		.matches = has_amu,
2417 		.cpu_enable = cpu_amu_enable,
2418 		ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, AMU, IMP)
2419 	},
2420 #endif /* CONFIG_ARM64_AMU_EXTN */
2421 	{
2422 		.desc = "Data cache clean to the PoU not required for I/D coherence",
2423 		.capability = ARM64_HAS_CACHE_IDC,
2424 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2425 		.matches = has_cache_idc,
2426 		.cpu_enable = cpu_emulate_effective_ctr,
2427 	},
2428 	{
2429 		.desc = "Instruction cache invalidation not required for I/D coherence",
2430 		.capability = ARM64_HAS_CACHE_DIC,
2431 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2432 		.matches = has_cache_dic,
2433 	},
2434 	{
2435 		.desc = "Stage-2 Force Write-Back",
2436 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2437 		.capability = ARM64_HAS_STAGE2_FWB,
2438 		.matches = has_cpuid_feature,
2439 		ARM64_CPUID_FIELDS(ID_AA64MMFR2_EL1, FWB, IMP)
2440 	},
2441 	{
2442 		.desc = "ARMv8.4 Translation Table Level",
2443 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2444 		.capability = ARM64_HAS_ARMv8_4_TTL,
2445 		.matches = has_cpuid_feature,
2446 		ARM64_CPUID_FIELDS(ID_AA64MMFR2_EL1, TTL, IMP)
2447 	},
2448 	{
2449 		.desc = "TLB range maintenance instructions",
2450 		.capability = ARM64_HAS_TLB_RANGE,
2451 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2452 		.matches = has_cpuid_feature,
2453 		ARM64_CPUID_FIELDS(ID_AA64ISAR0_EL1, TLB, RANGE)
2454 	},
2455 #ifdef CONFIG_ARM64_HW_AFDBM
2456 	{
2457 		/*
2458 		 * Since we turn this on always, we don't want the user to
2459 		 * think that the feature is available when it may not be.
2460 		 * So hide the description.
2461 		 *
2462 		 * .desc = "Hardware pagetable Dirty Bit Management",
2463 		 *
2464 		 */
2465 		.type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE,
2466 		.capability = ARM64_HW_DBM,
2467 		.matches = has_hw_dbm,
2468 		.cpu_enable = cpu_enable_hw_dbm,
2469 		ARM64_CPUID_FIELDS(ID_AA64MMFR1_EL1, HAFDBS, DBM)
2470 	},
2471 #endif
2472 	{
2473 		.desc = "CRC32 instructions",
2474 		.capability = ARM64_HAS_CRC32,
2475 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2476 		.matches = has_cpuid_feature,
2477 		ARM64_CPUID_FIELDS(ID_AA64ISAR0_EL1, CRC32, IMP)
2478 	},
2479 	{
2480 		.desc = "Speculative Store Bypassing Safe (SSBS)",
2481 		.capability = ARM64_SSBS,
2482 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2483 		.matches = has_cpuid_feature,
2484 		ARM64_CPUID_FIELDS(ID_AA64PFR1_EL1, SSBS, IMP)
2485 	},
2486 #ifdef CONFIG_ARM64_CNP
2487 	{
2488 		.desc = "Common not Private translations",
2489 		.capability = ARM64_HAS_CNP,
2490 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2491 		.matches = has_useable_cnp,
2492 		.cpu_enable = cpu_enable_cnp,
2493 		ARM64_CPUID_FIELDS(ID_AA64MMFR2_EL1, CnP, IMP)
2494 	},
2495 #endif
2496 	{
2497 		.desc = "Speculation barrier (SB)",
2498 		.capability = ARM64_HAS_SB,
2499 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2500 		.matches = has_cpuid_feature,
2501 		ARM64_CPUID_FIELDS(ID_AA64ISAR1_EL1, SB, IMP)
2502 	},
2503 #ifdef CONFIG_ARM64_PTR_AUTH
2504 	{
2505 		.desc = "Address authentication (architected QARMA5 algorithm)",
2506 		.capability = ARM64_HAS_ADDRESS_AUTH_ARCH_QARMA5,
2507 		.type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
2508 		.matches = has_address_auth_cpucap,
2509 		ARM64_CPUID_FIELDS(ID_AA64ISAR1_EL1, APA, PAuth)
2510 	},
2511 	{
2512 		.desc = "Address authentication (architected QARMA3 algorithm)",
2513 		.capability = ARM64_HAS_ADDRESS_AUTH_ARCH_QARMA3,
2514 		.type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
2515 		.matches = has_address_auth_cpucap,
2516 		ARM64_CPUID_FIELDS(ID_AA64ISAR2_EL1, APA3, PAuth)
2517 	},
2518 	{
2519 		.desc = "Address authentication (IMP DEF algorithm)",
2520 		.capability = ARM64_HAS_ADDRESS_AUTH_IMP_DEF,
2521 		.type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
2522 		.matches = has_address_auth_cpucap,
2523 		ARM64_CPUID_FIELDS(ID_AA64ISAR1_EL1, API, PAuth)
2524 	},
2525 	{
2526 		.capability = ARM64_HAS_ADDRESS_AUTH,
2527 		.type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
2528 		.matches = has_address_auth_metacap,
2529 	},
2530 	{
2531 		.desc = "Generic authentication (architected QARMA5 algorithm)",
2532 		.capability = ARM64_HAS_GENERIC_AUTH_ARCH_QARMA5,
2533 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2534 		.matches = has_cpuid_feature,
2535 		ARM64_CPUID_FIELDS(ID_AA64ISAR1_EL1, GPA, IMP)
2536 	},
2537 	{
2538 		.desc = "Generic authentication (architected QARMA3 algorithm)",
2539 		.capability = ARM64_HAS_GENERIC_AUTH_ARCH_QARMA3,
2540 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2541 		.matches = has_cpuid_feature,
2542 		ARM64_CPUID_FIELDS(ID_AA64ISAR2_EL1, GPA3, IMP)
2543 	},
2544 	{
2545 		.desc = "Generic authentication (IMP DEF algorithm)",
2546 		.capability = ARM64_HAS_GENERIC_AUTH_IMP_DEF,
2547 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2548 		.matches = has_cpuid_feature,
2549 		ARM64_CPUID_FIELDS(ID_AA64ISAR1_EL1, GPI, IMP)
2550 	},
2551 	{
2552 		.capability = ARM64_HAS_GENERIC_AUTH,
2553 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2554 		.matches = has_generic_auth,
2555 	},
2556 #endif /* CONFIG_ARM64_PTR_AUTH */
2557 #ifdef CONFIG_ARM64_PSEUDO_NMI
2558 	{
2559 		/*
2560 		 * Depends on having GICv3
2561 		 */
2562 		.desc = "IRQ priority masking",
2563 		.capability = ARM64_HAS_GIC_PRIO_MASKING,
2564 		.type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
2565 		.matches = can_use_gic_priorities,
2566 	},
2567 	{
2568 		/*
2569 		 * Depends on ARM64_HAS_GIC_PRIO_MASKING
2570 		 */
2571 		.capability = ARM64_HAS_GIC_PRIO_RELAXED_SYNC,
2572 		.type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
2573 		.matches = has_gic_prio_relaxed_sync,
2574 	},
2575 #endif
2576 #ifdef CONFIG_ARM64_E0PD
2577 	{
2578 		.desc = "E0PD",
2579 		.capability = ARM64_HAS_E0PD,
2580 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2581 		.cpu_enable = cpu_enable_e0pd,
2582 		.matches = has_cpuid_feature,
2583 		ARM64_CPUID_FIELDS(ID_AA64MMFR2_EL1, E0PD, IMP)
2584 	},
2585 #endif
2586 	{
2587 		.desc = "Random Number Generator",
2588 		.capability = ARM64_HAS_RNG,
2589 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2590 		.matches = has_cpuid_feature,
2591 		ARM64_CPUID_FIELDS(ID_AA64ISAR0_EL1, RNDR, IMP)
2592 	},
2593 #ifdef CONFIG_ARM64_BTI
2594 	{
2595 		.desc = "Branch Target Identification",
2596 		.capability = ARM64_BTI,
2597 #ifdef CONFIG_ARM64_BTI_KERNEL
2598 		.type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
2599 #else
2600 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2601 #endif
2602 		.matches = has_cpuid_feature,
2603 		.cpu_enable = bti_enable,
2604 		ARM64_CPUID_FIELDS(ID_AA64PFR1_EL1, BT, IMP)
2605 	},
2606 #endif
2607 #ifdef CONFIG_ARM64_MTE
2608 	{
2609 		.desc = "Memory Tagging Extension",
2610 		.capability = ARM64_MTE,
2611 		.type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
2612 		.matches = has_cpuid_feature,
2613 		.cpu_enable = cpu_enable_mte,
2614 		ARM64_CPUID_FIELDS(ID_AA64PFR1_EL1, MTE, MTE2)
2615 	},
2616 	{
2617 		.desc = "Asymmetric MTE Tag Check Fault",
2618 		.capability = ARM64_MTE_ASYMM,
2619 		.type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
2620 		.matches = has_cpuid_feature,
2621 		ARM64_CPUID_FIELDS(ID_AA64PFR1_EL1, MTE, MTE3)
2622 	},
2623 #endif /* CONFIG_ARM64_MTE */
2624 	{
2625 		.desc = "RCpc load-acquire (LDAPR)",
2626 		.capability = ARM64_HAS_LDAPR,
2627 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2628 		.matches = has_cpuid_feature,
2629 		ARM64_CPUID_FIELDS(ID_AA64ISAR1_EL1, LRCPC, IMP)
2630 	},
2631 	{
2632 		.desc = "Fine Grained Traps",
2633 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2634 		.capability = ARM64_HAS_FGT,
2635 		.matches = has_cpuid_feature,
2636 		ARM64_CPUID_FIELDS(ID_AA64MMFR0_EL1, FGT, IMP)
2637 	},
2638 #ifdef CONFIG_ARM64_SME
2639 	{
2640 		.desc = "Scalable Matrix Extension",
2641 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2642 		.capability = ARM64_SME,
2643 		.matches = has_cpuid_feature,
2644 		.cpu_enable = sme_kernel_enable,
2645 		ARM64_CPUID_FIELDS(ID_AA64PFR1_EL1, SME, IMP)
2646 	},
2647 	/* FA64 should be sorted after the base SME capability */
2648 	{
2649 		.desc = "FA64",
2650 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2651 		.capability = ARM64_SME_FA64,
2652 		.matches = has_cpuid_feature,
2653 		.cpu_enable = fa64_kernel_enable,
2654 		ARM64_CPUID_FIELDS(ID_AA64SMFR0_EL1, FA64, IMP)
2655 	},
2656 	{
2657 		.desc = "SME2",
2658 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2659 		.capability = ARM64_SME2,
2660 		.matches = has_cpuid_feature,
2661 		.cpu_enable = sme2_kernel_enable,
2662 		ARM64_CPUID_FIELDS(ID_AA64PFR1_EL1, SME, SME2)
2663 	},
2664 #endif /* CONFIG_ARM64_SME */
2665 	{
2666 		.desc = "WFx with timeout",
2667 		.capability = ARM64_HAS_WFXT,
2668 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2669 		.matches = has_cpuid_feature,
2670 		ARM64_CPUID_FIELDS(ID_AA64ISAR2_EL1, WFxT, IMP)
2671 	},
2672 	{
2673 		.desc = "Trap EL0 IMPLEMENTATION DEFINED functionality",
2674 		.capability = ARM64_HAS_TIDCP1,
2675 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2676 		.matches = has_cpuid_feature,
2677 		.cpu_enable = cpu_trap_el0_impdef,
2678 		ARM64_CPUID_FIELDS(ID_AA64MMFR1_EL1, TIDCP1, IMP)
2679 	},
2680 	{
2681 		.desc = "Data independent timing control (DIT)",
2682 		.capability = ARM64_HAS_DIT,
2683 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2684 		.matches = has_cpuid_feature,
2685 		.cpu_enable = cpu_enable_dit,
2686 		ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, DIT, IMP)
2687 	},
2688 	{
2689 		.desc = "Memory Copy and Memory Set instructions",
2690 		.capability = ARM64_HAS_MOPS,
2691 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2692 		.matches = has_cpuid_feature,
2693 		.cpu_enable = cpu_enable_mops,
2694 		ARM64_CPUID_FIELDS(ID_AA64ISAR2_EL1, MOPS, IMP)
2695 	},
2696 	{
2697 		.capability = ARM64_HAS_TCR2,
2698 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2699 		.matches = has_cpuid_feature,
2700 		ARM64_CPUID_FIELDS(ID_AA64MMFR3_EL1, TCRX, IMP)
2701 	},
2702 	{
2703 		.desc = "Stage-1 Permission Indirection Extension (S1PIE)",
2704 		.capability = ARM64_HAS_S1PIE,
2705 		.type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
2706 		.matches = has_cpuid_feature,
2707 		ARM64_CPUID_FIELDS(ID_AA64MMFR3_EL1, S1PIE, IMP)
2708 	},
2709 	{
2710 		.desc = "VHE for hypervisor only",
2711 		.capability = ARM64_KVM_HVHE,
2712 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2713 		.matches = hvhe_possible,
2714 	},
2715 	{
2716 		.desc = "Enhanced Virtualization Traps",
2717 		.capability = ARM64_HAS_EVT,
2718 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2719 		.matches = has_cpuid_feature,
2720 		ARM64_CPUID_FIELDS(ID_AA64MMFR2_EL1, EVT, IMP)
2721 	},
2722 	{},
2723 };
2724 
2725 #define HWCAP_CPUID_MATCH(reg, field, min_value)			\
2726 		.matches = has_user_cpuid_feature,			\
2727 		ARM64_CPUID_FIELDS(reg, field, min_value)
2728 
2729 #define __HWCAP_CAP(name, cap_type, cap)					\
2730 		.desc = name,							\
2731 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,				\
2732 		.hwcap_type = cap_type,						\
2733 		.hwcap = cap,							\
2734 
2735 #define HWCAP_CAP(reg, field, min_value, cap_type, cap)		\
2736 	{									\
2737 		__HWCAP_CAP(#cap, cap_type, cap)				\
2738 		HWCAP_CPUID_MATCH(reg, field, min_value) 		\
2739 	}
2740 
2741 #define HWCAP_MULTI_CAP(list, cap_type, cap)					\
2742 	{									\
2743 		__HWCAP_CAP(#cap, cap_type, cap)				\
2744 		.matches = cpucap_multi_entry_cap_matches,			\
2745 		.match_list = list,						\
2746 	}
2747 
2748 #define HWCAP_CAP_MATCH(match, cap_type, cap)					\
2749 	{									\
2750 		__HWCAP_CAP(#cap, cap_type, cap)				\
2751 		.matches = match,						\
2752 	}
2753 
2754 #ifdef CONFIG_ARM64_PTR_AUTH
2755 static const struct arm64_cpu_capabilities ptr_auth_hwcap_addr_matches[] = {
2756 	{
2757 		HWCAP_CPUID_MATCH(ID_AA64ISAR1_EL1, APA, PAuth)
2758 	},
2759 	{
2760 		HWCAP_CPUID_MATCH(ID_AA64ISAR2_EL1, APA3, PAuth)
2761 	},
2762 	{
2763 		HWCAP_CPUID_MATCH(ID_AA64ISAR1_EL1, API, PAuth)
2764 	},
2765 	{},
2766 };
2767 
2768 static const struct arm64_cpu_capabilities ptr_auth_hwcap_gen_matches[] = {
2769 	{
2770 		HWCAP_CPUID_MATCH(ID_AA64ISAR1_EL1, GPA, IMP)
2771 	},
2772 	{
2773 		HWCAP_CPUID_MATCH(ID_AA64ISAR2_EL1, GPA3, IMP)
2774 	},
2775 	{
2776 		HWCAP_CPUID_MATCH(ID_AA64ISAR1_EL1, GPI, IMP)
2777 	},
2778 	{},
2779 };
2780 #endif
2781 
2782 static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {
2783 	HWCAP_CAP(ID_AA64ISAR0_EL1, AES, PMULL, CAP_HWCAP, KERNEL_HWCAP_PMULL),
2784 	HWCAP_CAP(ID_AA64ISAR0_EL1, AES, AES, CAP_HWCAP, KERNEL_HWCAP_AES),
2785 	HWCAP_CAP(ID_AA64ISAR0_EL1, SHA1, IMP, CAP_HWCAP, KERNEL_HWCAP_SHA1),
2786 	HWCAP_CAP(ID_AA64ISAR0_EL1, SHA2, SHA256, CAP_HWCAP, KERNEL_HWCAP_SHA2),
2787 	HWCAP_CAP(ID_AA64ISAR0_EL1, SHA2, SHA512, CAP_HWCAP, KERNEL_HWCAP_SHA512),
2788 	HWCAP_CAP(ID_AA64ISAR0_EL1, CRC32, IMP, CAP_HWCAP, KERNEL_HWCAP_CRC32),
2789 	HWCAP_CAP(ID_AA64ISAR0_EL1, ATOMIC, IMP, CAP_HWCAP, KERNEL_HWCAP_ATOMICS),
2790 	HWCAP_CAP(ID_AA64ISAR0_EL1, RDM, IMP, CAP_HWCAP, KERNEL_HWCAP_ASIMDRDM),
2791 	HWCAP_CAP(ID_AA64ISAR0_EL1, SHA3, IMP, CAP_HWCAP, KERNEL_HWCAP_SHA3),
2792 	HWCAP_CAP(ID_AA64ISAR0_EL1, SM3, IMP, CAP_HWCAP, KERNEL_HWCAP_SM3),
2793 	HWCAP_CAP(ID_AA64ISAR0_EL1, SM4, IMP, CAP_HWCAP, KERNEL_HWCAP_SM4),
2794 	HWCAP_CAP(ID_AA64ISAR0_EL1, DP, IMP, CAP_HWCAP, KERNEL_HWCAP_ASIMDDP),
2795 	HWCAP_CAP(ID_AA64ISAR0_EL1, FHM, IMP, CAP_HWCAP, KERNEL_HWCAP_ASIMDFHM),
2796 	HWCAP_CAP(ID_AA64ISAR0_EL1, TS, FLAGM, CAP_HWCAP, KERNEL_HWCAP_FLAGM),
2797 	HWCAP_CAP(ID_AA64ISAR0_EL1, TS, FLAGM2, CAP_HWCAP, KERNEL_HWCAP_FLAGM2),
2798 	HWCAP_CAP(ID_AA64ISAR0_EL1, RNDR, IMP, CAP_HWCAP, KERNEL_HWCAP_RNG),
2799 	HWCAP_CAP(ID_AA64PFR0_EL1, FP, IMP, CAP_HWCAP, KERNEL_HWCAP_FP),
2800 	HWCAP_CAP(ID_AA64PFR0_EL1, FP, FP16, CAP_HWCAP, KERNEL_HWCAP_FPHP),
2801 	HWCAP_CAP(ID_AA64PFR0_EL1, AdvSIMD, IMP, CAP_HWCAP, KERNEL_HWCAP_ASIMD),
2802 	HWCAP_CAP(ID_AA64PFR0_EL1, AdvSIMD, FP16, CAP_HWCAP, KERNEL_HWCAP_ASIMDHP),
2803 	HWCAP_CAP(ID_AA64PFR0_EL1, DIT, IMP, CAP_HWCAP, KERNEL_HWCAP_DIT),
2804 	HWCAP_CAP(ID_AA64ISAR1_EL1, DPB, IMP, CAP_HWCAP, KERNEL_HWCAP_DCPOP),
2805 	HWCAP_CAP(ID_AA64ISAR1_EL1, DPB, DPB2, CAP_HWCAP, KERNEL_HWCAP_DCPODP),
2806 	HWCAP_CAP(ID_AA64ISAR1_EL1, JSCVT, IMP, CAP_HWCAP, KERNEL_HWCAP_JSCVT),
2807 	HWCAP_CAP(ID_AA64ISAR1_EL1, FCMA, IMP, CAP_HWCAP, KERNEL_HWCAP_FCMA),
2808 	HWCAP_CAP(ID_AA64ISAR1_EL1, LRCPC, IMP, CAP_HWCAP, KERNEL_HWCAP_LRCPC),
2809 	HWCAP_CAP(ID_AA64ISAR1_EL1, LRCPC, LRCPC2, CAP_HWCAP, KERNEL_HWCAP_ILRCPC),
2810 	HWCAP_CAP(ID_AA64ISAR1_EL1, FRINTTS, IMP, CAP_HWCAP, KERNEL_HWCAP_FRINT),
2811 	HWCAP_CAP(ID_AA64ISAR1_EL1, SB, IMP, CAP_HWCAP, KERNEL_HWCAP_SB),
2812 	HWCAP_CAP(ID_AA64ISAR1_EL1, BF16, IMP, CAP_HWCAP, KERNEL_HWCAP_BF16),
2813 	HWCAP_CAP(ID_AA64ISAR1_EL1, BF16, EBF16, CAP_HWCAP, KERNEL_HWCAP_EBF16),
2814 	HWCAP_CAP(ID_AA64ISAR1_EL1, DGH, IMP, CAP_HWCAP, KERNEL_HWCAP_DGH),
2815 	HWCAP_CAP(ID_AA64ISAR1_EL1, I8MM, IMP, CAP_HWCAP, KERNEL_HWCAP_I8MM),
2816 	HWCAP_CAP(ID_AA64MMFR2_EL1, AT, IMP, CAP_HWCAP, KERNEL_HWCAP_USCAT),
2817 #ifdef CONFIG_ARM64_SVE
2818 	HWCAP_CAP(ID_AA64PFR0_EL1, SVE, IMP, CAP_HWCAP, KERNEL_HWCAP_SVE),
2819 	HWCAP_CAP(ID_AA64ZFR0_EL1, SVEver, SVE2p1, CAP_HWCAP, KERNEL_HWCAP_SVE2P1),
2820 	HWCAP_CAP(ID_AA64ZFR0_EL1, SVEver, SVE2, CAP_HWCAP, KERNEL_HWCAP_SVE2),
2821 	HWCAP_CAP(ID_AA64ZFR0_EL1, AES, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEAES),
2822 	HWCAP_CAP(ID_AA64ZFR0_EL1, AES, PMULL128, CAP_HWCAP, KERNEL_HWCAP_SVEPMULL),
2823 	HWCAP_CAP(ID_AA64ZFR0_EL1, BitPerm, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEBITPERM),
2824 	HWCAP_CAP(ID_AA64ZFR0_EL1, BF16, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEBF16),
2825 	HWCAP_CAP(ID_AA64ZFR0_EL1, BF16, EBF16, CAP_HWCAP, KERNEL_HWCAP_SVE_EBF16),
2826 	HWCAP_CAP(ID_AA64ZFR0_EL1, SHA3, IMP, CAP_HWCAP, KERNEL_HWCAP_SVESHA3),
2827 	HWCAP_CAP(ID_AA64ZFR0_EL1, SM4, IMP, CAP_HWCAP, KERNEL_HWCAP_SVESM4),
2828 	HWCAP_CAP(ID_AA64ZFR0_EL1, I8MM, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEI8MM),
2829 	HWCAP_CAP(ID_AA64ZFR0_EL1, F32MM, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEF32MM),
2830 	HWCAP_CAP(ID_AA64ZFR0_EL1, F64MM, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEF64MM),
2831 #endif
2832 	HWCAP_CAP(ID_AA64PFR1_EL1, SSBS, SSBS2, CAP_HWCAP, KERNEL_HWCAP_SSBS),
2833 #ifdef CONFIG_ARM64_BTI
2834 	HWCAP_CAP(ID_AA64PFR1_EL1, BT, IMP, CAP_HWCAP, KERNEL_HWCAP_BTI),
2835 #endif
2836 #ifdef CONFIG_ARM64_PTR_AUTH
2837 	HWCAP_MULTI_CAP(ptr_auth_hwcap_addr_matches, CAP_HWCAP, KERNEL_HWCAP_PACA),
2838 	HWCAP_MULTI_CAP(ptr_auth_hwcap_gen_matches, CAP_HWCAP, KERNEL_HWCAP_PACG),
2839 #endif
2840 #ifdef CONFIG_ARM64_MTE
2841 	HWCAP_CAP(ID_AA64PFR1_EL1, MTE, MTE2, CAP_HWCAP, KERNEL_HWCAP_MTE),
2842 	HWCAP_CAP(ID_AA64PFR1_EL1, MTE, MTE3, CAP_HWCAP, KERNEL_HWCAP_MTE3),
2843 #endif /* CONFIG_ARM64_MTE */
2844 	HWCAP_CAP(ID_AA64MMFR0_EL1, ECV, IMP, CAP_HWCAP, KERNEL_HWCAP_ECV),
2845 	HWCAP_CAP(ID_AA64MMFR1_EL1, AFP, IMP, CAP_HWCAP, KERNEL_HWCAP_AFP),
2846 	HWCAP_CAP(ID_AA64ISAR2_EL1, CSSC, IMP, CAP_HWCAP, KERNEL_HWCAP_CSSC),
2847 	HWCAP_CAP(ID_AA64ISAR2_EL1, RPRFM, IMP, CAP_HWCAP, KERNEL_HWCAP_RPRFM),
2848 	HWCAP_CAP(ID_AA64ISAR2_EL1, RPRES, IMP, CAP_HWCAP, KERNEL_HWCAP_RPRES),
2849 	HWCAP_CAP(ID_AA64ISAR2_EL1, WFxT, IMP, CAP_HWCAP, KERNEL_HWCAP_WFXT),
2850 	HWCAP_CAP(ID_AA64ISAR2_EL1, MOPS, IMP, CAP_HWCAP, KERNEL_HWCAP_MOPS),
2851 	HWCAP_CAP(ID_AA64ISAR2_EL1, BC, IMP, CAP_HWCAP, KERNEL_HWCAP_HBC),
2852 #ifdef CONFIG_ARM64_SME
2853 	HWCAP_CAP(ID_AA64PFR1_EL1, SME, IMP, CAP_HWCAP, KERNEL_HWCAP_SME),
2854 	HWCAP_CAP(ID_AA64SMFR0_EL1, FA64, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_FA64),
2855 	HWCAP_CAP(ID_AA64SMFR0_EL1, SMEver, SME2p1, CAP_HWCAP, KERNEL_HWCAP_SME2P1),
2856 	HWCAP_CAP(ID_AA64SMFR0_EL1, SMEver, SME2, CAP_HWCAP, KERNEL_HWCAP_SME2),
2857 	HWCAP_CAP(ID_AA64SMFR0_EL1, I16I64, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_I16I64),
2858 	HWCAP_CAP(ID_AA64SMFR0_EL1, F64F64, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_F64F64),
2859 	HWCAP_CAP(ID_AA64SMFR0_EL1, I16I32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_I16I32),
2860 	HWCAP_CAP(ID_AA64SMFR0_EL1, B16B16, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_B16B16),
2861 	HWCAP_CAP(ID_AA64SMFR0_EL1, F16F16, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_F16F16),
2862 	HWCAP_CAP(ID_AA64SMFR0_EL1, I8I32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_I8I32),
2863 	HWCAP_CAP(ID_AA64SMFR0_EL1, F16F32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_F16F32),
2864 	HWCAP_CAP(ID_AA64SMFR0_EL1, B16F32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_B16F32),
2865 	HWCAP_CAP(ID_AA64SMFR0_EL1, BI32I32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_BI32I32),
2866 	HWCAP_CAP(ID_AA64SMFR0_EL1, F32F32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_F32F32),
2867 #endif /* CONFIG_ARM64_SME */
2868 	{},
2869 };
2870 
2871 #ifdef CONFIG_COMPAT
2872 static bool compat_has_neon(const struct arm64_cpu_capabilities *cap, int scope)
2873 {
2874 	/*
2875 	 * Check that all of MVFR1_EL1.{SIMDSP, SIMDInt, SIMDLS} are available,
2876 	 * in line with that of arm32 as in vfp_init(). We make sure that the
2877 	 * check is future proof, by making sure value is non-zero.
2878 	 */
2879 	u32 mvfr1;
2880 
2881 	WARN_ON(scope == SCOPE_LOCAL_CPU && preemptible());
2882 	if (scope == SCOPE_SYSTEM)
2883 		mvfr1 = read_sanitised_ftr_reg(SYS_MVFR1_EL1);
2884 	else
2885 		mvfr1 = read_sysreg_s(SYS_MVFR1_EL1);
2886 
2887 	return cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_EL1_SIMDSP_SHIFT) &&
2888 		cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_EL1_SIMDInt_SHIFT) &&
2889 		cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_EL1_SIMDLS_SHIFT);
2890 }
2891 #endif
2892 
2893 static const struct arm64_cpu_capabilities compat_elf_hwcaps[] = {
2894 #ifdef CONFIG_COMPAT
2895 	HWCAP_CAP_MATCH(compat_has_neon, CAP_COMPAT_HWCAP, COMPAT_HWCAP_NEON),
2896 	HWCAP_CAP(MVFR1_EL1, SIMDFMAC, IMP, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFPv4),
2897 	/* Arm v8 mandates MVFR0.FPDP == {0, 2}. So, piggy back on this for the presence of VFP support */
2898 	HWCAP_CAP(MVFR0_EL1, FPDP, VFPv3, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFP),
2899 	HWCAP_CAP(MVFR0_EL1, FPDP, VFPv3, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFPv3),
2900 	HWCAP_CAP(MVFR1_EL1, FPHP, FP16, CAP_COMPAT_HWCAP, COMPAT_HWCAP_FPHP),
2901 	HWCAP_CAP(MVFR1_EL1, SIMDHP, SIMDHP_FLOAT, CAP_COMPAT_HWCAP, COMPAT_HWCAP_ASIMDHP),
2902 	HWCAP_CAP(ID_ISAR5_EL1, AES, VMULL, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_PMULL),
2903 	HWCAP_CAP(ID_ISAR5_EL1, AES, IMP, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_AES),
2904 	HWCAP_CAP(ID_ISAR5_EL1, SHA1, IMP, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA1),
2905 	HWCAP_CAP(ID_ISAR5_EL1, SHA2, IMP, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA2),
2906 	HWCAP_CAP(ID_ISAR5_EL1, CRC32, IMP, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_CRC32),
2907 	HWCAP_CAP(ID_ISAR6_EL1, DP, IMP, CAP_COMPAT_HWCAP, COMPAT_HWCAP_ASIMDDP),
2908 	HWCAP_CAP(ID_ISAR6_EL1, FHM, IMP, CAP_COMPAT_HWCAP, COMPAT_HWCAP_ASIMDFHM),
2909 	HWCAP_CAP(ID_ISAR6_EL1, SB, IMP, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SB),
2910 	HWCAP_CAP(ID_ISAR6_EL1, BF16, IMP, CAP_COMPAT_HWCAP, COMPAT_HWCAP_ASIMDBF16),
2911 	HWCAP_CAP(ID_ISAR6_EL1, I8MM, IMP, CAP_COMPAT_HWCAP, COMPAT_HWCAP_I8MM),
2912 	HWCAP_CAP(ID_PFR2_EL1, SSBS, IMP, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SSBS),
2913 #endif
2914 	{},
2915 };
2916 
2917 static void cap_set_elf_hwcap(const struct arm64_cpu_capabilities *cap)
2918 {
2919 	switch (cap->hwcap_type) {
2920 	case CAP_HWCAP:
2921 		cpu_set_feature(cap->hwcap);
2922 		break;
2923 #ifdef CONFIG_COMPAT
2924 	case CAP_COMPAT_HWCAP:
2925 		compat_elf_hwcap |= (u32)cap->hwcap;
2926 		break;
2927 	case CAP_COMPAT_HWCAP2:
2928 		compat_elf_hwcap2 |= (u32)cap->hwcap;
2929 		break;
2930 #endif
2931 	default:
2932 		WARN_ON(1);
2933 		break;
2934 	}
2935 }
2936 
2937 /* Check if we have a particular HWCAP enabled */
2938 static bool cpus_have_elf_hwcap(const struct arm64_cpu_capabilities *cap)
2939 {
2940 	bool rc;
2941 
2942 	switch (cap->hwcap_type) {
2943 	case CAP_HWCAP:
2944 		rc = cpu_have_feature(cap->hwcap);
2945 		break;
2946 #ifdef CONFIG_COMPAT
2947 	case CAP_COMPAT_HWCAP:
2948 		rc = (compat_elf_hwcap & (u32)cap->hwcap) != 0;
2949 		break;
2950 	case CAP_COMPAT_HWCAP2:
2951 		rc = (compat_elf_hwcap2 & (u32)cap->hwcap) != 0;
2952 		break;
2953 #endif
2954 	default:
2955 		WARN_ON(1);
2956 		rc = false;
2957 	}
2958 
2959 	return rc;
2960 }
2961 
2962 static void setup_elf_hwcaps(const struct arm64_cpu_capabilities *hwcaps)
2963 {
2964 	/* We support emulation of accesses to CPU ID feature registers */
2965 	cpu_set_named_feature(CPUID);
2966 	for (; hwcaps->matches; hwcaps++)
2967 		if (hwcaps->matches(hwcaps, cpucap_default_scope(hwcaps)))
2968 			cap_set_elf_hwcap(hwcaps);
2969 }
2970 
2971 static void update_cpu_capabilities(u16 scope_mask)
2972 {
2973 	int i;
2974 	const struct arm64_cpu_capabilities *caps;
2975 
2976 	scope_mask &= ARM64_CPUCAP_SCOPE_MASK;
2977 	for (i = 0; i < ARM64_NCAPS; i++) {
2978 		caps = cpucap_ptrs[i];
2979 		if (!caps || !(caps->type & scope_mask) ||
2980 		    cpus_have_cap(caps->capability) ||
2981 		    !caps->matches(caps, cpucap_default_scope(caps)))
2982 			continue;
2983 
2984 		if (caps->desc)
2985 			pr_info("detected: %s\n", caps->desc);
2986 
2987 		__set_bit(caps->capability, system_cpucaps);
2988 
2989 		if ((scope_mask & SCOPE_BOOT_CPU) && (caps->type & SCOPE_BOOT_CPU))
2990 			set_bit(caps->capability, boot_cpucaps);
2991 	}
2992 }
2993 
2994 /*
2995  * Enable all the available capabilities on this CPU. The capabilities
2996  * with BOOT_CPU scope are handled separately and hence skipped here.
2997  */
2998 static int cpu_enable_non_boot_scope_capabilities(void *__unused)
2999 {
3000 	int i;
3001 	u16 non_boot_scope = SCOPE_ALL & ~SCOPE_BOOT_CPU;
3002 
3003 	for_each_available_cap(i) {
3004 		const struct arm64_cpu_capabilities *cap = cpucap_ptrs[i];
3005 
3006 		if (WARN_ON(!cap))
3007 			continue;
3008 
3009 		if (!(cap->type & non_boot_scope))
3010 			continue;
3011 
3012 		if (cap->cpu_enable)
3013 			cap->cpu_enable(cap);
3014 	}
3015 	return 0;
3016 }
3017 
3018 /*
3019  * Run through the enabled capabilities and enable() it on all active
3020  * CPUs
3021  */
3022 static void __init enable_cpu_capabilities(u16 scope_mask)
3023 {
3024 	int i;
3025 	const struct arm64_cpu_capabilities *caps;
3026 	bool boot_scope;
3027 
3028 	scope_mask &= ARM64_CPUCAP_SCOPE_MASK;
3029 	boot_scope = !!(scope_mask & SCOPE_BOOT_CPU);
3030 
3031 	for (i = 0; i < ARM64_NCAPS; i++) {
3032 		unsigned int num;
3033 
3034 		caps = cpucap_ptrs[i];
3035 		if (!caps || !(caps->type & scope_mask))
3036 			continue;
3037 		num = caps->capability;
3038 		if (!cpus_have_cap(num))
3039 			continue;
3040 
3041 		if (boot_scope && caps->cpu_enable)
3042 			/*
3043 			 * Capabilities with SCOPE_BOOT_CPU scope are finalised
3044 			 * before any secondary CPU boots. Thus, each secondary
3045 			 * will enable the capability as appropriate via
3046 			 * check_local_cpu_capabilities(). The only exception is
3047 			 * the boot CPU, for which the capability must be
3048 			 * enabled here. This approach avoids costly
3049 			 * stop_machine() calls for this case.
3050 			 */
3051 			caps->cpu_enable(caps);
3052 	}
3053 
3054 	/*
3055 	 * For all non-boot scope capabilities, use stop_machine()
3056 	 * as it schedules the work allowing us to modify PSTATE,
3057 	 * instead of on_each_cpu() which uses an IPI, giving us a
3058 	 * PSTATE that disappears when we return.
3059 	 */
3060 	if (!boot_scope)
3061 		stop_machine(cpu_enable_non_boot_scope_capabilities,
3062 			     NULL, cpu_online_mask);
3063 }
3064 
3065 /*
3066  * Run through the list of capabilities to check for conflicts.
3067  * If the system has already detected a capability, take necessary
3068  * action on this CPU.
3069  */
3070 static void verify_local_cpu_caps(u16 scope_mask)
3071 {
3072 	int i;
3073 	bool cpu_has_cap, system_has_cap;
3074 	const struct arm64_cpu_capabilities *caps;
3075 
3076 	scope_mask &= ARM64_CPUCAP_SCOPE_MASK;
3077 
3078 	for (i = 0; i < ARM64_NCAPS; i++) {
3079 		caps = cpucap_ptrs[i];
3080 		if (!caps || !(caps->type & scope_mask))
3081 			continue;
3082 
3083 		cpu_has_cap = caps->matches(caps, SCOPE_LOCAL_CPU);
3084 		system_has_cap = cpus_have_cap(caps->capability);
3085 
3086 		if (system_has_cap) {
3087 			/*
3088 			 * Check if the new CPU misses an advertised feature,
3089 			 * which is not safe to miss.
3090 			 */
3091 			if (!cpu_has_cap && !cpucap_late_cpu_optional(caps))
3092 				break;
3093 			/*
3094 			 * We have to issue cpu_enable() irrespective of
3095 			 * whether the CPU has it or not, as it is enabeld
3096 			 * system wide. It is upto the call back to take
3097 			 * appropriate action on this CPU.
3098 			 */
3099 			if (caps->cpu_enable)
3100 				caps->cpu_enable(caps);
3101 		} else {
3102 			/*
3103 			 * Check if the CPU has this capability if it isn't
3104 			 * safe to have when the system doesn't.
3105 			 */
3106 			if (cpu_has_cap && !cpucap_late_cpu_permitted(caps))
3107 				break;
3108 		}
3109 	}
3110 
3111 	if (i < ARM64_NCAPS) {
3112 		pr_crit("CPU%d: Detected conflict for capability %d (%s), System: %d, CPU: %d\n",
3113 			smp_processor_id(), caps->capability,
3114 			caps->desc, system_has_cap, cpu_has_cap);
3115 
3116 		if (cpucap_panic_on_conflict(caps))
3117 			cpu_panic_kernel();
3118 		else
3119 			cpu_die_early();
3120 	}
3121 }
3122 
3123 /*
3124  * Check for CPU features that are used in early boot
3125  * based on the Boot CPU value.
3126  */
3127 static void check_early_cpu_features(void)
3128 {
3129 	verify_cpu_asid_bits();
3130 
3131 	verify_local_cpu_caps(SCOPE_BOOT_CPU);
3132 }
3133 
3134 static void
3135 __verify_local_elf_hwcaps(const struct arm64_cpu_capabilities *caps)
3136 {
3137 
3138 	for (; caps->matches; caps++)
3139 		if (cpus_have_elf_hwcap(caps) && !caps->matches(caps, SCOPE_LOCAL_CPU)) {
3140 			pr_crit("CPU%d: missing HWCAP: %s\n",
3141 					smp_processor_id(), caps->desc);
3142 			cpu_die_early();
3143 		}
3144 }
3145 
3146 static void verify_local_elf_hwcaps(void)
3147 {
3148 	__verify_local_elf_hwcaps(arm64_elf_hwcaps);
3149 
3150 	if (id_aa64pfr0_32bit_el0(read_cpuid(ID_AA64PFR0_EL1)))
3151 		__verify_local_elf_hwcaps(compat_elf_hwcaps);
3152 }
3153 
3154 static void verify_sve_features(void)
3155 {
3156 	u64 safe_zcr = read_sanitised_ftr_reg(SYS_ZCR_EL1);
3157 	u64 zcr = read_zcr_features();
3158 
3159 	unsigned int safe_len = safe_zcr & ZCR_ELx_LEN_MASK;
3160 	unsigned int len = zcr & ZCR_ELx_LEN_MASK;
3161 
3162 	if (len < safe_len || vec_verify_vq_map(ARM64_VEC_SVE)) {
3163 		pr_crit("CPU%d: SVE: vector length support mismatch\n",
3164 			smp_processor_id());
3165 		cpu_die_early();
3166 	}
3167 
3168 	/* Add checks on other ZCR bits here if necessary */
3169 }
3170 
3171 static void verify_sme_features(void)
3172 {
3173 	u64 safe_smcr = read_sanitised_ftr_reg(SYS_SMCR_EL1);
3174 	u64 smcr = read_smcr_features();
3175 
3176 	unsigned int safe_len = safe_smcr & SMCR_ELx_LEN_MASK;
3177 	unsigned int len = smcr & SMCR_ELx_LEN_MASK;
3178 
3179 	if (len < safe_len || vec_verify_vq_map(ARM64_VEC_SME)) {
3180 		pr_crit("CPU%d: SME: vector length support mismatch\n",
3181 			smp_processor_id());
3182 		cpu_die_early();
3183 	}
3184 
3185 	/* Add checks on other SMCR bits here if necessary */
3186 }
3187 
3188 static void verify_hyp_capabilities(void)
3189 {
3190 	u64 safe_mmfr1, mmfr0, mmfr1;
3191 	int parange, ipa_max;
3192 	unsigned int safe_vmid_bits, vmid_bits;
3193 
3194 	if (!IS_ENABLED(CONFIG_KVM))
3195 		return;
3196 
3197 	safe_mmfr1 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
3198 	mmfr0 = read_cpuid(ID_AA64MMFR0_EL1);
3199 	mmfr1 = read_cpuid(ID_AA64MMFR1_EL1);
3200 
3201 	/* Verify VMID bits */
3202 	safe_vmid_bits = get_vmid_bits(safe_mmfr1);
3203 	vmid_bits = get_vmid_bits(mmfr1);
3204 	if (vmid_bits < safe_vmid_bits) {
3205 		pr_crit("CPU%d: VMID width mismatch\n", smp_processor_id());
3206 		cpu_die_early();
3207 	}
3208 
3209 	/* Verify IPA range */
3210 	parange = cpuid_feature_extract_unsigned_field(mmfr0,
3211 				ID_AA64MMFR0_EL1_PARANGE_SHIFT);
3212 	ipa_max = id_aa64mmfr0_parange_to_phys_shift(parange);
3213 	if (ipa_max < get_kvm_ipa_limit()) {
3214 		pr_crit("CPU%d: IPA range mismatch\n", smp_processor_id());
3215 		cpu_die_early();
3216 	}
3217 }
3218 
3219 /*
3220  * Run through the enabled system capabilities and enable() it on this CPU.
3221  * The capabilities were decided based on the available CPUs at the boot time.
3222  * Any new CPU should match the system wide status of the capability. If the
3223  * new CPU doesn't have a capability which the system now has enabled, we
3224  * cannot do anything to fix it up and could cause unexpected failures. So
3225  * we park the CPU.
3226  */
3227 static void verify_local_cpu_capabilities(void)
3228 {
3229 	/*
3230 	 * The capabilities with SCOPE_BOOT_CPU are checked from
3231 	 * check_early_cpu_features(), as they need to be verified
3232 	 * on all secondary CPUs.
3233 	 */
3234 	verify_local_cpu_caps(SCOPE_ALL & ~SCOPE_BOOT_CPU);
3235 	verify_local_elf_hwcaps();
3236 
3237 	if (system_supports_sve())
3238 		verify_sve_features();
3239 
3240 	if (system_supports_sme())
3241 		verify_sme_features();
3242 
3243 	if (is_hyp_mode_available())
3244 		verify_hyp_capabilities();
3245 }
3246 
3247 void check_local_cpu_capabilities(void)
3248 {
3249 	/*
3250 	 * All secondary CPUs should conform to the early CPU features
3251 	 * in use by the kernel based on boot CPU.
3252 	 */
3253 	check_early_cpu_features();
3254 
3255 	/*
3256 	 * If we haven't finalised the system capabilities, this CPU gets
3257 	 * a chance to update the errata work arounds and local features.
3258 	 * Otherwise, this CPU should verify that it has all the system
3259 	 * advertised capabilities.
3260 	 */
3261 	if (!system_capabilities_finalized())
3262 		update_cpu_capabilities(SCOPE_LOCAL_CPU);
3263 	else
3264 		verify_local_cpu_capabilities();
3265 }
3266 
3267 static void __init setup_boot_cpu_capabilities(void)
3268 {
3269 	/* Detect capabilities with either SCOPE_BOOT_CPU or SCOPE_LOCAL_CPU */
3270 	update_cpu_capabilities(SCOPE_BOOT_CPU | SCOPE_LOCAL_CPU);
3271 	/* Enable the SCOPE_BOOT_CPU capabilities alone right away */
3272 	enable_cpu_capabilities(SCOPE_BOOT_CPU);
3273 }
3274 
3275 bool this_cpu_has_cap(unsigned int n)
3276 {
3277 	if (!WARN_ON(preemptible()) && n < ARM64_NCAPS) {
3278 		const struct arm64_cpu_capabilities *cap = cpucap_ptrs[n];
3279 
3280 		if (cap)
3281 			return cap->matches(cap, SCOPE_LOCAL_CPU);
3282 	}
3283 
3284 	return false;
3285 }
3286 EXPORT_SYMBOL_GPL(this_cpu_has_cap);
3287 
3288 /*
3289  * This helper function is used in a narrow window when,
3290  * - The system wide safe registers are set with all the SMP CPUs and,
3291  * - The SYSTEM_FEATURE system_cpucaps may not have been set.
3292  * In all other cases cpus_have_{const_}cap() should be used.
3293  */
3294 static bool __maybe_unused __system_matches_cap(unsigned int n)
3295 {
3296 	if (n < ARM64_NCAPS) {
3297 		const struct arm64_cpu_capabilities *cap = cpucap_ptrs[n];
3298 
3299 		if (cap)
3300 			return cap->matches(cap, SCOPE_SYSTEM);
3301 	}
3302 	return false;
3303 }
3304 
3305 void cpu_set_feature(unsigned int num)
3306 {
3307 	set_bit(num, elf_hwcap);
3308 }
3309 
3310 bool cpu_have_feature(unsigned int num)
3311 {
3312 	return test_bit(num, elf_hwcap);
3313 }
3314 EXPORT_SYMBOL_GPL(cpu_have_feature);
3315 
3316 unsigned long cpu_get_elf_hwcap(void)
3317 {
3318 	/*
3319 	 * We currently only populate the first 32 bits of AT_HWCAP. Please
3320 	 * note that for userspace compatibility we guarantee that bits 62
3321 	 * and 63 will always be returned as 0.
3322 	 */
3323 	return elf_hwcap[0];
3324 }
3325 
3326 unsigned long cpu_get_elf_hwcap2(void)
3327 {
3328 	return elf_hwcap[1];
3329 }
3330 
3331 static void __init setup_system_capabilities(void)
3332 {
3333 	/*
3334 	 * We have finalised the system-wide safe feature
3335 	 * registers, finalise the capabilities that depend
3336 	 * on it. Also enable all the available capabilities,
3337 	 * that are not enabled already.
3338 	 */
3339 	update_cpu_capabilities(SCOPE_SYSTEM);
3340 	enable_cpu_capabilities(SCOPE_ALL & ~SCOPE_BOOT_CPU);
3341 }
3342 
3343 void __init setup_cpu_features(void)
3344 {
3345 	u32 cwg;
3346 
3347 	setup_system_capabilities();
3348 	setup_elf_hwcaps(arm64_elf_hwcaps);
3349 
3350 	if (system_supports_32bit_el0()) {
3351 		setup_elf_hwcaps(compat_elf_hwcaps);
3352 		elf_hwcap_fixup();
3353 	}
3354 
3355 	if (system_uses_ttbr0_pan())
3356 		pr_info("emulated: Privileged Access Never (PAN) using TTBR0_EL1 switching\n");
3357 
3358 	sve_setup();
3359 	sme_setup();
3360 	minsigstksz_setup();
3361 
3362 	/*
3363 	 * Check for sane CTR_EL0.CWG value.
3364 	 */
3365 	cwg = cache_type_cwg();
3366 	if (!cwg)
3367 		pr_warn("No Cache Writeback Granule information, assuming %d\n",
3368 			ARCH_DMA_MINALIGN);
3369 }
3370 
3371 static int enable_mismatched_32bit_el0(unsigned int cpu)
3372 {
3373 	/*
3374 	 * The first 32-bit-capable CPU we detected and so can no longer
3375 	 * be offlined by userspace. -1 indicates we haven't yet onlined
3376 	 * a 32-bit-capable CPU.
3377 	 */
3378 	static int lucky_winner = -1;
3379 
3380 	struct cpuinfo_arm64 *info = &per_cpu(cpu_data, cpu);
3381 	bool cpu_32bit = id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0);
3382 
3383 	if (cpu_32bit) {
3384 		cpumask_set_cpu(cpu, cpu_32bit_el0_mask);
3385 		static_branch_enable_cpuslocked(&arm64_mismatched_32bit_el0);
3386 	}
3387 
3388 	if (cpumask_test_cpu(0, cpu_32bit_el0_mask) == cpu_32bit)
3389 		return 0;
3390 
3391 	if (lucky_winner >= 0)
3392 		return 0;
3393 
3394 	/*
3395 	 * We've detected a mismatch. We need to keep one of our CPUs with
3396 	 * 32-bit EL0 online so that is_cpu_allowed() doesn't end up rejecting
3397 	 * every CPU in the system for a 32-bit task.
3398 	 */
3399 	lucky_winner = cpu_32bit ? cpu : cpumask_any_and(cpu_32bit_el0_mask,
3400 							 cpu_active_mask);
3401 	get_cpu_device(lucky_winner)->offline_disabled = true;
3402 	setup_elf_hwcaps(compat_elf_hwcaps);
3403 	elf_hwcap_fixup();
3404 	pr_info("Asymmetric 32-bit EL0 support detected on CPU %u; CPU hot-unplug disabled on CPU %u\n",
3405 		cpu, lucky_winner);
3406 	return 0;
3407 }
3408 
3409 static int __init init_32bit_el0_mask(void)
3410 {
3411 	if (!allow_mismatched_32bit_el0)
3412 		return 0;
3413 
3414 	if (!zalloc_cpumask_var(&cpu_32bit_el0_mask, GFP_KERNEL))
3415 		return -ENOMEM;
3416 
3417 	return cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
3418 				 "arm64/mismatched_32bit_el0:online",
3419 				 enable_mismatched_32bit_el0, NULL);
3420 }
3421 subsys_initcall_sync(init_32bit_el0_mask);
3422 
3423 static void __maybe_unused cpu_enable_cnp(struct arm64_cpu_capabilities const *cap)
3424 {
3425 	cpu_replace_ttbr1(lm_alias(swapper_pg_dir), idmap_pg_dir);
3426 }
3427 
3428 /*
3429  * We emulate only the following system register space.
3430  * Op0 = 0x3, CRn = 0x0, Op1 = 0x0, CRm = [0, 2 - 7]
3431  * See Table C5-6 System instruction encodings for System register accesses,
3432  * ARMv8 ARM(ARM DDI 0487A.f) for more details.
3433  */
3434 static inline bool __attribute_const__ is_emulated(u32 id)
3435 {
3436 	return (sys_reg_Op0(id) == 0x3 &&
3437 		sys_reg_CRn(id) == 0x0 &&
3438 		sys_reg_Op1(id) == 0x0 &&
3439 		(sys_reg_CRm(id) == 0 ||
3440 		 ((sys_reg_CRm(id) >= 2) && (sys_reg_CRm(id) <= 7))));
3441 }
3442 
3443 /*
3444  * With CRm == 0, reg should be one of :
3445  * MIDR_EL1, MPIDR_EL1 or REVIDR_EL1.
3446  */
3447 static inline int emulate_id_reg(u32 id, u64 *valp)
3448 {
3449 	switch (id) {
3450 	case SYS_MIDR_EL1:
3451 		*valp = read_cpuid_id();
3452 		break;
3453 	case SYS_MPIDR_EL1:
3454 		*valp = SYS_MPIDR_SAFE_VAL;
3455 		break;
3456 	case SYS_REVIDR_EL1:
3457 		/* IMPLEMENTATION DEFINED values are emulated with 0 */
3458 		*valp = 0;
3459 		break;
3460 	default:
3461 		return -EINVAL;
3462 	}
3463 
3464 	return 0;
3465 }
3466 
3467 static int emulate_sys_reg(u32 id, u64 *valp)
3468 {
3469 	struct arm64_ftr_reg *regp;
3470 
3471 	if (!is_emulated(id))
3472 		return -EINVAL;
3473 
3474 	if (sys_reg_CRm(id) == 0)
3475 		return emulate_id_reg(id, valp);
3476 
3477 	regp = get_arm64_ftr_reg_nowarn(id);
3478 	if (regp)
3479 		*valp = arm64_ftr_reg_user_value(regp);
3480 	else
3481 		/*
3482 		 * The untracked registers are either IMPLEMENTATION DEFINED
3483 		 * (e.g, ID_AFR0_EL1) or reserved RAZ.
3484 		 */
3485 		*valp = 0;
3486 	return 0;
3487 }
3488 
3489 int do_emulate_mrs(struct pt_regs *regs, u32 sys_reg, u32 rt)
3490 {
3491 	int rc;
3492 	u64 val;
3493 
3494 	rc = emulate_sys_reg(sys_reg, &val);
3495 	if (!rc) {
3496 		pt_regs_write_reg(regs, rt, val);
3497 		arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
3498 	}
3499 	return rc;
3500 }
3501 
3502 bool try_emulate_mrs(struct pt_regs *regs, u32 insn)
3503 {
3504 	u32 sys_reg, rt;
3505 
3506 	if (compat_user_mode(regs) || !aarch64_insn_is_mrs(insn))
3507 		return false;
3508 
3509 	/*
3510 	 * sys_reg values are defined as used in mrs/msr instruction.
3511 	 * shift the imm value to get the encoding.
3512 	 */
3513 	sys_reg = (u32)aarch64_insn_decode_immediate(AARCH64_INSN_IMM_16, insn) << 5;
3514 	rt = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RT, insn);
3515 	return do_emulate_mrs(regs, sys_reg, rt) == 0;
3516 }
3517 
3518 enum mitigation_state arm64_get_meltdown_state(void)
3519 {
3520 	if (__meltdown_safe)
3521 		return SPECTRE_UNAFFECTED;
3522 
3523 	if (arm64_kernel_unmapped_at_el0())
3524 		return SPECTRE_MITIGATED;
3525 
3526 	return SPECTRE_VULNERABLE;
3527 }
3528 
3529 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr,
3530 			  char *buf)
3531 {
3532 	switch (arm64_get_meltdown_state()) {
3533 	case SPECTRE_UNAFFECTED:
3534 		return sprintf(buf, "Not affected\n");
3535 
3536 	case SPECTRE_MITIGATED:
3537 		return sprintf(buf, "Mitigation: PTI\n");
3538 
3539 	default:
3540 		return sprintf(buf, "Vulnerable\n");
3541 	}
3542 }
3543