1 /* 2 * Contains CPU feature definitions 3 * 4 * Copyright (C) 2015 ARM Ltd. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program. If not, see <http://www.gnu.org/licenses/>. 17 */ 18 19 #define pr_fmt(fmt) "CPU features: " fmt 20 21 #include <linux/bsearch.h> 22 #include <linux/cpumask.h> 23 #include <linux/sort.h> 24 #include <linux/stop_machine.h> 25 #include <linux/types.h> 26 #include <linux/mm.h> 27 #include <asm/cpu.h> 28 #include <asm/cpufeature.h> 29 #include <asm/cpu_ops.h> 30 #include <asm/mmu_context.h> 31 #include <asm/processor.h> 32 #include <asm/sysreg.h> 33 #include <asm/traps.h> 34 #include <asm/virt.h> 35 36 unsigned long elf_hwcap __read_mostly; 37 EXPORT_SYMBOL_GPL(elf_hwcap); 38 39 #ifdef CONFIG_COMPAT 40 #define COMPAT_ELF_HWCAP_DEFAULT \ 41 (COMPAT_HWCAP_HALF|COMPAT_HWCAP_THUMB|\ 42 COMPAT_HWCAP_FAST_MULT|COMPAT_HWCAP_EDSP|\ 43 COMPAT_HWCAP_TLS|COMPAT_HWCAP_VFP|\ 44 COMPAT_HWCAP_VFPv3|COMPAT_HWCAP_VFPv4|\ 45 COMPAT_HWCAP_NEON|COMPAT_HWCAP_IDIV|\ 46 COMPAT_HWCAP_LPAE) 47 unsigned int compat_elf_hwcap __read_mostly = COMPAT_ELF_HWCAP_DEFAULT; 48 unsigned int compat_elf_hwcap2 __read_mostly; 49 #endif 50 51 DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS); 52 EXPORT_SYMBOL(cpu_hwcaps); 53 54 DEFINE_STATIC_KEY_ARRAY_FALSE(cpu_hwcap_keys, ARM64_NCAPS); 55 EXPORT_SYMBOL(cpu_hwcap_keys); 56 57 #define __ARM64_FTR_BITS(SIGNED, VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \ 58 { \ 59 .sign = SIGNED, \ 60 .visible = VISIBLE, \ 61 .strict = STRICT, \ 62 .type = TYPE, \ 63 .shift = SHIFT, \ 64 .width = WIDTH, \ 65 .safe_val = SAFE_VAL, \ 66 } 67 68 /* Define a feature with unsigned values */ 69 #define ARM64_FTR_BITS(VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \ 70 __ARM64_FTR_BITS(FTR_UNSIGNED, VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) 71 72 /* Define a feature with a signed value */ 73 #define S_ARM64_FTR_BITS(VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \ 74 __ARM64_FTR_BITS(FTR_SIGNED, VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) 75 76 #define ARM64_FTR_END \ 77 { \ 78 .width = 0, \ 79 } 80 81 /* meta feature for alternatives */ 82 static bool __maybe_unused 83 cpufeature_pan_not_uao(const struct arm64_cpu_capabilities *entry, int __unused); 84 85 86 /* 87 * NOTE: Any changes to the visibility of features should be kept in 88 * sync with the documentation of the CPU feature register ABI. 89 */ 90 static const struct arm64_ftr_bits ftr_id_aa64isar0[] = { 91 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR0_RDM_SHIFT, 4, 0), 92 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_ATOMICS_SHIFT, 4, 0), 93 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_CRC32_SHIFT, 4, 0), 94 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SHA2_SHIFT, 4, 0), 95 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SHA1_SHIFT, 4, 0), 96 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_AES_SHIFT, 4, 0), 97 ARM64_FTR_END, 98 }; 99 100 static const struct arm64_ftr_bits ftr_id_aa64isar1[] = { 101 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_LRCPC_SHIFT, 4, 0), 102 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_FCMA_SHIFT, 4, 0), 103 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_JSCVT_SHIFT, 4, 0), 104 ARM64_FTR_END, 105 }; 106 107 static const struct arm64_ftr_bits ftr_id_aa64pfr0[] = { 108 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64PFR0_GIC_SHIFT, 4, 0), 109 S_ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_ASIMD_SHIFT, 4, ID_AA64PFR0_ASIMD_NI), 110 S_ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_FP_SHIFT, 4, ID_AA64PFR0_FP_NI), 111 /* Linux doesn't care about the EL3 */ 112 ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64PFR0_EL3_SHIFT, 4, 0), 113 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64PFR0_EL2_SHIFT, 4, 0), 114 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64PFR0_EL1_SHIFT, 4, ID_AA64PFR0_EL1_64BIT_ONLY), 115 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64PFR0_EL0_SHIFT, 4, ID_AA64PFR0_EL0_64BIT_ONLY), 116 ARM64_FTR_END, 117 }; 118 119 static const struct arm64_ftr_bits ftr_id_aa64mmfr0[] = { 120 S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_TGRAN4_SHIFT, 4, ID_AA64MMFR0_TGRAN4_NI), 121 S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_TGRAN64_SHIFT, 4, ID_AA64MMFR0_TGRAN64_NI), 122 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_TGRAN16_SHIFT, 4, ID_AA64MMFR0_TGRAN16_NI), 123 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_BIGENDEL0_SHIFT, 4, 0), 124 /* Linux shouldn't care about secure memory */ 125 ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64MMFR0_SNSMEM_SHIFT, 4, 0), 126 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_BIGENDEL_SHIFT, 4, 0), 127 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_ASID_SHIFT, 4, 0), 128 /* 129 * Differing PARange is fine as long as all peripherals and memory are mapped 130 * within the minimum PARange of all CPUs 131 */ 132 ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_PARANGE_SHIFT, 4, 0), 133 ARM64_FTR_END, 134 }; 135 136 static const struct arm64_ftr_bits ftr_id_aa64mmfr1[] = { 137 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_PAN_SHIFT, 4, 0), 138 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR1_LOR_SHIFT, 4, 0), 139 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR1_HPD_SHIFT, 4, 0), 140 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR1_VHE_SHIFT, 4, 0), 141 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR1_VMIDBITS_SHIFT, 4, 0), 142 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR1_HADBS_SHIFT, 4, 0), 143 ARM64_FTR_END, 144 }; 145 146 static const struct arm64_ftr_bits ftr_id_aa64mmfr2[] = { 147 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR2_LVA_SHIFT, 4, 0), 148 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR2_IESB_SHIFT, 4, 0), 149 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR2_LSM_SHIFT, 4, 0), 150 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR2_UAO_SHIFT, 4, 0), 151 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64MMFR2_CNP_SHIFT, 4, 0), 152 ARM64_FTR_END, 153 }; 154 155 static const struct arm64_ftr_bits ftr_ctr[] = { 156 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, 31, 1, 1), /* RAO */ 157 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_HIGHER_SAFE, 24, 4, 0), /* CWG */ 158 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, 20, 4, 0), /* ERG */ 159 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, 16, 4, 1), /* DminLine */ 160 /* 161 * Linux can handle differing I-cache policies. Userspace JITs will 162 * make use of *minLine. 163 * If we have differing I-cache policies, report it as the weakest - VIPT. 164 */ 165 ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_EXACT, 14, 2, ICACHE_POLICY_VIPT), /* L1Ip */ 166 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0), /* IminLine */ 167 ARM64_FTR_END, 168 }; 169 170 struct arm64_ftr_reg arm64_ftr_reg_ctrel0 = { 171 .name = "SYS_CTR_EL0", 172 .ftr_bits = ftr_ctr 173 }; 174 175 static const struct arm64_ftr_bits ftr_id_mmfr0[] = { 176 S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 28, 4, 0xf), /* InnerShr */ 177 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 24, 4, 0), /* FCSE */ 178 ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, 20, 4, 0), /* AuxReg */ 179 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 16, 4, 0), /* TCM */ 180 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 12, 4, 0), /* ShareLvl */ 181 S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 8, 4, 0xf), /* OuterShr */ 182 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 4, 4, 0), /* PMSA */ 183 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 0, 4, 0), /* VMSA */ 184 ARM64_FTR_END, 185 }; 186 187 static const struct arm64_ftr_bits ftr_id_aa64dfr0[] = { 188 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 36, 28, 0), 189 ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64DFR0_PMSVER_SHIFT, 4, 0), 190 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_CTX_CMPS_SHIFT, 4, 0), 191 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_WRPS_SHIFT, 4, 0), 192 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_BRPS_SHIFT, 4, 0), 193 /* 194 * We can instantiate multiple PMU instances with different levels 195 * of support. 196 */ 197 S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64DFR0_PMUVER_SHIFT, 4, 0), 198 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64DFR0_TRACEVER_SHIFT, 4, 0), 199 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64DFR0_DEBUGVER_SHIFT, 4, 0x6), 200 ARM64_FTR_END, 201 }; 202 203 static const struct arm64_ftr_bits ftr_mvfr2[] = { 204 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 4, 4, 0), /* FPMisc */ 205 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 0, 4, 0), /* SIMDMisc */ 206 ARM64_FTR_END, 207 }; 208 209 static const struct arm64_ftr_bits ftr_dczid[] = { 210 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, 4, 1, 1), /* DZP */ 211 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0), /* BS */ 212 ARM64_FTR_END, 213 }; 214 215 216 static const struct arm64_ftr_bits ftr_id_isar5[] = { 217 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_ISAR5_RDM_SHIFT, 4, 0), 218 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_ISAR5_CRC32_SHIFT, 4, 0), 219 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_ISAR5_SHA2_SHIFT, 4, 0), 220 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_ISAR5_SHA1_SHIFT, 4, 0), 221 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_ISAR5_AES_SHIFT, 4, 0), 222 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_ISAR5_SEVL_SHIFT, 4, 0), 223 ARM64_FTR_END, 224 }; 225 226 static const struct arm64_ftr_bits ftr_id_mmfr4[] = { 227 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 4, 4, 0), /* ac2 */ 228 ARM64_FTR_END, 229 }; 230 231 static const struct arm64_ftr_bits ftr_id_pfr0[] = { 232 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 12, 4, 0), /* State3 */ 233 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 8, 4, 0), /* State2 */ 234 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 4, 4, 0), /* State1 */ 235 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 0, 4, 0), /* State0 */ 236 ARM64_FTR_END, 237 }; 238 239 static const struct arm64_ftr_bits ftr_id_dfr0[] = { 240 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 28, 4, 0), 241 S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 24, 4, 0xf), /* PerfMon */ 242 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 20, 4, 0), 243 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 16, 4, 0), 244 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 12, 4, 0), 245 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 8, 4, 0), 246 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0), 247 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0), 248 ARM64_FTR_END, 249 }; 250 251 /* 252 * Common ftr bits for a 32bit register with all hidden, strict 253 * attributes, with 4bit feature fields and a default safe value of 254 * 0. Covers the following 32bit registers: 255 * id_isar[0-4], id_mmfr[1-3], id_pfr1, mvfr[0-1] 256 */ 257 static const struct arm64_ftr_bits ftr_generic_32bits[] = { 258 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 28, 4, 0), 259 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 24, 4, 0), 260 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 20, 4, 0), 261 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 16, 4, 0), 262 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 12, 4, 0), 263 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 8, 4, 0), 264 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0), 265 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0), 266 ARM64_FTR_END, 267 }; 268 269 /* Table for a single 32bit feature value */ 270 static const struct arm64_ftr_bits ftr_single32[] = { 271 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 0, 32, 0), 272 ARM64_FTR_END, 273 }; 274 275 static const struct arm64_ftr_bits ftr_raz[] = { 276 ARM64_FTR_END, 277 }; 278 279 #define ARM64_FTR_REG(id, table) { \ 280 .sys_id = id, \ 281 .reg = &(struct arm64_ftr_reg){ \ 282 .name = #id, \ 283 .ftr_bits = &((table)[0]), \ 284 }} 285 286 static const struct __ftr_reg_entry { 287 u32 sys_id; 288 struct arm64_ftr_reg *reg; 289 } arm64_ftr_regs[] = { 290 291 /* Op1 = 0, CRn = 0, CRm = 1 */ 292 ARM64_FTR_REG(SYS_ID_PFR0_EL1, ftr_id_pfr0), 293 ARM64_FTR_REG(SYS_ID_PFR1_EL1, ftr_generic_32bits), 294 ARM64_FTR_REG(SYS_ID_DFR0_EL1, ftr_id_dfr0), 295 ARM64_FTR_REG(SYS_ID_MMFR0_EL1, ftr_id_mmfr0), 296 ARM64_FTR_REG(SYS_ID_MMFR1_EL1, ftr_generic_32bits), 297 ARM64_FTR_REG(SYS_ID_MMFR2_EL1, ftr_generic_32bits), 298 ARM64_FTR_REG(SYS_ID_MMFR3_EL1, ftr_generic_32bits), 299 300 /* Op1 = 0, CRn = 0, CRm = 2 */ 301 ARM64_FTR_REG(SYS_ID_ISAR0_EL1, ftr_generic_32bits), 302 ARM64_FTR_REG(SYS_ID_ISAR1_EL1, ftr_generic_32bits), 303 ARM64_FTR_REG(SYS_ID_ISAR2_EL1, ftr_generic_32bits), 304 ARM64_FTR_REG(SYS_ID_ISAR3_EL1, ftr_generic_32bits), 305 ARM64_FTR_REG(SYS_ID_ISAR4_EL1, ftr_generic_32bits), 306 ARM64_FTR_REG(SYS_ID_ISAR5_EL1, ftr_id_isar5), 307 ARM64_FTR_REG(SYS_ID_MMFR4_EL1, ftr_id_mmfr4), 308 309 /* Op1 = 0, CRn = 0, CRm = 3 */ 310 ARM64_FTR_REG(SYS_MVFR0_EL1, ftr_generic_32bits), 311 ARM64_FTR_REG(SYS_MVFR1_EL1, ftr_generic_32bits), 312 ARM64_FTR_REG(SYS_MVFR2_EL1, ftr_mvfr2), 313 314 /* Op1 = 0, CRn = 0, CRm = 4 */ 315 ARM64_FTR_REG(SYS_ID_AA64PFR0_EL1, ftr_id_aa64pfr0), 316 ARM64_FTR_REG(SYS_ID_AA64PFR1_EL1, ftr_raz), 317 318 /* Op1 = 0, CRn = 0, CRm = 5 */ 319 ARM64_FTR_REG(SYS_ID_AA64DFR0_EL1, ftr_id_aa64dfr0), 320 ARM64_FTR_REG(SYS_ID_AA64DFR1_EL1, ftr_raz), 321 322 /* Op1 = 0, CRn = 0, CRm = 6 */ 323 ARM64_FTR_REG(SYS_ID_AA64ISAR0_EL1, ftr_id_aa64isar0), 324 ARM64_FTR_REG(SYS_ID_AA64ISAR1_EL1, ftr_id_aa64isar1), 325 326 /* Op1 = 0, CRn = 0, CRm = 7 */ 327 ARM64_FTR_REG(SYS_ID_AA64MMFR0_EL1, ftr_id_aa64mmfr0), 328 ARM64_FTR_REG(SYS_ID_AA64MMFR1_EL1, ftr_id_aa64mmfr1), 329 ARM64_FTR_REG(SYS_ID_AA64MMFR2_EL1, ftr_id_aa64mmfr2), 330 331 /* Op1 = 3, CRn = 0, CRm = 0 */ 332 { SYS_CTR_EL0, &arm64_ftr_reg_ctrel0 }, 333 ARM64_FTR_REG(SYS_DCZID_EL0, ftr_dczid), 334 335 /* Op1 = 3, CRn = 14, CRm = 0 */ 336 ARM64_FTR_REG(SYS_CNTFRQ_EL0, ftr_single32), 337 }; 338 339 static int search_cmp_ftr_reg(const void *id, const void *regp) 340 { 341 return (int)(unsigned long)id - (int)((const struct __ftr_reg_entry *)regp)->sys_id; 342 } 343 344 /* 345 * get_arm64_ftr_reg - Lookup a feature register entry using its 346 * sys_reg() encoding. With the array arm64_ftr_regs sorted in the 347 * ascending order of sys_id , we use binary search to find a matching 348 * entry. 349 * 350 * returns - Upon success, matching ftr_reg entry for id. 351 * - NULL on failure. It is upto the caller to decide 352 * the impact of a failure. 353 */ 354 static struct arm64_ftr_reg *get_arm64_ftr_reg(u32 sys_id) 355 { 356 const struct __ftr_reg_entry *ret; 357 358 ret = bsearch((const void *)(unsigned long)sys_id, 359 arm64_ftr_regs, 360 ARRAY_SIZE(arm64_ftr_regs), 361 sizeof(arm64_ftr_regs[0]), 362 search_cmp_ftr_reg); 363 if (ret) 364 return ret->reg; 365 return NULL; 366 } 367 368 static u64 arm64_ftr_set_value(const struct arm64_ftr_bits *ftrp, s64 reg, 369 s64 ftr_val) 370 { 371 u64 mask = arm64_ftr_mask(ftrp); 372 373 reg &= ~mask; 374 reg |= (ftr_val << ftrp->shift) & mask; 375 return reg; 376 } 377 378 static s64 arm64_ftr_safe_value(const struct arm64_ftr_bits *ftrp, s64 new, 379 s64 cur) 380 { 381 s64 ret = 0; 382 383 switch (ftrp->type) { 384 case FTR_EXACT: 385 ret = ftrp->safe_val; 386 break; 387 case FTR_LOWER_SAFE: 388 ret = new < cur ? new : cur; 389 break; 390 case FTR_HIGHER_SAFE: 391 ret = new > cur ? new : cur; 392 break; 393 default: 394 BUG(); 395 } 396 397 return ret; 398 } 399 400 static void __init sort_ftr_regs(void) 401 { 402 int i; 403 404 /* Check that the array is sorted so that we can do the binary search */ 405 for (i = 1; i < ARRAY_SIZE(arm64_ftr_regs); i++) 406 BUG_ON(arm64_ftr_regs[i].sys_id < arm64_ftr_regs[i - 1].sys_id); 407 } 408 409 /* 410 * Initialise the CPU feature register from Boot CPU values. 411 * Also initiliases the strict_mask for the register. 412 * Any bits that are not covered by an arm64_ftr_bits entry are considered 413 * RES0 for the system-wide value, and must strictly match. 414 */ 415 static void __init init_cpu_ftr_reg(u32 sys_reg, u64 new) 416 { 417 u64 val = 0; 418 u64 strict_mask = ~0x0ULL; 419 u64 user_mask = 0; 420 u64 valid_mask = 0; 421 422 const struct arm64_ftr_bits *ftrp; 423 struct arm64_ftr_reg *reg = get_arm64_ftr_reg(sys_reg); 424 425 BUG_ON(!reg); 426 427 for (ftrp = reg->ftr_bits; ftrp->width; ftrp++) { 428 u64 ftr_mask = arm64_ftr_mask(ftrp); 429 s64 ftr_new = arm64_ftr_value(ftrp, new); 430 431 val = arm64_ftr_set_value(ftrp, val, ftr_new); 432 433 valid_mask |= ftr_mask; 434 if (!ftrp->strict) 435 strict_mask &= ~ftr_mask; 436 if (ftrp->visible) 437 user_mask |= ftr_mask; 438 else 439 reg->user_val = arm64_ftr_set_value(ftrp, 440 reg->user_val, 441 ftrp->safe_val); 442 } 443 444 val &= valid_mask; 445 446 reg->sys_val = val; 447 reg->strict_mask = strict_mask; 448 reg->user_mask = user_mask; 449 } 450 451 void __init init_cpu_features(struct cpuinfo_arm64 *info) 452 { 453 /* Before we start using the tables, make sure it is sorted */ 454 sort_ftr_regs(); 455 456 init_cpu_ftr_reg(SYS_CTR_EL0, info->reg_ctr); 457 init_cpu_ftr_reg(SYS_DCZID_EL0, info->reg_dczid); 458 init_cpu_ftr_reg(SYS_CNTFRQ_EL0, info->reg_cntfrq); 459 init_cpu_ftr_reg(SYS_ID_AA64DFR0_EL1, info->reg_id_aa64dfr0); 460 init_cpu_ftr_reg(SYS_ID_AA64DFR1_EL1, info->reg_id_aa64dfr1); 461 init_cpu_ftr_reg(SYS_ID_AA64ISAR0_EL1, info->reg_id_aa64isar0); 462 init_cpu_ftr_reg(SYS_ID_AA64ISAR1_EL1, info->reg_id_aa64isar1); 463 init_cpu_ftr_reg(SYS_ID_AA64MMFR0_EL1, info->reg_id_aa64mmfr0); 464 init_cpu_ftr_reg(SYS_ID_AA64MMFR1_EL1, info->reg_id_aa64mmfr1); 465 init_cpu_ftr_reg(SYS_ID_AA64MMFR2_EL1, info->reg_id_aa64mmfr2); 466 init_cpu_ftr_reg(SYS_ID_AA64PFR0_EL1, info->reg_id_aa64pfr0); 467 init_cpu_ftr_reg(SYS_ID_AA64PFR1_EL1, info->reg_id_aa64pfr1); 468 469 if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) { 470 init_cpu_ftr_reg(SYS_ID_DFR0_EL1, info->reg_id_dfr0); 471 init_cpu_ftr_reg(SYS_ID_ISAR0_EL1, info->reg_id_isar0); 472 init_cpu_ftr_reg(SYS_ID_ISAR1_EL1, info->reg_id_isar1); 473 init_cpu_ftr_reg(SYS_ID_ISAR2_EL1, info->reg_id_isar2); 474 init_cpu_ftr_reg(SYS_ID_ISAR3_EL1, info->reg_id_isar3); 475 init_cpu_ftr_reg(SYS_ID_ISAR4_EL1, info->reg_id_isar4); 476 init_cpu_ftr_reg(SYS_ID_ISAR5_EL1, info->reg_id_isar5); 477 init_cpu_ftr_reg(SYS_ID_MMFR0_EL1, info->reg_id_mmfr0); 478 init_cpu_ftr_reg(SYS_ID_MMFR1_EL1, info->reg_id_mmfr1); 479 init_cpu_ftr_reg(SYS_ID_MMFR2_EL1, info->reg_id_mmfr2); 480 init_cpu_ftr_reg(SYS_ID_MMFR3_EL1, info->reg_id_mmfr3); 481 init_cpu_ftr_reg(SYS_ID_PFR0_EL1, info->reg_id_pfr0); 482 init_cpu_ftr_reg(SYS_ID_PFR1_EL1, info->reg_id_pfr1); 483 init_cpu_ftr_reg(SYS_MVFR0_EL1, info->reg_mvfr0); 484 init_cpu_ftr_reg(SYS_MVFR1_EL1, info->reg_mvfr1); 485 init_cpu_ftr_reg(SYS_MVFR2_EL1, info->reg_mvfr2); 486 } 487 488 } 489 490 static void update_cpu_ftr_reg(struct arm64_ftr_reg *reg, u64 new) 491 { 492 const struct arm64_ftr_bits *ftrp; 493 494 for (ftrp = reg->ftr_bits; ftrp->width; ftrp++) { 495 s64 ftr_cur = arm64_ftr_value(ftrp, reg->sys_val); 496 s64 ftr_new = arm64_ftr_value(ftrp, new); 497 498 if (ftr_cur == ftr_new) 499 continue; 500 /* Find a safe value */ 501 ftr_new = arm64_ftr_safe_value(ftrp, ftr_new, ftr_cur); 502 reg->sys_val = arm64_ftr_set_value(ftrp, reg->sys_val, ftr_new); 503 } 504 505 } 506 507 static int check_update_ftr_reg(u32 sys_id, int cpu, u64 val, u64 boot) 508 { 509 struct arm64_ftr_reg *regp = get_arm64_ftr_reg(sys_id); 510 511 BUG_ON(!regp); 512 update_cpu_ftr_reg(regp, val); 513 if ((boot & regp->strict_mask) == (val & regp->strict_mask)) 514 return 0; 515 pr_warn("SANITY CHECK: Unexpected variation in %s. Boot CPU: %#016llx, CPU%d: %#016llx\n", 516 regp->name, boot, cpu, val); 517 return 1; 518 } 519 520 /* 521 * Update system wide CPU feature registers with the values from a 522 * non-boot CPU. Also performs SANITY checks to make sure that there 523 * aren't any insane variations from that of the boot CPU. 524 */ 525 void update_cpu_features(int cpu, 526 struct cpuinfo_arm64 *info, 527 struct cpuinfo_arm64 *boot) 528 { 529 int taint = 0; 530 531 /* 532 * The kernel can handle differing I-cache policies, but otherwise 533 * caches should look identical. Userspace JITs will make use of 534 * *minLine. 535 */ 536 taint |= check_update_ftr_reg(SYS_CTR_EL0, cpu, 537 info->reg_ctr, boot->reg_ctr); 538 539 /* 540 * Userspace may perform DC ZVA instructions. Mismatched block sizes 541 * could result in too much or too little memory being zeroed if a 542 * process is preempted and migrated between CPUs. 543 */ 544 taint |= check_update_ftr_reg(SYS_DCZID_EL0, cpu, 545 info->reg_dczid, boot->reg_dczid); 546 547 /* If different, timekeeping will be broken (especially with KVM) */ 548 taint |= check_update_ftr_reg(SYS_CNTFRQ_EL0, cpu, 549 info->reg_cntfrq, boot->reg_cntfrq); 550 551 /* 552 * The kernel uses self-hosted debug features and expects CPUs to 553 * support identical debug features. We presently need CTX_CMPs, WRPs, 554 * and BRPs to be identical. 555 * ID_AA64DFR1 is currently RES0. 556 */ 557 taint |= check_update_ftr_reg(SYS_ID_AA64DFR0_EL1, cpu, 558 info->reg_id_aa64dfr0, boot->reg_id_aa64dfr0); 559 taint |= check_update_ftr_reg(SYS_ID_AA64DFR1_EL1, cpu, 560 info->reg_id_aa64dfr1, boot->reg_id_aa64dfr1); 561 /* 562 * Even in big.LITTLE, processors should be identical instruction-set 563 * wise. 564 */ 565 taint |= check_update_ftr_reg(SYS_ID_AA64ISAR0_EL1, cpu, 566 info->reg_id_aa64isar0, boot->reg_id_aa64isar0); 567 taint |= check_update_ftr_reg(SYS_ID_AA64ISAR1_EL1, cpu, 568 info->reg_id_aa64isar1, boot->reg_id_aa64isar1); 569 570 /* 571 * Differing PARange support is fine as long as all peripherals and 572 * memory are mapped within the minimum PARange of all CPUs. 573 * Linux should not care about secure memory. 574 */ 575 taint |= check_update_ftr_reg(SYS_ID_AA64MMFR0_EL1, cpu, 576 info->reg_id_aa64mmfr0, boot->reg_id_aa64mmfr0); 577 taint |= check_update_ftr_reg(SYS_ID_AA64MMFR1_EL1, cpu, 578 info->reg_id_aa64mmfr1, boot->reg_id_aa64mmfr1); 579 taint |= check_update_ftr_reg(SYS_ID_AA64MMFR2_EL1, cpu, 580 info->reg_id_aa64mmfr2, boot->reg_id_aa64mmfr2); 581 582 /* 583 * EL3 is not our concern. 584 * ID_AA64PFR1 is currently RES0. 585 */ 586 taint |= check_update_ftr_reg(SYS_ID_AA64PFR0_EL1, cpu, 587 info->reg_id_aa64pfr0, boot->reg_id_aa64pfr0); 588 taint |= check_update_ftr_reg(SYS_ID_AA64PFR1_EL1, cpu, 589 info->reg_id_aa64pfr1, boot->reg_id_aa64pfr1); 590 591 /* 592 * If we have AArch32, we care about 32-bit features for compat. 593 * If the system doesn't support AArch32, don't update them. 594 */ 595 if (id_aa64pfr0_32bit_el0(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1)) && 596 id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) { 597 598 taint |= check_update_ftr_reg(SYS_ID_DFR0_EL1, cpu, 599 info->reg_id_dfr0, boot->reg_id_dfr0); 600 taint |= check_update_ftr_reg(SYS_ID_ISAR0_EL1, cpu, 601 info->reg_id_isar0, boot->reg_id_isar0); 602 taint |= check_update_ftr_reg(SYS_ID_ISAR1_EL1, cpu, 603 info->reg_id_isar1, boot->reg_id_isar1); 604 taint |= check_update_ftr_reg(SYS_ID_ISAR2_EL1, cpu, 605 info->reg_id_isar2, boot->reg_id_isar2); 606 taint |= check_update_ftr_reg(SYS_ID_ISAR3_EL1, cpu, 607 info->reg_id_isar3, boot->reg_id_isar3); 608 taint |= check_update_ftr_reg(SYS_ID_ISAR4_EL1, cpu, 609 info->reg_id_isar4, boot->reg_id_isar4); 610 taint |= check_update_ftr_reg(SYS_ID_ISAR5_EL1, cpu, 611 info->reg_id_isar5, boot->reg_id_isar5); 612 613 /* 614 * Regardless of the value of the AuxReg field, the AIFSR, ADFSR, and 615 * ACTLR formats could differ across CPUs and therefore would have to 616 * be trapped for virtualization anyway. 617 */ 618 taint |= check_update_ftr_reg(SYS_ID_MMFR0_EL1, cpu, 619 info->reg_id_mmfr0, boot->reg_id_mmfr0); 620 taint |= check_update_ftr_reg(SYS_ID_MMFR1_EL1, cpu, 621 info->reg_id_mmfr1, boot->reg_id_mmfr1); 622 taint |= check_update_ftr_reg(SYS_ID_MMFR2_EL1, cpu, 623 info->reg_id_mmfr2, boot->reg_id_mmfr2); 624 taint |= check_update_ftr_reg(SYS_ID_MMFR3_EL1, cpu, 625 info->reg_id_mmfr3, boot->reg_id_mmfr3); 626 taint |= check_update_ftr_reg(SYS_ID_PFR0_EL1, cpu, 627 info->reg_id_pfr0, boot->reg_id_pfr0); 628 taint |= check_update_ftr_reg(SYS_ID_PFR1_EL1, cpu, 629 info->reg_id_pfr1, boot->reg_id_pfr1); 630 taint |= check_update_ftr_reg(SYS_MVFR0_EL1, cpu, 631 info->reg_mvfr0, boot->reg_mvfr0); 632 taint |= check_update_ftr_reg(SYS_MVFR1_EL1, cpu, 633 info->reg_mvfr1, boot->reg_mvfr1); 634 taint |= check_update_ftr_reg(SYS_MVFR2_EL1, cpu, 635 info->reg_mvfr2, boot->reg_mvfr2); 636 } 637 638 /* 639 * Mismatched CPU features are a recipe for disaster. Don't even 640 * pretend to support them. 641 */ 642 WARN_TAINT_ONCE(taint, TAINT_CPU_OUT_OF_SPEC, 643 "Unsupported CPU feature variation.\n"); 644 } 645 646 u64 read_sanitised_ftr_reg(u32 id) 647 { 648 struct arm64_ftr_reg *regp = get_arm64_ftr_reg(id); 649 650 /* We shouldn't get a request for an unsupported register */ 651 BUG_ON(!regp); 652 return regp->sys_val; 653 } 654 655 #define read_sysreg_case(r) \ 656 case r: return read_sysreg_s(r) 657 658 /* 659 * __read_sysreg_by_encoding() - Used by a STARTING cpu before cpuinfo is populated. 660 * Read the system register on the current CPU 661 */ 662 static u64 __read_sysreg_by_encoding(u32 sys_id) 663 { 664 switch (sys_id) { 665 read_sysreg_case(SYS_ID_PFR0_EL1); 666 read_sysreg_case(SYS_ID_PFR1_EL1); 667 read_sysreg_case(SYS_ID_DFR0_EL1); 668 read_sysreg_case(SYS_ID_MMFR0_EL1); 669 read_sysreg_case(SYS_ID_MMFR1_EL1); 670 read_sysreg_case(SYS_ID_MMFR2_EL1); 671 read_sysreg_case(SYS_ID_MMFR3_EL1); 672 read_sysreg_case(SYS_ID_ISAR0_EL1); 673 read_sysreg_case(SYS_ID_ISAR1_EL1); 674 read_sysreg_case(SYS_ID_ISAR2_EL1); 675 read_sysreg_case(SYS_ID_ISAR3_EL1); 676 read_sysreg_case(SYS_ID_ISAR4_EL1); 677 read_sysreg_case(SYS_ID_ISAR5_EL1); 678 read_sysreg_case(SYS_MVFR0_EL1); 679 read_sysreg_case(SYS_MVFR1_EL1); 680 read_sysreg_case(SYS_MVFR2_EL1); 681 682 read_sysreg_case(SYS_ID_AA64PFR0_EL1); 683 read_sysreg_case(SYS_ID_AA64PFR1_EL1); 684 read_sysreg_case(SYS_ID_AA64DFR0_EL1); 685 read_sysreg_case(SYS_ID_AA64DFR1_EL1); 686 read_sysreg_case(SYS_ID_AA64MMFR0_EL1); 687 read_sysreg_case(SYS_ID_AA64MMFR1_EL1); 688 read_sysreg_case(SYS_ID_AA64MMFR2_EL1); 689 read_sysreg_case(SYS_ID_AA64ISAR0_EL1); 690 read_sysreg_case(SYS_ID_AA64ISAR1_EL1); 691 692 read_sysreg_case(SYS_CNTFRQ_EL0); 693 read_sysreg_case(SYS_CTR_EL0); 694 read_sysreg_case(SYS_DCZID_EL0); 695 696 default: 697 BUG(); 698 return 0; 699 } 700 } 701 702 #include <linux/irqchip/arm-gic-v3.h> 703 704 static bool 705 feature_matches(u64 reg, const struct arm64_cpu_capabilities *entry) 706 { 707 int val = cpuid_feature_extract_field(reg, entry->field_pos, entry->sign); 708 709 return val >= entry->min_field_value; 710 } 711 712 static bool 713 has_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope) 714 { 715 u64 val; 716 717 WARN_ON(scope == SCOPE_LOCAL_CPU && preemptible()); 718 if (scope == SCOPE_SYSTEM) 719 val = read_sanitised_ftr_reg(entry->sys_reg); 720 else 721 val = __read_sysreg_by_encoding(entry->sys_reg); 722 723 return feature_matches(val, entry); 724 } 725 726 static bool has_useable_gicv3_cpuif(const struct arm64_cpu_capabilities *entry, int scope) 727 { 728 bool has_sre; 729 730 if (!has_cpuid_feature(entry, scope)) 731 return false; 732 733 has_sre = gic_enable_sre(); 734 if (!has_sre) 735 pr_warn_once("%s present but disabled by higher exception level\n", 736 entry->desc); 737 738 return has_sre; 739 } 740 741 static bool has_no_hw_prefetch(const struct arm64_cpu_capabilities *entry, int __unused) 742 { 743 u32 midr = read_cpuid_id(); 744 745 /* Cavium ThunderX pass 1.x and 2.x */ 746 return MIDR_IS_CPU_MODEL_RANGE(midr, MIDR_THUNDERX, 747 MIDR_CPU_VAR_REV(0, 0), 748 MIDR_CPU_VAR_REV(1, MIDR_REVISION_MASK)); 749 } 750 751 static bool runs_at_el2(const struct arm64_cpu_capabilities *entry, int __unused) 752 { 753 return is_kernel_in_hyp_mode(); 754 } 755 756 static bool hyp_offset_low(const struct arm64_cpu_capabilities *entry, 757 int __unused) 758 { 759 phys_addr_t idmap_addr = __pa_symbol(__hyp_idmap_text_start); 760 761 /* 762 * Activate the lower HYP offset only if: 763 * - the idmap doesn't clash with it, 764 * - the kernel is not running at EL2. 765 */ 766 return idmap_addr > GENMASK(VA_BITS - 2, 0) && !is_kernel_in_hyp_mode(); 767 } 768 769 static bool has_no_fpsimd(const struct arm64_cpu_capabilities *entry, int __unused) 770 { 771 u64 pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1); 772 773 return cpuid_feature_extract_signed_field(pfr0, 774 ID_AA64PFR0_FP_SHIFT) < 0; 775 } 776 777 static const struct arm64_cpu_capabilities arm64_features[] = { 778 { 779 .desc = "GIC system register CPU interface", 780 .capability = ARM64_HAS_SYSREG_GIC_CPUIF, 781 .def_scope = SCOPE_SYSTEM, 782 .matches = has_useable_gicv3_cpuif, 783 .sys_reg = SYS_ID_AA64PFR0_EL1, 784 .field_pos = ID_AA64PFR0_GIC_SHIFT, 785 .sign = FTR_UNSIGNED, 786 .min_field_value = 1, 787 }, 788 #ifdef CONFIG_ARM64_PAN 789 { 790 .desc = "Privileged Access Never", 791 .capability = ARM64_HAS_PAN, 792 .def_scope = SCOPE_SYSTEM, 793 .matches = has_cpuid_feature, 794 .sys_reg = SYS_ID_AA64MMFR1_EL1, 795 .field_pos = ID_AA64MMFR1_PAN_SHIFT, 796 .sign = FTR_UNSIGNED, 797 .min_field_value = 1, 798 .enable = cpu_enable_pan, 799 }, 800 #endif /* CONFIG_ARM64_PAN */ 801 #if defined(CONFIG_AS_LSE) && defined(CONFIG_ARM64_LSE_ATOMICS) 802 { 803 .desc = "LSE atomic instructions", 804 .capability = ARM64_HAS_LSE_ATOMICS, 805 .def_scope = SCOPE_SYSTEM, 806 .matches = has_cpuid_feature, 807 .sys_reg = SYS_ID_AA64ISAR0_EL1, 808 .field_pos = ID_AA64ISAR0_ATOMICS_SHIFT, 809 .sign = FTR_UNSIGNED, 810 .min_field_value = 2, 811 }, 812 #endif /* CONFIG_AS_LSE && CONFIG_ARM64_LSE_ATOMICS */ 813 { 814 .desc = "Software prefetching using PRFM", 815 .capability = ARM64_HAS_NO_HW_PREFETCH, 816 .def_scope = SCOPE_SYSTEM, 817 .matches = has_no_hw_prefetch, 818 }, 819 #ifdef CONFIG_ARM64_UAO 820 { 821 .desc = "User Access Override", 822 .capability = ARM64_HAS_UAO, 823 .def_scope = SCOPE_SYSTEM, 824 .matches = has_cpuid_feature, 825 .sys_reg = SYS_ID_AA64MMFR2_EL1, 826 .field_pos = ID_AA64MMFR2_UAO_SHIFT, 827 .min_field_value = 1, 828 /* 829 * We rely on stop_machine() calling uao_thread_switch() to set 830 * UAO immediately after patching. 831 */ 832 }, 833 #endif /* CONFIG_ARM64_UAO */ 834 #ifdef CONFIG_ARM64_PAN 835 { 836 .capability = ARM64_ALT_PAN_NOT_UAO, 837 .def_scope = SCOPE_SYSTEM, 838 .matches = cpufeature_pan_not_uao, 839 }, 840 #endif /* CONFIG_ARM64_PAN */ 841 { 842 .desc = "Virtualization Host Extensions", 843 .capability = ARM64_HAS_VIRT_HOST_EXTN, 844 .def_scope = SCOPE_SYSTEM, 845 .matches = runs_at_el2, 846 }, 847 { 848 .desc = "32-bit EL0 Support", 849 .capability = ARM64_HAS_32BIT_EL0, 850 .def_scope = SCOPE_SYSTEM, 851 .matches = has_cpuid_feature, 852 .sys_reg = SYS_ID_AA64PFR0_EL1, 853 .sign = FTR_UNSIGNED, 854 .field_pos = ID_AA64PFR0_EL0_SHIFT, 855 .min_field_value = ID_AA64PFR0_EL0_32BIT_64BIT, 856 }, 857 { 858 .desc = "Reduced HYP mapping offset", 859 .capability = ARM64_HYP_OFFSET_LOW, 860 .def_scope = SCOPE_SYSTEM, 861 .matches = hyp_offset_low, 862 }, 863 { 864 /* FP/SIMD is not implemented */ 865 .capability = ARM64_HAS_NO_FPSIMD, 866 .def_scope = SCOPE_SYSTEM, 867 .min_field_value = 0, 868 .matches = has_no_fpsimd, 869 }, 870 {}, 871 }; 872 873 #define HWCAP_CAP(reg, field, s, min_value, type, cap) \ 874 { \ 875 .desc = #cap, \ 876 .def_scope = SCOPE_SYSTEM, \ 877 .matches = has_cpuid_feature, \ 878 .sys_reg = reg, \ 879 .field_pos = field, \ 880 .sign = s, \ 881 .min_field_value = min_value, \ 882 .hwcap_type = type, \ 883 .hwcap = cap, \ 884 } 885 886 static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { 887 HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_AES_SHIFT, FTR_UNSIGNED, 2, CAP_HWCAP, HWCAP_PMULL), 888 HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_AES_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_AES), 889 HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA1_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SHA1), 890 HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA2_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SHA2), 891 HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_CRC32_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_CRC32), 892 HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_ATOMICS_SHIFT, FTR_UNSIGNED, 2, CAP_HWCAP, HWCAP_ATOMICS), 893 HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_RDM_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_ASIMDRDM), 894 HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, FTR_SIGNED, 0, CAP_HWCAP, HWCAP_FP), 895 HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, HWCAP_FPHP), 896 HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, FTR_SIGNED, 0, CAP_HWCAP, HWCAP_ASIMD), 897 HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, HWCAP_ASIMDHP), 898 HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_JSCVT_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_JSCVT), 899 HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_FCMA_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_FCMA), 900 HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_LRCPC_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_LRCPC), 901 {}, 902 }; 903 904 static const struct arm64_cpu_capabilities compat_elf_hwcaps[] = { 905 #ifdef CONFIG_COMPAT 906 HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_AES_SHIFT, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_PMULL), 907 HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_AES_SHIFT, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_AES), 908 HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_SHA1_SHIFT, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA1), 909 HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_SHA2_SHIFT, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA2), 910 HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_CRC32_SHIFT, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_CRC32), 911 #endif 912 {}, 913 }; 914 915 static void __init cap_set_elf_hwcap(const struct arm64_cpu_capabilities *cap) 916 { 917 switch (cap->hwcap_type) { 918 case CAP_HWCAP: 919 elf_hwcap |= cap->hwcap; 920 break; 921 #ifdef CONFIG_COMPAT 922 case CAP_COMPAT_HWCAP: 923 compat_elf_hwcap |= (u32)cap->hwcap; 924 break; 925 case CAP_COMPAT_HWCAP2: 926 compat_elf_hwcap2 |= (u32)cap->hwcap; 927 break; 928 #endif 929 default: 930 WARN_ON(1); 931 break; 932 } 933 } 934 935 /* Check if we have a particular HWCAP enabled */ 936 static bool cpus_have_elf_hwcap(const struct arm64_cpu_capabilities *cap) 937 { 938 bool rc; 939 940 switch (cap->hwcap_type) { 941 case CAP_HWCAP: 942 rc = (elf_hwcap & cap->hwcap) != 0; 943 break; 944 #ifdef CONFIG_COMPAT 945 case CAP_COMPAT_HWCAP: 946 rc = (compat_elf_hwcap & (u32)cap->hwcap) != 0; 947 break; 948 case CAP_COMPAT_HWCAP2: 949 rc = (compat_elf_hwcap2 & (u32)cap->hwcap) != 0; 950 break; 951 #endif 952 default: 953 WARN_ON(1); 954 rc = false; 955 } 956 957 return rc; 958 } 959 960 static void __init setup_elf_hwcaps(const struct arm64_cpu_capabilities *hwcaps) 961 { 962 /* We support emulation of accesses to CPU ID feature registers */ 963 elf_hwcap |= HWCAP_CPUID; 964 for (; hwcaps->matches; hwcaps++) 965 if (hwcaps->matches(hwcaps, hwcaps->def_scope)) 966 cap_set_elf_hwcap(hwcaps); 967 } 968 969 void update_cpu_capabilities(const struct arm64_cpu_capabilities *caps, 970 const char *info) 971 { 972 for (; caps->matches; caps++) { 973 if (!caps->matches(caps, caps->def_scope)) 974 continue; 975 976 if (!cpus_have_cap(caps->capability) && caps->desc) 977 pr_info("%s %s\n", info, caps->desc); 978 cpus_set_cap(caps->capability); 979 } 980 } 981 982 /* 983 * Run through the enabled capabilities and enable() it on all active 984 * CPUs 985 */ 986 void __init enable_cpu_capabilities(const struct arm64_cpu_capabilities *caps) 987 { 988 for (; caps->matches; caps++) 989 if (caps->enable && cpus_have_cap(caps->capability)) 990 /* 991 * Use stop_machine() as it schedules the work allowing 992 * us to modify PSTATE, instead of on_each_cpu() which 993 * uses an IPI, giving us a PSTATE that disappears when 994 * we return. 995 */ 996 stop_machine(caps->enable, NULL, cpu_online_mask); 997 } 998 999 /* 1000 * Flag to indicate if we have computed the system wide 1001 * capabilities based on the boot time active CPUs. This 1002 * will be used to determine if a new booting CPU should 1003 * go through the verification process to make sure that it 1004 * supports the system capabilities, without using a hotplug 1005 * notifier. 1006 */ 1007 static bool sys_caps_initialised; 1008 1009 static inline void set_sys_caps_initialised(void) 1010 { 1011 sys_caps_initialised = true; 1012 } 1013 1014 /* 1015 * Check for CPU features that are used in early boot 1016 * based on the Boot CPU value. 1017 */ 1018 static void check_early_cpu_features(void) 1019 { 1020 verify_cpu_run_el(); 1021 verify_cpu_asid_bits(); 1022 } 1023 1024 static void 1025 verify_local_elf_hwcaps(const struct arm64_cpu_capabilities *caps) 1026 { 1027 1028 for (; caps->matches; caps++) 1029 if (cpus_have_elf_hwcap(caps) && !caps->matches(caps, SCOPE_LOCAL_CPU)) { 1030 pr_crit("CPU%d: missing HWCAP: %s\n", 1031 smp_processor_id(), caps->desc); 1032 cpu_die_early(); 1033 } 1034 } 1035 1036 static void 1037 verify_local_cpu_features(const struct arm64_cpu_capabilities *caps) 1038 { 1039 for (; caps->matches; caps++) { 1040 if (!cpus_have_cap(caps->capability)) 1041 continue; 1042 /* 1043 * If the new CPU misses an advertised feature, we cannot proceed 1044 * further, park the cpu. 1045 */ 1046 if (!caps->matches(caps, SCOPE_LOCAL_CPU)) { 1047 pr_crit("CPU%d: missing feature: %s\n", 1048 smp_processor_id(), caps->desc); 1049 cpu_die_early(); 1050 } 1051 if (caps->enable) 1052 caps->enable(NULL); 1053 } 1054 } 1055 1056 /* 1057 * Run through the enabled system capabilities and enable() it on this CPU. 1058 * The capabilities were decided based on the available CPUs at the boot time. 1059 * Any new CPU should match the system wide status of the capability. If the 1060 * new CPU doesn't have a capability which the system now has enabled, we 1061 * cannot do anything to fix it up and could cause unexpected failures. So 1062 * we park the CPU. 1063 */ 1064 static void verify_local_cpu_capabilities(void) 1065 { 1066 verify_local_cpu_errata_workarounds(); 1067 verify_local_cpu_features(arm64_features); 1068 verify_local_elf_hwcaps(arm64_elf_hwcaps); 1069 if (system_supports_32bit_el0()) 1070 verify_local_elf_hwcaps(compat_elf_hwcaps); 1071 } 1072 1073 void check_local_cpu_capabilities(void) 1074 { 1075 /* 1076 * All secondary CPUs should conform to the early CPU features 1077 * in use by the kernel based on boot CPU. 1078 */ 1079 check_early_cpu_features(); 1080 1081 /* 1082 * If we haven't finalised the system capabilities, this CPU gets 1083 * a chance to update the errata work arounds. 1084 * Otherwise, this CPU should verify that it has all the system 1085 * advertised capabilities. 1086 */ 1087 if (!sys_caps_initialised) 1088 update_cpu_errata_workarounds(); 1089 else 1090 verify_local_cpu_capabilities(); 1091 } 1092 1093 static void __init setup_feature_capabilities(void) 1094 { 1095 update_cpu_capabilities(arm64_features, "detected feature:"); 1096 enable_cpu_capabilities(arm64_features); 1097 } 1098 1099 /* 1100 * Check if the current CPU has a given feature capability. 1101 * Should be called from non-preemptible context. 1102 */ 1103 static bool __this_cpu_has_cap(const struct arm64_cpu_capabilities *cap_array, 1104 unsigned int cap) 1105 { 1106 const struct arm64_cpu_capabilities *caps; 1107 1108 if (WARN_ON(preemptible())) 1109 return false; 1110 1111 for (caps = cap_array; caps->desc; caps++) 1112 if (caps->capability == cap && caps->matches) 1113 return caps->matches(caps, SCOPE_LOCAL_CPU); 1114 1115 return false; 1116 } 1117 1118 extern const struct arm64_cpu_capabilities arm64_errata[]; 1119 1120 bool this_cpu_has_cap(unsigned int cap) 1121 { 1122 return (__this_cpu_has_cap(arm64_features, cap) || 1123 __this_cpu_has_cap(arm64_errata, cap)); 1124 } 1125 1126 void __init setup_cpu_features(void) 1127 { 1128 u32 cwg; 1129 int cls; 1130 1131 /* Set the CPU feature capabilies */ 1132 setup_feature_capabilities(); 1133 enable_errata_workarounds(); 1134 setup_elf_hwcaps(arm64_elf_hwcaps); 1135 1136 if (system_supports_32bit_el0()) 1137 setup_elf_hwcaps(compat_elf_hwcaps); 1138 1139 /* Advertise that we have computed the system capabilities */ 1140 set_sys_caps_initialised(); 1141 1142 /* 1143 * Check for sane CTR_EL0.CWG value. 1144 */ 1145 cwg = cache_type_cwg(); 1146 cls = cache_line_size(); 1147 if (!cwg) 1148 pr_warn("No Cache Writeback Granule information, assuming cache line size %d\n", 1149 cls); 1150 if (L1_CACHE_BYTES < cls) 1151 pr_warn("L1_CACHE_BYTES smaller than the Cache Writeback Granule (%d < %d)\n", 1152 L1_CACHE_BYTES, cls); 1153 } 1154 1155 static bool __maybe_unused 1156 cpufeature_pan_not_uao(const struct arm64_cpu_capabilities *entry, int __unused) 1157 { 1158 return (cpus_have_const_cap(ARM64_HAS_PAN) && !cpus_have_const_cap(ARM64_HAS_UAO)); 1159 } 1160 1161 /* 1162 * We emulate only the following system register space. 1163 * Op0 = 0x3, CRn = 0x0, Op1 = 0x0, CRm = [0, 4 - 7] 1164 * See Table C5-6 System instruction encodings for System register accesses, 1165 * ARMv8 ARM(ARM DDI 0487A.f) for more details. 1166 */ 1167 static inline bool __attribute_const__ is_emulated(u32 id) 1168 { 1169 return (sys_reg_Op0(id) == 0x3 && 1170 sys_reg_CRn(id) == 0x0 && 1171 sys_reg_Op1(id) == 0x0 && 1172 (sys_reg_CRm(id) == 0 || 1173 ((sys_reg_CRm(id) >= 4) && (sys_reg_CRm(id) <= 7)))); 1174 } 1175 1176 /* 1177 * With CRm == 0, reg should be one of : 1178 * MIDR_EL1, MPIDR_EL1 or REVIDR_EL1. 1179 */ 1180 static inline int emulate_id_reg(u32 id, u64 *valp) 1181 { 1182 switch (id) { 1183 case SYS_MIDR_EL1: 1184 *valp = read_cpuid_id(); 1185 break; 1186 case SYS_MPIDR_EL1: 1187 *valp = SYS_MPIDR_SAFE_VAL; 1188 break; 1189 case SYS_REVIDR_EL1: 1190 /* IMPLEMENTATION DEFINED values are emulated with 0 */ 1191 *valp = 0; 1192 break; 1193 default: 1194 return -EINVAL; 1195 } 1196 1197 return 0; 1198 } 1199 1200 static int emulate_sys_reg(u32 id, u64 *valp) 1201 { 1202 struct arm64_ftr_reg *regp; 1203 1204 if (!is_emulated(id)) 1205 return -EINVAL; 1206 1207 if (sys_reg_CRm(id) == 0) 1208 return emulate_id_reg(id, valp); 1209 1210 regp = get_arm64_ftr_reg(id); 1211 if (regp) 1212 *valp = arm64_ftr_reg_user_value(regp); 1213 else 1214 /* 1215 * The untracked registers are either IMPLEMENTATION DEFINED 1216 * (e.g, ID_AFR0_EL1) or reserved RAZ. 1217 */ 1218 *valp = 0; 1219 return 0; 1220 } 1221 1222 static int emulate_mrs(struct pt_regs *regs, u32 insn) 1223 { 1224 int rc; 1225 u32 sys_reg, dst; 1226 u64 val; 1227 1228 /* 1229 * sys_reg values are defined as used in mrs/msr instruction. 1230 * shift the imm value to get the encoding. 1231 */ 1232 sys_reg = (u32)aarch64_insn_decode_immediate(AARCH64_INSN_IMM_16, insn) << 5; 1233 rc = emulate_sys_reg(sys_reg, &val); 1234 if (!rc) { 1235 dst = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RT, insn); 1236 pt_regs_write_reg(regs, dst, val); 1237 regs->pc += 4; 1238 } 1239 1240 return rc; 1241 } 1242 1243 static struct undef_hook mrs_hook = { 1244 .instr_mask = 0xfff00000, 1245 .instr_val = 0xd5300000, 1246 .pstate_mask = COMPAT_PSR_MODE_MASK, 1247 .pstate_val = PSR_MODE_EL0t, 1248 .fn = emulate_mrs, 1249 }; 1250 1251 static int __init enable_mrs_emulation(void) 1252 { 1253 register_undef_hook(&mrs_hook); 1254 return 0; 1255 } 1256 1257 late_initcall(enable_mrs_emulation); 1258