1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 1994 Linus Torvalds 4 * 5 * Cyrix stuff, June 1998 by: 6 * - Rafael R. Reilova (moved everything from head.S), 7 * <rreilova@ececs.uc.edu> 8 * - Channing Corn (tests & fixes), 9 * - Andrew D. Balsa (code cleanup). 10 */ 11 #include <linux/init.h> 12 #include <linux/utsname.h> 13 #include <linux/cpu.h> 14 #include <linux/module.h> 15 16 #include <asm/nospec-branch.h> 17 #include <asm/cmdline.h> 18 #include <asm/bugs.h> 19 #include <asm/processor.h> 20 #include <asm/processor-flags.h> 21 #include <asm/fpu/internal.h> 22 #include <asm/msr.h> 23 #include <asm/paravirt.h> 24 #include <asm/alternative.h> 25 #include <asm/pgtable.h> 26 #include <asm/set_memory.h> 27 #include <asm/intel-family.h> 28 29 static void __init spectre_v2_select_mitigation(void); 30 31 void __init check_bugs(void) 32 { 33 identify_boot_cpu(); 34 35 if (!IS_ENABLED(CONFIG_SMP)) { 36 pr_info("CPU: "); 37 print_cpu_info(&boot_cpu_data); 38 } 39 40 /* Select the proper spectre mitigation before patching alternatives */ 41 spectre_v2_select_mitigation(); 42 43 #ifdef CONFIG_X86_32 44 /* 45 * Check whether we are able to run this kernel safely on SMP. 46 * 47 * - i386 is no longer supported. 48 * - In order to run on anything without a TSC, we need to be 49 * compiled for a i486. 50 */ 51 if (boot_cpu_data.x86 < 4) 52 panic("Kernel requires i486+ for 'invlpg' and other features"); 53 54 init_utsname()->machine[1] = 55 '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86); 56 alternative_instructions(); 57 58 fpu__init_check_bugs(); 59 #else /* CONFIG_X86_64 */ 60 alternative_instructions(); 61 62 /* 63 * Make sure the first 2MB area is not mapped by huge pages 64 * There are typically fixed size MTRRs in there and overlapping 65 * MTRRs into large pages causes slow downs. 66 * 67 * Right now we don't do that with gbpages because there seems 68 * very little benefit for that case. 69 */ 70 if (!direct_gbpages) 71 set_memory_4k((unsigned long)__va(0), 1); 72 #endif 73 } 74 75 /* The kernel command line selection */ 76 enum spectre_v2_mitigation_cmd { 77 SPECTRE_V2_CMD_NONE, 78 SPECTRE_V2_CMD_AUTO, 79 SPECTRE_V2_CMD_FORCE, 80 SPECTRE_V2_CMD_RETPOLINE, 81 SPECTRE_V2_CMD_RETPOLINE_GENERIC, 82 SPECTRE_V2_CMD_RETPOLINE_AMD, 83 }; 84 85 static const char *spectre_v2_strings[] = { 86 [SPECTRE_V2_NONE] = "Vulnerable", 87 [SPECTRE_V2_RETPOLINE_MINIMAL] = "Vulnerable: Minimal generic ASM retpoline", 88 [SPECTRE_V2_RETPOLINE_MINIMAL_AMD] = "Vulnerable: Minimal AMD ASM retpoline", 89 [SPECTRE_V2_RETPOLINE_GENERIC] = "Mitigation: Full generic retpoline", 90 [SPECTRE_V2_RETPOLINE_AMD] = "Mitigation: Full AMD retpoline", 91 }; 92 93 #undef pr_fmt 94 #define pr_fmt(fmt) "Spectre V2 : " fmt 95 96 static enum spectre_v2_mitigation spectre_v2_enabled = SPECTRE_V2_NONE; 97 98 #ifdef RETPOLINE 99 static bool spectre_v2_bad_module; 100 101 bool retpoline_module_ok(bool has_retpoline) 102 { 103 if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline) 104 return true; 105 106 pr_err("System may be vulnerable to spectre v2\n"); 107 spectre_v2_bad_module = true; 108 return false; 109 } 110 111 static inline const char *spectre_v2_module_string(void) 112 { 113 return spectre_v2_bad_module ? " - vulnerable module loaded" : ""; 114 } 115 #else 116 static inline const char *spectre_v2_module_string(void) { return ""; } 117 #endif 118 119 static void __init spec2_print_if_insecure(const char *reason) 120 { 121 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2)) 122 pr_info("%s selected on command line.\n", reason); 123 } 124 125 static void __init spec2_print_if_secure(const char *reason) 126 { 127 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2)) 128 pr_info("%s selected on command line.\n", reason); 129 } 130 131 static inline bool retp_compiler(void) 132 { 133 return __is_defined(RETPOLINE); 134 } 135 136 static inline bool match_option(const char *arg, int arglen, const char *opt) 137 { 138 int len = strlen(opt); 139 140 return len == arglen && !strncmp(arg, opt, len); 141 } 142 143 static const struct { 144 const char *option; 145 enum spectre_v2_mitigation_cmd cmd; 146 bool secure; 147 } mitigation_options[] = { 148 { "off", SPECTRE_V2_CMD_NONE, false }, 149 { "on", SPECTRE_V2_CMD_FORCE, true }, 150 { "retpoline", SPECTRE_V2_CMD_RETPOLINE, false }, 151 { "retpoline,amd", SPECTRE_V2_CMD_RETPOLINE_AMD, false }, 152 { "retpoline,generic", SPECTRE_V2_CMD_RETPOLINE_GENERIC, false }, 153 { "auto", SPECTRE_V2_CMD_AUTO, false }, 154 }; 155 156 static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void) 157 { 158 char arg[20]; 159 int ret, i; 160 enum spectre_v2_mitigation_cmd cmd = SPECTRE_V2_CMD_AUTO; 161 162 if (cmdline_find_option_bool(boot_command_line, "nospectre_v2")) 163 return SPECTRE_V2_CMD_NONE; 164 else { 165 ret = cmdline_find_option(boot_command_line, "spectre_v2", arg, sizeof(arg)); 166 if (ret < 0) 167 return SPECTRE_V2_CMD_AUTO; 168 169 for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) { 170 if (!match_option(arg, ret, mitigation_options[i].option)) 171 continue; 172 cmd = mitigation_options[i].cmd; 173 break; 174 } 175 176 if (i >= ARRAY_SIZE(mitigation_options)) { 177 pr_err("unknown option (%s). Switching to AUTO select\n", arg); 178 return SPECTRE_V2_CMD_AUTO; 179 } 180 } 181 182 if ((cmd == SPECTRE_V2_CMD_RETPOLINE || 183 cmd == SPECTRE_V2_CMD_RETPOLINE_AMD || 184 cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC) && 185 !IS_ENABLED(CONFIG_RETPOLINE)) { 186 pr_err("%s selected but not compiled in. Switching to AUTO select\n", mitigation_options[i].option); 187 return SPECTRE_V2_CMD_AUTO; 188 } 189 190 if (cmd == SPECTRE_V2_CMD_RETPOLINE_AMD && 191 boot_cpu_data.x86_vendor != X86_VENDOR_AMD) { 192 pr_err("retpoline,amd selected but CPU is not AMD. Switching to AUTO select\n"); 193 return SPECTRE_V2_CMD_AUTO; 194 } 195 196 if (mitigation_options[i].secure) 197 spec2_print_if_secure(mitigation_options[i].option); 198 else 199 spec2_print_if_insecure(mitigation_options[i].option); 200 201 return cmd; 202 } 203 204 /* Check for Skylake-like CPUs (for RSB handling) */ 205 static bool __init is_skylake_era(void) 206 { 207 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && 208 boot_cpu_data.x86 == 6) { 209 switch (boot_cpu_data.x86_model) { 210 case INTEL_FAM6_SKYLAKE_MOBILE: 211 case INTEL_FAM6_SKYLAKE_DESKTOP: 212 case INTEL_FAM6_SKYLAKE_X: 213 case INTEL_FAM6_KABYLAKE_MOBILE: 214 case INTEL_FAM6_KABYLAKE_DESKTOP: 215 return true; 216 } 217 } 218 return false; 219 } 220 221 static void __init spectre_v2_select_mitigation(void) 222 { 223 enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline(); 224 enum spectre_v2_mitigation mode = SPECTRE_V2_NONE; 225 226 /* 227 * If the CPU is not affected and the command line mode is NONE or AUTO 228 * then nothing to do. 229 */ 230 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) && 231 (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO)) 232 return; 233 234 switch (cmd) { 235 case SPECTRE_V2_CMD_NONE: 236 return; 237 238 case SPECTRE_V2_CMD_FORCE: 239 case SPECTRE_V2_CMD_AUTO: 240 if (IS_ENABLED(CONFIG_RETPOLINE)) 241 goto retpoline_auto; 242 break; 243 case SPECTRE_V2_CMD_RETPOLINE_AMD: 244 if (IS_ENABLED(CONFIG_RETPOLINE)) 245 goto retpoline_amd; 246 break; 247 case SPECTRE_V2_CMD_RETPOLINE_GENERIC: 248 if (IS_ENABLED(CONFIG_RETPOLINE)) 249 goto retpoline_generic; 250 break; 251 case SPECTRE_V2_CMD_RETPOLINE: 252 if (IS_ENABLED(CONFIG_RETPOLINE)) 253 goto retpoline_auto; 254 break; 255 } 256 pr_err("Spectre mitigation: kernel not compiled with retpoline; no mitigation available!"); 257 return; 258 259 retpoline_auto: 260 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) { 261 retpoline_amd: 262 if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) { 263 pr_err("Spectre mitigation: LFENCE not serializing, switching to generic retpoline\n"); 264 goto retpoline_generic; 265 } 266 mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_AMD : 267 SPECTRE_V2_RETPOLINE_MINIMAL_AMD; 268 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD); 269 setup_force_cpu_cap(X86_FEATURE_RETPOLINE); 270 } else { 271 retpoline_generic: 272 mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_GENERIC : 273 SPECTRE_V2_RETPOLINE_MINIMAL; 274 setup_force_cpu_cap(X86_FEATURE_RETPOLINE); 275 } 276 277 spectre_v2_enabled = mode; 278 pr_info("%s\n", spectre_v2_strings[mode]); 279 280 /* 281 * If neither SMEP nor PTI are available, there is a risk of 282 * hitting userspace addresses in the RSB after a context switch 283 * from a shallow call stack to a deeper one. To prevent this fill 284 * the entire RSB, even when using IBRS. 285 * 286 * Skylake era CPUs have a separate issue with *underflow* of the 287 * RSB, when they will predict 'ret' targets from the generic BTB. 288 * The proper mitigation for this is IBRS. If IBRS is not supported 289 * or deactivated in favour of retpolines the RSB fill on context 290 * switch is required. 291 */ 292 if ((!boot_cpu_has(X86_FEATURE_PTI) && 293 !boot_cpu_has(X86_FEATURE_SMEP)) || is_skylake_era()) { 294 setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW); 295 pr_info("Spectre v2 mitigation: Filling RSB on context switch\n"); 296 } 297 298 /* Initialize Indirect Branch Prediction Barrier if supported */ 299 if (boot_cpu_has(X86_FEATURE_IBPB)) { 300 setup_force_cpu_cap(X86_FEATURE_USE_IBPB); 301 pr_info("Spectre v2 mitigation: Enabling Indirect Branch Prediction Barrier\n"); 302 } 303 304 /* 305 * Retpoline means the kernel is safe because it has no indirect 306 * branches. But firmware isn't, so use IBRS to protect that. 307 */ 308 if (boot_cpu_has(X86_FEATURE_IBRS)) { 309 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW); 310 pr_info("Enabling Restricted Speculation for firmware calls\n"); 311 } 312 } 313 314 #undef pr_fmt 315 316 #ifdef CONFIG_SYSFS 317 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf) 318 { 319 if (!boot_cpu_has_bug(X86_BUG_CPU_MELTDOWN)) 320 return sprintf(buf, "Not affected\n"); 321 if (boot_cpu_has(X86_FEATURE_PTI)) 322 return sprintf(buf, "Mitigation: PTI\n"); 323 return sprintf(buf, "Vulnerable\n"); 324 } 325 326 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf) 327 { 328 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1)) 329 return sprintf(buf, "Not affected\n"); 330 return sprintf(buf, "Mitigation: __user pointer sanitization\n"); 331 } 332 333 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf) 334 { 335 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2)) 336 return sprintf(buf, "Not affected\n"); 337 338 return sprintf(buf, "%s%s%s%s\n", spectre_v2_strings[spectre_v2_enabled], 339 boot_cpu_has(X86_FEATURE_USE_IBPB) ? ", IBPB" : "", 340 boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "", 341 spectre_v2_module_string()); 342 } 343 #endif 344