1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 1994 Linus Torvalds 4 * 5 * Cyrix stuff, June 1998 by: 6 * - Rafael R. Reilova (moved everything from head.S), 7 * <rreilova@ececs.uc.edu> 8 * - Channing Corn (tests & fixes), 9 * - Andrew D. Balsa (code cleanup). 10 */ 11 #include <linux/init.h> 12 #include <linux/utsname.h> 13 #include <linux/cpu.h> 14 #include <linux/module.h> 15 #include <linux/nospec.h> 16 #include <linux/prctl.h> 17 #include <linux/sched/smt.h> 18 19 #include <asm/spec-ctrl.h> 20 #include <asm/cmdline.h> 21 #include <asm/bugs.h> 22 #include <asm/processor.h> 23 #include <asm/processor-flags.h> 24 #include <asm/fpu/internal.h> 25 #include <asm/msr.h> 26 #include <asm/vmx.h> 27 #include <asm/paravirt.h> 28 #include <asm/alternative.h> 29 #include <asm/pgtable.h> 30 #include <asm/set_memory.h> 31 #include <asm/intel-family.h> 32 #include <asm/e820/api.h> 33 #include <asm/hypervisor.h> 34 35 #include "cpu.h" 36 37 static void __init spectre_v1_select_mitigation(void); 38 static void __init spectre_v2_select_mitigation(void); 39 static void __init ssb_select_mitigation(void); 40 static void __init l1tf_select_mitigation(void); 41 static void __init mds_select_mitigation(void); 42 static void __init mds_print_mitigation(void); 43 static void __init taa_select_mitigation(void); 44 45 /* The base value of the SPEC_CTRL MSR that always has to be preserved. */ 46 u64 x86_spec_ctrl_base; 47 EXPORT_SYMBOL_GPL(x86_spec_ctrl_base); 48 static DEFINE_MUTEX(spec_ctrl_mutex); 49 50 /* 51 * The vendor and possibly platform specific bits which can be modified in 52 * x86_spec_ctrl_base. 53 */ 54 static u64 __ro_after_init x86_spec_ctrl_mask = SPEC_CTRL_IBRS; 55 56 /* 57 * AMD specific MSR info for Speculative Store Bypass control. 58 * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu(). 59 */ 60 u64 __ro_after_init x86_amd_ls_cfg_base; 61 u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask; 62 63 /* Control conditional STIBP in switch_to() */ 64 DEFINE_STATIC_KEY_FALSE(switch_to_cond_stibp); 65 /* Control conditional IBPB in switch_mm() */ 66 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb); 67 /* Control unconditional IBPB in switch_mm() */ 68 DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb); 69 70 /* Control MDS CPU buffer clear before returning to user space */ 71 DEFINE_STATIC_KEY_FALSE(mds_user_clear); 72 EXPORT_SYMBOL_GPL(mds_user_clear); 73 /* Control MDS CPU buffer clear before idling (halt, mwait) */ 74 DEFINE_STATIC_KEY_FALSE(mds_idle_clear); 75 EXPORT_SYMBOL_GPL(mds_idle_clear); 76 77 void __init check_bugs(void) 78 { 79 identify_boot_cpu(); 80 81 /* 82 * identify_boot_cpu() initialized SMT support information, let the 83 * core code know. 84 */ 85 cpu_smt_check_topology(); 86 87 if (!IS_ENABLED(CONFIG_SMP)) { 88 pr_info("CPU: "); 89 print_cpu_info(&boot_cpu_data); 90 } 91 92 /* 93 * Read the SPEC_CTRL MSR to account for reserved bits which may 94 * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD 95 * init code as it is not enumerated and depends on the family. 96 */ 97 if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) 98 rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); 99 100 /* Allow STIBP in MSR_SPEC_CTRL if supported */ 101 if (boot_cpu_has(X86_FEATURE_STIBP)) 102 x86_spec_ctrl_mask |= SPEC_CTRL_STIBP; 103 104 /* Select the proper CPU mitigations before patching alternatives: */ 105 spectre_v1_select_mitigation(); 106 spectre_v2_select_mitigation(); 107 ssb_select_mitigation(); 108 l1tf_select_mitigation(); 109 mds_select_mitigation(); 110 taa_select_mitigation(); 111 112 /* 113 * As MDS and TAA mitigations are inter-related, print MDS 114 * mitigation until after TAA mitigation selection is done. 115 */ 116 mds_print_mitigation(); 117 118 arch_smt_update(); 119 120 #ifdef CONFIG_X86_32 121 /* 122 * Check whether we are able to run this kernel safely on SMP. 123 * 124 * - i386 is no longer supported. 125 * - In order to run on anything without a TSC, we need to be 126 * compiled for a i486. 127 */ 128 if (boot_cpu_data.x86 < 4) 129 panic("Kernel requires i486+ for 'invlpg' and other features"); 130 131 init_utsname()->machine[1] = 132 '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86); 133 alternative_instructions(); 134 135 fpu__init_check_bugs(); 136 #else /* CONFIG_X86_64 */ 137 alternative_instructions(); 138 139 /* 140 * Make sure the first 2MB area is not mapped by huge pages 141 * There are typically fixed size MTRRs in there and overlapping 142 * MTRRs into large pages causes slow downs. 143 * 144 * Right now we don't do that with gbpages because there seems 145 * very little benefit for that case. 146 */ 147 if (!direct_gbpages) 148 set_memory_4k((unsigned long)__va(0), 1); 149 #endif 150 } 151 152 void 153 x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest) 154 { 155 u64 msrval, guestval, hostval = x86_spec_ctrl_base; 156 struct thread_info *ti = current_thread_info(); 157 158 /* Is MSR_SPEC_CTRL implemented ? */ 159 if (static_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) { 160 /* 161 * Restrict guest_spec_ctrl to supported values. Clear the 162 * modifiable bits in the host base value and or the 163 * modifiable bits from the guest value. 164 */ 165 guestval = hostval & ~x86_spec_ctrl_mask; 166 guestval |= guest_spec_ctrl & x86_spec_ctrl_mask; 167 168 /* SSBD controlled in MSR_SPEC_CTRL */ 169 if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) || 170 static_cpu_has(X86_FEATURE_AMD_SSBD)) 171 hostval |= ssbd_tif_to_spec_ctrl(ti->flags); 172 173 /* Conditional STIBP enabled? */ 174 if (static_branch_unlikely(&switch_to_cond_stibp)) 175 hostval |= stibp_tif_to_spec_ctrl(ti->flags); 176 177 if (hostval != guestval) { 178 msrval = setguest ? guestval : hostval; 179 wrmsrl(MSR_IA32_SPEC_CTRL, msrval); 180 } 181 } 182 183 /* 184 * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update 185 * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported. 186 */ 187 if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD) && 188 !static_cpu_has(X86_FEATURE_VIRT_SSBD)) 189 return; 190 191 /* 192 * If the host has SSBD mitigation enabled, force it in the host's 193 * virtual MSR value. If its not permanently enabled, evaluate 194 * current's TIF_SSBD thread flag. 195 */ 196 if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE)) 197 hostval = SPEC_CTRL_SSBD; 198 else 199 hostval = ssbd_tif_to_spec_ctrl(ti->flags); 200 201 /* Sanitize the guest value */ 202 guestval = guest_virt_spec_ctrl & SPEC_CTRL_SSBD; 203 204 if (hostval != guestval) { 205 unsigned long tif; 206 207 tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) : 208 ssbd_spec_ctrl_to_tif(hostval); 209 210 speculation_ctrl_update(tif); 211 } 212 } 213 EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl); 214 215 static void x86_amd_ssb_disable(void) 216 { 217 u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask; 218 219 if (boot_cpu_has(X86_FEATURE_VIRT_SSBD)) 220 wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD); 221 else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD)) 222 wrmsrl(MSR_AMD64_LS_CFG, msrval); 223 } 224 225 #undef pr_fmt 226 #define pr_fmt(fmt) "MDS: " fmt 227 228 /* Default mitigation for MDS-affected CPUs */ 229 static enum mds_mitigations mds_mitigation __ro_after_init = MDS_MITIGATION_FULL; 230 static bool mds_nosmt __ro_after_init = false; 231 232 static const char * const mds_strings[] = { 233 [MDS_MITIGATION_OFF] = "Vulnerable", 234 [MDS_MITIGATION_FULL] = "Mitigation: Clear CPU buffers", 235 [MDS_MITIGATION_VMWERV] = "Vulnerable: Clear CPU buffers attempted, no microcode", 236 }; 237 238 static void __init mds_select_mitigation(void) 239 { 240 if (!boot_cpu_has_bug(X86_BUG_MDS) || cpu_mitigations_off()) { 241 mds_mitigation = MDS_MITIGATION_OFF; 242 return; 243 } 244 245 if (mds_mitigation == MDS_MITIGATION_FULL) { 246 if (!boot_cpu_has(X86_FEATURE_MD_CLEAR)) 247 mds_mitigation = MDS_MITIGATION_VMWERV; 248 249 static_branch_enable(&mds_user_clear); 250 251 if (!boot_cpu_has(X86_BUG_MSBDS_ONLY) && 252 (mds_nosmt || cpu_mitigations_auto_nosmt())) 253 cpu_smt_disable(false); 254 } 255 } 256 257 static void __init mds_print_mitigation(void) 258 { 259 if (!boot_cpu_has_bug(X86_BUG_MDS) || cpu_mitigations_off()) 260 return; 261 262 pr_info("%s\n", mds_strings[mds_mitigation]); 263 } 264 265 static int __init mds_cmdline(char *str) 266 { 267 if (!boot_cpu_has_bug(X86_BUG_MDS)) 268 return 0; 269 270 if (!str) 271 return -EINVAL; 272 273 if (!strcmp(str, "off")) 274 mds_mitigation = MDS_MITIGATION_OFF; 275 else if (!strcmp(str, "full")) 276 mds_mitigation = MDS_MITIGATION_FULL; 277 else if (!strcmp(str, "full,nosmt")) { 278 mds_mitigation = MDS_MITIGATION_FULL; 279 mds_nosmt = true; 280 } 281 282 return 0; 283 } 284 early_param("mds", mds_cmdline); 285 286 #undef pr_fmt 287 #define pr_fmt(fmt) "TAA: " fmt 288 289 enum taa_mitigations { 290 TAA_MITIGATION_OFF, 291 TAA_MITIGATION_UCODE_NEEDED, 292 TAA_MITIGATION_VERW, 293 TAA_MITIGATION_TSX_DISABLED, 294 }; 295 296 /* Default mitigation for TAA-affected CPUs */ 297 static enum taa_mitigations taa_mitigation __ro_after_init = TAA_MITIGATION_VERW; 298 static bool taa_nosmt __ro_after_init; 299 300 static const char * const taa_strings[] = { 301 [TAA_MITIGATION_OFF] = "Vulnerable", 302 [TAA_MITIGATION_UCODE_NEEDED] = "Vulnerable: Clear CPU buffers attempted, no microcode", 303 [TAA_MITIGATION_VERW] = "Mitigation: Clear CPU buffers", 304 [TAA_MITIGATION_TSX_DISABLED] = "Mitigation: TSX disabled", 305 }; 306 307 static void __init taa_select_mitigation(void) 308 { 309 u64 ia32_cap; 310 311 if (!boot_cpu_has_bug(X86_BUG_TAA)) { 312 taa_mitigation = TAA_MITIGATION_OFF; 313 return; 314 } 315 316 /* TSX previously disabled by tsx=off */ 317 if (!boot_cpu_has(X86_FEATURE_RTM)) { 318 taa_mitigation = TAA_MITIGATION_TSX_DISABLED; 319 goto out; 320 } 321 322 if (cpu_mitigations_off()) { 323 taa_mitigation = TAA_MITIGATION_OFF; 324 return; 325 } 326 327 /* 328 * TAA mitigation via VERW is turned off if both 329 * tsx_async_abort=off and mds=off are specified. 330 */ 331 if (taa_mitigation == TAA_MITIGATION_OFF && 332 mds_mitigation == MDS_MITIGATION_OFF) 333 goto out; 334 335 if (boot_cpu_has(X86_FEATURE_MD_CLEAR)) 336 taa_mitigation = TAA_MITIGATION_VERW; 337 else 338 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED; 339 340 /* 341 * VERW doesn't clear the CPU buffers when MD_CLEAR=1 and MDS_NO=1. 342 * A microcode update fixes this behavior to clear CPU buffers. It also 343 * adds support for MSR_IA32_TSX_CTRL which is enumerated by the 344 * ARCH_CAP_TSX_CTRL_MSR bit. 345 * 346 * On MDS_NO=1 CPUs if ARCH_CAP_TSX_CTRL_MSR is not set, microcode 347 * update is required. 348 */ 349 ia32_cap = x86_read_arch_cap_msr(); 350 if ( (ia32_cap & ARCH_CAP_MDS_NO) && 351 !(ia32_cap & ARCH_CAP_TSX_CTRL_MSR)) 352 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED; 353 354 /* 355 * TSX is enabled, select alternate mitigation for TAA which is 356 * the same as MDS. Enable MDS static branch to clear CPU buffers. 357 * 358 * For guests that can't determine whether the correct microcode is 359 * present on host, enable the mitigation for UCODE_NEEDED as well. 360 */ 361 static_branch_enable(&mds_user_clear); 362 363 if (taa_nosmt || cpu_mitigations_auto_nosmt()) 364 cpu_smt_disable(false); 365 366 /* 367 * Update MDS mitigation, if necessary, as the mds_user_clear is 368 * now enabled for TAA mitigation. 369 */ 370 if (mds_mitigation == MDS_MITIGATION_OFF && 371 boot_cpu_has_bug(X86_BUG_MDS)) { 372 mds_mitigation = MDS_MITIGATION_FULL; 373 mds_select_mitigation(); 374 } 375 out: 376 pr_info("%s\n", taa_strings[taa_mitigation]); 377 } 378 379 static int __init tsx_async_abort_parse_cmdline(char *str) 380 { 381 if (!boot_cpu_has_bug(X86_BUG_TAA)) 382 return 0; 383 384 if (!str) 385 return -EINVAL; 386 387 if (!strcmp(str, "off")) { 388 taa_mitigation = TAA_MITIGATION_OFF; 389 } else if (!strcmp(str, "full")) { 390 taa_mitigation = TAA_MITIGATION_VERW; 391 } else if (!strcmp(str, "full,nosmt")) { 392 taa_mitigation = TAA_MITIGATION_VERW; 393 taa_nosmt = true; 394 } 395 396 return 0; 397 } 398 early_param("tsx_async_abort", tsx_async_abort_parse_cmdline); 399 400 #undef pr_fmt 401 #define pr_fmt(fmt) "Spectre V1 : " fmt 402 403 enum spectre_v1_mitigation { 404 SPECTRE_V1_MITIGATION_NONE, 405 SPECTRE_V1_MITIGATION_AUTO, 406 }; 407 408 static enum spectre_v1_mitigation spectre_v1_mitigation __ro_after_init = 409 SPECTRE_V1_MITIGATION_AUTO; 410 411 static const char * const spectre_v1_strings[] = { 412 [SPECTRE_V1_MITIGATION_NONE] = "Vulnerable: __user pointer sanitization and usercopy barriers only; no swapgs barriers", 413 [SPECTRE_V1_MITIGATION_AUTO] = "Mitigation: usercopy/swapgs barriers and __user pointer sanitization", 414 }; 415 416 /* 417 * Does SMAP provide full mitigation against speculative kernel access to 418 * userspace? 419 */ 420 static bool smap_works_speculatively(void) 421 { 422 if (!boot_cpu_has(X86_FEATURE_SMAP)) 423 return false; 424 425 /* 426 * On CPUs which are vulnerable to Meltdown, SMAP does not 427 * prevent speculative access to user data in the L1 cache. 428 * Consider SMAP to be non-functional as a mitigation on these 429 * CPUs. 430 */ 431 if (boot_cpu_has(X86_BUG_CPU_MELTDOWN)) 432 return false; 433 434 return true; 435 } 436 437 static void __init spectre_v1_select_mitigation(void) 438 { 439 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1) || cpu_mitigations_off()) { 440 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE; 441 return; 442 } 443 444 if (spectre_v1_mitigation == SPECTRE_V1_MITIGATION_AUTO) { 445 /* 446 * With Spectre v1, a user can speculatively control either 447 * path of a conditional swapgs with a user-controlled GS 448 * value. The mitigation is to add lfences to both code paths. 449 * 450 * If FSGSBASE is enabled, the user can put a kernel address in 451 * GS, in which case SMAP provides no protection. 452 * 453 * [ NOTE: Don't check for X86_FEATURE_FSGSBASE until the 454 * FSGSBASE enablement patches have been merged. ] 455 * 456 * If FSGSBASE is disabled, the user can only put a user space 457 * address in GS. That makes an attack harder, but still 458 * possible if there's no SMAP protection. 459 */ 460 if (!smap_works_speculatively()) { 461 /* 462 * Mitigation can be provided from SWAPGS itself or 463 * PTI as the CR3 write in the Meltdown mitigation 464 * is serializing. 465 * 466 * If neither is there, mitigate with an LFENCE to 467 * stop speculation through swapgs. 468 */ 469 if (boot_cpu_has_bug(X86_BUG_SWAPGS) && 470 !boot_cpu_has(X86_FEATURE_PTI)) 471 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_USER); 472 473 /* 474 * Enable lfences in the kernel entry (non-swapgs) 475 * paths, to prevent user entry from speculatively 476 * skipping swapgs. 477 */ 478 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_KERNEL); 479 } 480 } 481 482 pr_info("%s\n", spectre_v1_strings[spectre_v1_mitigation]); 483 } 484 485 static int __init nospectre_v1_cmdline(char *str) 486 { 487 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE; 488 return 0; 489 } 490 early_param("nospectre_v1", nospectre_v1_cmdline); 491 492 #undef pr_fmt 493 #define pr_fmt(fmt) "Spectre V2 : " fmt 494 495 static enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init = 496 SPECTRE_V2_NONE; 497 498 static enum spectre_v2_user_mitigation spectre_v2_user __ro_after_init = 499 SPECTRE_V2_USER_NONE; 500 501 #ifdef CONFIG_RETPOLINE 502 static bool spectre_v2_bad_module; 503 504 bool retpoline_module_ok(bool has_retpoline) 505 { 506 if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline) 507 return true; 508 509 pr_err("System may be vulnerable to spectre v2\n"); 510 spectre_v2_bad_module = true; 511 return false; 512 } 513 514 static inline const char *spectre_v2_module_string(void) 515 { 516 return spectre_v2_bad_module ? " - vulnerable module loaded" : ""; 517 } 518 #else 519 static inline const char *spectre_v2_module_string(void) { return ""; } 520 #endif 521 522 static inline bool match_option(const char *arg, int arglen, const char *opt) 523 { 524 int len = strlen(opt); 525 526 return len == arglen && !strncmp(arg, opt, len); 527 } 528 529 /* The kernel command line selection for spectre v2 */ 530 enum spectre_v2_mitigation_cmd { 531 SPECTRE_V2_CMD_NONE, 532 SPECTRE_V2_CMD_AUTO, 533 SPECTRE_V2_CMD_FORCE, 534 SPECTRE_V2_CMD_RETPOLINE, 535 SPECTRE_V2_CMD_RETPOLINE_GENERIC, 536 SPECTRE_V2_CMD_RETPOLINE_AMD, 537 }; 538 539 enum spectre_v2_user_cmd { 540 SPECTRE_V2_USER_CMD_NONE, 541 SPECTRE_V2_USER_CMD_AUTO, 542 SPECTRE_V2_USER_CMD_FORCE, 543 SPECTRE_V2_USER_CMD_PRCTL, 544 SPECTRE_V2_USER_CMD_PRCTL_IBPB, 545 SPECTRE_V2_USER_CMD_SECCOMP, 546 SPECTRE_V2_USER_CMD_SECCOMP_IBPB, 547 }; 548 549 static const char * const spectre_v2_user_strings[] = { 550 [SPECTRE_V2_USER_NONE] = "User space: Vulnerable", 551 [SPECTRE_V2_USER_STRICT] = "User space: Mitigation: STIBP protection", 552 [SPECTRE_V2_USER_STRICT_PREFERRED] = "User space: Mitigation: STIBP always-on protection", 553 [SPECTRE_V2_USER_PRCTL] = "User space: Mitigation: STIBP via prctl", 554 [SPECTRE_V2_USER_SECCOMP] = "User space: Mitigation: STIBP via seccomp and prctl", 555 }; 556 557 static const struct { 558 const char *option; 559 enum spectre_v2_user_cmd cmd; 560 bool secure; 561 } v2_user_options[] __initconst = { 562 { "auto", SPECTRE_V2_USER_CMD_AUTO, false }, 563 { "off", SPECTRE_V2_USER_CMD_NONE, false }, 564 { "on", SPECTRE_V2_USER_CMD_FORCE, true }, 565 { "prctl", SPECTRE_V2_USER_CMD_PRCTL, false }, 566 { "prctl,ibpb", SPECTRE_V2_USER_CMD_PRCTL_IBPB, false }, 567 { "seccomp", SPECTRE_V2_USER_CMD_SECCOMP, false }, 568 { "seccomp,ibpb", SPECTRE_V2_USER_CMD_SECCOMP_IBPB, false }, 569 }; 570 571 static void __init spec_v2_user_print_cond(const char *reason, bool secure) 572 { 573 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure) 574 pr_info("spectre_v2_user=%s forced on command line.\n", reason); 575 } 576 577 static enum spectre_v2_user_cmd __init 578 spectre_v2_parse_user_cmdline(enum spectre_v2_mitigation_cmd v2_cmd) 579 { 580 char arg[20]; 581 int ret, i; 582 583 switch (v2_cmd) { 584 case SPECTRE_V2_CMD_NONE: 585 return SPECTRE_V2_USER_CMD_NONE; 586 case SPECTRE_V2_CMD_FORCE: 587 return SPECTRE_V2_USER_CMD_FORCE; 588 default: 589 break; 590 } 591 592 ret = cmdline_find_option(boot_command_line, "spectre_v2_user", 593 arg, sizeof(arg)); 594 if (ret < 0) 595 return SPECTRE_V2_USER_CMD_AUTO; 596 597 for (i = 0; i < ARRAY_SIZE(v2_user_options); i++) { 598 if (match_option(arg, ret, v2_user_options[i].option)) { 599 spec_v2_user_print_cond(v2_user_options[i].option, 600 v2_user_options[i].secure); 601 return v2_user_options[i].cmd; 602 } 603 } 604 605 pr_err("Unknown user space protection option (%s). Switching to AUTO select\n", arg); 606 return SPECTRE_V2_USER_CMD_AUTO; 607 } 608 609 static void __init 610 spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd) 611 { 612 enum spectre_v2_user_mitigation mode = SPECTRE_V2_USER_NONE; 613 bool smt_possible = IS_ENABLED(CONFIG_SMP); 614 enum spectre_v2_user_cmd cmd; 615 616 if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP)) 617 return; 618 619 if (cpu_smt_control == CPU_SMT_FORCE_DISABLED || 620 cpu_smt_control == CPU_SMT_NOT_SUPPORTED) 621 smt_possible = false; 622 623 cmd = spectre_v2_parse_user_cmdline(v2_cmd); 624 switch (cmd) { 625 case SPECTRE_V2_USER_CMD_NONE: 626 goto set_mode; 627 case SPECTRE_V2_USER_CMD_FORCE: 628 mode = SPECTRE_V2_USER_STRICT; 629 break; 630 case SPECTRE_V2_USER_CMD_PRCTL: 631 case SPECTRE_V2_USER_CMD_PRCTL_IBPB: 632 mode = SPECTRE_V2_USER_PRCTL; 633 break; 634 case SPECTRE_V2_USER_CMD_AUTO: 635 case SPECTRE_V2_USER_CMD_SECCOMP: 636 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB: 637 if (IS_ENABLED(CONFIG_SECCOMP)) 638 mode = SPECTRE_V2_USER_SECCOMP; 639 else 640 mode = SPECTRE_V2_USER_PRCTL; 641 break; 642 } 643 644 /* 645 * At this point, an STIBP mode other than "off" has been set. 646 * If STIBP support is not being forced, check if STIBP always-on 647 * is preferred. 648 */ 649 if (mode != SPECTRE_V2_USER_STRICT && 650 boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON)) 651 mode = SPECTRE_V2_USER_STRICT_PREFERRED; 652 653 /* Initialize Indirect Branch Prediction Barrier */ 654 if (boot_cpu_has(X86_FEATURE_IBPB)) { 655 setup_force_cpu_cap(X86_FEATURE_USE_IBPB); 656 657 switch (cmd) { 658 case SPECTRE_V2_USER_CMD_FORCE: 659 case SPECTRE_V2_USER_CMD_PRCTL_IBPB: 660 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB: 661 static_branch_enable(&switch_mm_always_ibpb); 662 break; 663 case SPECTRE_V2_USER_CMD_PRCTL: 664 case SPECTRE_V2_USER_CMD_AUTO: 665 case SPECTRE_V2_USER_CMD_SECCOMP: 666 static_branch_enable(&switch_mm_cond_ibpb); 667 break; 668 default: 669 break; 670 } 671 672 pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n", 673 static_key_enabled(&switch_mm_always_ibpb) ? 674 "always-on" : "conditional"); 675 } 676 677 /* If enhanced IBRS is enabled no STIBP required */ 678 if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED) 679 return; 680 681 /* 682 * If SMT is not possible or STIBP is not available clear the STIBP 683 * mode. 684 */ 685 if (!smt_possible || !boot_cpu_has(X86_FEATURE_STIBP)) 686 mode = SPECTRE_V2_USER_NONE; 687 set_mode: 688 spectre_v2_user = mode; 689 /* Only print the STIBP mode when SMT possible */ 690 if (smt_possible) 691 pr_info("%s\n", spectre_v2_user_strings[mode]); 692 } 693 694 static const char * const spectre_v2_strings[] = { 695 [SPECTRE_V2_NONE] = "Vulnerable", 696 [SPECTRE_V2_RETPOLINE_GENERIC] = "Mitigation: Full generic retpoline", 697 [SPECTRE_V2_RETPOLINE_AMD] = "Mitigation: Full AMD retpoline", 698 [SPECTRE_V2_IBRS_ENHANCED] = "Mitigation: Enhanced IBRS", 699 }; 700 701 static const struct { 702 const char *option; 703 enum spectre_v2_mitigation_cmd cmd; 704 bool secure; 705 } mitigation_options[] __initconst = { 706 { "off", SPECTRE_V2_CMD_NONE, false }, 707 { "on", SPECTRE_V2_CMD_FORCE, true }, 708 { "retpoline", SPECTRE_V2_CMD_RETPOLINE, false }, 709 { "retpoline,amd", SPECTRE_V2_CMD_RETPOLINE_AMD, false }, 710 { "retpoline,generic", SPECTRE_V2_CMD_RETPOLINE_GENERIC, false }, 711 { "auto", SPECTRE_V2_CMD_AUTO, false }, 712 }; 713 714 static void __init spec_v2_print_cond(const char *reason, bool secure) 715 { 716 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure) 717 pr_info("%s selected on command line.\n", reason); 718 } 719 720 static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void) 721 { 722 enum spectre_v2_mitigation_cmd cmd = SPECTRE_V2_CMD_AUTO; 723 char arg[20]; 724 int ret, i; 725 726 if (cmdline_find_option_bool(boot_command_line, "nospectre_v2") || 727 cpu_mitigations_off()) 728 return SPECTRE_V2_CMD_NONE; 729 730 ret = cmdline_find_option(boot_command_line, "spectre_v2", arg, sizeof(arg)); 731 if (ret < 0) 732 return SPECTRE_V2_CMD_AUTO; 733 734 for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) { 735 if (!match_option(arg, ret, mitigation_options[i].option)) 736 continue; 737 cmd = mitigation_options[i].cmd; 738 break; 739 } 740 741 if (i >= ARRAY_SIZE(mitigation_options)) { 742 pr_err("unknown option (%s). Switching to AUTO select\n", arg); 743 return SPECTRE_V2_CMD_AUTO; 744 } 745 746 if ((cmd == SPECTRE_V2_CMD_RETPOLINE || 747 cmd == SPECTRE_V2_CMD_RETPOLINE_AMD || 748 cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC) && 749 !IS_ENABLED(CONFIG_RETPOLINE)) { 750 pr_err("%s selected but not compiled in. Switching to AUTO select\n", mitigation_options[i].option); 751 return SPECTRE_V2_CMD_AUTO; 752 } 753 754 if (cmd == SPECTRE_V2_CMD_RETPOLINE_AMD && 755 boot_cpu_data.x86_vendor != X86_VENDOR_HYGON && 756 boot_cpu_data.x86_vendor != X86_VENDOR_AMD) { 757 pr_err("retpoline,amd selected but CPU is not AMD. Switching to AUTO select\n"); 758 return SPECTRE_V2_CMD_AUTO; 759 } 760 761 spec_v2_print_cond(mitigation_options[i].option, 762 mitigation_options[i].secure); 763 return cmd; 764 } 765 766 static void __init spectre_v2_select_mitigation(void) 767 { 768 enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline(); 769 enum spectre_v2_mitigation mode = SPECTRE_V2_NONE; 770 771 /* 772 * If the CPU is not affected and the command line mode is NONE or AUTO 773 * then nothing to do. 774 */ 775 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) && 776 (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO)) 777 return; 778 779 switch (cmd) { 780 case SPECTRE_V2_CMD_NONE: 781 return; 782 783 case SPECTRE_V2_CMD_FORCE: 784 case SPECTRE_V2_CMD_AUTO: 785 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { 786 mode = SPECTRE_V2_IBRS_ENHANCED; 787 /* Force it so VMEXIT will restore correctly */ 788 x86_spec_ctrl_base |= SPEC_CTRL_IBRS; 789 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); 790 goto specv2_set_mode; 791 } 792 if (IS_ENABLED(CONFIG_RETPOLINE)) 793 goto retpoline_auto; 794 break; 795 case SPECTRE_V2_CMD_RETPOLINE_AMD: 796 if (IS_ENABLED(CONFIG_RETPOLINE)) 797 goto retpoline_amd; 798 break; 799 case SPECTRE_V2_CMD_RETPOLINE_GENERIC: 800 if (IS_ENABLED(CONFIG_RETPOLINE)) 801 goto retpoline_generic; 802 break; 803 case SPECTRE_V2_CMD_RETPOLINE: 804 if (IS_ENABLED(CONFIG_RETPOLINE)) 805 goto retpoline_auto; 806 break; 807 } 808 pr_err("Spectre mitigation: kernel not compiled with retpoline; no mitigation available!"); 809 return; 810 811 retpoline_auto: 812 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || 813 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) { 814 retpoline_amd: 815 if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) { 816 pr_err("Spectre mitigation: LFENCE not serializing, switching to generic retpoline\n"); 817 goto retpoline_generic; 818 } 819 mode = SPECTRE_V2_RETPOLINE_AMD; 820 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD); 821 setup_force_cpu_cap(X86_FEATURE_RETPOLINE); 822 } else { 823 retpoline_generic: 824 mode = SPECTRE_V2_RETPOLINE_GENERIC; 825 setup_force_cpu_cap(X86_FEATURE_RETPOLINE); 826 } 827 828 specv2_set_mode: 829 spectre_v2_enabled = mode; 830 pr_info("%s\n", spectre_v2_strings[mode]); 831 832 /* 833 * If spectre v2 protection has been enabled, unconditionally fill 834 * RSB during a context switch; this protects against two independent 835 * issues: 836 * 837 * - RSB underflow (and switch to BTB) on Skylake+ 838 * - SpectreRSB variant of spectre v2 on X86_BUG_SPECTRE_V2 CPUs 839 */ 840 setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW); 841 pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n"); 842 843 /* 844 * Retpoline means the kernel is safe because it has no indirect 845 * branches. Enhanced IBRS protects firmware too, so, enable restricted 846 * speculation around firmware calls only when Enhanced IBRS isn't 847 * supported. 848 * 849 * Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because 850 * the user might select retpoline on the kernel command line and if 851 * the CPU supports Enhanced IBRS, kernel might un-intentionally not 852 * enable IBRS around firmware calls. 853 */ 854 if (boot_cpu_has(X86_FEATURE_IBRS) && mode != SPECTRE_V2_IBRS_ENHANCED) { 855 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW); 856 pr_info("Enabling Restricted Speculation for firmware calls\n"); 857 } 858 859 /* Set up IBPB and STIBP depending on the general spectre V2 command */ 860 spectre_v2_user_select_mitigation(cmd); 861 } 862 863 static void update_stibp_msr(void * __unused) 864 { 865 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); 866 } 867 868 /* Update x86_spec_ctrl_base in case SMT state changed. */ 869 static void update_stibp_strict(void) 870 { 871 u64 mask = x86_spec_ctrl_base & ~SPEC_CTRL_STIBP; 872 873 if (sched_smt_active()) 874 mask |= SPEC_CTRL_STIBP; 875 876 if (mask == x86_spec_ctrl_base) 877 return; 878 879 pr_info("Update user space SMT mitigation: STIBP %s\n", 880 mask & SPEC_CTRL_STIBP ? "always-on" : "off"); 881 x86_spec_ctrl_base = mask; 882 on_each_cpu(update_stibp_msr, NULL, 1); 883 } 884 885 /* Update the static key controlling the evaluation of TIF_SPEC_IB */ 886 static void update_indir_branch_cond(void) 887 { 888 if (sched_smt_active()) 889 static_branch_enable(&switch_to_cond_stibp); 890 else 891 static_branch_disable(&switch_to_cond_stibp); 892 } 893 894 #undef pr_fmt 895 #define pr_fmt(fmt) fmt 896 897 /* Update the static key controlling the MDS CPU buffer clear in idle */ 898 static void update_mds_branch_idle(void) 899 { 900 /* 901 * Enable the idle clearing if SMT is active on CPUs which are 902 * affected only by MSBDS and not any other MDS variant. 903 * 904 * The other variants cannot be mitigated when SMT is enabled, so 905 * clearing the buffers on idle just to prevent the Store Buffer 906 * repartitioning leak would be a window dressing exercise. 907 */ 908 if (!boot_cpu_has_bug(X86_BUG_MSBDS_ONLY)) 909 return; 910 911 if (sched_smt_active()) 912 static_branch_enable(&mds_idle_clear); 913 else 914 static_branch_disable(&mds_idle_clear); 915 } 916 917 #define MDS_MSG_SMT "MDS CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/mds.html for more details.\n" 918 #define TAA_MSG_SMT "TAA CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/tsx_async_abort.html for more details.\n" 919 920 void cpu_bugs_smt_update(void) 921 { 922 mutex_lock(&spec_ctrl_mutex); 923 924 switch (spectre_v2_user) { 925 case SPECTRE_V2_USER_NONE: 926 break; 927 case SPECTRE_V2_USER_STRICT: 928 case SPECTRE_V2_USER_STRICT_PREFERRED: 929 update_stibp_strict(); 930 break; 931 case SPECTRE_V2_USER_PRCTL: 932 case SPECTRE_V2_USER_SECCOMP: 933 update_indir_branch_cond(); 934 break; 935 } 936 937 switch (mds_mitigation) { 938 case MDS_MITIGATION_FULL: 939 case MDS_MITIGATION_VMWERV: 940 if (sched_smt_active() && !boot_cpu_has(X86_BUG_MSBDS_ONLY)) 941 pr_warn_once(MDS_MSG_SMT); 942 update_mds_branch_idle(); 943 break; 944 case MDS_MITIGATION_OFF: 945 break; 946 } 947 948 switch (taa_mitigation) { 949 case TAA_MITIGATION_VERW: 950 case TAA_MITIGATION_UCODE_NEEDED: 951 if (sched_smt_active()) 952 pr_warn_once(TAA_MSG_SMT); 953 break; 954 case TAA_MITIGATION_TSX_DISABLED: 955 case TAA_MITIGATION_OFF: 956 break; 957 } 958 959 mutex_unlock(&spec_ctrl_mutex); 960 } 961 962 #undef pr_fmt 963 #define pr_fmt(fmt) "Speculative Store Bypass: " fmt 964 965 static enum ssb_mitigation ssb_mode __ro_after_init = SPEC_STORE_BYPASS_NONE; 966 967 /* The kernel command line selection */ 968 enum ssb_mitigation_cmd { 969 SPEC_STORE_BYPASS_CMD_NONE, 970 SPEC_STORE_BYPASS_CMD_AUTO, 971 SPEC_STORE_BYPASS_CMD_ON, 972 SPEC_STORE_BYPASS_CMD_PRCTL, 973 SPEC_STORE_BYPASS_CMD_SECCOMP, 974 }; 975 976 static const char * const ssb_strings[] = { 977 [SPEC_STORE_BYPASS_NONE] = "Vulnerable", 978 [SPEC_STORE_BYPASS_DISABLE] = "Mitigation: Speculative Store Bypass disabled", 979 [SPEC_STORE_BYPASS_PRCTL] = "Mitigation: Speculative Store Bypass disabled via prctl", 980 [SPEC_STORE_BYPASS_SECCOMP] = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp", 981 }; 982 983 static const struct { 984 const char *option; 985 enum ssb_mitigation_cmd cmd; 986 } ssb_mitigation_options[] __initconst = { 987 { "auto", SPEC_STORE_BYPASS_CMD_AUTO }, /* Platform decides */ 988 { "on", SPEC_STORE_BYPASS_CMD_ON }, /* Disable Speculative Store Bypass */ 989 { "off", SPEC_STORE_BYPASS_CMD_NONE }, /* Don't touch Speculative Store Bypass */ 990 { "prctl", SPEC_STORE_BYPASS_CMD_PRCTL }, /* Disable Speculative Store Bypass via prctl */ 991 { "seccomp", SPEC_STORE_BYPASS_CMD_SECCOMP }, /* Disable Speculative Store Bypass via prctl and seccomp */ 992 }; 993 994 static enum ssb_mitigation_cmd __init ssb_parse_cmdline(void) 995 { 996 enum ssb_mitigation_cmd cmd = SPEC_STORE_BYPASS_CMD_AUTO; 997 char arg[20]; 998 int ret, i; 999 1000 if (cmdline_find_option_bool(boot_command_line, "nospec_store_bypass_disable") || 1001 cpu_mitigations_off()) { 1002 return SPEC_STORE_BYPASS_CMD_NONE; 1003 } else { 1004 ret = cmdline_find_option(boot_command_line, "spec_store_bypass_disable", 1005 arg, sizeof(arg)); 1006 if (ret < 0) 1007 return SPEC_STORE_BYPASS_CMD_AUTO; 1008 1009 for (i = 0; i < ARRAY_SIZE(ssb_mitigation_options); i++) { 1010 if (!match_option(arg, ret, ssb_mitigation_options[i].option)) 1011 continue; 1012 1013 cmd = ssb_mitigation_options[i].cmd; 1014 break; 1015 } 1016 1017 if (i >= ARRAY_SIZE(ssb_mitigation_options)) { 1018 pr_err("unknown option (%s). Switching to AUTO select\n", arg); 1019 return SPEC_STORE_BYPASS_CMD_AUTO; 1020 } 1021 } 1022 1023 return cmd; 1024 } 1025 1026 static enum ssb_mitigation __init __ssb_select_mitigation(void) 1027 { 1028 enum ssb_mitigation mode = SPEC_STORE_BYPASS_NONE; 1029 enum ssb_mitigation_cmd cmd; 1030 1031 if (!boot_cpu_has(X86_FEATURE_SSBD)) 1032 return mode; 1033 1034 cmd = ssb_parse_cmdline(); 1035 if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS) && 1036 (cmd == SPEC_STORE_BYPASS_CMD_NONE || 1037 cmd == SPEC_STORE_BYPASS_CMD_AUTO)) 1038 return mode; 1039 1040 switch (cmd) { 1041 case SPEC_STORE_BYPASS_CMD_AUTO: 1042 case SPEC_STORE_BYPASS_CMD_SECCOMP: 1043 /* 1044 * Choose prctl+seccomp as the default mode if seccomp is 1045 * enabled. 1046 */ 1047 if (IS_ENABLED(CONFIG_SECCOMP)) 1048 mode = SPEC_STORE_BYPASS_SECCOMP; 1049 else 1050 mode = SPEC_STORE_BYPASS_PRCTL; 1051 break; 1052 case SPEC_STORE_BYPASS_CMD_ON: 1053 mode = SPEC_STORE_BYPASS_DISABLE; 1054 break; 1055 case SPEC_STORE_BYPASS_CMD_PRCTL: 1056 mode = SPEC_STORE_BYPASS_PRCTL; 1057 break; 1058 case SPEC_STORE_BYPASS_CMD_NONE: 1059 break; 1060 } 1061 1062 /* 1063 * If SSBD is controlled by the SPEC_CTRL MSR, then set the proper 1064 * bit in the mask to allow guests to use the mitigation even in the 1065 * case where the host does not enable it. 1066 */ 1067 if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) || 1068 static_cpu_has(X86_FEATURE_AMD_SSBD)) { 1069 x86_spec_ctrl_mask |= SPEC_CTRL_SSBD; 1070 } 1071 1072 /* 1073 * We have three CPU feature flags that are in play here: 1074 * - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible. 1075 * - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass 1076 * - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation 1077 */ 1078 if (mode == SPEC_STORE_BYPASS_DISABLE) { 1079 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE); 1080 /* 1081 * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may 1082 * use a completely different MSR and bit dependent on family. 1083 */ 1084 if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) && 1085 !static_cpu_has(X86_FEATURE_AMD_SSBD)) { 1086 x86_amd_ssb_disable(); 1087 } else { 1088 x86_spec_ctrl_base |= SPEC_CTRL_SSBD; 1089 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); 1090 } 1091 } 1092 1093 return mode; 1094 } 1095 1096 static void ssb_select_mitigation(void) 1097 { 1098 ssb_mode = __ssb_select_mitigation(); 1099 1100 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS)) 1101 pr_info("%s\n", ssb_strings[ssb_mode]); 1102 } 1103 1104 #undef pr_fmt 1105 #define pr_fmt(fmt) "Speculation prctl: " fmt 1106 1107 static void task_update_spec_tif(struct task_struct *tsk) 1108 { 1109 /* Force the update of the real TIF bits */ 1110 set_tsk_thread_flag(tsk, TIF_SPEC_FORCE_UPDATE); 1111 1112 /* 1113 * Immediately update the speculation control MSRs for the current 1114 * task, but for a non-current task delay setting the CPU 1115 * mitigation until it is scheduled next. 1116 * 1117 * This can only happen for SECCOMP mitigation. For PRCTL it's 1118 * always the current task. 1119 */ 1120 if (tsk == current) 1121 speculation_ctrl_update_current(); 1122 } 1123 1124 static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl) 1125 { 1126 if (ssb_mode != SPEC_STORE_BYPASS_PRCTL && 1127 ssb_mode != SPEC_STORE_BYPASS_SECCOMP) 1128 return -ENXIO; 1129 1130 switch (ctrl) { 1131 case PR_SPEC_ENABLE: 1132 /* If speculation is force disabled, enable is not allowed */ 1133 if (task_spec_ssb_force_disable(task)) 1134 return -EPERM; 1135 task_clear_spec_ssb_disable(task); 1136 task_clear_spec_ssb_noexec(task); 1137 task_update_spec_tif(task); 1138 break; 1139 case PR_SPEC_DISABLE: 1140 task_set_spec_ssb_disable(task); 1141 task_clear_spec_ssb_noexec(task); 1142 task_update_spec_tif(task); 1143 break; 1144 case PR_SPEC_FORCE_DISABLE: 1145 task_set_spec_ssb_disable(task); 1146 task_set_spec_ssb_force_disable(task); 1147 task_clear_spec_ssb_noexec(task); 1148 task_update_spec_tif(task); 1149 break; 1150 case PR_SPEC_DISABLE_NOEXEC: 1151 if (task_spec_ssb_force_disable(task)) 1152 return -EPERM; 1153 task_set_spec_ssb_disable(task); 1154 task_set_spec_ssb_noexec(task); 1155 task_update_spec_tif(task); 1156 break; 1157 default: 1158 return -ERANGE; 1159 } 1160 return 0; 1161 } 1162 1163 static int ib_prctl_set(struct task_struct *task, unsigned long ctrl) 1164 { 1165 switch (ctrl) { 1166 case PR_SPEC_ENABLE: 1167 if (spectre_v2_user == SPECTRE_V2_USER_NONE) 1168 return 0; 1169 /* 1170 * Indirect branch speculation is always disabled in strict 1171 * mode. 1172 */ 1173 if (spectre_v2_user == SPECTRE_V2_USER_STRICT || 1174 spectre_v2_user == SPECTRE_V2_USER_STRICT_PREFERRED) 1175 return -EPERM; 1176 task_clear_spec_ib_disable(task); 1177 task_update_spec_tif(task); 1178 break; 1179 case PR_SPEC_DISABLE: 1180 case PR_SPEC_FORCE_DISABLE: 1181 /* 1182 * Indirect branch speculation is always allowed when 1183 * mitigation is force disabled. 1184 */ 1185 if (spectre_v2_user == SPECTRE_V2_USER_NONE) 1186 return -EPERM; 1187 if (spectre_v2_user == SPECTRE_V2_USER_STRICT || 1188 spectre_v2_user == SPECTRE_V2_USER_STRICT_PREFERRED) 1189 return 0; 1190 task_set_spec_ib_disable(task); 1191 if (ctrl == PR_SPEC_FORCE_DISABLE) 1192 task_set_spec_ib_force_disable(task); 1193 task_update_spec_tif(task); 1194 break; 1195 default: 1196 return -ERANGE; 1197 } 1198 return 0; 1199 } 1200 1201 int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which, 1202 unsigned long ctrl) 1203 { 1204 switch (which) { 1205 case PR_SPEC_STORE_BYPASS: 1206 return ssb_prctl_set(task, ctrl); 1207 case PR_SPEC_INDIRECT_BRANCH: 1208 return ib_prctl_set(task, ctrl); 1209 default: 1210 return -ENODEV; 1211 } 1212 } 1213 1214 #ifdef CONFIG_SECCOMP 1215 void arch_seccomp_spec_mitigate(struct task_struct *task) 1216 { 1217 if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP) 1218 ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE); 1219 if (spectre_v2_user == SPECTRE_V2_USER_SECCOMP) 1220 ib_prctl_set(task, PR_SPEC_FORCE_DISABLE); 1221 } 1222 #endif 1223 1224 static int ssb_prctl_get(struct task_struct *task) 1225 { 1226 switch (ssb_mode) { 1227 case SPEC_STORE_BYPASS_DISABLE: 1228 return PR_SPEC_DISABLE; 1229 case SPEC_STORE_BYPASS_SECCOMP: 1230 case SPEC_STORE_BYPASS_PRCTL: 1231 if (task_spec_ssb_force_disable(task)) 1232 return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE; 1233 if (task_spec_ssb_noexec(task)) 1234 return PR_SPEC_PRCTL | PR_SPEC_DISABLE_NOEXEC; 1235 if (task_spec_ssb_disable(task)) 1236 return PR_SPEC_PRCTL | PR_SPEC_DISABLE; 1237 return PR_SPEC_PRCTL | PR_SPEC_ENABLE; 1238 default: 1239 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS)) 1240 return PR_SPEC_ENABLE; 1241 return PR_SPEC_NOT_AFFECTED; 1242 } 1243 } 1244 1245 static int ib_prctl_get(struct task_struct *task) 1246 { 1247 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2)) 1248 return PR_SPEC_NOT_AFFECTED; 1249 1250 switch (spectre_v2_user) { 1251 case SPECTRE_V2_USER_NONE: 1252 return PR_SPEC_ENABLE; 1253 case SPECTRE_V2_USER_PRCTL: 1254 case SPECTRE_V2_USER_SECCOMP: 1255 if (task_spec_ib_force_disable(task)) 1256 return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE; 1257 if (task_spec_ib_disable(task)) 1258 return PR_SPEC_PRCTL | PR_SPEC_DISABLE; 1259 return PR_SPEC_PRCTL | PR_SPEC_ENABLE; 1260 case SPECTRE_V2_USER_STRICT: 1261 case SPECTRE_V2_USER_STRICT_PREFERRED: 1262 return PR_SPEC_DISABLE; 1263 default: 1264 return PR_SPEC_NOT_AFFECTED; 1265 } 1266 } 1267 1268 int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which) 1269 { 1270 switch (which) { 1271 case PR_SPEC_STORE_BYPASS: 1272 return ssb_prctl_get(task); 1273 case PR_SPEC_INDIRECT_BRANCH: 1274 return ib_prctl_get(task); 1275 default: 1276 return -ENODEV; 1277 } 1278 } 1279 1280 void x86_spec_ctrl_setup_ap(void) 1281 { 1282 if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) 1283 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); 1284 1285 if (ssb_mode == SPEC_STORE_BYPASS_DISABLE) 1286 x86_amd_ssb_disable(); 1287 } 1288 1289 bool itlb_multihit_kvm_mitigation; 1290 EXPORT_SYMBOL_GPL(itlb_multihit_kvm_mitigation); 1291 1292 #undef pr_fmt 1293 #define pr_fmt(fmt) "L1TF: " fmt 1294 1295 /* Default mitigation for L1TF-affected CPUs */ 1296 enum l1tf_mitigations l1tf_mitigation __ro_after_init = L1TF_MITIGATION_FLUSH; 1297 #if IS_ENABLED(CONFIG_KVM_INTEL) 1298 EXPORT_SYMBOL_GPL(l1tf_mitigation); 1299 #endif 1300 enum vmx_l1d_flush_state l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO; 1301 EXPORT_SYMBOL_GPL(l1tf_vmx_mitigation); 1302 1303 /* 1304 * These CPUs all support 44bits physical address space internally in the 1305 * cache but CPUID can report a smaller number of physical address bits. 1306 * 1307 * The L1TF mitigation uses the top most address bit for the inversion of 1308 * non present PTEs. When the installed memory reaches into the top most 1309 * address bit due to memory holes, which has been observed on machines 1310 * which report 36bits physical address bits and have 32G RAM installed, 1311 * then the mitigation range check in l1tf_select_mitigation() triggers. 1312 * This is a false positive because the mitigation is still possible due to 1313 * the fact that the cache uses 44bit internally. Use the cache bits 1314 * instead of the reported physical bits and adjust them on the affected 1315 * machines to 44bit if the reported bits are less than 44. 1316 */ 1317 static void override_cache_bits(struct cpuinfo_x86 *c) 1318 { 1319 if (c->x86 != 6) 1320 return; 1321 1322 switch (c->x86_model) { 1323 case INTEL_FAM6_NEHALEM: 1324 case INTEL_FAM6_WESTMERE: 1325 case INTEL_FAM6_SANDYBRIDGE: 1326 case INTEL_FAM6_IVYBRIDGE: 1327 case INTEL_FAM6_HASWELL: 1328 case INTEL_FAM6_HASWELL_L: 1329 case INTEL_FAM6_HASWELL_G: 1330 case INTEL_FAM6_BROADWELL: 1331 case INTEL_FAM6_BROADWELL_G: 1332 case INTEL_FAM6_SKYLAKE_L: 1333 case INTEL_FAM6_SKYLAKE: 1334 case INTEL_FAM6_KABYLAKE_L: 1335 case INTEL_FAM6_KABYLAKE: 1336 if (c->x86_cache_bits < 44) 1337 c->x86_cache_bits = 44; 1338 break; 1339 } 1340 } 1341 1342 static void __init l1tf_select_mitigation(void) 1343 { 1344 u64 half_pa; 1345 1346 if (!boot_cpu_has_bug(X86_BUG_L1TF)) 1347 return; 1348 1349 if (cpu_mitigations_off()) 1350 l1tf_mitigation = L1TF_MITIGATION_OFF; 1351 else if (cpu_mitigations_auto_nosmt()) 1352 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT; 1353 1354 override_cache_bits(&boot_cpu_data); 1355 1356 switch (l1tf_mitigation) { 1357 case L1TF_MITIGATION_OFF: 1358 case L1TF_MITIGATION_FLUSH_NOWARN: 1359 case L1TF_MITIGATION_FLUSH: 1360 break; 1361 case L1TF_MITIGATION_FLUSH_NOSMT: 1362 case L1TF_MITIGATION_FULL: 1363 cpu_smt_disable(false); 1364 break; 1365 case L1TF_MITIGATION_FULL_FORCE: 1366 cpu_smt_disable(true); 1367 break; 1368 } 1369 1370 #if CONFIG_PGTABLE_LEVELS == 2 1371 pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n"); 1372 return; 1373 #endif 1374 1375 half_pa = (u64)l1tf_pfn_limit() << PAGE_SHIFT; 1376 if (l1tf_mitigation != L1TF_MITIGATION_OFF && 1377 e820__mapped_any(half_pa, ULLONG_MAX - half_pa, E820_TYPE_RAM)) { 1378 pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n"); 1379 pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n", 1380 half_pa); 1381 pr_info("However, doing so will make a part of your RAM unusable.\n"); 1382 pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html might help you decide.\n"); 1383 return; 1384 } 1385 1386 setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV); 1387 } 1388 1389 static int __init l1tf_cmdline(char *str) 1390 { 1391 if (!boot_cpu_has_bug(X86_BUG_L1TF)) 1392 return 0; 1393 1394 if (!str) 1395 return -EINVAL; 1396 1397 if (!strcmp(str, "off")) 1398 l1tf_mitigation = L1TF_MITIGATION_OFF; 1399 else if (!strcmp(str, "flush,nowarn")) 1400 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOWARN; 1401 else if (!strcmp(str, "flush")) 1402 l1tf_mitigation = L1TF_MITIGATION_FLUSH; 1403 else if (!strcmp(str, "flush,nosmt")) 1404 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT; 1405 else if (!strcmp(str, "full")) 1406 l1tf_mitigation = L1TF_MITIGATION_FULL; 1407 else if (!strcmp(str, "full,force")) 1408 l1tf_mitigation = L1TF_MITIGATION_FULL_FORCE; 1409 1410 return 0; 1411 } 1412 early_param("l1tf", l1tf_cmdline); 1413 1414 #undef pr_fmt 1415 #define pr_fmt(fmt) fmt 1416 1417 #ifdef CONFIG_SYSFS 1418 1419 #define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion" 1420 1421 #if IS_ENABLED(CONFIG_KVM_INTEL) 1422 static const char * const l1tf_vmx_states[] = { 1423 [VMENTER_L1D_FLUSH_AUTO] = "auto", 1424 [VMENTER_L1D_FLUSH_NEVER] = "vulnerable", 1425 [VMENTER_L1D_FLUSH_COND] = "conditional cache flushes", 1426 [VMENTER_L1D_FLUSH_ALWAYS] = "cache flushes", 1427 [VMENTER_L1D_FLUSH_EPT_DISABLED] = "EPT disabled", 1428 [VMENTER_L1D_FLUSH_NOT_REQUIRED] = "flush not necessary" 1429 }; 1430 1431 static ssize_t l1tf_show_state(char *buf) 1432 { 1433 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO) 1434 return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG); 1435 1436 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED || 1437 (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER && 1438 sched_smt_active())) { 1439 return sprintf(buf, "%s; VMX: %s\n", L1TF_DEFAULT_MSG, 1440 l1tf_vmx_states[l1tf_vmx_mitigation]); 1441 } 1442 1443 return sprintf(buf, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG, 1444 l1tf_vmx_states[l1tf_vmx_mitigation], 1445 sched_smt_active() ? "vulnerable" : "disabled"); 1446 } 1447 1448 static ssize_t itlb_multihit_show_state(char *buf) 1449 { 1450 if (itlb_multihit_kvm_mitigation) 1451 return sprintf(buf, "KVM: Mitigation: Split huge pages\n"); 1452 else 1453 return sprintf(buf, "KVM: Vulnerable\n"); 1454 } 1455 #else 1456 static ssize_t l1tf_show_state(char *buf) 1457 { 1458 return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG); 1459 } 1460 1461 static ssize_t itlb_multihit_show_state(char *buf) 1462 { 1463 return sprintf(buf, "Processor vulnerable\n"); 1464 } 1465 #endif 1466 1467 static ssize_t mds_show_state(char *buf) 1468 { 1469 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 1470 return sprintf(buf, "%s; SMT Host state unknown\n", 1471 mds_strings[mds_mitigation]); 1472 } 1473 1474 if (boot_cpu_has(X86_BUG_MSBDS_ONLY)) { 1475 return sprintf(buf, "%s; SMT %s\n", mds_strings[mds_mitigation], 1476 (mds_mitigation == MDS_MITIGATION_OFF ? "vulnerable" : 1477 sched_smt_active() ? "mitigated" : "disabled")); 1478 } 1479 1480 return sprintf(buf, "%s; SMT %s\n", mds_strings[mds_mitigation], 1481 sched_smt_active() ? "vulnerable" : "disabled"); 1482 } 1483 1484 static ssize_t tsx_async_abort_show_state(char *buf) 1485 { 1486 if ((taa_mitigation == TAA_MITIGATION_TSX_DISABLED) || 1487 (taa_mitigation == TAA_MITIGATION_OFF)) 1488 return sprintf(buf, "%s\n", taa_strings[taa_mitigation]); 1489 1490 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 1491 return sprintf(buf, "%s; SMT Host state unknown\n", 1492 taa_strings[taa_mitigation]); 1493 } 1494 1495 return sprintf(buf, "%s; SMT %s\n", taa_strings[taa_mitigation], 1496 sched_smt_active() ? "vulnerable" : "disabled"); 1497 } 1498 1499 static char *stibp_state(void) 1500 { 1501 if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED) 1502 return ""; 1503 1504 switch (spectre_v2_user) { 1505 case SPECTRE_V2_USER_NONE: 1506 return ", STIBP: disabled"; 1507 case SPECTRE_V2_USER_STRICT: 1508 return ", STIBP: forced"; 1509 case SPECTRE_V2_USER_STRICT_PREFERRED: 1510 return ", STIBP: always-on"; 1511 case SPECTRE_V2_USER_PRCTL: 1512 case SPECTRE_V2_USER_SECCOMP: 1513 if (static_key_enabled(&switch_to_cond_stibp)) 1514 return ", STIBP: conditional"; 1515 } 1516 return ""; 1517 } 1518 1519 static char *ibpb_state(void) 1520 { 1521 if (boot_cpu_has(X86_FEATURE_IBPB)) { 1522 if (static_key_enabled(&switch_mm_always_ibpb)) 1523 return ", IBPB: always-on"; 1524 if (static_key_enabled(&switch_mm_cond_ibpb)) 1525 return ", IBPB: conditional"; 1526 return ", IBPB: disabled"; 1527 } 1528 return ""; 1529 } 1530 1531 static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr, 1532 char *buf, unsigned int bug) 1533 { 1534 if (!boot_cpu_has_bug(bug)) 1535 return sprintf(buf, "Not affected\n"); 1536 1537 switch (bug) { 1538 case X86_BUG_CPU_MELTDOWN: 1539 if (boot_cpu_has(X86_FEATURE_PTI)) 1540 return sprintf(buf, "Mitigation: PTI\n"); 1541 1542 if (hypervisor_is_type(X86_HYPER_XEN_PV)) 1543 return sprintf(buf, "Unknown (XEN PV detected, hypervisor mitigation required)\n"); 1544 1545 break; 1546 1547 case X86_BUG_SPECTRE_V1: 1548 return sprintf(buf, "%s\n", spectre_v1_strings[spectre_v1_mitigation]); 1549 1550 case X86_BUG_SPECTRE_V2: 1551 return sprintf(buf, "%s%s%s%s%s%s\n", spectre_v2_strings[spectre_v2_enabled], 1552 ibpb_state(), 1553 boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "", 1554 stibp_state(), 1555 boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "", 1556 spectre_v2_module_string()); 1557 1558 case X86_BUG_SPEC_STORE_BYPASS: 1559 return sprintf(buf, "%s\n", ssb_strings[ssb_mode]); 1560 1561 case X86_BUG_L1TF: 1562 if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV)) 1563 return l1tf_show_state(buf); 1564 break; 1565 1566 case X86_BUG_MDS: 1567 return mds_show_state(buf); 1568 1569 case X86_BUG_TAA: 1570 return tsx_async_abort_show_state(buf); 1571 1572 case X86_BUG_ITLB_MULTIHIT: 1573 return itlb_multihit_show_state(buf); 1574 1575 default: 1576 break; 1577 } 1578 1579 return sprintf(buf, "Vulnerable\n"); 1580 } 1581 1582 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf) 1583 { 1584 return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN); 1585 } 1586 1587 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf) 1588 { 1589 return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1); 1590 } 1591 1592 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf) 1593 { 1594 return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2); 1595 } 1596 1597 ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf) 1598 { 1599 return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS); 1600 } 1601 1602 ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf) 1603 { 1604 return cpu_show_common(dev, attr, buf, X86_BUG_L1TF); 1605 } 1606 1607 ssize_t cpu_show_mds(struct device *dev, struct device_attribute *attr, char *buf) 1608 { 1609 return cpu_show_common(dev, attr, buf, X86_BUG_MDS); 1610 } 1611 1612 ssize_t cpu_show_tsx_async_abort(struct device *dev, struct device_attribute *attr, char *buf) 1613 { 1614 return cpu_show_common(dev, attr, buf, X86_BUG_TAA); 1615 } 1616 1617 ssize_t cpu_show_itlb_multihit(struct device *dev, struct device_attribute *attr, char *buf) 1618 { 1619 return cpu_show_common(dev, attr, buf, X86_BUG_ITLB_MULTIHIT); 1620 } 1621 #endif 1622