1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 1994 Linus Torvalds 4 * 5 * Cyrix stuff, June 1998 by: 6 * - Rafael R. Reilova (moved everything from head.S), 7 * <rreilova@ececs.uc.edu> 8 * - Channing Corn (tests & fixes), 9 * - Andrew D. Balsa (code cleanup). 10 */ 11 #include <linux/init.h> 12 #include <linux/utsname.h> 13 #include <linux/cpu.h> 14 #include <linux/module.h> 15 #include <linux/nospec.h> 16 #include <linux/prctl.h> 17 #include <linux/sched/smt.h> 18 #include <linux/pgtable.h> 19 #include <linux/bpf.h> 20 21 #include <asm/spec-ctrl.h> 22 #include <asm/cmdline.h> 23 #include <asm/bugs.h> 24 #include <asm/processor.h> 25 #include <asm/processor-flags.h> 26 #include <asm/fpu/api.h> 27 #include <asm/msr.h> 28 #include <asm/vmx.h> 29 #include <asm/paravirt.h> 30 #include <asm/alternative.h> 31 #include <asm/set_memory.h> 32 #include <asm/intel-family.h> 33 #include <asm/e820/api.h> 34 #include <asm/hypervisor.h> 35 #include <asm/tlbflush.h> 36 37 #include "cpu.h" 38 39 static void __init spectre_v1_select_mitigation(void); 40 static void __init spectre_v2_select_mitigation(void); 41 static void __init retbleed_select_mitigation(void); 42 static void __init spectre_v2_user_select_mitigation(void); 43 static void __init ssb_select_mitigation(void); 44 static void __init l1tf_select_mitigation(void); 45 static void __init mds_select_mitigation(void); 46 static void __init md_clear_update_mitigation(void); 47 static void __init md_clear_select_mitigation(void); 48 static void __init taa_select_mitigation(void); 49 static void __init mmio_select_mitigation(void); 50 static void __init srbds_select_mitigation(void); 51 static void __init l1d_flush_select_mitigation(void); 52 53 /* The base value of the SPEC_CTRL MSR without task-specific bits set */ 54 u64 x86_spec_ctrl_base; 55 EXPORT_SYMBOL_GPL(x86_spec_ctrl_base); 56 57 /* The current value of the SPEC_CTRL MSR with task-specific bits set */ 58 DEFINE_PER_CPU(u64, x86_spec_ctrl_current); 59 EXPORT_SYMBOL_GPL(x86_spec_ctrl_current); 60 61 static DEFINE_MUTEX(spec_ctrl_mutex); 62 63 /* Update SPEC_CTRL MSR and its cached copy unconditionally */ 64 static void update_spec_ctrl(u64 val) 65 { 66 this_cpu_write(x86_spec_ctrl_current, val); 67 wrmsrl(MSR_IA32_SPEC_CTRL, val); 68 } 69 70 /* 71 * Keep track of the SPEC_CTRL MSR value for the current task, which may differ 72 * from x86_spec_ctrl_base due to STIBP/SSB in __speculation_ctrl_update(). 73 */ 74 void update_spec_ctrl_cond(u64 val) 75 { 76 if (this_cpu_read(x86_spec_ctrl_current) == val) 77 return; 78 79 this_cpu_write(x86_spec_ctrl_current, val); 80 81 /* 82 * When KERNEL_IBRS this MSR is written on return-to-user, unless 83 * forced the update can be delayed until that time. 84 */ 85 if (!cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS)) 86 wrmsrl(MSR_IA32_SPEC_CTRL, val); 87 } 88 89 u64 spec_ctrl_current(void) 90 { 91 return this_cpu_read(x86_spec_ctrl_current); 92 } 93 EXPORT_SYMBOL_GPL(spec_ctrl_current); 94 95 /* 96 * AMD specific MSR info for Speculative Store Bypass control. 97 * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu(). 98 */ 99 u64 __ro_after_init x86_amd_ls_cfg_base; 100 u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask; 101 102 /* Control conditional STIBP in switch_to() */ 103 DEFINE_STATIC_KEY_FALSE(switch_to_cond_stibp); 104 /* Control conditional IBPB in switch_mm() */ 105 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb); 106 /* Control unconditional IBPB in switch_mm() */ 107 DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb); 108 109 /* Control MDS CPU buffer clear before returning to user space */ 110 DEFINE_STATIC_KEY_FALSE(mds_user_clear); 111 EXPORT_SYMBOL_GPL(mds_user_clear); 112 /* Control MDS CPU buffer clear before idling (halt, mwait) */ 113 DEFINE_STATIC_KEY_FALSE(mds_idle_clear); 114 EXPORT_SYMBOL_GPL(mds_idle_clear); 115 116 /* 117 * Controls whether l1d flush based mitigations are enabled, 118 * based on hw features and admin setting via boot parameter 119 * defaults to false 120 */ 121 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_l1d_flush); 122 123 /* Controls CPU Fill buffer clear before KVM guest MMIO accesses */ 124 DEFINE_STATIC_KEY_FALSE(mmio_stale_data_clear); 125 EXPORT_SYMBOL_GPL(mmio_stale_data_clear); 126 127 void __init check_bugs(void) 128 { 129 identify_boot_cpu(); 130 131 /* 132 * identify_boot_cpu() initialized SMT support information, let the 133 * core code know. 134 */ 135 cpu_smt_check_topology(); 136 137 if (!IS_ENABLED(CONFIG_SMP)) { 138 pr_info("CPU: "); 139 print_cpu_info(&boot_cpu_data); 140 } 141 142 /* 143 * Read the SPEC_CTRL MSR to account for reserved bits which may 144 * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD 145 * init code as it is not enumerated and depends on the family. 146 */ 147 if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) 148 rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); 149 150 /* Select the proper CPU mitigations before patching alternatives: */ 151 spectre_v1_select_mitigation(); 152 spectre_v2_select_mitigation(); 153 /* 154 * retbleed_select_mitigation() relies on the state set by 155 * spectre_v2_select_mitigation(); specifically it wants to know about 156 * spectre_v2=ibrs. 157 */ 158 retbleed_select_mitigation(); 159 /* 160 * spectre_v2_user_select_mitigation() relies on the state set by 161 * retbleed_select_mitigation(); specifically the STIBP selection is 162 * forced for UNRET or IBPB. 163 */ 164 spectre_v2_user_select_mitigation(); 165 ssb_select_mitigation(); 166 l1tf_select_mitigation(); 167 md_clear_select_mitigation(); 168 srbds_select_mitigation(); 169 l1d_flush_select_mitigation(); 170 171 arch_smt_update(); 172 173 #ifdef CONFIG_X86_32 174 /* 175 * Check whether we are able to run this kernel safely on SMP. 176 * 177 * - i386 is no longer supported. 178 * - In order to run on anything without a TSC, we need to be 179 * compiled for a i486. 180 */ 181 if (boot_cpu_data.x86 < 4) 182 panic("Kernel requires i486+ for 'invlpg' and other features"); 183 184 init_utsname()->machine[1] = 185 '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86); 186 alternative_instructions(); 187 188 fpu__init_check_bugs(); 189 #else /* CONFIG_X86_64 */ 190 alternative_instructions(); 191 192 /* 193 * Make sure the first 2MB area is not mapped by huge pages 194 * There are typically fixed size MTRRs in there and overlapping 195 * MTRRs into large pages causes slow downs. 196 * 197 * Right now we don't do that with gbpages because there seems 198 * very little benefit for that case. 199 */ 200 if (!direct_gbpages) 201 set_memory_4k((unsigned long)__va(0), 1); 202 #endif 203 } 204 205 /* 206 * NOTE: This function is *only* called for SVM, since Intel uses 207 * MSR_IA32_SPEC_CTRL for SSBD. 208 */ 209 void 210 x86_virt_spec_ctrl(u64 guest_virt_spec_ctrl, bool setguest) 211 { 212 u64 guestval, hostval; 213 struct thread_info *ti = current_thread_info(); 214 215 /* 216 * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update 217 * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported. 218 */ 219 if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD) && 220 !static_cpu_has(X86_FEATURE_VIRT_SSBD)) 221 return; 222 223 /* 224 * If the host has SSBD mitigation enabled, force it in the host's 225 * virtual MSR value. If its not permanently enabled, evaluate 226 * current's TIF_SSBD thread flag. 227 */ 228 if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE)) 229 hostval = SPEC_CTRL_SSBD; 230 else 231 hostval = ssbd_tif_to_spec_ctrl(ti->flags); 232 233 /* Sanitize the guest value */ 234 guestval = guest_virt_spec_ctrl & SPEC_CTRL_SSBD; 235 236 if (hostval != guestval) { 237 unsigned long tif; 238 239 tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) : 240 ssbd_spec_ctrl_to_tif(hostval); 241 242 speculation_ctrl_update(tif); 243 } 244 } 245 EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl); 246 247 static void x86_amd_ssb_disable(void) 248 { 249 u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask; 250 251 if (boot_cpu_has(X86_FEATURE_VIRT_SSBD)) 252 wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD); 253 else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD)) 254 wrmsrl(MSR_AMD64_LS_CFG, msrval); 255 } 256 257 #undef pr_fmt 258 #define pr_fmt(fmt) "MDS: " fmt 259 260 /* Default mitigation for MDS-affected CPUs */ 261 static enum mds_mitigations mds_mitigation __ro_after_init = MDS_MITIGATION_FULL; 262 static bool mds_nosmt __ro_after_init = false; 263 264 static const char * const mds_strings[] = { 265 [MDS_MITIGATION_OFF] = "Vulnerable", 266 [MDS_MITIGATION_FULL] = "Mitigation: Clear CPU buffers", 267 [MDS_MITIGATION_VMWERV] = "Vulnerable: Clear CPU buffers attempted, no microcode", 268 }; 269 270 static void __init mds_select_mitigation(void) 271 { 272 if (!boot_cpu_has_bug(X86_BUG_MDS) || cpu_mitigations_off()) { 273 mds_mitigation = MDS_MITIGATION_OFF; 274 return; 275 } 276 277 if (mds_mitigation == MDS_MITIGATION_FULL) { 278 if (!boot_cpu_has(X86_FEATURE_MD_CLEAR)) 279 mds_mitigation = MDS_MITIGATION_VMWERV; 280 281 static_branch_enable(&mds_user_clear); 282 283 if (!boot_cpu_has(X86_BUG_MSBDS_ONLY) && 284 (mds_nosmt || cpu_mitigations_auto_nosmt())) 285 cpu_smt_disable(false); 286 } 287 } 288 289 static int __init mds_cmdline(char *str) 290 { 291 if (!boot_cpu_has_bug(X86_BUG_MDS)) 292 return 0; 293 294 if (!str) 295 return -EINVAL; 296 297 if (!strcmp(str, "off")) 298 mds_mitigation = MDS_MITIGATION_OFF; 299 else if (!strcmp(str, "full")) 300 mds_mitigation = MDS_MITIGATION_FULL; 301 else if (!strcmp(str, "full,nosmt")) { 302 mds_mitigation = MDS_MITIGATION_FULL; 303 mds_nosmt = true; 304 } 305 306 return 0; 307 } 308 early_param("mds", mds_cmdline); 309 310 #undef pr_fmt 311 #define pr_fmt(fmt) "TAA: " fmt 312 313 enum taa_mitigations { 314 TAA_MITIGATION_OFF, 315 TAA_MITIGATION_UCODE_NEEDED, 316 TAA_MITIGATION_VERW, 317 TAA_MITIGATION_TSX_DISABLED, 318 }; 319 320 /* Default mitigation for TAA-affected CPUs */ 321 static enum taa_mitigations taa_mitigation __ro_after_init = TAA_MITIGATION_VERW; 322 static bool taa_nosmt __ro_after_init; 323 324 static const char * const taa_strings[] = { 325 [TAA_MITIGATION_OFF] = "Vulnerable", 326 [TAA_MITIGATION_UCODE_NEEDED] = "Vulnerable: Clear CPU buffers attempted, no microcode", 327 [TAA_MITIGATION_VERW] = "Mitigation: Clear CPU buffers", 328 [TAA_MITIGATION_TSX_DISABLED] = "Mitigation: TSX disabled", 329 }; 330 331 static void __init taa_select_mitigation(void) 332 { 333 u64 ia32_cap; 334 335 if (!boot_cpu_has_bug(X86_BUG_TAA)) { 336 taa_mitigation = TAA_MITIGATION_OFF; 337 return; 338 } 339 340 /* TSX previously disabled by tsx=off */ 341 if (!boot_cpu_has(X86_FEATURE_RTM)) { 342 taa_mitigation = TAA_MITIGATION_TSX_DISABLED; 343 return; 344 } 345 346 if (cpu_mitigations_off()) { 347 taa_mitigation = TAA_MITIGATION_OFF; 348 return; 349 } 350 351 /* 352 * TAA mitigation via VERW is turned off if both 353 * tsx_async_abort=off and mds=off are specified. 354 */ 355 if (taa_mitigation == TAA_MITIGATION_OFF && 356 mds_mitigation == MDS_MITIGATION_OFF) 357 return; 358 359 if (boot_cpu_has(X86_FEATURE_MD_CLEAR)) 360 taa_mitigation = TAA_MITIGATION_VERW; 361 else 362 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED; 363 364 /* 365 * VERW doesn't clear the CPU buffers when MD_CLEAR=1 and MDS_NO=1. 366 * A microcode update fixes this behavior to clear CPU buffers. It also 367 * adds support for MSR_IA32_TSX_CTRL which is enumerated by the 368 * ARCH_CAP_TSX_CTRL_MSR bit. 369 * 370 * On MDS_NO=1 CPUs if ARCH_CAP_TSX_CTRL_MSR is not set, microcode 371 * update is required. 372 */ 373 ia32_cap = x86_read_arch_cap_msr(); 374 if ( (ia32_cap & ARCH_CAP_MDS_NO) && 375 !(ia32_cap & ARCH_CAP_TSX_CTRL_MSR)) 376 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED; 377 378 /* 379 * TSX is enabled, select alternate mitigation for TAA which is 380 * the same as MDS. Enable MDS static branch to clear CPU buffers. 381 * 382 * For guests that can't determine whether the correct microcode is 383 * present on host, enable the mitigation for UCODE_NEEDED as well. 384 */ 385 static_branch_enable(&mds_user_clear); 386 387 if (taa_nosmt || cpu_mitigations_auto_nosmt()) 388 cpu_smt_disable(false); 389 } 390 391 static int __init tsx_async_abort_parse_cmdline(char *str) 392 { 393 if (!boot_cpu_has_bug(X86_BUG_TAA)) 394 return 0; 395 396 if (!str) 397 return -EINVAL; 398 399 if (!strcmp(str, "off")) { 400 taa_mitigation = TAA_MITIGATION_OFF; 401 } else if (!strcmp(str, "full")) { 402 taa_mitigation = TAA_MITIGATION_VERW; 403 } else if (!strcmp(str, "full,nosmt")) { 404 taa_mitigation = TAA_MITIGATION_VERW; 405 taa_nosmt = true; 406 } 407 408 return 0; 409 } 410 early_param("tsx_async_abort", tsx_async_abort_parse_cmdline); 411 412 #undef pr_fmt 413 #define pr_fmt(fmt) "MMIO Stale Data: " fmt 414 415 enum mmio_mitigations { 416 MMIO_MITIGATION_OFF, 417 MMIO_MITIGATION_UCODE_NEEDED, 418 MMIO_MITIGATION_VERW, 419 }; 420 421 /* Default mitigation for Processor MMIO Stale Data vulnerabilities */ 422 static enum mmio_mitigations mmio_mitigation __ro_after_init = MMIO_MITIGATION_VERW; 423 static bool mmio_nosmt __ro_after_init = false; 424 425 static const char * const mmio_strings[] = { 426 [MMIO_MITIGATION_OFF] = "Vulnerable", 427 [MMIO_MITIGATION_UCODE_NEEDED] = "Vulnerable: Clear CPU buffers attempted, no microcode", 428 [MMIO_MITIGATION_VERW] = "Mitigation: Clear CPU buffers", 429 }; 430 431 static void __init mmio_select_mitigation(void) 432 { 433 u64 ia32_cap; 434 435 if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA) || 436 boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN) || 437 cpu_mitigations_off()) { 438 mmio_mitigation = MMIO_MITIGATION_OFF; 439 return; 440 } 441 442 if (mmio_mitigation == MMIO_MITIGATION_OFF) 443 return; 444 445 ia32_cap = x86_read_arch_cap_msr(); 446 447 /* 448 * Enable CPU buffer clear mitigation for host and VMM, if also affected 449 * by MDS or TAA. Otherwise, enable mitigation for VMM only. 450 */ 451 if (boot_cpu_has_bug(X86_BUG_MDS) || (boot_cpu_has_bug(X86_BUG_TAA) && 452 boot_cpu_has(X86_FEATURE_RTM))) 453 static_branch_enable(&mds_user_clear); 454 else 455 static_branch_enable(&mmio_stale_data_clear); 456 457 /* 458 * If Processor-MMIO-Stale-Data bug is present and Fill Buffer data can 459 * be propagated to uncore buffers, clearing the Fill buffers on idle 460 * is required irrespective of SMT state. 461 */ 462 if (!(ia32_cap & ARCH_CAP_FBSDP_NO)) 463 static_branch_enable(&mds_idle_clear); 464 465 /* 466 * Check if the system has the right microcode. 467 * 468 * CPU Fill buffer clear mitigation is enumerated by either an explicit 469 * FB_CLEAR or by the presence of both MD_CLEAR and L1D_FLUSH on MDS 470 * affected systems. 471 */ 472 if ((ia32_cap & ARCH_CAP_FB_CLEAR) || 473 (boot_cpu_has(X86_FEATURE_MD_CLEAR) && 474 boot_cpu_has(X86_FEATURE_FLUSH_L1D) && 475 !(ia32_cap & ARCH_CAP_MDS_NO))) 476 mmio_mitigation = MMIO_MITIGATION_VERW; 477 else 478 mmio_mitigation = MMIO_MITIGATION_UCODE_NEEDED; 479 480 if (mmio_nosmt || cpu_mitigations_auto_nosmt()) 481 cpu_smt_disable(false); 482 } 483 484 static int __init mmio_stale_data_parse_cmdline(char *str) 485 { 486 if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) 487 return 0; 488 489 if (!str) 490 return -EINVAL; 491 492 if (!strcmp(str, "off")) { 493 mmio_mitigation = MMIO_MITIGATION_OFF; 494 } else if (!strcmp(str, "full")) { 495 mmio_mitigation = MMIO_MITIGATION_VERW; 496 } else if (!strcmp(str, "full,nosmt")) { 497 mmio_mitigation = MMIO_MITIGATION_VERW; 498 mmio_nosmt = true; 499 } 500 501 return 0; 502 } 503 early_param("mmio_stale_data", mmio_stale_data_parse_cmdline); 504 505 #undef pr_fmt 506 #define pr_fmt(fmt) "" fmt 507 508 static void __init md_clear_update_mitigation(void) 509 { 510 if (cpu_mitigations_off()) 511 return; 512 513 if (!static_key_enabled(&mds_user_clear)) 514 goto out; 515 516 /* 517 * mds_user_clear is now enabled. Update MDS, TAA and MMIO Stale Data 518 * mitigation, if necessary. 519 */ 520 if (mds_mitigation == MDS_MITIGATION_OFF && 521 boot_cpu_has_bug(X86_BUG_MDS)) { 522 mds_mitigation = MDS_MITIGATION_FULL; 523 mds_select_mitigation(); 524 } 525 if (taa_mitigation == TAA_MITIGATION_OFF && 526 boot_cpu_has_bug(X86_BUG_TAA)) { 527 taa_mitigation = TAA_MITIGATION_VERW; 528 taa_select_mitigation(); 529 } 530 if (mmio_mitigation == MMIO_MITIGATION_OFF && 531 boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) { 532 mmio_mitigation = MMIO_MITIGATION_VERW; 533 mmio_select_mitigation(); 534 } 535 out: 536 if (boot_cpu_has_bug(X86_BUG_MDS)) 537 pr_info("MDS: %s\n", mds_strings[mds_mitigation]); 538 if (boot_cpu_has_bug(X86_BUG_TAA)) 539 pr_info("TAA: %s\n", taa_strings[taa_mitigation]); 540 if (boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) 541 pr_info("MMIO Stale Data: %s\n", mmio_strings[mmio_mitigation]); 542 else if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN)) 543 pr_info("MMIO Stale Data: Unknown: No mitigations\n"); 544 } 545 546 static void __init md_clear_select_mitigation(void) 547 { 548 mds_select_mitigation(); 549 taa_select_mitigation(); 550 mmio_select_mitigation(); 551 552 /* 553 * As MDS, TAA and MMIO Stale Data mitigations are inter-related, update 554 * and print their mitigation after MDS, TAA and MMIO Stale Data 555 * mitigation selection is done. 556 */ 557 md_clear_update_mitigation(); 558 } 559 560 #undef pr_fmt 561 #define pr_fmt(fmt) "SRBDS: " fmt 562 563 enum srbds_mitigations { 564 SRBDS_MITIGATION_OFF, 565 SRBDS_MITIGATION_UCODE_NEEDED, 566 SRBDS_MITIGATION_FULL, 567 SRBDS_MITIGATION_TSX_OFF, 568 SRBDS_MITIGATION_HYPERVISOR, 569 }; 570 571 static enum srbds_mitigations srbds_mitigation __ro_after_init = SRBDS_MITIGATION_FULL; 572 573 static const char * const srbds_strings[] = { 574 [SRBDS_MITIGATION_OFF] = "Vulnerable", 575 [SRBDS_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode", 576 [SRBDS_MITIGATION_FULL] = "Mitigation: Microcode", 577 [SRBDS_MITIGATION_TSX_OFF] = "Mitigation: TSX disabled", 578 [SRBDS_MITIGATION_HYPERVISOR] = "Unknown: Dependent on hypervisor status", 579 }; 580 581 static bool srbds_off; 582 583 void update_srbds_msr(void) 584 { 585 u64 mcu_ctrl; 586 587 if (!boot_cpu_has_bug(X86_BUG_SRBDS)) 588 return; 589 590 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) 591 return; 592 593 if (srbds_mitigation == SRBDS_MITIGATION_UCODE_NEEDED) 594 return; 595 596 /* 597 * A MDS_NO CPU for which SRBDS mitigation is not needed due to TSX 598 * being disabled and it hasn't received the SRBDS MSR microcode. 599 */ 600 if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL)) 601 return; 602 603 rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 604 605 switch (srbds_mitigation) { 606 case SRBDS_MITIGATION_OFF: 607 case SRBDS_MITIGATION_TSX_OFF: 608 mcu_ctrl |= RNGDS_MITG_DIS; 609 break; 610 case SRBDS_MITIGATION_FULL: 611 mcu_ctrl &= ~RNGDS_MITG_DIS; 612 break; 613 default: 614 break; 615 } 616 617 wrmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 618 } 619 620 static void __init srbds_select_mitigation(void) 621 { 622 u64 ia32_cap; 623 624 if (!boot_cpu_has_bug(X86_BUG_SRBDS)) 625 return; 626 627 /* 628 * Check to see if this is one of the MDS_NO systems supporting TSX that 629 * are only exposed to SRBDS when TSX is enabled or when CPU is affected 630 * by Processor MMIO Stale Data vulnerability. 631 */ 632 ia32_cap = x86_read_arch_cap_msr(); 633 if ((ia32_cap & ARCH_CAP_MDS_NO) && !boot_cpu_has(X86_FEATURE_RTM) && 634 !boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) 635 srbds_mitigation = SRBDS_MITIGATION_TSX_OFF; 636 else if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) 637 srbds_mitigation = SRBDS_MITIGATION_HYPERVISOR; 638 else if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL)) 639 srbds_mitigation = SRBDS_MITIGATION_UCODE_NEEDED; 640 else if (cpu_mitigations_off() || srbds_off) 641 srbds_mitigation = SRBDS_MITIGATION_OFF; 642 643 update_srbds_msr(); 644 pr_info("%s\n", srbds_strings[srbds_mitigation]); 645 } 646 647 static int __init srbds_parse_cmdline(char *str) 648 { 649 if (!str) 650 return -EINVAL; 651 652 if (!boot_cpu_has_bug(X86_BUG_SRBDS)) 653 return 0; 654 655 srbds_off = !strcmp(str, "off"); 656 return 0; 657 } 658 early_param("srbds", srbds_parse_cmdline); 659 660 #undef pr_fmt 661 #define pr_fmt(fmt) "L1D Flush : " fmt 662 663 enum l1d_flush_mitigations { 664 L1D_FLUSH_OFF = 0, 665 L1D_FLUSH_ON, 666 }; 667 668 static enum l1d_flush_mitigations l1d_flush_mitigation __initdata = L1D_FLUSH_OFF; 669 670 static void __init l1d_flush_select_mitigation(void) 671 { 672 if (!l1d_flush_mitigation || !boot_cpu_has(X86_FEATURE_FLUSH_L1D)) 673 return; 674 675 static_branch_enable(&switch_mm_cond_l1d_flush); 676 pr_info("Conditional flush on switch_mm() enabled\n"); 677 } 678 679 static int __init l1d_flush_parse_cmdline(char *str) 680 { 681 if (!strcmp(str, "on")) 682 l1d_flush_mitigation = L1D_FLUSH_ON; 683 684 return 0; 685 } 686 early_param("l1d_flush", l1d_flush_parse_cmdline); 687 688 #undef pr_fmt 689 #define pr_fmt(fmt) "Spectre V1 : " fmt 690 691 enum spectre_v1_mitigation { 692 SPECTRE_V1_MITIGATION_NONE, 693 SPECTRE_V1_MITIGATION_AUTO, 694 }; 695 696 static enum spectre_v1_mitigation spectre_v1_mitigation __ro_after_init = 697 SPECTRE_V1_MITIGATION_AUTO; 698 699 static const char * const spectre_v1_strings[] = { 700 [SPECTRE_V1_MITIGATION_NONE] = "Vulnerable: __user pointer sanitization and usercopy barriers only; no swapgs barriers", 701 [SPECTRE_V1_MITIGATION_AUTO] = "Mitigation: usercopy/swapgs barriers and __user pointer sanitization", 702 }; 703 704 /* 705 * Does SMAP provide full mitigation against speculative kernel access to 706 * userspace? 707 */ 708 static bool smap_works_speculatively(void) 709 { 710 if (!boot_cpu_has(X86_FEATURE_SMAP)) 711 return false; 712 713 /* 714 * On CPUs which are vulnerable to Meltdown, SMAP does not 715 * prevent speculative access to user data in the L1 cache. 716 * Consider SMAP to be non-functional as a mitigation on these 717 * CPUs. 718 */ 719 if (boot_cpu_has(X86_BUG_CPU_MELTDOWN)) 720 return false; 721 722 return true; 723 } 724 725 static void __init spectre_v1_select_mitigation(void) 726 { 727 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1) || cpu_mitigations_off()) { 728 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE; 729 return; 730 } 731 732 if (spectre_v1_mitigation == SPECTRE_V1_MITIGATION_AUTO) { 733 /* 734 * With Spectre v1, a user can speculatively control either 735 * path of a conditional swapgs with a user-controlled GS 736 * value. The mitigation is to add lfences to both code paths. 737 * 738 * If FSGSBASE is enabled, the user can put a kernel address in 739 * GS, in which case SMAP provides no protection. 740 * 741 * If FSGSBASE is disabled, the user can only put a user space 742 * address in GS. That makes an attack harder, but still 743 * possible if there's no SMAP protection. 744 */ 745 if (boot_cpu_has(X86_FEATURE_FSGSBASE) || 746 !smap_works_speculatively()) { 747 /* 748 * Mitigation can be provided from SWAPGS itself or 749 * PTI as the CR3 write in the Meltdown mitigation 750 * is serializing. 751 * 752 * If neither is there, mitigate with an LFENCE to 753 * stop speculation through swapgs. 754 */ 755 if (boot_cpu_has_bug(X86_BUG_SWAPGS) && 756 !boot_cpu_has(X86_FEATURE_PTI)) 757 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_USER); 758 759 /* 760 * Enable lfences in the kernel entry (non-swapgs) 761 * paths, to prevent user entry from speculatively 762 * skipping swapgs. 763 */ 764 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_KERNEL); 765 } 766 } 767 768 pr_info("%s\n", spectre_v1_strings[spectre_v1_mitigation]); 769 } 770 771 static int __init nospectre_v1_cmdline(char *str) 772 { 773 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE; 774 return 0; 775 } 776 early_param("nospectre_v1", nospectre_v1_cmdline); 777 778 static enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init = 779 SPECTRE_V2_NONE; 780 781 #undef pr_fmt 782 #define pr_fmt(fmt) "RETBleed: " fmt 783 784 enum retbleed_mitigation { 785 RETBLEED_MITIGATION_NONE, 786 RETBLEED_MITIGATION_UNRET, 787 RETBLEED_MITIGATION_IBPB, 788 RETBLEED_MITIGATION_IBRS, 789 RETBLEED_MITIGATION_EIBRS, 790 RETBLEED_MITIGATION_STUFF, 791 }; 792 793 enum retbleed_mitigation_cmd { 794 RETBLEED_CMD_OFF, 795 RETBLEED_CMD_AUTO, 796 RETBLEED_CMD_UNRET, 797 RETBLEED_CMD_IBPB, 798 RETBLEED_CMD_STUFF, 799 }; 800 801 static const char * const retbleed_strings[] = { 802 [RETBLEED_MITIGATION_NONE] = "Vulnerable", 803 [RETBLEED_MITIGATION_UNRET] = "Mitigation: untrained return thunk", 804 [RETBLEED_MITIGATION_IBPB] = "Mitigation: IBPB", 805 [RETBLEED_MITIGATION_IBRS] = "Mitigation: IBRS", 806 [RETBLEED_MITIGATION_EIBRS] = "Mitigation: Enhanced IBRS", 807 [RETBLEED_MITIGATION_STUFF] = "Mitigation: Stuffing", 808 }; 809 810 static enum retbleed_mitigation retbleed_mitigation __ro_after_init = 811 RETBLEED_MITIGATION_NONE; 812 static enum retbleed_mitigation_cmd retbleed_cmd __ro_after_init = 813 RETBLEED_CMD_AUTO; 814 815 static int __ro_after_init retbleed_nosmt = false; 816 817 static int __init retbleed_parse_cmdline(char *str) 818 { 819 if (!str) 820 return -EINVAL; 821 822 while (str) { 823 char *next = strchr(str, ','); 824 if (next) { 825 *next = 0; 826 next++; 827 } 828 829 if (!strcmp(str, "off")) { 830 retbleed_cmd = RETBLEED_CMD_OFF; 831 } else if (!strcmp(str, "auto")) { 832 retbleed_cmd = RETBLEED_CMD_AUTO; 833 } else if (!strcmp(str, "unret")) { 834 retbleed_cmd = RETBLEED_CMD_UNRET; 835 } else if (!strcmp(str, "ibpb")) { 836 retbleed_cmd = RETBLEED_CMD_IBPB; 837 } else if (!strcmp(str, "stuff")) { 838 retbleed_cmd = RETBLEED_CMD_STUFF; 839 } else if (!strcmp(str, "nosmt")) { 840 retbleed_nosmt = true; 841 } else if (!strcmp(str, "force")) { 842 setup_force_cpu_bug(X86_BUG_RETBLEED); 843 } else { 844 pr_err("Ignoring unknown retbleed option (%s).", str); 845 } 846 847 str = next; 848 } 849 850 return 0; 851 } 852 early_param("retbleed", retbleed_parse_cmdline); 853 854 #define RETBLEED_UNTRAIN_MSG "WARNING: BTB untrained return thunk mitigation is only effective on AMD/Hygon!\n" 855 #define RETBLEED_INTEL_MSG "WARNING: Spectre v2 mitigation leaves CPU vulnerable to RETBleed attacks, data leaks possible!\n" 856 857 static void __init retbleed_select_mitigation(void) 858 { 859 bool mitigate_smt = false; 860 861 if (!boot_cpu_has_bug(X86_BUG_RETBLEED) || cpu_mitigations_off()) 862 return; 863 864 switch (retbleed_cmd) { 865 case RETBLEED_CMD_OFF: 866 return; 867 868 case RETBLEED_CMD_UNRET: 869 if (IS_ENABLED(CONFIG_CPU_UNRET_ENTRY)) { 870 retbleed_mitigation = RETBLEED_MITIGATION_UNRET; 871 } else { 872 pr_err("WARNING: kernel not compiled with CPU_UNRET_ENTRY.\n"); 873 goto do_cmd_auto; 874 } 875 break; 876 877 case RETBLEED_CMD_IBPB: 878 if (!boot_cpu_has(X86_FEATURE_IBPB)) { 879 pr_err("WARNING: CPU does not support IBPB.\n"); 880 goto do_cmd_auto; 881 } else if (IS_ENABLED(CONFIG_CPU_IBPB_ENTRY)) { 882 retbleed_mitigation = RETBLEED_MITIGATION_IBPB; 883 } else { 884 pr_err("WARNING: kernel not compiled with CPU_IBPB_ENTRY.\n"); 885 goto do_cmd_auto; 886 } 887 break; 888 889 case RETBLEED_CMD_STUFF: 890 if (IS_ENABLED(CONFIG_CALL_DEPTH_TRACKING) && 891 spectre_v2_enabled == SPECTRE_V2_RETPOLINE) { 892 retbleed_mitigation = RETBLEED_MITIGATION_STUFF; 893 894 } else { 895 if (IS_ENABLED(CONFIG_CALL_DEPTH_TRACKING)) 896 pr_err("WARNING: retbleed=stuff depends on spectre_v2=retpoline\n"); 897 else 898 pr_err("WARNING: kernel not compiled with CALL_DEPTH_TRACKING.\n"); 899 900 goto do_cmd_auto; 901 } 902 break; 903 904 do_cmd_auto: 905 case RETBLEED_CMD_AUTO: 906 default: 907 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || 908 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) { 909 if (IS_ENABLED(CONFIG_CPU_UNRET_ENTRY)) 910 retbleed_mitigation = RETBLEED_MITIGATION_UNRET; 911 else if (IS_ENABLED(CONFIG_CPU_IBPB_ENTRY) && boot_cpu_has(X86_FEATURE_IBPB)) 912 retbleed_mitigation = RETBLEED_MITIGATION_IBPB; 913 } 914 915 /* 916 * The Intel mitigation (IBRS or eIBRS) was already selected in 917 * spectre_v2_select_mitigation(). 'retbleed_mitigation' will 918 * be set accordingly below. 919 */ 920 921 break; 922 } 923 924 switch (retbleed_mitigation) { 925 case RETBLEED_MITIGATION_UNRET: 926 setup_force_cpu_cap(X86_FEATURE_RETHUNK); 927 setup_force_cpu_cap(X86_FEATURE_UNRET); 928 929 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && 930 boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) 931 pr_err(RETBLEED_UNTRAIN_MSG); 932 933 mitigate_smt = true; 934 break; 935 936 case RETBLEED_MITIGATION_IBPB: 937 setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB); 938 mitigate_smt = true; 939 break; 940 941 case RETBLEED_MITIGATION_STUFF: 942 setup_force_cpu_cap(X86_FEATURE_RETHUNK); 943 setup_force_cpu_cap(X86_FEATURE_CALL_DEPTH); 944 x86_set_skl_return_thunk(); 945 break; 946 947 default: 948 break; 949 } 950 951 if (mitigate_smt && !boot_cpu_has(X86_FEATURE_STIBP) && 952 (retbleed_nosmt || cpu_mitigations_auto_nosmt())) 953 cpu_smt_disable(false); 954 955 /* 956 * Let IBRS trump all on Intel without affecting the effects of the 957 * retbleed= cmdline option except for call depth based stuffing 958 */ 959 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { 960 switch (spectre_v2_enabled) { 961 case SPECTRE_V2_IBRS: 962 retbleed_mitigation = RETBLEED_MITIGATION_IBRS; 963 break; 964 case SPECTRE_V2_EIBRS: 965 case SPECTRE_V2_EIBRS_RETPOLINE: 966 case SPECTRE_V2_EIBRS_LFENCE: 967 retbleed_mitigation = RETBLEED_MITIGATION_EIBRS; 968 break; 969 default: 970 if (retbleed_mitigation != RETBLEED_MITIGATION_STUFF) 971 pr_err(RETBLEED_INTEL_MSG); 972 } 973 } 974 975 pr_info("%s\n", retbleed_strings[retbleed_mitigation]); 976 } 977 978 #undef pr_fmt 979 #define pr_fmt(fmt) "Spectre V2 : " fmt 980 981 static enum spectre_v2_user_mitigation spectre_v2_user_stibp __ro_after_init = 982 SPECTRE_V2_USER_NONE; 983 static enum spectre_v2_user_mitigation spectre_v2_user_ibpb __ro_after_init = 984 SPECTRE_V2_USER_NONE; 985 986 #ifdef CONFIG_RETPOLINE 987 static bool spectre_v2_bad_module; 988 989 bool retpoline_module_ok(bool has_retpoline) 990 { 991 if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline) 992 return true; 993 994 pr_err("System may be vulnerable to spectre v2\n"); 995 spectre_v2_bad_module = true; 996 return false; 997 } 998 999 static inline const char *spectre_v2_module_string(void) 1000 { 1001 return spectre_v2_bad_module ? " - vulnerable module loaded" : ""; 1002 } 1003 #else 1004 static inline const char *spectre_v2_module_string(void) { return ""; } 1005 #endif 1006 1007 #define SPECTRE_V2_LFENCE_MSG "WARNING: LFENCE mitigation is not recommended for this CPU, data leaks possible!\n" 1008 #define SPECTRE_V2_EIBRS_EBPF_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS on, data leaks possible via Spectre v2 BHB attacks!\n" 1009 #define SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS+LFENCE mitigation and SMT, data leaks possible via Spectre v2 BHB attacks!\n" 1010 #define SPECTRE_V2_IBRS_PERF_MSG "WARNING: IBRS mitigation selected on Enhanced IBRS CPU, this may cause unnecessary performance loss\n" 1011 1012 #ifdef CONFIG_BPF_SYSCALL 1013 void unpriv_ebpf_notify(int new_state) 1014 { 1015 if (new_state) 1016 return; 1017 1018 /* Unprivileged eBPF is enabled */ 1019 1020 switch (spectre_v2_enabled) { 1021 case SPECTRE_V2_EIBRS: 1022 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG); 1023 break; 1024 case SPECTRE_V2_EIBRS_LFENCE: 1025 if (sched_smt_active()) 1026 pr_err(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG); 1027 break; 1028 default: 1029 break; 1030 } 1031 } 1032 #endif 1033 1034 static inline bool match_option(const char *arg, int arglen, const char *opt) 1035 { 1036 int len = strlen(opt); 1037 1038 return len == arglen && !strncmp(arg, opt, len); 1039 } 1040 1041 /* The kernel command line selection for spectre v2 */ 1042 enum spectre_v2_mitigation_cmd { 1043 SPECTRE_V2_CMD_NONE, 1044 SPECTRE_V2_CMD_AUTO, 1045 SPECTRE_V2_CMD_FORCE, 1046 SPECTRE_V2_CMD_RETPOLINE, 1047 SPECTRE_V2_CMD_RETPOLINE_GENERIC, 1048 SPECTRE_V2_CMD_RETPOLINE_LFENCE, 1049 SPECTRE_V2_CMD_EIBRS, 1050 SPECTRE_V2_CMD_EIBRS_RETPOLINE, 1051 SPECTRE_V2_CMD_EIBRS_LFENCE, 1052 SPECTRE_V2_CMD_IBRS, 1053 }; 1054 1055 enum spectre_v2_user_cmd { 1056 SPECTRE_V2_USER_CMD_NONE, 1057 SPECTRE_V2_USER_CMD_AUTO, 1058 SPECTRE_V2_USER_CMD_FORCE, 1059 SPECTRE_V2_USER_CMD_PRCTL, 1060 SPECTRE_V2_USER_CMD_PRCTL_IBPB, 1061 SPECTRE_V2_USER_CMD_SECCOMP, 1062 SPECTRE_V2_USER_CMD_SECCOMP_IBPB, 1063 }; 1064 1065 static const char * const spectre_v2_user_strings[] = { 1066 [SPECTRE_V2_USER_NONE] = "User space: Vulnerable", 1067 [SPECTRE_V2_USER_STRICT] = "User space: Mitigation: STIBP protection", 1068 [SPECTRE_V2_USER_STRICT_PREFERRED] = "User space: Mitigation: STIBP always-on protection", 1069 [SPECTRE_V2_USER_PRCTL] = "User space: Mitigation: STIBP via prctl", 1070 [SPECTRE_V2_USER_SECCOMP] = "User space: Mitigation: STIBP via seccomp and prctl", 1071 }; 1072 1073 static const struct { 1074 const char *option; 1075 enum spectre_v2_user_cmd cmd; 1076 bool secure; 1077 } v2_user_options[] __initconst = { 1078 { "auto", SPECTRE_V2_USER_CMD_AUTO, false }, 1079 { "off", SPECTRE_V2_USER_CMD_NONE, false }, 1080 { "on", SPECTRE_V2_USER_CMD_FORCE, true }, 1081 { "prctl", SPECTRE_V2_USER_CMD_PRCTL, false }, 1082 { "prctl,ibpb", SPECTRE_V2_USER_CMD_PRCTL_IBPB, false }, 1083 { "seccomp", SPECTRE_V2_USER_CMD_SECCOMP, false }, 1084 { "seccomp,ibpb", SPECTRE_V2_USER_CMD_SECCOMP_IBPB, false }, 1085 }; 1086 1087 static void __init spec_v2_user_print_cond(const char *reason, bool secure) 1088 { 1089 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure) 1090 pr_info("spectre_v2_user=%s forced on command line.\n", reason); 1091 } 1092 1093 static __ro_after_init enum spectre_v2_mitigation_cmd spectre_v2_cmd; 1094 1095 static enum spectre_v2_user_cmd __init 1096 spectre_v2_parse_user_cmdline(void) 1097 { 1098 char arg[20]; 1099 int ret, i; 1100 1101 switch (spectre_v2_cmd) { 1102 case SPECTRE_V2_CMD_NONE: 1103 return SPECTRE_V2_USER_CMD_NONE; 1104 case SPECTRE_V2_CMD_FORCE: 1105 return SPECTRE_V2_USER_CMD_FORCE; 1106 default: 1107 break; 1108 } 1109 1110 ret = cmdline_find_option(boot_command_line, "spectre_v2_user", 1111 arg, sizeof(arg)); 1112 if (ret < 0) 1113 return SPECTRE_V2_USER_CMD_AUTO; 1114 1115 for (i = 0; i < ARRAY_SIZE(v2_user_options); i++) { 1116 if (match_option(arg, ret, v2_user_options[i].option)) { 1117 spec_v2_user_print_cond(v2_user_options[i].option, 1118 v2_user_options[i].secure); 1119 return v2_user_options[i].cmd; 1120 } 1121 } 1122 1123 pr_err("Unknown user space protection option (%s). Switching to AUTO select\n", arg); 1124 return SPECTRE_V2_USER_CMD_AUTO; 1125 } 1126 1127 static inline bool spectre_v2_in_ibrs_mode(enum spectre_v2_mitigation mode) 1128 { 1129 return mode == SPECTRE_V2_IBRS || 1130 mode == SPECTRE_V2_EIBRS || 1131 mode == SPECTRE_V2_EIBRS_RETPOLINE || 1132 mode == SPECTRE_V2_EIBRS_LFENCE; 1133 } 1134 1135 static void __init 1136 spectre_v2_user_select_mitigation(void) 1137 { 1138 enum spectre_v2_user_mitigation mode = SPECTRE_V2_USER_NONE; 1139 bool smt_possible = IS_ENABLED(CONFIG_SMP); 1140 enum spectre_v2_user_cmd cmd; 1141 1142 if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP)) 1143 return; 1144 1145 if (cpu_smt_control == CPU_SMT_FORCE_DISABLED || 1146 cpu_smt_control == CPU_SMT_NOT_SUPPORTED) 1147 smt_possible = false; 1148 1149 cmd = spectre_v2_parse_user_cmdline(); 1150 switch (cmd) { 1151 case SPECTRE_V2_USER_CMD_NONE: 1152 goto set_mode; 1153 case SPECTRE_V2_USER_CMD_FORCE: 1154 mode = SPECTRE_V2_USER_STRICT; 1155 break; 1156 case SPECTRE_V2_USER_CMD_AUTO: 1157 case SPECTRE_V2_USER_CMD_PRCTL: 1158 case SPECTRE_V2_USER_CMD_PRCTL_IBPB: 1159 mode = SPECTRE_V2_USER_PRCTL; 1160 break; 1161 case SPECTRE_V2_USER_CMD_SECCOMP: 1162 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB: 1163 if (IS_ENABLED(CONFIG_SECCOMP)) 1164 mode = SPECTRE_V2_USER_SECCOMP; 1165 else 1166 mode = SPECTRE_V2_USER_PRCTL; 1167 break; 1168 } 1169 1170 /* Initialize Indirect Branch Prediction Barrier */ 1171 if (boot_cpu_has(X86_FEATURE_IBPB)) { 1172 setup_force_cpu_cap(X86_FEATURE_USE_IBPB); 1173 1174 spectre_v2_user_ibpb = mode; 1175 switch (cmd) { 1176 case SPECTRE_V2_USER_CMD_FORCE: 1177 case SPECTRE_V2_USER_CMD_PRCTL_IBPB: 1178 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB: 1179 static_branch_enable(&switch_mm_always_ibpb); 1180 spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT; 1181 break; 1182 case SPECTRE_V2_USER_CMD_PRCTL: 1183 case SPECTRE_V2_USER_CMD_AUTO: 1184 case SPECTRE_V2_USER_CMD_SECCOMP: 1185 static_branch_enable(&switch_mm_cond_ibpb); 1186 break; 1187 default: 1188 break; 1189 } 1190 1191 pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n", 1192 static_key_enabled(&switch_mm_always_ibpb) ? 1193 "always-on" : "conditional"); 1194 } 1195 1196 /* 1197 * If no STIBP, IBRS or enhanced IBRS is enabled, or SMT impossible, 1198 * STIBP is not required. 1199 */ 1200 if (!boot_cpu_has(X86_FEATURE_STIBP) || 1201 !smt_possible || 1202 spectre_v2_in_ibrs_mode(spectre_v2_enabled)) 1203 return; 1204 1205 /* 1206 * At this point, an STIBP mode other than "off" has been set. 1207 * If STIBP support is not being forced, check if STIBP always-on 1208 * is preferred. 1209 */ 1210 if (mode != SPECTRE_V2_USER_STRICT && 1211 boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON)) 1212 mode = SPECTRE_V2_USER_STRICT_PREFERRED; 1213 1214 if (retbleed_mitigation == RETBLEED_MITIGATION_UNRET || 1215 retbleed_mitigation == RETBLEED_MITIGATION_IBPB) { 1216 if (mode != SPECTRE_V2_USER_STRICT && 1217 mode != SPECTRE_V2_USER_STRICT_PREFERRED) 1218 pr_info("Selecting STIBP always-on mode to complement retbleed mitigation\n"); 1219 mode = SPECTRE_V2_USER_STRICT_PREFERRED; 1220 } 1221 1222 spectre_v2_user_stibp = mode; 1223 1224 set_mode: 1225 pr_info("%s\n", spectre_v2_user_strings[mode]); 1226 } 1227 1228 static const char * const spectre_v2_strings[] = { 1229 [SPECTRE_V2_NONE] = "Vulnerable", 1230 [SPECTRE_V2_RETPOLINE] = "Mitigation: Retpolines", 1231 [SPECTRE_V2_LFENCE] = "Mitigation: LFENCE", 1232 [SPECTRE_V2_EIBRS] = "Mitigation: Enhanced IBRS", 1233 [SPECTRE_V2_EIBRS_LFENCE] = "Mitigation: Enhanced IBRS + LFENCE", 1234 [SPECTRE_V2_EIBRS_RETPOLINE] = "Mitigation: Enhanced IBRS + Retpolines", 1235 [SPECTRE_V2_IBRS] = "Mitigation: IBRS", 1236 }; 1237 1238 static const struct { 1239 const char *option; 1240 enum spectre_v2_mitigation_cmd cmd; 1241 bool secure; 1242 } mitigation_options[] __initconst = { 1243 { "off", SPECTRE_V2_CMD_NONE, false }, 1244 { "on", SPECTRE_V2_CMD_FORCE, true }, 1245 { "retpoline", SPECTRE_V2_CMD_RETPOLINE, false }, 1246 { "retpoline,amd", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false }, 1247 { "retpoline,lfence", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false }, 1248 { "retpoline,generic", SPECTRE_V2_CMD_RETPOLINE_GENERIC, false }, 1249 { "eibrs", SPECTRE_V2_CMD_EIBRS, false }, 1250 { "eibrs,lfence", SPECTRE_V2_CMD_EIBRS_LFENCE, false }, 1251 { "eibrs,retpoline", SPECTRE_V2_CMD_EIBRS_RETPOLINE, false }, 1252 { "auto", SPECTRE_V2_CMD_AUTO, false }, 1253 { "ibrs", SPECTRE_V2_CMD_IBRS, false }, 1254 }; 1255 1256 static void __init spec_v2_print_cond(const char *reason, bool secure) 1257 { 1258 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure) 1259 pr_info("%s selected on command line.\n", reason); 1260 } 1261 1262 static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void) 1263 { 1264 enum spectre_v2_mitigation_cmd cmd = SPECTRE_V2_CMD_AUTO; 1265 char arg[20]; 1266 int ret, i; 1267 1268 if (cmdline_find_option_bool(boot_command_line, "nospectre_v2") || 1269 cpu_mitigations_off()) 1270 return SPECTRE_V2_CMD_NONE; 1271 1272 ret = cmdline_find_option(boot_command_line, "spectre_v2", arg, sizeof(arg)); 1273 if (ret < 0) 1274 return SPECTRE_V2_CMD_AUTO; 1275 1276 for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) { 1277 if (!match_option(arg, ret, mitigation_options[i].option)) 1278 continue; 1279 cmd = mitigation_options[i].cmd; 1280 break; 1281 } 1282 1283 if (i >= ARRAY_SIZE(mitigation_options)) { 1284 pr_err("unknown option (%s). Switching to AUTO select\n", arg); 1285 return SPECTRE_V2_CMD_AUTO; 1286 } 1287 1288 if ((cmd == SPECTRE_V2_CMD_RETPOLINE || 1289 cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE || 1290 cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC || 1291 cmd == SPECTRE_V2_CMD_EIBRS_LFENCE || 1292 cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) && 1293 !IS_ENABLED(CONFIG_RETPOLINE)) { 1294 pr_err("%s selected but not compiled in. Switching to AUTO select\n", 1295 mitigation_options[i].option); 1296 return SPECTRE_V2_CMD_AUTO; 1297 } 1298 1299 if ((cmd == SPECTRE_V2_CMD_EIBRS || 1300 cmd == SPECTRE_V2_CMD_EIBRS_LFENCE || 1301 cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) && 1302 !boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { 1303 pr_err("%s selected but CPU doesn't have eIBRS. Switching to AUTO select\n", 1304 mitigation_options[i].option); 1305 return SPECTRE_V2_CMD_AUTO; 1306 } 1307 1308 if ((cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE || 1309 cmd == SPECTRE_V2_CMD_EIBRS_LFENCE) && 1310 !boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) { 1311 pr_err("%s selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n", 1312 mitigation_options[i].option); 1313 return SPECTRE_V2_CMD_AUTO; 1314 } 1315 1316 if (cmd == SPECTRE_V2_CMD_IBRS && !IS_ENABLED(CONFIG_CPU_IBRS_ENTRY)) { 1317 pr_err("%s selected but not compiled in. Switching to AUTO select\n", 1318 mitigation_options[i].option); 1319 return SPECTRE_V2_CMD_AUTO; 1320 } 1321 1322 if (cmd == SPECTRE_V2_CMD_IBRS && boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) { 1323 pr_err("%s selected but not Intel CPU. Switching to AUTO select\n", 1324 mitigation_options[i].option); 1325 return SPECTRE_V2_CMD_AUTO; 1326 } 1327 1328 if (cmd == SPECTRE_V2_CMD_IBRS && !boot_cpu_has(X86_FEATURE_IBRS)) { 1329 pr_err("%s selected but CPU doesn't have IBRS. Switching to AUTO select\n", 1330 mitigation_options[i].option); 1331 return SPECTRE_V2_CMD_AUTO; 1332 } 1333 1334 if (cmd == SPECTRE_V2_CMD_IBRS && cpu_feature_enabled(X86_FEATURE_XENPV)) { 1335 pr_err("%s selected but running as XenPV guest. Switching to AUTO select\n", 1336 mitigation_options[i].option); 1337 return SPECTRE_V2_CMD_AUTO; 1338 } 1339 1340 spec_v2_print_cond(mitigation_options[i].option, 1341 mitigation_options[i].secure); 1342 return cmd; 1343 } 1344 1345 static enum spectre_v2_mitigation __init spectre_v2_select_retpoline(void) 1346 { 1347 if (!IS_ENABLED(CONFIG_RETPOLINE)) { 1348 pr_err("Kernel not compiled with retpoline; no mitigation available!"); 1349 return SPECTRE_V2_NONE; 1350 } 1351 1352 return SPECTRE_V2_RETPOLINE; 1353 } 1354 1355 /* Disable in-kernel use of non-RSB RET predictors */ 1356 static void __init spec_ctrl_disable_kernel_rrsba(void) 1357 { 1358 u64 ia32_cap; 1359 1360 if (!boot_cpu_has(X86_FEATURE_RRSBA_CTRL)) 1361 return; 1362 1363 ia32_cap = x86_read_arch_cap_msr(); 1364 1365 if (ia32_cap & ARCH_CAP_RRSBA) { 1366 x86_spec_ctrl_base |= SPEC_CTRL_RRSBA_DIS_S; 1367 update_spec_ctrl(x86_spec_ctrl_base); 1368 } 1369 } 1370 1371 static void __init spectre_v2_determine_rsb_fill_type_at_vmexit(enum spectre_v2_mitigation mode) 1372 { 1373 /* 1374 * Similar to context switches, there are two types of RSB attacks 1375 * after VM exit: 1376 * 1377 * 1) RSB underflow 1378 * 1379 * 2) Poisoned RSB entry 1380 * 1381 * When retpoline is enabled, both are mitigated by filling/clearing 1382 * the RSB. 1383 * 1384 * When IBRS is enabled, while #1 would be mitigated by the IBRS branch 1385 * prediction isolation protections, RSB still needs to be cleared 1386 * because of #2. Note that SMEP provides no protection here, unlike 1387 * user-space-poisoned RSB entries. 1388 * 1389 * eIBRS should protect against RSB poisoning, but if the EIBRS_PBRSB 1390 * bug is present then a LITE version of RSB protection is required, 1391 * just a single call needs to retire before a RET is executed. 1392 */ 1393 switch (mode) { 1394 case SPECTRE_V2_NONE: 1395 return; 1396 1397 case SPECTRE_V2_EIBRS_LFENCE: 1398 case SPECTRE_V2_EIBRS: 1399 if (boot_cpu_has_bug(X86_BUG_EIBRS_PBRSB)) { 1400 setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT_LITE); 1401 pr_info("Spectre v2 / PBRSB-eIBRS: Retire a single CALL on VMEXIT\n"); 1402 } 1403 return; 1404 1405 case SPECTRE_V2_EIBRS_RETPOLINE: 1406 case SPECTRE_V2_RETPOLINE: 1407 case SPECTRE_V2_LFENCE: 1408 case SPECTRE_V2_IBRS: 1409 setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT); 1410 pr_info("Spectre v2 / SpectreRSB : Filling RSB on VMEXIT\n"); 1411 return; 1412 } 1413 1414 pr_warn_once("Unknown Spectre v2 mode, disabling RSB mitigation at VM exit"); 1415 dump_stack(); 1416 } 1417 1418 static void __init spectre_v2_select_mitigation(void) 1419 { 1420 enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline(); 1421 enum spectre_v2_mitigation mode = SPECTRE_V2_NONE; 1422 1423 /* 1424 * If the CPU is not affected and the command line mode is NONE or AUTO 1425 * then nothing to do. 1426 */ 1427 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) && 1428 (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO)) 1429 return; 1430 1431 switch (cmd) { 1432 case SPECTRE_V2_CMD_NONE: 1433 return; 1434 1435 case SPECTRE_V2_CMD_FORCE: 1436 case SPECTRE_V2_CMD_AUTO: 1437 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { 1438 mode = SPECTRE_V2_EIBRS; 1439 break; 1440 } 1441 1442 if (IS_ENABLED(CONFIG_CPU_IBRS_ENTRY) && 1443 boot_cpu_has_bug(X86_BUG_RETBLEED) && 1444 retbleed_cmd != RETBLEED_CMD_OFF && 1445 retbleed_cmd != RETBLEED_CMD_STUFF && 1446 boot_cpu_has(X86_FEATURE_IBRS) && 1447 boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { 1448 mode = SPECTRE_V2_IBRS; 1449 break; 1450 } 1451 1452 mode = spectre_v2_select_retpoline(); 1453 break; 1454 1455 case SPECTRE_V2_CMD_RETPOLINE_LFENCE: 1456 pr_err(SPECTRE_V2_LFENCE_MSG); 1457 mode = SPECTRE_V2_LFENCE; 1458 break; 1459 1460 case SPECTRE_V2_CMD_RETPOLINE_GENERIC: 1461 mode = SPECTRE_V2_RETPOLINE; 1462 break; 1463 1464 case SPECTRE_V2_CMD_RETPOLINE: 1465 mode = spectre_v2_select_retpoline(); 1466 break; 1467 1468 case SPECTRE_V2_CMD_IBRS: 1469 mode = SPECTRE_V2_IBRS; 1470 break; 1471 1472 case SPECTRE_V2_CMD_EIBRS: 1473 mode = SPECTRE_V2_EIBRS; 1474 break; 1475 1476 case SPECTRE_V2_CMD_EIBRS_LFENCE: 1477 mode = SPECTRE_V2_EIBRS_LFENCE; 1478 break; 1479 1480 case SPECTRE_V2_CMD_EIBRS_RETPOLINE: 1481 mode = SPECTRE_V2_EIBRS_RETPOLINE; 1482 break; 1483 } 1484 1485 if (mode == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) 1486 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG); 1487 1488 if (spectre_v2_in_ibrs_mode(mode)) { 1489 x86_spec_ctrl_base |= SPEC_CTRL_IBRS; 1490 update_spec_ctrl(x86_spec_ctrl_base); 1491 } 1492 1493 switch (mode) { 1494 case SPECTRE_V2_NONE: 1495 case SPECTRE_V2_EIBRS: 1496 break; 1497 1498 case SPECTRE_V2_IBRS: 1499 setup_force_cpu_cap(X86_FEATURE_KERNEL_IBRS); 1500 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) 1501 pr_warn(SPECTRE_V2_IBRS_PERF_MSG); 1502 break; 1503 1504 case SPECTRE_V2_LFENCE: 1505 case SPECTRE_V2_EIBRS_LFENCE: 1506 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_LFENCE); 1507 fallthrough; 1508 1509 case SPECTRE_V2_RETPOLINE: 1510 case SPECTRE_V2_EIBRS_RETPOLINE: 1511 setup_force_cpu_cap(X86_FEATURE_RETPOLINE); 1512 break; 1513 } 1514 1515 /* 1516 * Disable alternate RSB predictions in kernel when indirect CALLs and 1517 * JMPs gets protection against BHI and Intramode-BTI, but RET 1518 * prediction from a non-RSB predictor is still a risk. 1519 */ 1520 if (mode == SPECTRE_V2_EIBRS_LFENCE || 1521 mode == SPECTRE_V2_EIBRS_RETPOLINE || 1522 mode == SPECTRE_V2_RETPOLINE) 1523 spec_ctrl_disable_kernel_rrsba(); 1524 1525 spectre_v2_enabled = mode; 1526 pr_info("%s\n", spectre_v2_strings[mode]); 1527 1528 /* 1529 * If Spectre v2 protection has been enabled, fill the RSB during a 1530 * context switch. In general there are two types of RSB attacks 1531 * across context switches, for which the CALLs/RETs may be unbalanced. 1532 * 1533 * 1) RSB underflow 1534 * 1535 * Some Intel parts have "bottomless RSB". When the RSB is empty, 1536 * speculated return targets may come from the branch predictor, 1537 * which could have a user-poisoned BTB or BHB entry. 1538 * 1539 * AMD has it even worse: *all* returns are speculated from the BTB, 1540 * regardless of the state of the RSB. 1541 * 1542 * When IBRS or eIBRS is enabled, the "user -> kernel" attack 1543 * scenario is mitigated by the IBRS branch prediction isolation 1544 * properties, so the RSB buffer filling wouldn't be necessary to 1545 * protect against this type of attack. 1546 * 1547 * The "user -> user" attack scenario is mitigated by RSB filling. 1548 * 1549 * 2) Poisoned RSB entry 1550 * 1551 * If the 'next' in-kernel return stack is shorter than 'prev', 1552 * 'next' could be tricked into speculating with a user-poisoned RSB 1553 * entry. 1554 * 1555 * The "user -> kernel" attack scenario is mitigated by SMEP and 1556 * eIBRS. 1557 * 1558 * The "user -> user" scenario, also known as SpectreBHB, requires 1559 * RSB clearing. 1560 * 1561 * So to mitigate all cases, unconditionally fill RSB on context 1562 * switches. 1563 * 1564 * FIXME: Is this pointless for retbleed-affected AMD? 1565 */ 1566 setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW); 1567 pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n"); 1568 1569 spectre_v2_determine_rsb_fill_type_at_vmexit(mode); 1570 1571 /* 1572 * Retpoline protects the kernel, but doesn't protect firmware. IBRS 1573 * and Enhanced IBRS protect firmware too, so enable IBRS around 1574 * firmware calls only when IBRS / Enhanced IBRS aren't otherwise 1575 * enabled. 1576 * 1577 * Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because 1578 * the user might select retpoline on the kernel command line and if 1579 * the CPU supports Enhanced IBRS, kernel might un-intentionally not 1580 * enable IBRS around firmware calls. 1581 */ 1582 if (boot_cpu_has_bug(X86_BUG_RETBLEED) && 1583 boot_cpu_has(X86_FEATURE_IBPB) && 1584 (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || 1585 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)) { 1586 1587 if (retbleed_cmd != RETBLEED_CMD_IBPB) { 1588 setup_force_cpu_cap(X86_FEATURE_USE_IBPB_FW); 1589 pr_info("Enabling Speculation Barrier for firmware calls\n"); 1590 } 1591 1592 } else if (boot_cpu_has(X86_FEATURE_IBRS) && !spectre_v2_in_ibrs_mode(mode)) { 1593 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW); 1594 pr_info("Enabling Restricted Speculation for firmware calls\n"); 1595 } 1596 1597 /* Set up IBPB and STIBP depending on the general spectre V2 command */ 1598 spectre_v2_cmd = cmd; 1599 } 1600 1601 static void update_stibp_msr(void * __unused) 1602 { 1603 u64 val = spec_ctrl_current() | (x86_spec_ctrl_base & SPEC_CTRL_STIBP); 1604 update_spec_ctrl(val); 1605 } 1606 1607 /* Update x86_spec_ctrl_base in case SMT state changed. */ 1608 static void update_stibp_strict(void) 1609 { 1610 u64 mask = x86_spec_ctrl_base & ~SPEC_CTRL_STIBP; 1611 1612 if (sched_smt_active()) 1613 mask |= SPEC_CTRL_STIBP; 1614 1615 if (mask == x86_spec_ctrl_base) 1616 return; 1617 1618 pr_info("Update user space SMT mitigation: STIBP %s\n", 1619 mask & SPEC_CTRL_STIBP ? "always-on" : "off"); 1620 x86_spec_ctrl_base = mask; 1621 on_each_cpu(update_stibp_msr, NULL, 1); 1622 } 1623 1624 /* Update the static key controlling the evaluation of TIF_SPEC_IB */ 1625 static void update_indir_branch_cond(void) 1626 { 1627 if (sched_smt_active()) 1628 static_branch_enable(&switch_to_cond_stibp); 1629 else 1630 static_branch_disable(&switch_to_cond_stibp); 1631 } 1632 1633 #undef pr_fmt 1634 #define pr_fmt(fmt) fmt 1635 1636 /* Update the static key controlling the MDS CPU buffer clear in idle */ 1637 static void update_mds_branch_idle(void) 1638 { 1639 u64 ia32_cap = x86_read_arch_cap_msr(); 1640 1641 /* 1642 * Enable the idle clearing if SMT is active on CPUs which are 1643 * affected only by MSBDS and not any other MDS variant. 1644 * 1645 * The other variants cannot be mitigated when SMT is enabled, so 1646 * clearing the buffers on idle just to prevent the Store Buffer 1647 * repartitioning leak would be a window dressing exercise. 1648 */ 1649 if (!boot_cpu_has_bug(X86_BUG_MSBDS_ONLY)) 1650 return; 1651 1652 if (sched_smt_active()) { 1653 static_branch_enable(&mds_idle_clear); 1654 } else if (mmio_mitigation == MMIO_MITIGATION_OFF || 1655 (ia32_cap & ARCH_CAP_FBSDP_NO)) { 1656 static_branch_disable(&mds_idle_clear); 1657 } 1658 } 1659 1660 #define MDS_MSG_SMT "MDS CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/mds.html for more details.\n" 1661 #define TAA_MSG_SMT "TAA CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/tsx_async_abort.html for more details.\n" 1662 #define MMIO_MSG_SMT "MMIO Stale Data CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/processor_mmio_stale_data.html for more details.\n" 1663 1664 void cpu_bugs_smt_update(void) 1665 { 1666 mutex_lock(&spec_ctrl_mutex); 1667 1668 if (sched_smt_active() && unprivileged_ebpf_enabled() && 1669 spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE) 1670 pr_warn_once(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG); 1671 1672 switch (spectre_v2_user_stibp) { 1673 case SPECTRE_V2_USER_NONE: 1674 break; 1675 case SPECTRE_V2_USER_STRICT: 1676 case SPECTRE_V2_USER_STRICT_PREFERRED: 1677 update_stibp_strict(); 1678 break; 1679 case SPECTRE_V2_USER_PRCTL: 1680 case SPECTRE_V2_USER_SECCOMP: 1681 update_indir_branch_cond(); 1682 break; 1683 } 1684 1685 switch (mds_mitigation) { 1686 case MDS_MITIGATION_FULL: 1687 case MDS_MITIGATION_VMWERV: 1688 if (sched_smt_active() && !boot_cpu_has(X86_BUG_MSBDS_ONLY)) 1689 pr_warn_once(MDS_MSG_SMT); 1690 update_mds_branch_idle(); 1691 break; 1692 case MDS_MITIGATION_OFF: 1693 break; 1694 } 1695 1696 switch (taa_mitigation) { 1697 case TAA_MITIGATION_VERW: 1698 case TAA_MITIGATION_UCODE_NEEDED: 1699 if (sched_smt_active()) 1700 pr_warn_once(TAA_MSG_SMT); 1701 break; 1702 case TAA_MITIGATION_TSX_DISABLED: 1703 case TAA_MITIGATION_OFF: 1704 break; 1705 } 1706 1707 switch (mmio_mitigation) { 1708 case MMIO_MITIGATION_VERW: 1709 case MMIO_MITIGATION_UCODE_NEEDED: 1710 if (sched_smt_active()) 1711 pr_warn_once(MMIO_MSG_SMT); 1712 break; 1713 case MMIO_MITIGATION_OFF: 1714 break; 1715 } 1716 1717 mutex_unlock(&spec_ctrl_mutex); 1718 } 1719 1720 #undef pr_fmt 1721 #define pr_fmt(fmt) "Speculative Store Bypass: " fmt 1722 1723 static enum ssb_mitigation ssb_mode __ro_after_init = SPEC_STORE_BYPASS_NONE; 1724 1725 /* The kernel command line selection */ 1726 enum ssb_mitigation_cmd { 1727 SPEC_STORE_BYPASS_CMD_NONE, 1728 SPEC_STORE_BYPASS_CMD_AUTO, 1729 SPEC_STORE_BYPASS_CMD_ON, 1730 SPEC_STORE_BYPASS_CMD_PRCTL, 1731 SPEC_STORE_BYPASS_CMD_SECCOMP, 1732 }; 1733 1734 static const char * const ssb_strings[] = { 1735 [SPEC_STORE_BYPASS_NONE] = "Vulnerable", 1736 [SPEC_STORE_BYPASS_DISABLE] = "Mitigation: Speculative Store Bypass disabled", 1737 [SPEC_STORE_BYPASS_PRCTL] = "Mitigation: Speculative Store Bypass disabled via prctl", 1738 [SPEC_STORE_BYPASS_SECCOMP] = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp", 1739 }; 1740 1741 static const struct { 1742 const char *option; 1743 enum ssb_mitigation_cmd cmd; 1744 } ssb_mitigation_options[] __initconst = { 1745 { "auto", SPEC_STORE_BYPASS_CMD_AUTO }, /* Platform decides */ 1746 { "on", SPEC_STORE_BYPASS_CMD_ON }, /* Disable Speculative Store Bypass */ 1747 { "off", SPEC_STORE_BYPASS_CMD_NONE }, /* Don't touch Speculative Store Bypass */ 1748 { "prctl", SPEC_STORE_BYPASS_CMD_PRCTL }, /* Disable Speculative Store Bypass via prctl */ 1749 { "seccomp", SPEC_STORE_BYPASS_CMD_SECCOMP }, /* Disable Speculative Store Bypass via prctl and seccomp */ 1750 }; 1751 1752 static enum ssb_mitigation_cmd __init ssb_parse_cmdline(void) 1753 { 1754 enum ssb_mitigation_cmd cmd = SPEC_STORE_BYPASS_CMD_AUTO; 1755 char arg[20]; 1756 int ret, i; 1757 1758 if (cmdline_find_option_bool(boot_command_line, "nospec_store_bypass_disable") || 1759 cpu_mitigations_off()) { 1760 return SPEC_STORE_BYPASS_CMD_NONE; 1761 } else { 1762 ret = cmdline_find_option(boot_command_line, "spec_store_bypass_disable", 1763 arg, sizeof(arg)); 1764 if (ret < 0) 1765 return SPEC_STORE_BYPASS_CMD_AUTO; 1766 1767 for (i = 0; i < ARRAY_SIZE(ssb_mitigation_options); i++) { 1768 if (!match_option(arg, ret, ssb_mitigation_options[i].option)) 1769 continue; 1770 1771 cmd = ssb_mitigation_options[i].cmd; 1772 break; 1773 } 1774 1775 if (i >= ARRAY_SIZE(ssb_mitigation_options)) { 1776 pr_err("unknown option (%s). Switching to AUTO select\n", arg); 1777 return SPEC_STORE_BYPASS_CMD_AUTO; 1778 } 1779 } 1780 1781 return cmd; 1782 } 1783 1784 static enum ssb_mitigation __init __ssb_select_mitigation(void) 1785 { 1786 enum ssb_mitigation mode = SPEC_STORE_BYPASS_NONE; 1787 enum ssb_mitigation_cmd cmd; 1788 1789 if (!boot_cpu_has(X86_FEATURE_SSBD)) 1790 return mode; 1791 1792 cmd = ssb_parse_cmdline(); 1793 if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS) && 1794 (cmd == SPEC_STORE_BYPASS_CMD_NONE || 1795 cmd == SPEC_STORE_BYPASS_CMD_AUTO)) 1796 return mode; 1797 1798 switch (cmd) { 1799 case SPEC_STORE_BYPASS_CMD_SECCOMP: 1800 /* 1801 * Choose prctl+seccomp as the default mode if seccomp is 1802 * enabled. 1803 */ 1804 if (IS_ENABLED(CONFIG_SECCOMP)) 1805 mode = SPEC_STORE_BYPASS_SECCOMP; 1806 else 1807 mode = SPEC_STORE_BYPASS_PRCTL; 1808 break; 1809 case SPEC_STORE_BYPASS_CMD_ON: 1810 mode = SPEC_STORE_BYPASS_DISABLE; 1811 break; 1812 case SPEC_STORE_BYPASS_CMD_AUTO: 1813 case SPEC_STORE_BYPASS_CMD_PRCTL: 1814 mode = SPEC_STORE_BYPASS_PRCTL; 1815 break; 1816 case SPEC_STORE_BYPASS_CMD_NONE: 1817 break; 1818 } 1819 1820 /* 1821 * We have three CPU feature flags that are in play here: 1822 * - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible. 1823 * - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass 1824 * - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation 1825 */ 1826 if (mode == SPEC_STORE_BYPASS_DISABLE) { 1827 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE); 1828 /* 1829 * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may 1830 * use a completely different MSR and bit dependent on family. 1831 */ 1832 if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) && 1833 !static_cpu_has(X86_FEATURE_AMD_SSBD)) { 1834 x86_amd_ssb_disable(); 1835 } else { 1836 x86_spec_ctrl_base |= SPEC_CTRL_SSBD; 1837 update_spec_ctrl(x86_spec_ctrl_base); 1838 } 1839 } 1840 1841 return mode; 1842 } 1843 1844 static void ssb_select_mitigation(void) 1845 { 1846 ssb_mode = __ssb_select_mitigation(); 1847 1848 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS)) 1849 pr_info("%s\n", ssb_strings[ssb_mode]); 1850 } 1851 1852 #undef pr_fmt 1853 #define pr_fmt(fmt) "Speculation prctl: " fmt 1854 1855 static void task_update_spec_tif(struct task_struct *tsk) 1856 { 1857 /* Force the update of the real TIF bits */ 1858 set_tsk_thread_flag(tsk, TIF_SPEC_FORCE_UPDATE); 1859 1860 /* 1861 * Immediately update the speculation control MSRs for the current 1862 * task, but for a non-current task delay setting the CPU 1863 * mitigation until it is scheduled next. 1864 * 1865 * This can only happen for SECCOMP mitigation. For PRCTL it's 1866 * always the current task. 1867 */ 1868 if (tsk == current) 1869 speculation_ctrl_update_current(); 1870 } 1871 1872 static int l1d_flush_prctl_set(struct task_struct *task, unsigned long ctrl) 1873 { 1874 1875 if (!static_branch_unlikely(&switch_mm_cond_l1d_flush)) 1876 return -EPERM; 1877 1878 switch (ctrl) { 1879 case PR_SPEC_ENABLE: 1880 set_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH); 1881 return 0; 1882 case PR_SPEC_DISABLE: 1883 clear_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH); 1884 return 0; 1885 default: 1886 return -ERANGE; 1887 } 1888 } 1889 1890 static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl) 1891 { 1892 if (ssb_mode != SPEC_STORE_BYPASS_PRCTL && 1893 ssb_mode != SPEC_STORE_BYPASS_SECCOMP) 1894 return -ENXIO; 1895 1896 switch (ctrl) { 1897 case PR_SPEC_ENABLE: 1898 /* If speculation is force disabled, enable is not allowed */ 1899 if (task_spec_ssb_force_disable(task)) 1900 return -EPERM; 1901 task_clear_spec_ssb_disable(task); 1902 task_clear_spec_ssb_noexec(task); 1903 task_update_spec_tif(task); 1904 break; 1905 case PR_SPEC_DISABLE: 1906 task_set_spec_ssb_disable(task); 1907 task_clear_spec_ssb_noexec(task); 1908 task_update_spec_tif(task); 1909 break; 1910 case PR_SPEC_FORCE_DISABLE: 1911 task_set_spec_ssb_disable(task); 1912 task_set_spec_ssb_force_disable(task); 1913 task_clear_spec_ssb_noexec(task); 1914 task_update_spec_tif(task); 1915 break; 1916 case PR_SPEC_DISABLE_NOEXEC: 1917 if (task_spec_ssb_force_disable(task)) 1918 return -EPERM; 1919 task_set_spec_ssb_disable(task); 1920 task_set_spec_ssb_noexec(task); 1921 task_update_spec_tif(task); 1922 break; 1923 default: 1924 return -ERANGE; 1925 } 1926 return 0; 1927 } 1928 1929 static bool is_spec_ib_user_controlled(void) 1930 { 1931 return spectre_v2_user_ibpb == SPECTRE_V2_USER_PRCTL || 1932 spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP || 1933 spectre_v2_user_stibp == SPECTRE_V2_USER_PRCTL || 1934 spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP; 1935 } 1936 1937 static int ib_prctl_set(struct task_struct *task, unsigned long ctrl) 1938 { 1939 switch (ctrl) { 1940 case PR_SPEC_ENABLE: 1941 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && 1942 spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) 1943 return 0; 1944 1945 /* 1946 * With strict mode for both IBPB and STIBP, the instruction 1947 * code paths avoid checking this task flag and instead, 1948 * unconditionally run the instruction. However, STIBP and IBPB 1949 * are independent and either can be set to conditionally 1950 * enabled regardless of the mode of the other. 1951 * 1952 * If either is set to conditional, allow the task flag to be 1953 * updated, unless it was force-disabled by a previous prctl 1954 * call. Currently, this is possible on an AMD CPU which has the 1955 * feature X86_FEATURE_AMD_STIBP_ALWAYS_ON. In this case, if the 1956 * kernel is booted with 'spectre_v2_user=seccomp', then 1957 * spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP and 1958 * spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED. 1959 */ 1960 if (!is_spec_ib_user_controlled() || 1961 task_spec_ib_force_disable(task)) 1962 return -EPERM; 1963 1964 task_clear_spec_ib_disable(task); 1965 task_update_spec_tif(task); 1966 break; 1967 case PR_SPEC_DISABLE: 1968 case PR_SPEC_FORCE_DISABLE: 1969 /* 1970 * Indirect branch speculation is always allowed when 1971 * mitigation is force disabled. 1972 */ 1973 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && 1974 spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) 1975 return -EPERM; 1976 1977 if (!is_spec_ib_user_controlled()) 1978 return 0; 1979 1980 task_set_spec_ib_disable(task); 1981 if (ctrl == PR_SPEC_FORCE_DISABLE) 1982 task_set_spec_ib_force_disable(task); 1983 task_update_spec_tif(task); 1984 if (task == current) 1985 indirect_branch_prediction_barrier(); 1986 break; 1987 default: 1988 return -ERANGE; 1989 } 1990 return 0; 1991 } 1992 1993 int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which, 1994 unsigned long ctrl) 1995 { 1996 switch (which) { 1997 case PR_SPEC_STORE_BYPASS: 1998 return ssb_prctl_set(task, ctrl); 1999 case PR_SPEC_INDIRECT_BRANCH: 2000 return ib_prctl_set(task, ctrl); 2001 case PR_SPEC_L1D_FLUSH: 2002 return l1d_flush_prctl_set(task, ctrl); 2003 default: 2004 return -ENODEV; 2005 } 2006 } 2007 2008 #ifdef CONFIG_SECCOMP 2009 void arch_seccomp_spec_mitigate(struct task_struct *task) 2010 { 2011 if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP) 2012 ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE); 2013 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP || 2014 spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP) 2015 ib_prctl_set(task, PR_SPEC_FORCE_DISABLE); 2016 } 2017 #endif 2018 2019 static int l1d_flush_prctl_get(struct task_struct *task) 2020 { 2021 if (!static_branch_unlikely(&switch_mm_cond_l1d_flush)) 2022 return PR_SPEC_FORCE_DISABLE; 2023 2024 if (test_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH)) 2025 return PR_SPEC_PRCTL | PR_SPEC_ENABLE; 2026 else 2027 return PR_SPEC_PRCTL | PR_SPEC_DISABLE; 2028 } 2029 2030 static int ssb_prctl_get(struct task_struct *task) 2031 { 2032 switch (ssb_mode) { 2033 case SPEC_STORE_BYPASS_DISABLE: 2034 return PR_SPEC_DISABLE; 2035 case SPEC_STORE_BYPASS_SECCOMP: 2036 case SPEC_STORE_BYPASS_PRCTL: 2037 if (task_spec_ssb_force_disable(task)) 2038 return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE; 2039 if (task_spec_ssb_noexec(task)) 2040 return PR_SPEC_PRCTL | PR_SPEC_DISABLE_NOEXEC; 2041 if (task_spec_ssb_disable(task)) 2042 return PR_SPEC_PRCTL | PR_SPEC_DISABLE; 2043 return PR_SPEC_PRCTL | PR_SPEC_ENABLE; 2044 default: 2045 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS)) 2046 return PR_SPEC_ENABLE; 2047 return PR_SPEC_NOT_AFFECTED; 2048 } 2049 } 2050 2051 static int ib_prctl_get(struct task_struct *task) 2052 { 2053 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2)) 2054 return PR_SPEC_NOT_AFFECTED; 2055 2056 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && 2057 spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) 2058 return PR_SPEC_ENABLE; 2059 else if (is_spec_ib_user_controlled()) { 2060 if (task_spec_ib_force_disable(task)) 2061 return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE; 2062 if (task_spec_ib_disable(task)) 2063 return PR_SPEC_PRCTL | PR_SPEC_DISABLE; 2064 return PR_SPEC_PRCTL | PR_SPEC_ENABLE; 2065 } else if (spectre_v2_user_ibpb == SPECTRE_V2_USER_STRICT || 2066 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT || 2067 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED) 2068 return PR_SPEC_DISABLE; 2069 else 2070 return PR_SPEC_NOT_AFFECTED; 2071 } 2072 2073 int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which) 2074 { 2075 switch (which) { 2076 case PR_SPEC_STORE_BYPASS: 2077 return ssb_prctl_get(task); 2078 case PR_SPEC_INDIRECT_BRANCH: 2079 return ib_prctl_get(task); 2080 case PR_SPEC_L1D_FLUSH: 2081 return l1d_flush_prctl_get(task); 2082 default: 2083 return -ENODEV; 2084 } 2085 } 2086 2087 void x86_spec_ctrl_setup_ap(void) 2088 { 2089 if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) 2090 update_spec_ctrl(x86_spec_ctrl_base); 2091 2092 if (ssb_mode == SPEC_STORE_BYPASS_DISABLE) 2093 x86_amd_ssb_disable(); 2094 } 2095 2096 bool itlb_multihit_kvm_mitigation; 2097 EXPORT_SYMBOL_GPL(itlb_multihit_kvm_mitigation); 2098 2099 #undef pr_fmt 2100 #define pr_fmt(fmt) "L1TF: " fmt 2101 2102 /* Default mitigation for L1TF-affected CPUs */ 2103 enum l1tf_mitigations l1tf_mitigation __ro_after_init = L1TF_MITIGATION_FLUSH; 2104 #if IS_ENABLED(CONFIG_KVM_INTEL) 2105 EXPORT_SYMBOL_GPL(l1tf_mitigation); 2106 #endif 2107 enum vmx_l1d_flush_state l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO; 2108 EXPORT_SYMBOL_GPL(l1tf_vmx_mitigation); 2109 2110 /* 2111 * These CPUs all support 44bits physical address space internally in the 2112 * cache but CPUID can report a smaller number of physical address bits. 2113 * 2114 * The L1TF mitigation uses the top most address bit for the inversion of 2115 * non present PTEs. When the installed memory reaches into the top most 2116 * address bit due to memory holes, which has been observed on machines 2117 * which report 36bits physical address bits and have 32G RAM installed, 2118 * then the mitigation range check in l1tf_select_mitigation() triggers. 2119 * This is a false positive because the mitigation is still possible due to 2120 * the fact that the cache uses 44bit internally. Use the cache bits 2121 * instead of the reported physical bits and adjust them on the affected 2122 * machines to 44bit if the reported bits are less than 44. 2123 */ 2124 static void override_cache_bits(struct cpuinfo_x86 *c) 2125 { 2126 if (c->x86 != 6) 2127 return; 2128 2129 switch (c->x86_model) { 2130 case INTEL_FAM6_NEHALEM: 2131 case INTEL_FAM6_WESTMERE: 2132 case INTEL_FAM6_SANDYBRIDGE: 2133 case INTEL_FAM6_IVYBRIDGE: 2134 case INTEL_FAM6_HASWELL: 2135 case INTEL_FAM6_HASWELL_L: 2136 case INTEL_FAM6_HASWELL_G: 2137 case INTEL_FAM6_BROADWELL: 2138 case INTEL_FAM6_BROADWELL_G: 2139 case INTEL_FAM6_SKYLAKE_L: 2140 case INTEL_FAM6_SKYLAKE: 2141 case INTEL_FAM6_KABYLAKE_L: 2142 case INTEL_FAM6_KABYLAKE: 2143 if (c->x86_cache_bits < 44) 2144 c->x86_cache_bits = 44; 2145 break; 2146 } 2147 } 2148 2149 static void __init l1tf_select_mitigation(void) 2150 { 2151 u64 half_pa; 2152 2153 if (!boot_cpu_has_bug(X86_BUG_L1TF)) 2154 return; 2155 2156 if (cpu_mitigations_off()) 2157 l1tf_mitigation = L1TF_MITIGATION_OFF; 2158 else if (cpu_mitigations_auto_nosmt()) 2159 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT; 2160 2161 override_cache_bits(&boot_cpu_data); 2162 2163 switch (l1tf_mitigation) { 2164 case L1TF_MITIGATION_OFF: 2165 case L1TF_MITIGATION_FLUSH_NOWARN: 2166 case L1TF_MITIGATION_FLUSH: 2167 break; 2168 case L1TF_MITIGATION_FLUSH_NOSMT: 2169 case L1TF_MITIGATION_FULL: 2170 cpu_smt_disable(false); 2171 break; 2172 case L1TF_MITIGATION_FULL_FORCE: 2173 cpu_smt_disable(true); 2174 break; 2175 } 2176 2177 #if CONFIG_PGTABLE_LEVELS == 2 2178 pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n"); 2179 return; 2180 #endif 2181 2182 half_pa = (u64)l1tf_pfn_limit() << PAGE_SHIFT; 2183 if (l1tf_mitigation != L1TF_MITIGATION_OFF && 2184 e820__mapped_any(half_pa, ULLONG_MAX - half_pa, E820_TYPE_RAM)) { 2185 pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n"); 2186 pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n", 2187 half_pa); 2188 pr_info("However, doing so will make a part of your RAM unusable.\n"); 2189 pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html might help you decide.\n"); 2190 return; 2191 } 2192 2193 setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV); 2194 } 2195 2196 static int __init l1tf_cmdline(char *str) 2197 { 2198 if (!boot_cpu_has_bug(X86_BUG_L1TF)) 2199 return 0; 2200 2201 if (!str) 2202 return -EINVAL; 2203 2204 if (!strcmp(str, "off")) 2205 l1tf_mitigation = L1TF_MITIGATION_OFF; 2206 else if (!strcmp(str, "flush,nowarn")) 2207 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOWARN; 2208 else if (!strcmp(str, "flush")) 2209 l1tf_mitigation = L1TF_MITIGATION_FLUSH; 2210 else if (!strcmp(str, "flush,nosmt")) 2211 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT; 2212 else if (!strcmp(str, "full")) 2213 l1tf_mitigation = L1TF_MITIGATION_FULL; 2214 else if (!strcmp(str, "full,force")) 2215 l1tf_mitigation = L1TF_MITIGATION_FULL_FORCE; 2216 2217 return 0; 2218 } 2219 early_param("l1tf", l1tf_cmdline); 2220 2221 #undef pr_fmt 2222 #define pr_fmt(fmt) fmt 2223 2224 #ifdef CONFIG_SYSFS 2225 2226 #define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion" 2227 2228 #if IS_ENABLED(CONFIG_KVM_INTEL) 2229 static const char * const l1tf_vmx_states[] = { 2230 [VMENTER_L1D_FLUSH_AUTO] = "auto", 2231 [VMENTER_L1D_FLUSH_NEVER] = "vulnerable", 2232 [VMENTER_L1D_FLUSH_COND] = "conditional cache flushes", 2233 [VMENTER_L1D_FLUSH_ALWAYS] = "cache flushes", 2234 [VMENTER_L1D_FLUSH_EPT_DISABLED] = "EPT disabled", 2235 [VMENTER_L1D_FLUSH_NOT_REQUIRED] = "flush not necessary" 2236 }; 2237 2238 static ssize_t l1tf_show_state(char *buf) 2239 { 2240 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO) 2241 return sysfs_emit(buf, "%s\n", L1TF_DEFAULT_MSG); 2242 2243 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED || 2244 (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER && 2245 sched_smt_active())) { 2246 return sysfs_emit(buf, "%s; VMX: %s\n", L1TF_DEFAULT_MSG, 2247 l1tf_vmx_states[l1tf_vmx_mitigation]); 2248 } 2249 2250 return sysfs_emit(buf, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG, 2251 l1tf_vmx_states[l1tf_vmx_mitigation], 2252 sched_smt_active() ? "vulnerable" : "disabled"); 2253 } 2254 2255 static ssize_t itlb_multihit_show_state(char *buf) 2256 { 2257 if (!boot_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) || 2258 !boot_cpu_has(X86_FEATURE_VMX)) 2259 return sysfs_emit(buf, "KVM: Mitigation: VMX unsupported\n"); 2260 else if (!(cr4_read_shadow() & X86_CR4_VMXE)) 2261 return sysfs_emit(buf, "KVM: Mitigation: VMX disabled\n"); 2262 else if (itlb_multihit_kvm_mitigation) 2263 return sysfs_emit(buf, "KVM: Mitigation: Split huge pages\n"); 2264 else 2265 return sysfs_emit(buf, "KVM: Vulnerable\n"); 2266 } 2267 #else 2268 static ssize_t l1tf_show_state(char *buf) 2269 { 2270 return sysfs_emit(buf, "%s\n", L1TF_DEFAULT_MSG); 2271 } 2272 2273 static ssize_t itlb_multihit_show_state(char *buf) 2274 { 2275 return sysfs_emit(buf, "Processor vulnerable\n"); 2276 } 2277 #endif 2278 2279 static ssize_t mds_show_state(char *buf) 2280 { 2281 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 2282 return sysfs_emit(buf, "%s; SMT Host state unknown\n", 2283 mds_strings[mds_mitigation]); 2284 } 2285 2286 if (boot_cpu_has(X86_BUG_MSBDS_ONLY)) { 2287 return sysfs_emit(buf, "%s; SMT %s\n", mds_strings[mds_mitigation], 2288 (mds_mitigation == MDS_MITIGATION_OFF ? "vulnerable" : 2289 sched_smt_active() ? "mitigated" : "disabled")); 2290 } 2291 2292 return sysfs_emit(buf, "%s; SMT %s\n", mds_strings[mds_mitigation], 2293 sched_smt_active() ? "vulnerable" : "disabled"); 2294 } 2295 2296 static ssize_t tsx_async_abort_show_state(char *buf) 2297 { 2298 if ((taa_mitigation == TAA_MITIGATION_TSX_DISABLED) || 2299 (taa_mitigation == TAA_MITIGATION_OFF)) 2300 return sysfs_emit(buf, "%s\n", taa_strings[taa_mitigation]); 2301 2302 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 2303 return sysfs_emit(buf, "%s; SMT Host state unknown\n", 2304 taa_strings[taa_mitigation]); 2305 } 2306 2307 return sysfs_emit(buf, "%s; SMT %s\n", taa_strings[taa_mitigation], 2308 sched_smt_active() ? "vulnerable" : "disabled"); 2309 } 2310 2311 static ssize_t mmio_stale_data_show_state(char *buf) 2312 { 2313 if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN)) 2314 return sysfs_emit(buf, "Unknown: No mitigations\n"); 2315 2316 if (mmio_mitigation == MMIO_MITIGATION_OFF) 2317 return sysfs_emit(buf, "%s\n", mmio_strings[mmio_mitigation]); 2318 2319 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 2320 return sysfs_emit(buf, "%s; SMT Host state unknown\n", 2321 mmio_strings[mmio_mitigation]); 2322 } 2323 2324 return sysfs_emit(buf, "%s; SMT %s\n", mmio_strings[mmio_mitigation], 2325 sched_smt_active() ? "vulnerable" : "disabled"); 2326 } 2327 2328 static char *stibp_state(void) 2329 { 2330 if (spectre_v2_in_ibrs_mode(spectre_v2_enabled)) 2331 return ""; 2332 2333 switch (spectre_v2_user_stibp) { 2334 case SPECTRE_V2_USER_NONE: 2335 return ", STIBP: disabled"; 2336 case SPECTRE_V2_USER_STRICT: 2337 return ", STIBP: forced"; 2338 case SPECTRE_V2_USER_STRICT_PREFERRED: 2339 return ", STIBP: always-on"; 2340 case SPECTRE_V2_USER_PRCTL: 2341 case SPECTRE_V2_USER_SECCOMP: 2342 if (static_key_enabled(&switch_to_cond_stibp)) 2343 return ", STIBP: conditional"; 2344 } 2345 return ""; 2346 } 2347 2348 static char *ibpb_state(void) 2349 { 2350 if (boot_cpu_has(X86_FEATURE_IBPB)) { 2351 if (static_key_enabled(&switch_mm_always_ibpb)) 2352 return ", IBPB: always-on"; 2353 if (static_key_enabled(&switch_mm_cond_ibpb)) 2354 return ", IBPB: conditional"; 2355 return ", IBPB: disabled"; 2356 } 2357 return ""; 2358 } 2359 2360 static char *pbrsb_eibrs_state(void) 2361 { 2362 if (boot_cpu_has_bug(X86_BUG_EIBRS_PBRSB)) { 2363 if (boot_cpu_has(X86_FEATURE_RSB_VMEXIT_LITE) || 2364 boot_cpu_has(X86_FEATURE_RSB_VMEXIT)) 2365 return ", PBRSB-eIBRS: SW sequence"; 2366 else 2367 return ", PBRSB-eIBRS: Vulnerable"; 2368 } else { 2369 return ", PBRSB-eIBRS: Not affected"; 2370 } 2371 } 2372 2373 static ssize_t spectre_v2_show_state(char *buf) 2374 { 2375 if (spectre_v2_enabled == SPECTRE_V2_LFENCE) 2376 return sysfs_emit(buf, "Vulnerable: LFENCE\n"); 2377 2378 if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) 2379 return sysfs_emit(buf, "Vulnerable: eIBRS with unprivileged eBPF\n"); 2380 2381 if (sched_smt_active() && unprivileged_ebpf_enabled() && 2382 spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE) 2383 return sysfs_emit(buf, "Vulnerable: eIBRS+LFENCE with unprivileged eBPF and SMT\n"); 2384 2385 return sysfs_emit(buf, "%s%s%s%s%s%s%s\n", 2386 spectre_v2_strings[spectre_v2_enabled], 2387 ibpb_state(), 2388 boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "", 2389 stibp_state(), 2390 boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "", 2391 pbrsb_eibrs_state(), 2392 spectre_v2_module_string()); 2393 } 2394 2395 static ssize_t srbds_show_state(char *buf) 2396 { 2397 return sysfs_emit(buf, "%s\n", srbds_strings[srbds_mitigation]); 2398 } 2399 2400 static ssize_t retbleed_show_state(char *buf) 2401 { 2402 if (retbleed_mitigation == RETBLEED_MITIGATION_UNRET || 2403 retbleed_mitigation == RETBLEED_MITIGATION_IBPB) { 2404 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && 2405 boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) 2406 return sysfs_emit(buf, "Vulnerable: untrained return thunk / IBPB on non-AMD based uarch\n"); 2407 2408 return sysfs_emit(buf, "%s; SMT %s\n", retbleed_strings[retbleed_mitigation], 2409 !sched_smt_active() ? "disabled" : 2410 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT || 2411 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED ? 2412 "enabled with STIBP protection" : "vulnerable"); 2413 } 2414 2415 return sysfs_emit(buf, "%s\n", retbleed_strings[retbleed_mitigation]); 2416 } 2417 2418 static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr, 2419 char *buf, unsigned int bug) 2420 { 2421 if (!boot_cpu_has_bug(bug)) 2422 return sysfs_emit(buf, "Not affected\n"); 2423 2424 switch (bug) { 2425 case X86_BUG_CPU_MELTDOWN: 2426 if (boot_cpu_has(X86_FEATURE_PTI)) 2427 return sysfs_emit(buf, "Mitigation: PTI\n"); 2428 2429 if (hypervisor_is_type(X86_HYPER_XEN_PV)) 2430 return sysfs_emit(buf, "Unknown (XEN PV detected, hypervisor mitigation required)\n"); 2431 2432 break; 2433 2434 case X86_BUG_SPECTRE_V1: 2435 return sysfs_emit(buf, "%s\n", spectre_v1_strings[spectre_v1_mitigation]); 2436 2437 case X86_BUG_SPECTRE_V2: 2438 return spectre_v2_show_state(buf); 2439 2440 case X86_BUG_SPEC_STORE_BYPASS: 2441 return sysfs_emit(buf, "%s\n", ssb_strings[ssb_mode]); 2442 2443 case X86_BUG_L1TF: 2444 if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV)) 2445 return l1tf_show_state(buf); 2446 break; 2447 2448 case X86_BUG_MDS: 2449 return mds_show_state(buf); 2450 2451 case X86_BUG_TAA: 2452 return tsx_async_abort_show_state(buf); 2453 2454 case X86_BUG_ITLB_MULTIHIT: 2455 return itlb_multihit_show_state(buf); 2456 2457 case X86_BUG_SRBDS: 2458 return srbds_show_state(buf); 2459 2460 case X86_BUG_MMIO_STALE_DATA: 2461 case X86_BUG_MMIO_UNKNOWN: 2462 return mmio_stale_data_show_state(buf); 2463 2464 case X86_BUG_RETBLEED: 2465 return retbleed_show_state(buf); 2466 2467 default: 2468 break; 2469 } 2470 2471 return sysfs_emit(buf, "Vulnerable\n"); 2472 } 2473 2474 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf) 2475 { 2476 return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN); 2477 } 2478 2479 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf) 2480 { 2481 return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1); 2482 } 2483 2484 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf) 2485 { 2486 return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2); 2487 } 2488 2489 ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf) 2490 { 2491 return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS); 2492 } 2493 2494 ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf) 2495 { 2496 return cpu_show_common(dev, attr, buf, X86_BUG_L1TF); 2497 } 2498 2499 ssize_t cpu_show_mds(struct device *dev, struct device_attribute *attr, char *buf) 2500 { 2501 return cpu_show_common(dev, attr, buf, X86_BUG_MDS); 2502 } 2503 2504 ssize_t cpu_show_tsx_async_abort(struct device *dev, struct device_attribute *attr, char *buf) 2505 { 2506 return cpu_show_common(dev, attr, buf, X86_BUG_TAA); 2507 } 2508 2509 ssize_t cpu_show_itlb_multihit(struct device *dev, struct device_attribute *attr, char *buf) 2510 { 2511 return cpu_show_common(dev, attr, buf, X86_BUG_ITLB_MULTIHIT); 2512 } 2513 2514 ssize_t cpu_show_srbds(struct device *dev, struct device_attribute *attr, char *buf) 2515 { 2516 return cpu_show_common(dev, attr, buf, X86_BUG_SRBDS); 2517 } 2518 2519 ssize_t cpu_show_mmio_stale_data(struct device *dev, struct device_attribute *attr, char *buf) 2520 { 2521 if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN)) 2522 return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_UNKNOWN); 2523 else 2524 return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_STALE_DATA); 2525 } 2526 2527 ssize_t cpu_show_retbleed(struct device *dev, struct device_attribute *attr, char *buf) 2528 { 2529 return cpu_show_common(dev, attr, buf, X86_BUG_RETBLEED); 2530 } 2531 #endif 2532