1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 1994 Linus Torvalds 4 * 5 * Cyrix stuff, June 1998 by: 6 * - Rafael R. Reilova (moved everything from head.S), 7 * <rreilova@ececs.uc.edu> 8 * - Channing Corn (tests & fixes), 9 * - Andrew D. Balsa (code cleanup). 10 */ 11 #include <linux/init.h> 12 #include <linux/utsname.h> 13 #include <linux/cpu.h> 14 #include <linux/module.h> 15 #include <linux/nospec.h> 16 #include <linux/prctl.h> 17 #include <linux/sched/smt.h> 18 #include <linux/pgtable.h> 19 #include <linux/bpf.h> 20 21 #include <asm/spec-ctrl.h> 22 #include <asm/cmdline.h> 23 #include <asm/bugs.h> 24 #include <asm/processor.h> 25 #include <asm/processor-flags.h> 26 #include <asm/fpu/api.h> 27 #include <asm/msr.h> 28 #include <asm/vmx.h> 29 #include <asm/paravirt.h> 30 #include <asm/alternative.h> 31 #include <asm/set_memory.h> 32 #include <asm/intel-family.h> 33 #include <asm/e820/api.h> 34 #include <asm/hypervisor.h> 35 #include <asm/tlbflush.h> 36 #include <asm/cpu.h> 37 38 #include "cpu.h" 39 40 static void __init spectre_v1_select_mitigation(void); 41 static void __init spectre_v2_select_mitigation(void); 42 static void __init retbleed_select_mitigation(void); 43 static void __init spectre_v2_user_select_mitigation(void); 44 static void __init ssb_select_mitigation(void); 45 static void __init l1tf_select_mitigation(void); 46 static void __init mds_select_mitigation(void); 47 static void __init md_clear_update_mitigation(void); 48 static void __init md_clear_select_mitigation(void); 49 static void __init taa_select_mitigation(void); 50 static void __init mmio_select_mitigation(void); 51 static void __init srbds_select_mitigation(void); 52 static void __init l1d_flush_select_mitigation(void); 53 54 /* The base value of the SPEC_CTRL MSR without task-specific bits set */ 55 u64 x86_spec_ctrl_base; 56 EXPORT_SYMBOL_GPL(x86_spec_ctrl_base); 57 58 /* The current value of the SPEC_CTRL MSR with task-specific bits set */ 59 DEFINE_PER_CPU(u64, x86_spec_ctrl_current); 60 EXPORT_SYMBOL_GPL(x86_spec_ctrl_current); 61 62 static DEFINE_MUTEX(spec_ctrl_mutex); 63 64 /* Update SPEC_CTRL MSR and its cached copy unconditionally */ 65 static void update_spec_ctrl(u64 val) 66 { 67 this_cpu_write(x86_spec_ctrl_current, val); 68 wrmsrl(MSR_IA32_SPEC_CTRL, val); 69 } 70 71 /* 72 * Keep track of the SPEC_CTRL MSR value for the current task, which may differ 73 * from x86_spec_ctrl_base due to STIBP/SSB in __speculation_ctrl_update(). 74 */ 75 void update_spec_ctrl_cond(u64 val) 76 { 77 if (this_cpu_read(x86_spec_ctrl_current) == val) 78 return; 79 80 this_cpu_write(x86_spec_ctrl_current, val); 81 82 /* 83 * When KERNEL_IBRS this MSR is written on return-to-user, unless 84 * forced the update can be delayed until that time. 85 */ 86 if (!cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS)) 87 wrmsrl(MSR_IA32_SPEC_CTRL, val); 88 } 89 90 noinstr u64 spec_ctrl_current(void) 91 { 92 return this_cpu_read(x86_spec_ctrl_current); 93 } 94 EXPORT_SYMBOL_GPL(spec_ctrl_current); 95 96 /* 97 * AMD specific MSR info for Speculative Store Bypass control. 98 * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu(). 99 */ 100 u64 __ro_after_init x86_amd_ls_cfg_base; 101 u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask; 102 103 /* Control conditional STIBP in switch_to() */ 104 DEFINE_STATIC_KEY_FALSE(switch_to_cond_stibp); 105 /* Control conditional IBPB in switch_mm() */ 106 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb); 107 /* Control unconditional IBPB in switch_mm() */ 108 DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb); 109 110 /* Control MDS CPU buffer clear before returning to user space */ 111 DEFINE_STATIC_KEY_FALSE(mds_user_clear); 112 EXPORT_SYMBOL_GPL(mds_user_clear); 113 /* Control MDS CPU buffer clear before idling (halt, mwait) */ 114 DEFINE_STATIC_KEY_FALSE(mds_idle_clear); 115 EXPORT_SYMBOL_GPL(mds_idle_clear); 116 117 /* 118 * Controls whether l1d flush based mitigations are enabled, 119 * based on hw features and admin setting via boot parameter 120 * defaults to false 121 */ 122 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_l1d_flush); 123 124 /* Controls CPU Fill buffer clear before KVM guest MMIO accesses */ 125 DEFINE_STATIC_KEY_FALSE(mmio_stale_data_clear); 126 EXPORT_SYMBOL_GPL(mmio_stale_data_clear); 127 128 void __init check_bugs(void) 129 { 130 identify_boot_cpu(); 131 132 /* 133 * identify_boot_cpu() initialized SMT support information, let the 134 * core code know. 135 */ 136 cpu_smt_check_topology(); 137 138 if (!IS_ENABLED(CONFIG_SMP)) { 139 pr_info("CPU: "); 140 print_cpu_info(&boot_cpu_data); 141 } 142 143 /* 144 * Read the SPEC_CTRL MSR to account for reserved bits which may 145 * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD 146 * init code as it is not enumerated and depends on the family. 147 */ 148 if (cpu_feature_enabled(X86_FEATURE_MSR_SPEC_CTRL)) { 149 rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); 150 151 /* 152 * Previously running kernel (kexec), may have some controls 153 * turned ON. Clear them and let the mitigations setup below 154 * rediscover them based on configuration. 155 */ 156 x86_spec_ctrl_base &= ~SPEC_CTRL_MITIGATIONS_MASK; 157 } 158 159 /* Select the proper CPU mitigations before patching alternatives: */ 160 spectre_v1_select_mitigation(); 161 spectre_v2_select_mitigation(); 162 /* 163 * retbleed_select_mitigation() relies on the state set by 164 * spectre_v2_select_mitigation(); specifically it wants to know about 165 * spectre_v2=ibrs. 166 */ 167 retbleed_select_mitigation(); 168 /* 169 * spectre_v2_user_select_mitigation() relies on the state set by 170 * retbleed_select_mitigation(); specifically the STIBP selection is 171 * forced for UNRET or IBPB. 172 */ 173 spectre_v2_user_select_mitigation(); 174 ssb_select_mitigation(); 175 l1tf_select_mitigation(); 176 md_clear_select_mitigation(); 177 srbds_select_mitigation(); 178 l1d_flush_select_mitigation(); 179 180 arch_smt_update(); 181 182 #ifdef CONFIG_X86_32 183 /* 184 * Check whether we are able to run this kernel safely on SMP. 185 * 186 * - i386 is no longer supported. 187 * - In order to run on anything without a TSC, we need to be 188 * compiled for a i486. 189 */ 190 if (boot_cpu_data.x86 < 4) 191 panic("Kernel requires i486+ for 'invlpg' and other features"); 192 193 init_utsname()->machine[1] = 194 '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86); 195 alternative_instructions(); 196 197 fpu__init_check_bugs(); 198 #else /* CONFIG_X86_64 */ 199 alternative_instructions(); 200 201 /* 202 * Make sure the first 2MB area is not mapped by huge pages 203 * There are typically fixed size MTRRs in there and overlapping 204 * MTRRs into large pages causes slow downs. 205 * 206 * Right now we don't do that with gbpages because there seems 207 * very little benefit for that case. 208 */ 209 if (!direct_gbpages) 210 set_memory_4k((unsigned long)__va(0), 1); 211 #endif 212 } 213 214 /* 215 * NOTE: This function is *only* called for SVM, since Intel uses 216 * MSR_IA32_SPEC_CTRL for SSBD. 217 */ 218 void 219 x86_virt_spec_ctrl(u64 guest_virt_spec_ctrl, bool setguest) 220 { 221 u64 guestval, hostval; 222 struct thread_info *ti = current_thread_info(); 223 224 /* 225 * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update 226 * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported. 227 */ 228 if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD) && 229 !static_cpu_has(X86_FEATURE_VIRT_SSBD)) 230 return; 231 232 /* 233 * If the host has SSBD mitigation enabled, force it in the host's 234 * virtual MSR value. If its not permanently enabled, evaluate 235 * current's TIF_SSBD thread flag. 236 */ 237 if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE)) 238 hostval = SPEC_CTRL_SSBD; 239 else 240 hostval = ssbd_tif_to_spec_ctrl(ti->flags); 241 242 /* Sanitize the guest value */ 243 guestval = guest_virt_spec_ctrl & SPEC_CTRL_SSBD; 244 245 if (hostval != guestval) { 246 unsigned long tif; 247 248 tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) : 249 ssbd_spec_ctrl_to_tif(hostval); 250 251 speculation_ctrl_update(tif); 252 } 253 } 254 EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl); 255 256 static void x86_amd_ssb_disable(void) 257 { 258 u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask; 259 260 if (boot_cpu_has(X86_FEATURE_VIRT_SSBD)) 261 wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD); 262 else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD)) 263 wrmsrl(MSR_AMD64_LS_CFG, msrval); 264 } 265 266 #undef pr_fmt 267 #define pr_fmt(fmt) "MDS: " fmt 268 269 /* Default mitigation for MDS-affected CPUs */ 270 static enum mds_mitigations mds_mitigation __ro_after_init = MDS_MITIGATION_FULL; 271 static bool mds_nosmt __ro_after_init = false; 272 273 static const char * const mds_strings[] = { 274 [MDS_MITIGATION_OFF] = "Vulnerable", 275 [MDS_MITIGATION_FULL] = "Mitigation: Clear CPU buffers", 276 [MDS_MITIGATION_VMWERV] = "Vulnerable: Clear CPU buffers attempted, no microcode", 277 }; 278 279 static void __init mds_select_mitigation(void) 280 { 281 if (!boot_cpu_has_bug(X86_BUG_MDS) || cpu_mitigations_off()) { 282 mds_mitigation = MDS_MITIGATION_OFF; 283 return; 284 } 285 286 if (mds_mitigation == MDS_MITIGATION_FULL) { 287 if (!boot_cpu_has(X86_FEATURE_MD_CLEAR)) 288 mds_mitigation = MDS_MITIGATION_VMWERV; 289 290 static_branch_enable(&mds_user_clear); 291 292 if (!boot_cpu_has(X86_BUG_MSBDS_ONLY) && 293 (mds_nosmt || cpu_mitigations_auto_nosmt())) 294 cpu_smt_disable(false); 295 } 296 } 297 298 static int __init mds_cmdline(char *str) 299 { 300 if (!boot_cpu_has_bug(X86_BUG_MDS)) 301 return 0; 302 303 if (!str) 304 return -EINVAL; 305 306 if (!strcmp(str, "off")) 307 mds_mitigation = MDS_MITIGATION_OFF; 308 else if (!strcmp(str, "full")) 309 mds_mitigation = MDS_MITIGATION_FULL; 310 else if (!strcmp(str, "full,nosmt")) { 311 mds_mitigation = MDS_MITIGATION_FULL; 312 mds_nosmt = true; 313 } 314 315 return 0; 316 } 317 early_param("mds", mds_cmdline); 318 319 #undef pr_fmt 320 #define pr_fmt(fmt) "TAA: " fmt 321 322 enum taa_mitigations { 323 TAA_MITIGATION_OFF, 324 TAA_MITIGATION_UCODE_NEEDED, 325 TAA_MITIGATION_VERW, 326 TAA_MITIGATION_TSX_DISABLED, 327 }; 328 329 /* Default mitigation for TAA-affected CPUs */ 330 static enum taa_mitigations taa_mitigation __ro_after_init = TAA_MITIGATION_VERW; 331 static bool taa_nosmt __ro_after_init; 332 333 static const char * const taa_strings[] = { 334 [TAA_MITIGATION_OFF] = "Vulnerable", 335 [TAA_MITIGATION_UCODE_NEEDED] = "Vulnerable: Clear CPU buffers attempted, no microcode", 336 [TAA_MITIGATION_VERW] = "Mitigation: Clear CPU buffers", 337 [TAA_MITIGATION_TSX_DISABLED] = "Mitigation: TSX disabled", 338 }; 339 340 static void __init taa_select_mitigation(void) 341 { 342 u64 ia32_cap; 343 344 if (!boot_cpu_has_bug(X86_BUG_TAA)) { 345 taa_mitigation = TAA_MITIGATION_OFF; 346 return; 347 } 348 349 /* TSX previously disabled by tsx=off */ 350 if (!boot_cpu_has(X86_FEATURE_RTM)) { 351 taa_mitigation = TAA_MITIGATION_TSX_DISABLED; 352 return; 353 } 354 355 if (cpu_mitigations_off()) { 356 taa_mitigation = TAA_MITIGATION_OFF; 357 return; 358 } 359 360 /* 361 * TAA mitigation via VERW is turned off if both 362 * tsx_async_abort=off and mds=off are specified. 363 */ 364 if (taa_mitigation == TAA_MITIGATION_OFF && 365 mds_mitigation == MDS_MITIGATION_OFF) 366 return; 367 368 if (boot_cpu_has(X86_FEATURE_MD_CLEAR)) 369 taa_mitigation = TAA_MITIGATION_VERW; 370 else 371 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED; 372 373 /* 374 * VERW doesn't clear the CPU buffers when MD_CLEAR=1 and MDS_NO=1. 375 * A microcode update fixes this behavior to clear CPU buffers. It also 376 * adds support for MSR_IA32_TSX_CTRL which is enumerated by the 377 * ARCH_CAP_TSX_CTRL_MSR bit. 378 * 379 * On MDS_NO=1 CPUs if ARCH_CAP_TSX_CTRL_MSR is not set, microcode 380 * update is required. 381 */ 382 ia32_cap = x86_read_arch_cap_msr(); 383 if ( (ia32_cap & ARCH_CAP_MDS_NO) && 384 !(ia32_cap & ARCH_CAP_TSX_CTRL_MSR)) 385 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED; 386 387 /* 388 * TSX is enabled, select alternate mitigation for TAA which is 389 * the same as MDS. Enable MDS static branch to clear CPU buffers. 390 * 391 * For guests that can't determine whether the correct microcode is 392 * present on host, enable the mitigation for UCODE_NEEDED as well. 393 */ 394 static_branch_enable(&mds_user_clear); 395 396 if (taa_nosmt || cpu_mitigations_auto_nosmt()) 397 cpu_smt_disable(false); 398 } 399 400 static int __init tsx_async_abort_parse_cmdline(char *str) 401 { 402 if (!boot_cpu_has_bug(X86_BUG_TAA)) 403 return 0; 404 405 if (!str) 406 return -EINVAL; 407 408 if (!strcmp(str, "off")) { 409 taa_mitigation = TAA_MITIGATION_OFF; 410 } else if (!strcmp(str, "full")) { 411 taa_mitigation = TAA_MITIGATION_VERW; 412 } else if (!strcmp(str, "full,nosmt")) { 413 taa_mitigation = TAA_MITIGATION_VERW; 414 taa_nosmt = true; 415 } 416 417 return 0; 418 } 419 early_param("tsx_async_abort", tsx_async_abort_parse_cmdline); 420 421 #undef pr_fmt 422 #define pr_fmt(fmt) "MMIO Stale Data: " fmt 423 424 enum mmio_mitigations { 425 MMIO_MITIGATION_OFF, 426 MMIO_MITIGATION_UCODE_NEEDED, 427 MMIO_MITIGATION_VERW, 428 }; 429 430 /* Default mitigation for Processor MMIO Stale Data vulnerabilities */ 431 static enum mmio_mitigations mmio_mitigation __ro_after_init = MMIO_MITIGATION_VERW; 432 static bool mmio_nosmt __ro_after_init = false; 433 434 static const char * const mmio_strings[] = { 435 [MMIO_MITIGATION_OFF] = "Vulnerable", 436 [MMIO_MITIGATION_UCODE_NEEDED] = "Vulnerable: Clear CPU buffers attempted, no microcode", 437 [MMIO_MITIGATION_VERW] = "Mitigation: Clear CPU buffers", 438 }; 439 440 static void __init mmio_select_mitigation(void) 441 { 442 u64 ia32_cap; 443 444 if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA) || 445 boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN) || 446 cpu_mitigations_off()) { 447 mmio_mitigation = MMIO_MITIGATION_OFF; 448 return; 449 } 450 451 if (mmio_mitigation == MMIO_MITIGATION_OFF) 452 return; 453 454 ia32_cap = x86_read_arch_cap_msr(); 455 456 /* 457 * Enable CPU buffer clear mitigation for host and VMM, if also affected 458 * by MDS or TAA. Otherwise, enable mitigation for VMM only. 459 */ 460 if (boot_cpu_has_bug(X86_BUG_MDS) || (boot_cpu_has_bug(X86_BUG_TAA) && 461 boot_cpu_has(X86_FEATURE_RTM))) 462 static_branch_enable(&mds_user_clear); 463 else 464 static_branch_enable(&mmio_stale_data_clear); 465 466 /* 467 * If Processor-MMIO-Stale-Data bug is present and Fill Buffer data can 468 * be propagated to uncore buffers, clearing the Fill buffers on idle 469 * is required irrespective of SMT state. 470 */ 471 if (!(ia32_cap & ARCH_CAP_FBSDP_NO)) 472 static_branch_enable(&mds_idle_clear); 473 474 /* 475 * Check if the system has the right microcode. 476 * 477 * CPU Fill buffer clear mitigation is enumerated by either an explicit 478 * FB_CLEAR or by the presence of both MD_CLEAR and L1D_FLUSH on MDS 479 * affected systems. 480 */ 481 if ((ia32_cap & ARCH_CAP_FB_CLEAR) || 482 (boot_cpu_has(X86_FEATURE_MD_CLEAR) && 483 boot_cpu_has(X86_FEATURE_FLUSH_L1D) && 484 !(ia32_cap & ARCH_CAP_MDS_NO))) 485 mmio_mitigation = MMIO_MITIGATION_VERW; 486 else 487 mmio_mitigation = MMIO_MITIGATION_UCODE_NEEDED; 488 489 if (mmio_nosmt || cpu_mitigations_auto_nosmt()) 490 cpu_smt_disable(false); 491 } 492 493 static int __init mmio_stale_data_parse_cmdline(char *str) 494 { 495 if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) 496 return 0; 497 498 if (!str) 499 return -EINVAL; 500 501 if (!strcmp(str, "off")) { 502 mmio_mitigation = MMIO_MITIGATION_OFF; 503 } else if (!strcmp(str, "full")) { 504 mmio_mitigation = MMIO_MITIGATION_VERW; 505 } else if (!strcmp(str, "full,nosmt")) { 506 mmio_mitigation = MMIO_MITIGATION_VERW; 507 mmio_nosmt = true; 508 } 509 510 return 0; 511 } 512 early_param("mmio_stale_data", mmio_stale_data_parse_cmdline); 513 514 #undef pr_fmt 515 #define pr_fmt(fmt) "" fmt 516 517 static void __init md_clear_update_mitigation(void) 518 { 519 if (cpu_mitigations_off()) 520 return; 521 522 if (!static_key_enabled(&mds_user_clear)) 523 goto out; 524 525 /* 526 * mds_user_clear is now enabled. Update MDS, TAA and MMIO Stale Data 527 * mitigation, if necessary. 528 */ 529 if (mds_mitigation == MDS_MITIGATION_OFF && 530 boot_cpu_has_bug(X86_BUG_MDS)) { 531 mds_mitigation = MDS_MITIGATION_FULL; 532 mds_select_mitigation(); 533 } 534 if (taa_mitigation == TAA_MITIGATION_OFF && 535 boot_cpu_has_bug(X86_BUG_TAA)) { 536 taa_mitigation = TAA_MITIGATION_VERW; 537 taa_select_mitigation(); 538 } 539 if (mmio_mitigation == MMIO_MITIGATION_OFF && 540 boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) { 541 mmio_mitigation = MMIO_MITIGATION_VERW; 542 mmio_select_mitigation(); 543 } 544 out: 545 if (boot_cpu_has_bug(X86_BUG_MDS)) 546 pr_info("MDS: %s\n", mds_strings[mds_mitigation]); 547 if (boot_cpu_has_bug(X86_BUG_TAA)) 548 pr_info("TAA: %s\n", taa_strings[taa_mitigation]); 549 if (boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) 550 pr_info("MMIO Stale Data: %s\n", mmio_strings[mmio_mitigation]); 551 else if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN)) 552 pr_info("MMIO Stale Data: Unknown: No mitigations\n"); 553 } 554 555 static void __init md_clear_select_mitigation(void) 556 { 557 mds_select_mitigation(); 558 taa_select_mitigation(); 559 mmio_select_mitigation(); 560 561 /* 562 * As MDS, TAA and MMIO Stale Data mitigations are inter-related, update 563 * and print their mitigation after MDS, TAA and MMIO Stale Data 564 * mitigation selection is done. 565 */ 566 md_clear_update_mitigation(); 567 } 568 569 #undef pr_fmt 570 #define pr_fmt(fmt) "SRBDS: " fmt 571 572 enum srbds_mitigations { 573 SRBDS_MITIGATION_OFF, 574 SRBDS_MITIGATION_UCODE_NEEDED, 575 SRBDS_MITIGATION_FULL, 576 SRBDS_MITIGATION_TSX_OFF, 577 SRBDS_MITIGATION_HYPERVISOR, 578 }; 579 580 static enum srbds_mitigations srbds_mitigation __ro_after_init = SRBDS_MITIGATION_FULL; 581 582 static const char * const srbds_strings[] = { 583 [SRBDS_MITIGATION_OFF] = "Vulnerable", 584 [SRBDS_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode", 585 [SRBDS_MITIGATION_FULL] = "Mitigation: Microcode", 586 [SRBDS_MITIGATION_TSX_OFF] = "Mitigation: TSX disabled", 587 [SRBDS_MITIGATION_HYPERVISOR] = "Unknown: Dependent on hypervisor status", 588 }; 589 590 static bool srbds_off; 591 592 void update_srbds_msr(void) 593 { 594 u64 mcu_ctrl; 595 596 if (!boot_cpu_has_bug(X86_BUG_SRBDS)) 597 return; 598 599 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) 600 return; 601 602 if (srbds_mitigation == SRBDS_MITIGATION_UCODE_NEEDED) 603 return; 604 605 /* 606 * A MDS_NO CPU for which SRBDS mitigation is not needed due to TSX 607 * being disabled and it hasn't received the SRBDS MSR microcode. 608 */ 609 if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL)) 610 return; 611 612 rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 613 614 switch (srbds_mitigation) { 615 case SRBDS_MITIGATION_OFF: 616 case SRBDS_MITIGATION_TSX_OFF: 617 mcu_ctrl |= RNGDS_MITG_DIS; 618 break; 619 case SRBDS_MITIGATION_FULL: 620 mcu_ctrl &= ~RNGDS_MITG_DIS; 621 break; 622 default: 623 break; 624 } 625 626 wrmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl); 627 } 628 629 static void __init srbds_select_mitigation(void) 630 { 631 u64 ia32_cap; 632 633 if (!boot_cpu_has_bug(X86_BUG_SRBDS)) 634 return; 635 636 /* 637 * Check to see if this is one of the MDS_NO systems supporting TSX that 638 * are only exposed to SRBDS when TSX is enabled or when CPU is affected 639 * by Processor MMIO Stale Data vulnerability. 640 */ 641 ia32_cap = x86_read_arch_cap_msr(); 642 if ((ia32_cap & ARCH_CAP_MDS_NO) && !boot_cpu_has(X86_FEATURE_RTM) && 643 !boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) 644 srbds_mitigation = SRBDS_MITIGATION_TSX_OFF; 645 else if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) 646 srbds_mitigation = SRBDS_MITIGATION_HYPERVISOR; 647 else if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL)) 648 srbds_mitigation = SRBDS_MITIGATION_UCODE_NEEDED; 649 else if (cpu_mitigations_off() || srbds_off) 650 srbds_mitigation = SRBDS_MITIGATION_OFF; 651 652 update_srbds_msr(); 653 pr_info("%s\n", srbds_strings[srbds_mitigation]); 654 } 655 656 static int __init srbds_parse_cmdline(char *str) 657 { 658 if (!str) 659 return -EINVAL; 660 661 if (!boot_cpu_has_bug(X86_BUG_SRBDS)) 662 return 0; 663 664 srbds_off = !strcmp(str, "off"); 665 return 0; 666 } 667 early_param("srbds", srbds_parse_cmdline); 668 669 #undef pr_fmt 670 #define pr_fmt(fmt) "L1D Flush : " fmt 671 672 enum l1d_flush_mitigations { 673 L1D_FLUSH_OFF = 0, 674 L1D_FLUSH_ON, 675 }; 676 677 static enum l1d_flush_mitigations l1d_flush_mitigation __initdata = L1D_FLUSH_OFF; 678 679 static void __init l1d_flush_select_mitigation(void) 680 { 681 if (!l1d_flush_mitigation || !boot_cpu_has(X86_FEATURE_FLUSH_L1D)) 682 return; 683 684 static_branch_enable(&switch_mm_cond_l1d_flush); 685 pr_info("Conditional flush on switch_mm() enabled\n"); 686 } 687 688 static int __init l1d_flush_parse_cmdline(char *str) 689 { 690 if (!strcmp(str, "on")) 691 l1d_flush_mitigation = L1D_FLUSH_ON; 692 693 return 0; 694 } 695 early_param("l1d_flush", l1d_flush_parse_cmdline); 696 697 #undef pr_fmt 698 #define pr_fmt(fmt) "Spectre V1 : " fmt 699 700 enum spectre_v1_mitigation { 701 SPECTRE_V1_MITIGATION_NONE, 702 SPECTRE_V1_MITIGATION_AUTO, 703 }; 704 705 static enum spectre_v1_mitigation spectre_v1_mitigation __ro_after_init = 706 SPECTRE_V1_MITIGATION_AUTO; 707 708 static const char * const spectre_v1_strings[] = { 709 [SPECTRE_V1_MITIGATION_NONE] = "Vulnerable: __user pointer sanitization and usercopy barriers only; no swapgs barriers", 710 [SPECTRE_V1_MITIGATION_AUTO] = "Mitigation: usercopy/swapgs barriers and __user pointer sanitization", 711 }; 712 713 /* 714 * Does SMAP provide full mitigation against speculative kernel access to 715 * userspace? 716 */ 717 static bool smap_works_speculatively(void) 718 { 719 if (!boot_cpu_has(X86_FEATURE_SMAP)) 720 return false; 721 722 /* 723 * On CPUs which are vulnerable to Meltdown, SMAP does not 724 * prevent speculative access to user data in the L1 cache. 725 * Consider SMAP to be non-functional as a mitigation on these 726 * CPUs. 727 */ 728 if (boot_cpu_has(X86_BUG_CPU_MELTDOWN)) 729 return false; 730 731 return true; 732 } 733 734 static void __init spectre_v1_select_mitigation(void) 735 { 736 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1) || cpu_mitigations_off()) { 737 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE; 738 return; 739 } 740 741 if (spectre_v1_mitigation == SPECTRE_V1_MITIGATION_AUTO) { 742 /* 743 * With Spectre v1, a user can speculatively control either 744 * path of a conditional swapgs with a user-controlled GS 745 * value. The mitigation is to add lfences to both code paths. 746 * 747 * If FSGSBASE is enabled, the user can put a kernel address in 748 * GS, in which case SMAP provides no protection. 749 * 750 * If FSGSBASE is disabled, the user can only put a user space 751 * address in GS. That makes an attack harder, but still 752 * possible if there's no SMAP protection. 753 */ 754 if (boot_cpu_has(X86_FEATURE_FSGSBASE) || 755 !smap_works_speculatively()) { 756 /* 757 * Mitigation can be provided from SWAPGS itself or 758 * PTI as the CR3 write in the Meltdown mitigation 759 * is serializing. 760 * 761 * If neither is there, mitigate with an LFENCE to 762 * stop speculation through swapgs. 763 */ 764 if (boot_cpu_has_bug(X86_BUG_SWAPGS) && 765 !boot_cpu_has(X86_FEATURE_PTI)) 766 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_USER); 767 768 /* 769 * Enable lfences in the kernel entry (non-swapgs) 770 * paths, to prevent user entry from speculatively 771 * skipping swapgs. 772 */ 773 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_KERNEL); 774 } 775 } 776 777 pr_info("%s\n", spectre_v1_strings[spectre_v1_mitigation]); 778 } 779 780 static int __init nospectre_v1_cmdline(char *str) 781 { 782 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE; 783 return 0; 784 } 785 early_param("nospectre_v1", nospectre_v1_cmdline); 786 787 enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init = SPECTRE_V2_NONE; 788 789 #undef pr_fmt 790 #define pr_fmt(fmt) "RETBleed: " fmt 791 792 enum retbleed_mitigation { 793 RETBLEED_MITIGATION_NONE, 794 RETBLEED_MITIGATION_UNRET, 795 RETBLEED_MITIGATION_IBPB, 796 RETBLEED_MITIGATION_IBRS, 797 RETBLEED_MITIGATION_EIBRS, 798 RETBLEED_MITIGATION_STUFF, 799 }; 800 801 enum retbleed_mitigation_cmd { 802 RETBLEED_CMD_OFF, 803 RETBLEED_CMD_AUTO, 804 RETBLEED_CMD_UNRET, 805 RETBLEED_CMD_IBPB, 806 RETBLEED_CMD_STUFF, 807 }; 808 809 static const char * const retbleed_strings[] = { 810 [RETBLEED_MITIGATION_NONE] = "Vulnerable", 811 [RETBLEED_MITIGATION_UNRET] = "Mitigation: untrained return thunk", 812 [RETBLEED_MITIGATION_IBPB] = "Mitigation: IBPB", 813 [RETBLEED_MITIGATION_IBRS] = "Mitigation: IBRS", 814 [RETBLEED_MITIGATION_EIBRS] = "Mitigation: Enhanced IBRS", 815 [RETBLEED_MITIGATION_STUFF] = "Mitigation: Stuffing", 816 }; 817 818 static enum retbleed_mitigation retbleed_mitigation __ro_after_init = 819 RETBLEED_MITIGATION_NONE; 820 static enum retbleed_mitigation_cmd retbleed_cmd __ro_after_init = 821 RETBLEED_CMD_AUTO; 822 823 static int __ro_after_init retbleed_nosmt = false; 824 825 static int __init retbleed_parse_cmdline(char *str) 826 { 827 if (!str) 828 return -EINVAL; 829 830 while (str) { 831 char *next = strchr(str, ','); 832 if (next) { 833 *next = 0; 834 next++; 835 } 836 837 if (!strcmp(str, "off")) { 838 retbleed_cmd = RETBLEED_CMD_OFF; 839 } else if (!strcmp(str, "auto")) { 840 retbleed_cmd = RETBLEED_CMD_AUTO; 841 } else if (!strcmp(str, "unret")) { 842 retbleed_cmd = RETBLEED_CMD_UNRET; 843 } else if (!strcmp(str, "ibpb")) { 844 retbleed_cmd = RETBLEED_CMD_IBPB; 845 } else if (!strcmp(str, "stuff")) { 846 retbleed_cmd = RETBLEED_CMD_STUFF; 847 } else if (!strcmp(str, "nosmt")) { 848 retbleed_nosmt = true; 849 } else if (!strcmp(str, "force")) { 850 setup_force_cpu_bug(X86_BUG_RETBLEED); 851 } else { 852 pr_err("Ignoring unknown retbleed option (%s).", str); 853 } 854 855 str = next; 856 } 857 858 return 0; 859 } 860 early_param("retbleed", retbleed_parse_cmdline); 861 862 #define RETBLEED_UNTRAIN_MSG "WARNING: BTB untrained return thunk mitigation is only effective on AMD/Hygon!\n" 863 #define RETBLEED_INTEL_MSG "WARNING: Spectre v2 mitigation leaves CPU vulnerable to RETBleed attacks, data leaks possible!\n" 864 865 static void __init retbleed_select_mitigation(void) 866 { 867 bool mitigate_smt = false; 868 869 if (!boot_cpu_has_bug(X86_BUG_RETBLEED) || cpu_mitigations_off()) 870 return; 871 872 switch (retbleed_cmd) { 873 case RETBLEED_CMD_OFF: 874 return; 875 876 case RETBLEED_CMD_UNRET: 877 if (IS_ENABLED(CONFIG_CPU_UNRET_ENTRY)) { 878 retbleed_mitigation = RETBLEED_MITIGATION_UNRET; 879 } else { 880 pr_err("WARNING: kernel not compiled with CPU_UNRET_ENTRY.\n"); 881 goto do_cmd_auto; 882 } 883 break; 884 885 case RETBLEED_CMD_IBPB: 886 if (!boot_cpu_has(X86_FEATURE_IBPB)) { 887 pr_err("WARNING: CPU does not support IBPB.\n"); 888 goto do_cmd_auto; 889 } else if (IS_ENABLED(CONFIG_CPU_IBPB_ENTRY)) { 890 retbleed_mitigation = RETBLEED_MITIGATION_IBPB; 891 } else { 892 pr_err("WARNING: kernel not compiled with CPU_IBPB_ENTRY.\n"); 893 goto do_cmd_auto; 894 } 895 break; 896 897 case RETBLEED_CMD_STUFF: 898 if (IS_ENABLED(CONFIG_CALL_DEPTH_TRACKING) && 899 spectre_v2_enabled == SPECTRE_V2_RETPOLINE) { 900 retbleed_mitigation = RETBLEED_MITIGATION_STUFF; 901 902 } else { 903 if (IS_ENABLED(CONFIG_CALL_DEPTH_TRACKING)) 904 pr_err("WARNING: retbleed=stuff depends on spectre_v2=retpoline\n"); 905 else 906 pr_err("WARNING: kernel not compiled with CALL_DEPTH_TRACKING.\n"); 907 908 goto do_cmd_auto; 909 } 910 break; 911 912 do_cmd_auto: 913 case RETBLEED_CMD_AUTO: 914 default: 915 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || 916 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) { 917 if (IS_ENABLED(CONFIG_CPU_UNRET_ENTRY)) 918 retbleed_mitigation = RETBLEED_MITIGATION_UNRET; 919 else if (IS_ENABLED(CONFIG_CPU_IBPB_ENTRY) && boot_cpu_has(X86_FEATURE_IBPB)) 920 retbleed_mitigation = RETBLEED_MITIGATION_IBPB; 921 } 922 923 /* 924 * The Intel mitigation (IBRS or eIBRS) was already selected in 925 * spectre_v2_select_mitigation(). 'retbleed_mitigation' will 926 * be set accordingly below. 927 */ 928 929 break; 930 } 931 932 switch (retbleed_mitigation) { 933 case RETBLEED_MITIGATION_UNRET: 934 setup_force_cpu_cap(X86_FEATURE_RETHUNK); 935 setup_force_cpu_cap(X86_FEATURE_UNRET); 936 937 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && 938 boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) 939 pr_err(RETBLEED_UNTRAIN_MSG); 940 941 mitigate_smt = true; 942 break; 943 944 case RETBLEED_MITIGATION_IBPB: 945 setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB); 946 mitigate_smt = true; 947 break; 948 949 case RETBLEED_MITIGATION_STUFF: 950 setup_force_cpu_cap(X86_FEATURE_RETHUNK); 951 setup_force_cpu_cap(X86_FEATURE_CALL_DEPTH); 952 x86_set_skl_return_thunk(); 953 break; 954 955 default: 956 break; 957 } 958 959 if (mitigate_smt && !boot_cpu_has(X86_FEATURE_STIBP) && 960 (retbleed_nosmt || cpu_mitigations_auto_nosmt())) 961 cpu_smt_disable(false); 962 963 /* 964 * Let IBRS trump all on Intel without affecting the effects of the 965 * retbleed= cmdline option except for call depth based stuffing 966 */ 967 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { 968 switch (spectre_v2_enabled) { 969 case SPECTRE_V2_IBRS: 970 retbleed_mitigation = RETBLEED_MITIGATION_IBRS; 971 break; 972 case SPECTRE_V2_EIBRS: 973 case SPECTRE_V2_EIBRS_RETPOLINE: 974 case SPECTRE_V2_EIBRS_LFENCE: 975 retbleed_mitigation = RETBLEED_MITIGATION_EIBRS; 976 break; 977 default: 978 if (retbleed_mitigation != RETBLEED_MITIGATION_STUFF) 979 pr_err(RETBLEED_INTEL_MSG); 980 } 981 } 982 983 pr_info("%s\n", retbleed_strings[retbleed_mitigation]); 984 } 985 986 #undef pr_fmt 987 #define pr_fmt(fmt) "Spectre V2 : " fmt 988 989 static enum spectre_v2_user_mitigation spectre_v2_user_stibp __ro_after_init = 990 SPECTRE_V2_USER_NONE; 991 static enum spectre_v2_user_mitigation spectre_v2_user_ibpb __ro_after_init = 992 SPECTRE_V2_USER_NONE; 993 994 #ifdef CONFIG_RETPOLINE 995 static bool spectre_v2_bad_module; 996 997 bool retpoline_module_ok(bool has_retpoline) 998 { 999 if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline) 1000 return true; 1001 1002 pr_err("System may be vulnerable to spectre v2\n"); 1003 spectre_v2_bad_module = true; 1004 return false; 1005 } 1006 1007 static inline const char *spectre_v2_module_string(void) 1008 { 1009 return spectre_v2_bad_module ? " - vulnerable module loaded" : ""; 1010 } 1011 #else 1012 static inline const char *spectre_v2_module_string(void) { return ""; } 1013 #endif 1014 1015 #define SPECTRE_V2_LFENCE_MSG "WARNING: LFENCE mitigation is not recommended for this CPU, data leaks possible!\n" 1016 #define SPECTRE_V2_EIBRS_EBPF_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS on, data leaks possible via Spectre v2 BHB attacks!\n" 1017 #define SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS+LFENCE mitigation and SMT, data leaks possible via Spectre v2 BHB attacks!\n" 1018 #define SPECTRE_V2_IBRS_PERF_MSG "WARNING: IBRS mitigation selected on Enhanced IBRS CPU, this may cause unnecessary performance loss\n" 1019 1020 #ifdef CONFIG_BPF_SYSCALL 1021 void unpriv_ebpf_notify(int new_state) 1022 { 1023 if (new_state) 1024 return; 1025 1026 /* Unprivileged eBPF is enabled */ 1027 1028 switch (spectre_v2_enabled) { 1029 case SPECTRE_V2_EIBRS: 1030 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG); 1031 break; 1032 case SPECTRE_V2_EIBRS_LFENCE: 1033 if (sched_smt_active()) 1034 pr_err(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG); 1035 break; 1036 default: 1037 break; 1038 } 1039 } 1040 #endif 1041 1042 static inline bool match_option(const char *arg, int arglen, const char *opt) 1043 { 1044 int len = strlen(opt); 1045 1046 return len == arglen && !strncmp(arg, opt, len); 1047 } 1048 1049 /* The kernel command line selection for spectre v2 */ 1050 enum spectre_v2_mitigation_cmd { 1051 SPECTRE_V2_CMD_NONE, 1052 SPECTRE_V2_CMD_AUTO, 1053 SPECTRE_V2_CMD_FORCE, 1054 SPECTRE_V2_CMD_RETPOLINE, 1055 SPECTRE_V2_CMD_RETPOLINE_GENERIC, 1056 SPECTRE_V2_CMD_RETPOLINE_LFENCE, 1057 SPECTRE_V2_CMD_EIBRS, 1058 SPECTRE_V2_CMD_EIBRS_RETPOLINE, 1059 SPECTRE_V2_CMD_EIBRS_LFENCE, 1060 SPECTRE_V2_CMD_IBRS, 1061 }; 1062 1063 enum spectre_v2_user_cmd { 1064 SPECTRE_V2_USER_CMD_NONE, 1065 SPECTRE_V2_USER_CMD_AUTO, 1066 SPECTRE_V2_USER_CMD_FORCE, 1067 SPECTRE_V2_USER_CMD_PRCTL, 1068 SPECTRE_V2_USER_CMD_PRCTL_IBPB, 1069 SPECTRE_V2_USER_CMD_SECCOMP, 1070 SPECTRE_V2_USER_CMD_SECCOMP_IBPB, 1071 }; 1072 1073 static const char * const spectre_v2_user_strings[] = { 1074 [SPECTRE_V2_USER_NONE] = "User space: Vulnerable", 1075 [SPECTRE_V2_USER_STRICT] = "User space: Mitigation: STIBP protection", 1076 [SPECTRE_V2_USER_STRICT_PREFERRED] = "User space: Mitigation: STIBP always-on protection", 1077 [SPECTRE_V2_USER_PRCTL] = "User space: Mitigation: STIBP via prctl", 1078 [SPECTRE_V2_USER_SECCOMP] = "User space: Mitigation: STIBP via seccomp and prctl", 1079 }; 1080 1081 static const struct { 1082 const char *option; 1083 enum spectre_v2_user_cmd cmd; 1084 bool secure; 1085 } v2_user_options[] __initconst = { 1086 { "auto", SPECTRE_V2_USER_CMD_AUTO, false }, 1087 { "off", SPECTRE_V2_USER_CMD_NONE, false }, 1088 { "on", SPECTRE_V2_USER_CMD_FORCE, true }, 1089 { "prctl", SPECTRE_V2_USER_CMD_PRCTL, false }, 1090 { "prctl,ibpb", SPECTRE_V2_USER_CMD_PRCTL_IBPB, false }, 1091 { "seccomp", SPECTRE_V2_USER_CMD_SECCOMP, false }, 1092 { "seccomp,ibpb", SPECTRE_V2_USER_CMD_SECCOMP_IBPB, false }, 1093 }; 1094 1095 static void __init spec_v2_user_print_cond(const char *reason, bool secure) 1096 { 1097 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure) 1098 pr_info("spectre_v2_user=%s forced on command line.\n", reason); 1099 } 1100 1101 static __ro_after_init enum spectre_v2_mitigation_cmd spectre_v2_cmd; 1102 1103 static enum spectre_v2_user_cmd __init 1104 spectre_v2_parse_user_cmdline(void) 1105 { 1106 char arg[20]; 1107 int ret, i; 1108 1109 switch (spectre_v2_cmd) { 1110 case SPECTRE_V2_CMD_NONE: 1111 return SPECTRE_V2_USER_CMD_NONE; 1112 case SPECTRE_V2_CMD_FORCE: 1113 return SPECTRE_V2_USER_CMD_FORCE; 1114 default: 1115 break; 1116 } 1117 1118 ret = cmdline_find_option(boot_command_line, "spectre_v2_user", 1119 arg, sizeof(arg)); 1120 if (ret < 0) 1121 return SPECTRE_V2_USER_CMD_AUTO; 1122 1123 for (i = 0; i < ARRAY_SIZE(v2_user_options); i++) { 1124 if (match_option(arg, ret, v2_user_options[i].option)) { 1125 spec_v2_user_print_cond(v2_user_options[i].option, 1126 v2_user_options[i].secure); 1127 return v2_user_options[i].cmd; 1128 } 1129 } 1130 1131 pr_err("Unknown user space protection option (%s). Switching to AUTO select\n", arg); 1132 return SPECTRE_V2_USER_CMD_AUTO; 1133 } 1134 1135 static inline bool spectre_v2_in_ibrs_mode(enum spectre_v2_mitigation mode) 1136 { 1137 return spectre_v2_in_eibrs_mode(mode) || mode == SPECTRE_V2_IBRS; 1138 } 1139 1140 static void __init 1141 spectre_v2_user_select_mitigation(void) 1142 { 1143 enum spectre_v2_user_mitigation mode = SPECTRE_V2_USER_NONE; 1144 bool smt_possible = IS_ENABLED(CONFIG_SMP); 1145 enum spectre_v2_user_cmd cmd; 1146 1147 if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP)) 1148 return; 1149 1150 if (cpu_smt_control == CPU_SMT_FORCE_DISABLED || 1151 cpu_smt_control == CPU_SMT_NOT_SUPPORTED) 1152 smt_possible = false; 1153 1154 cmd = spectre_v2_parse_user_cmdline(); 1155 switch (cmd) { 1156 case SPECTRE_V2_USER_CMD_NONE: 1157 goto set_mode; 1158 case SPECTRE_V2_USER_CMD_FORCE: 1159 mode = SPECTRE_V2_USER_STRICT; 1160 break; 1161 case SPECTRE_V2_USER_CMD_AUTO: 1162 case SPECTRE_V2_USER_CMD_PRCTL: 1163 case SPECTRE_V2_USER_CMD_PRCTL_IBPB: 1164 mode = SPECTRE_V2_USER_PRCTL; 1165 break; 1166 case SPECTRE_V2_USER_CMD_SECCOMP: 1167 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB: 1168 if (IS_ENABLED(CONFIG_SECCOMP)) 1169 mode = SPECTRE_V2_USER_SECCOMP; 1170 else 1171 mode = SPECTRE_V2_USER_PRCTL; 1172 break; 1173 } 1174 1175 /* Initialize Indirect Branch Prediction Barrier */ 1176 if (boot_cpu_has(X86_FEATURE_IBPB)) { 1177 setup_force_cpu_cap(X86_FEATURE_USE_IBPB); 1178 1179 spectre_v2_user_ibpb = mode; 1180 switch (cmd) { 1181 case SPECTRE_V2_USER_CMD_FORCE: 1182 case SPECTRE_V2_USER_CMD_PRCTL_IBPB: 1183 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB: 1184 static_branch_enable(&switch_mm_always_ibpb); 1185 spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT; 1186 break; 1187 case SPECTRE_V2_USER_CMD_PRCTL: 1188 case SPECTRE_V2_USER_CMD_AUTO: 1189 case SPECTRE_V2_USER_CMD_SECCOMP: 1190 static_branch_enable(&switch_mm_cond_ibpb); 1191 break; 1192 default: 1193 break; 1194 } 1195 1196 pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n", 1197 static_key_enabled(&switch_mm_always_ibpb) ? 1198 "always-on" : "conditional"); 1199 } 1200 1201 /* 1202 * If no STIBP, enhanced IBRS is enabled, or SMT impossible, STIBP 1203 * is not required. 1204 * 1205 * Enhanced IBRS also protects against cross-thread branch target 1206 * injection in user-mode as the IBRS bit remains always set which 1207 * implicitly enables cross-thread protections. However, in legacy IBRS 1208 * mode, the IBRS bit is set only on kernel entry and cleared on return 1209 * to userspace. This disables the implicit cross-thread protection, 1210 * so allow for STIBP to be selected in that case. 1211 */ 1212 if (!boot_cpu_has(X86_FEATURE_STIBP) || 1213 !smt_possible || 1214 spectre_v2_in_eibrs_mode(spectre_v2_enabled)) 1215 return; 1216 1217 /* 1218 * At this point, an STIBP mode other than "off" has been set. 1219 * If STIBP support is not being forced, check if STIBP always-on 1220 * is preferred. 1221 */ 1222 if (mode != SPECTRE_V2_USER_STRICT && 1223 boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON)) 1224 mode = SPECTRE_V2_USER_STRICT_PREFERRED; 1225 1226 if (retbleed_mitigation == RETBLEED_MITIGATION_UNRET || 1227 retbleed_mitigation == RETBLEED_MITIGATION_IBPB) { 1228 if (mode != SPECTRE_V2_USER_STRICT && 1229 mode != SPECTRE_V2_USER_STRICT_PREFERRED) 1230 pr_info("Selecting STIBP always-on mode to complement retbleed mitigation\n"); 1231 mode = SPECTRE_V2_USER_STRICT_PREFERRED; 1232 } 1233 1234 spectre_v2_user_stibp = mode; 1235 1236 set_mode: 1237 pr_info("%s\n", spectre_v2_user_strings[mode]); 1238 } 1239 1240 static const char * const spectre_v2_strings[] = { 1241 [SPECTRE_V2_NONE] = "Vulnerable", 1242 [SPECTRE_V2_RETPOLINE] = "Mitigation: Retpolines", 1243 [SPECTRE_V2_LFENCE] = "Mitigation: LFENCE", 1244 [SPECTRE_V2_EIBRS] = "Mitigation: Enhanced / Automatic IBRS", 1245 [SPECTRE_V2_EIBRS_LFENCE] = "Mitigation: Enhanced / Automatic IBRS + LFENCE", 1246 [SPECTRE_V2_EIBRS_RETPOLINE] = "Mitigation: Enhanced / Automatic IBRS + Retpolines", 1247 [SPECTRE_V2_IBRS] = "Mitigation: IBRS", 1248 }; 1249 1250 static const struct { 1251 const char *option; 1252 enum spectre_v2_mitigation_cmd cmd; 1253 bool secure; 1254 } mitigation_options[] __initconst = { 1255 { "off", SPECTRE_V2_CMD_NONE, false }, 1256 { "on", SPECTRE_V2_CMD_FORCE, true }, 1257 { "retpoline", SPECTRE_V2_CMD_RETPOLINE, false }, 1258 { "retpoline,amd", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false }, 1259 { "retpoline,lfence", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false }, 1260 { "retpoline,generic", SPECTRE_V2_CMD_RETPOLINE_GENERIC, false }, 1261 { "eibrs", SPECTRE_V2_CMD_EIBRS, false }, 1262 { "eibrs,lfence", SPECTRE_V2_CMD_EIBRS_LFENCE, false }, 1263 { "eibrs,retpoline", SPECTRE_V2_CMD_EIBRS_RETPOLINE, false }, 1264 { "auto", SPECTRE_V2_CMD_AUTO, false }, 1265 { "ibrs", SPECTRE_V2_CMD_IBRS, false }, 1266 }; 1267 1268 static void __init spec_v2_print_cond(const char *reason, bool secure) 1269 { 1270 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure) 1271 pr_info("%s selected on command line.\n", reason); 1272 } 1273 1274 static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void) 1275 { 1276 enum spectre_v2_mitigation_cmd cmd = SPECTRE_V2_CMD_AUTO; 1277 char arg[20]; 1278 int ret, i; 1279 1280 if (cmdline_find_option_bool(boot_command_line, "nospectre_v2") || 1281 cpu_mitigations_off()) 1282 return SPECTRE_V2_CMD_NONE; 1283 1284 ret = cmdline_find_option(boot_command_line, "spectre_v2", arg, sizeof(arg)); 1285 if (ret < 0) 1286 return SPECTRE_V2_CMD_AUTO; 1287 1288 for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) { 1289 if (!match_option(arg, ret, mitigation_options[i].option)) 1290 continue; 1291 cmd = mitigation_options[i].cmd; 1292 break; 1293 } 1294 1295 if (i >= ARRAY_SIZE(mitigation_options)) { 1296 pr_err("unknown option (%s). Switching to AUTO select\n", arg); 1297 return SPECTRE_V2_CMD_AUTO; 1298 } 1299 1300 if ((cmd == SPECTRE_V2_CMD_RETPOLINE || 1301 cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE || 1302 cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC || 1303 cmd == SPECTRE_V2_CMD_EIBRS_LFENCE || 1304 cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) && 1305 !IS_ENABLED(CONFIG_RETPOLINE)) { 1306 pr_err("%s selected but not compiled in. Switching to AUTO select\n", 1307 mitigation_options[i].option); 1308 return SPECTRE_V2_CMD_AUTO; 1309 } 1310 1311 if ((cmd == SPECTRE_V2_CMD_EIBRS || 1312 cmd == SPECTRE_V2_CMD_EIBRS_LFENCE || 1313 cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) && 1314 !boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { 1315 pr_err("%s selected but CPU doesn't have Enhanced or Automatic IBRS. Switching to AUTO select\n", 1316 mitigation_options[i].option); 1317 return SPECTRE_V2_CMD_AUTO; 1318 } 1319 1320 if ((cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE || 1321 cmd == SPECTRE_V2_CMD_EIBRS_LFENCE) && 1322 !boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) { 1323 pr_err("%s selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n", 1324 mitigation_options[i].option); 1325 return SPECTRE_V2_CMD_AUTO; 1326 } 1327 1328 if (cmd == SPECTRE_V2_CMD_IBRS && !IS_ENABLED(CONFIG_CPU_IBRS_ENTRY)) { 1329 pr_err("%s selected but not compiled in. Switching to AUTO select\n", 1330 mitigation_options[i].option); 1331 return SPECTRE_V2_CMD_AUTO; 1332 } 1333 1334 if (cmd == SPECTRE_V2_CMD_IBRS && boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) { 1335 pr_err("%s selected but not Intel CPU. Switching to AUTO select\n", 1336 mitigation_options[i].option); 1337 return SPECTRE_V2_CMD_AUTO; 1338 } 1339 1340 if (cmd == SPECTRE_V2_CMD_IBRS && !boot_cpu_has(X86_FEATURE_IBRS)) { 1341 pr_err("%s selected but CPU doesn't have IBRS. Switching to AUTO select\n", 1342 mitigation_options[i].option); 1343 return SPECTRE_V2_CMD_AUTO; 1344 } 1345 1346 if (cmd == SPECTRE_V2_CMD_IBRS && cpu_feature_enabled(X86_FEATURE_XENPV)) { 1347 pr_err("%s selected but running as XenPV guest. Switching to AUTO select\n", 1348 mitigation_options[i].option); 1349 return SPECTRE_V2_CMD_AUTO; 1350 } 1351 1352 spec_v2_print_cond(mitigation_options[i].option, 1353 mitigation_options[i].secure); 1354 return cmd; 1355 } 1356 1357 static enum spectre_v2_mitigation __init spectre_v2_select_retpoline(void) 1358 { 1359 if (!IS_ENABLED(CONFIG_RETPOLINE)) { 1360 pr_err("Kernel not compiled with retpoline; no mitigation available!"); 1361 return SPECTRE_V2_NONE; 1362 } 1363 1364 return SPECTRE_V2_RETPOLINE; 1365 } 1366 1367 /* Disable in-kernel use of non-RSB RET predictors */ 1368 static void __init spec_ctrl_disable_kernel_rrsba(void) 1369 { 1370 u64 ia32_cap; 1371 1372 if (!boot_cpu_has(X86_FEATURE_RRSBA_CTRL)) 1373 return; 1374 1375 ia32_cap = x86_read_arch_cap_msr(); 1376 1377 if (ia32_cap & ARCH_CAP_RRSBA) { 1378 x86_spec_ctrl_base |= SPEC_CTRL_RRSBA_DIS_S; 1379 update_spec_ctrl(x86_spec_ctrl_base); 1380 } 1381 } 1382 1383 static void __init spectre_v2_determine_rsb_fill_type_at_vmexit(enum spectre_v2_mitigation mode) 1384 { 1385 /* 1386 * Similar to context switches, there are two types of RSB attacks 1387 * after VM exit: 1388 * 1389 * 1) RSB underflow 1390 * 1391 * 2) Poisoned RSB entry 1392 * 1393 * When retpoline is enabled, both are mitigated by filling/clearing 1394 * the RSB. 1395 * 1396 * When IBRS is enabled, while #1 would be mitigated by the IBRS branch 1397 * prediction isolation protections, RSB still needs to be cleared 1398 * because of #2. Note that SMEP provides no protection here, unlike 1399 * user-space-poisoned RSB entries. 1400 * 1401 * eIBRS should protect against RSB poisoning, but if the EIBRS_PBRSB 1402 * bug is present then a LITE version of RSB protection is required, 1403 * just a single call needs to retire before a RET is executed. 1404 */ 1405 switch (mode) { 1406 case SPECTRE_V2_NONE: 1407 return; 1408 1409 case SPECTRE_V2_EIBRS_LFENCE: 1410 case SPECTRE_V2_EIBRS: 1411 if (boot_cpu_has_bug(X86_BUG_EIBRS_PBRSB)) { 1412 setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT_LITE); 1413 pr_info("Spectre v2 / PBRSB-eIBRS: Retire a single CALL on VMEXIT\n"); 1414 } 1415 return; 1416 1417 case SPECTRE_V2_EIBRS_RETPOLINE: 1418 case SPECTRE_V2_RETPOLINE: 1419 case SPECTRE_V2_LFENCE: 1420 case SPECTRE_V2_IBRS: 1421 setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT); 1422 pr_info("Spectre v2 / SpectreRSB : Filling RSB on VMEXIT\n"); 1423 return; 1424 } 1425 1426 pr_warn_once("Unknown Spectre v2 mode, disabling RSB mitigation at VM exit"); 1427 dump_stack(); 1428 } 1429 1430 static void __init spectre_v2_select_mitigation(void) 1431 { 1432 enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline(); 1433 enum spectre_v2_mitigation mode = SPECTRE_V2_NONE; 1434 1435 /* 1436 * If the CPU is not affected and the command line mode is NONE or AUTO 1437 * then nothing to do. 1438 */ 1439 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) && 1440 (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO)) 1441 return; 1442 1443 switch (cmd) { 1444 case SPECTRE_V2_CMD_NONE: 1445 return; 1446 1447 case SPECTRE_V2_CMD_FORCE: 1448 case SPECTRE_V2_CMD_AUTO: 1449 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) { 1450 mode = SPECTRE_V2_EIBRS; 1451 break; 1452 } 1453 1454 if (IS_ENABLED(CONFIG_CPU_IBRS_ENTRY) && 1455 boot_cpu_has_bug(X86_BUG_RETBLEED) && 1456 retbleed_cmd != RETBLEED_CMD_OFF && 1457 retbleed_cmd != RETBLEED_CMD_STUFF && 1458 boot_cpu_has(X86_FEATURE_IBRS) && 1459 boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { 1460 mode = SPECTRE_V2_IBRS; 1461 break; 1462 } 1463 1464 mode = spectre_v2_select_retpoline(); 1465 break; 1466 1467 case SPECTRE_V2_CMD_RETPOLINE_LFENCE: 1468 pr_err(SPECTRE_V2_LFENCE_MSG); 1469 mode = SPECTRE_V2_LFENCE; 1470 break; 1471 1472 case SPECTRE_V2_CMD_RETPOLINE_GENERIC: 1473 mode = SPECTRE_V2_RETPOLINE; 1474 break; 1475 1476 case SPECTRE_V2_CMD_RETPOLINE: 1477 mode = spectre_v2_select_retpoline(); 1478 break; 1479 1480 case SPECTRE_V2_CMD_IBRS: 1481 mode = SPECTRE_V2_IBRS; 1482 break; 1483 1484 case SPECTRE_V2_CMD_EIBRS: 1485 mode = SPECTRE_V2_EIBRS; 1486 break; 1487 1488 case SPECTRE_V2_CMD_EIBRS_LFENCE: 1489 mode = SPECTRE_V2_EIBRS_LFENCE; 1490 break; 1491 1492 case SPECTRE_V2_CMD_EIBRS_RETPOLINE: 1493 mode = SPECTRE_V2_EIBRS_RETPOLINE; 1494 break; 1495 } 1496 1497 if (mode == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) 1498 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG); 1499 1500 if (spectre_v2_in_ibrs_mode(mode)) { 1501 if (boot_cpu_has(X86_FEATURE_AUTOIBRS)) { 1502 msr_set_bit(MSR_EFER, _EFER_AUTOIBRS); 1503 } else { 1504 x86_spec_ctrl_base |= SPEC_CTRL_IBRS; 1505 update_spec_ctrl(x86_spec_ctrl_base); 1506 } 1507 } 1508 1509 switch (mode) { 1510 case SPECTRE_V2_NONE: 1511 case SPECTRE_V2_EIBRS: 1512 break; 1513 1514 case SPECTRE_V2_IBRS: 1515 setup_force_cpu_cap(X86_FEATURE_KERNEL_IBRS); 1516 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) 1517 pr_warn(SPECTRE_V2_IBRS_PERF_MSG); 1518 break; 1519 1520 case SPECTRE_V2_LFENCE: 1521 case SPECTRE_V2_EIBRS_LFENCE: 1522 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_LFENCE); 1523 fallthrough; 1524 1525 case SPECTRE_V2_RETPOLINE: 1526 case SPECTRE_V2_EIBRS_RETPOLINE: 1527 setup_force_cpu_cap(X86_FEATURE_RETPOLINE); 1528 break; 1529 } 1530 1531 /* 1532 * Disable alternate RSB predictions in kernel when indirect CALLs and 1533 * JMPs gets protection against BHI and Intramode-BTI, but RET 1534 * prediction from a non-RSB predictor is still a risk. 1535 */ 1536 if (mode == SPECTRE_V2_EIBRS_LFENCE || 1537 mode == SPECTRE_V2_EIBRS_RETPOLINE || 1538 mode == SPECTRE_V2_RETPOLINE) 1539 spec_ctrl_disable_kernel_rrsba(); 1540 1541 spectre_v2_enabled = mode; 1542 pr_info("%s\n", spectre_v2_strings[mode]); 1543 1544 /* 1545 * If Spectre v2 protection has been enabled, fill the RSB during a 1546 * context switch. In general there are two types of RSB attacks 1547 * across context switches, for which the CALLs/RETs may be unbalanced. 1548 * 1549 * 1) RSB underflow 1550 * 1551 * Some Intel parts have "bottomless RSB". When the RSB is empty, 1552 * speculated return targets may come from the branch predictor, 1553 * which could have a user-poisoned BTB or BHB entry. 1554 * 1555 * AMD has it even worse: *all* returns are speculated from the BTB, 1556 * regardless of the state of the RSB. 1557 * 1558 * When IBRS or eIBRS is enabled, the "user -> kernel" attack 1559 * scenario is mitigated by the IBRS branch prediction isolation 1560 * properties, so the RSB buffer filling wouldn't be necessary to 1561 * protect against this type of attack. 1562 * 1563 * The "user -> user" attack scenario is mitigated by RSB filling. 1564 * 1565 * 2) Poisoned RSB entry 1566 * 1567 * If the 'next' in-kernel return stack is shorter than 'prev', 1568 * 'next' could be tricked into speculating with a user-poisoned RSB 1569 * entry. 1570 * 1571 * The "user -> kernel" attack scenario is mitigated by SMEP and 1572 * eIBRS. 1573 * 1574 * The "user -> user" scenario, also known as SpectreBHB, requires 1575 * RSB clearing. 1576 * 1577 * So to mitigate all cases, unconditionally fill RSB on context 1578 * switches. 1579 * 1580 * FIXME: Is this pointless for retbleed-affected AMD? 1581 */ 1582 setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW); 1583 pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n"); 1584 1585 spectre_v2_determine_rsb_fill_type_at_vmexit(mode); 1586 1587 /* 1588 * Retpoline protects the kernel, but doesn't protect firmware. IBRS 1589 * and Enhanced IBRS protect firmware too, so enable IBRS around 1590 * firmware calls only when IBRS / Enhanced / Automatic IBRS aren't 1591 * otherwise enabled. 1592 * 1593 * Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because 1594 * the user might select retpoline on the kernel command line and if 1595 * the CPU supports Enhanced IBRS, kernel might un-intentionally not 1596 * enable IBRS around firmware calls. 1597 */ 1598 if (boot_cpu_has_bug(X86_BUG_RETBLEED) && 1599 boot_cpu_has(X86_FEATURE_IBPB) && 1600 (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || 1601 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)) { 1602 1603 if (retbleed_cmd != RETBLEED_CMD_IBPB) { 1604 setup_force_cpu_cap(X86_FEATURE_USE_IBPB_FW); 1605 pr_info("Enabling Speculation Barrier for firmware calls\n"); 1606 } 1607 1608 } else if (boot_cpu_has(X86_FEATURE_IBRS) && !spectre_v2_in_ibrs_mode(mode)) { 1609 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW); 1610 pr_info("Enabling Restricted Speculation for firmware calls\n"); 1611 } 1612 1613 /* Set up IBPB and STIBP depending on the general spectre V2 command */ 1614 spectre_v2_cmd = cmd; 1615 } 1616 1617 static void update_stibp_msr(void * __unused) 1618 { 1619 u64 val = spec_ctrl_current() | (x86_spec_ctrl_base & SPEC_CTRL_STIBP); 1620 update_spec_ctrl(val); 1621 } 1622 1623 /* Update x86_spec_ctrl_base in case SMT state changed. */ 1624 static void update_stibp_strict(void) 1625 { 1626 u64 mask = x86_spec_ctrl_base & ~SPEC_CTRL_STIBP; 1627 1628 if (sched_smt_active()) 1629 mask |= SPEC_CTRL_STIBP; 1630 1631 if (mask == x86_spec_ctrl_base) 1632 return; 1633 1634 pr_info("Update user space SMT mitigation: STIBP %s\n", 1635 mask & SPEC_CTRL_STIBP ? "always-on" : "off"); 1636 x86_spec_ctrl_base = mask; 1637 on_each_cpu(update_stibp_msr, NULL, 1); 1638 } 1639 1640 /* Update the static key controlling the evaluation of TIF_SPEC_IB */ 1641 static void update_indir_branch_cond(void) 1642 { 1643 if (sched_smt_active()) 1644 static_branch_enable(&switch_to_cond_stibp); 1645 else 1646 static_branch_disable(&switch_to_cond_stibp); 1647 } 1648 1649 #undef pr_fmt 1650 #define pr_fmt(fmt) fmt 1651 1652 /* Update the static key controlling the MDS CPU buffer clear in idle */ 1653 static void update_mds_branch_idle(void) 1654 { 1655 u64 ia32_cap = x86_read_arch_cap_msr(); 1656 1657 /* 1658 * Enable the idle clearing if SMT is active on CPUs which are 1659 * affected only by MSBDS and not any other MDS variant. 1660 * 1661 * The other variants cannot be mitigated when SMT is enabled, so 1662 * clearing the buffers on idle just to prevent the Store Buffer 1663 * repartitioning leak would be a window dressing exercise. 1664 */ 1665 if (!boot_cpu_has_bug(X86_BUG_MSBDS_ONLY)) 1666 return; 1667 1668 if (sched_smt_active()) { 1669 static_branch_enable(&mds_idle_clear); 1670 } else if (mmio_mitigation == MMIO_MITIGATION_OFF || 1671 (ia32_cap & ARCH_CAP_FBSDP_NO)) { 1672 static_branch_disable(&mds_idle_clear); 1673 } 1674 } 1675 1676 #define MDS_MSG_SMT "MDS CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/mds.html for more details.\n" 1677 #define TAA_MSG_SMT "TAA CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/tsx_async_abort.html for more details.\n" 1678 #define MMIO_MSG_SMT "MMIO Stale Data CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/processor_mmio_stale_data.html for more details.\n" 1679 1680 void cpu_bugs_smt_update(void) 1681 { 1682 mutex_lock(&spec_ctrl_mutex); 1683 1684 if (sched_smt_active() && unprivileged_ebpf_enabled() && 1685 spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE) 1686 pr_warn_once(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG); 1687 1688 switch (spectre_v2_user_stibp) { 1689 case SPECTRE_V2_USER_NONE: 1690 break; 1691 case SPECTRE_V2_USER_STRICT: 1692 case SPECTRE_V2_USER_STRICT_PREFERRED: 1693 update_stibp_strict(); 1694 break; 1695 case SPECTRE_V2_USER_PRCTL: 1696 case SPECTRE_V2_USER_SECCOMP: 1697 update_indir_branch_cond(); 1698 break; 1699 } 1700 1701 switch (mds_mitigation) { 1702 case MDS_MITIGATION_FULL: 1703 case MDS_MITIGATION_VMWERV: 1704 if (sched_smt_active() && !boot_cpu_has(X86_BUG_MSBDS_ONLY)) 1705 pr_warn_once(MDS_MSG_SMT); 1706 update_mds_branch_idle(); 1707 break; 1708 case MDS_MITIGATION_OFF: 1709 break; 1710 } 1711 1712 switch (taa_mitigation) { 1713 case TAA_MITIGATION_VERW: 1714 case TAA_MITIGATION_UCODE_NEEDED: 1715 if (sched_smt_active()) 1716 pr_warn_once(TAA_MSG_SMT); 1717 break; 1718 case TAA_MITIGATION_TSX_DISABLED: 1719 case TAA_MITIGATION_OFF: 1720 break; 1721 } 1722 1723 switch (mmio_mitigation) { 1724 case MMIO_MITIGATION_VERW: 1725 case MMIO_MITIGATION_UCODE_NEEDED: 1726 if (sched_smt_active()) 1727 pr_warn_once(MMIO_MSG_SMT); 1728 break; 1729 case MMIO_MITIGATION_OFF: 1730 break; 1731 } 1732 1733 mutex_unlock(&spec_ctrl_mutex); 1734 } 1735 1736 #undef pr_fmt 1737 #define pr_fmt(fmt) "Speculative Store Bypass: " fmt 1738 1739 static enum ssb_mitigation ssb_mode __ro_after_init = SPEC_STORE_BYPASS_NONE; 1740 1741 /* The kernel command line selection */ 1742 enum ssb_mitigation_cmd { 1743 SPEC_STORE_BYPASS_CMD_NONE, 1744 SPEC_STORE_BYPASS_CMD_AUTO, 1745 SPEC_STORE_BYPASS_CMD_ON, 1746 SPEC_STORE_BYPASS_CMD_PRCTL, 1747 SPEC_STORE_BYPASS_CMD_SECCOMP, 1748 }; 1749 1750 static const char * const ssb_strings[] = { 1751 [SPEC_STORE_BYPASS_NONE] = "Vulnerable", 1752 [SPEC_STORE_BYPASS_DISABLE] = "Mitigation: Speculative Store Bypass disabled", 1753 [SPEC_STORE_BYPASS_PRCTL] = "Mitigation: Speculative Store Bypass disabled via prctl", 1754 [SPEC_STORE_BYPASS_SECCOMP] = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp", 1755 }; 1756 1757 static const struct { 1758 const char *option; 1759 enum ssb_mitigation_cmd cmd; 1760 } ssb_mitigation_options[] __initconst = { 1761 { "auto", SPEC_STORE_BYPASS_CMD_AUTO }, /* Platform decides */ 1762 { "on", SPEC_STORE_BYPASS_CMD_ON }, /* Disable Speculative Store Bypass */ 1763 { "off", SPEC_STORE_BYPASS_CMD_NONE }, /* Don't touch Speculative Store Bypass */ 1764 { "prctl", SPEC_STORE_BYPASS_CMD_PRCTL }, /* Disable Speculative Store Bypass via prctl */ 1765 { "seccomp", SPEC_STORE_BYPASS_CMD_SECCOMP }, /* Disable Speculative Store Bypass via prctl and seccomp */ 1766 }; 1767 1768 static enum ssb_mitigation_cmd __init ssb_parse_cmdline(void) 1769 { 1770 enum ssb_mitigation_cmd cmd = SPEC_STORE_BYPASS_CMD_AUTO; 1771 char arg[20]; 1772 int ret, i; 1773 1774 if (cmdline_find_option_bool(boot_command_line, "nospec_store_bypass_disable") || 1775 cpu_mitigations_off()) { 1776 return SPEC_STORE_BYPASS_CMD_NONE; 1777 } else { 1778 ret = cmdline_find_option(boot_command_line, "spec_store_bypass_disable", 1779 arg, sizeof(arg)); 1780 if (ret < 0) 1781 return SPEC_STORE_BYPASS_CMD_AUTO; 1782 1783 for (i = 0; i < ARRAY_SIZE(ssb_mitigation_options); i++) { 1784 if (!match_option(arg, ret, ssb_mitigation_options[i].option)) 1785 continue; 1786 1787 cmd = ssb_mitigation_options[i].cmd; 1788 break; 1789 } 1790 1791 if (i >= ARRAY_SIZE(ssb_mitigation_options)) { 1792 pr_err("unknown option (%s). Switching to AUTO select\n", arg); 1793 return SPEC_STORE_BYPASS_CMD_AUTO; 1794 } 1795 } 1796 1797 return cmd; 1798 } 1799 1800 static enum ssb_mitigation __init __ssb_select_mitigation(void) 1801 { 1802 enum ssb_mitigation mode = SPEC_STORE_BYPASS_NONE; 1803 enum ssb_mitigation_cmd cmd; 1804 1805 if (!boot_cpu_has(X86_FEATURE_SSBD)) 1806 return mode; 1807 1808 cmd = ssb_parse_cmdline(); 1809 if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS) && 1810 (cmd == SPEC_STORE_BYPASS_CMD_NONE || 1811 cmd == SPEC_STORE_BYPASS_CMD_AUTO)) 1812 return mode; 1813 1814 switch (cmd) { 1815 case SPEC_STORE_BYPASS_CMD_SECCOMP: 1816 /* 1817 * Choose prctl+seccomp as the default mode if seccomp is 1818 * enabled. 1819 */ 1820 if (IS_ENABLED(CONFIG_SECCOMP)) 1821 mode = SPEC_STORE_BYPASS_SECCOMP; 1822 else 1823 mode = SPEC_STORE_BYPASS_PRCTL; 1824 break; 1825 case SPEC_STORE_BYPASS_CMD_ON: 1826 mode = SPEC_STORE_BYPASS_DISABLE; 1827 break; 1828 case SPEC_STORE_BYPASS_CMD_AUTO: 1829 case SPEC_STORE_BYPASS_CMD_PRCTL: 1830 mode = SPEC_STORE_BYPASS_PRCTL; 1831 break; 1832 case SPEC_STORE_BYPASS_CMD_NONE: 1833 break; 1834 } 1835 1836 /* 1837 * We have three CPU feature flags that are in play here: 1838 * - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible. 1839 * - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass 1840 * - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation 1841 */ 1842 if (mode == SPEC_STORE_BYPASS_DISABLE) { 1843 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE); 1844 /* 1845 * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may 1846 * use a completely different MSR and bit dependent on family. 1847 */ 1848 if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) && 1849 !static_cpu_has(X86_FEATURE_AMD_SSBD)) { 1850 x86_amd_ssb_disable(); 1851 } else { 1852 x86_spec_ctrl_base |= SPEC_CTRL_SSBD; 1853 update_spec_ctrl(x86_spec_ctrl_base); 1854 } 1855 } 1856 1857 return mode; 1858 } 1859 1860 static void ssb_select_mitigation(void) 1861 { 1862 ssb_mode = __ssb_select_mitigation(); 1863 1864 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS)) 1865 pr_info("%s\n", ssb_strings[ssb_mode]); 1866 } 1867 1868 #undef pr_fmt 1869 #define pr_fmt(fmt) "Speculation prctl: " fmt 1870 1871 static void task_update_spec_tif(struct task_struct *tsk) 1872 { 1873 /* Force the update of the real TIF bits */ 1874 set_tsk_thread_flag(tsk, TIF_SPEC_FORCE_UPDATE); 1875 1876 /* 1877 * Immediately update the speculation control MSRs for the current 1878 * task, but for a non-current task delay setting the CPU 1879 * mitigation until it is scheduled next. 1880 * 1881 * This can only happen for SECCOMP mitigation. For PRCTL it's 1882 * always the current task. 1883 */ 1884 if (tsk == current) 1885 speculation_ctrl_update_current(); 1886 } 1887 1888 static int l1d_flush_prctl_set(struct task_struct *task, unsigned long ctrl) 1889 { 1890 1891 if (!static_branch_unlikely(&switch_mm_cond_l1d_flush)) 1892 return -EPERM; 1893 1894 switch (ctrl) { 1895 case PR_SPEC_ENABLE: 1896 set_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH); 1897 return 0; 1898 case PR_SPEC_DISABLE: 1899 clear_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH); 1900 return 0; 1901 default: 1902 return -ERANGE; 1903 } 1904 } 1905 1906 static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl) 1907 { 1908 if (ssb_mode != SPEC_STORE_BYPASS_PRCTL && 1909 ssb_mode != SPEC_STORE_BYPASS_SECCOMP) 1910 return -ENXIO; 1911 1912 switch (ctrl) { 1913 case PR_SPEC_ENABLE: 1914 /* If speculation is force disabled, enable is not allowed */ 1915 if (task_spec_ssb_force_disable(task)) 1916 return -EPERM; 1917 task_clear_spec_ssb_disable(task); 1918 task_clear_spec_ssb_noexec(task); 1919 task_update_spec_tif(task); 1920 break; 1921 case PR_SPEC_DISABLE: 1922 task_set_spec_ssb_disable(task); 1923 task_clear_spec_ssb_noexec(task); 1924 task_update_spec_tif(task); 1925 break; 1926 case PR_SPEC_FORCE_DISABLE: 1927 task_set_spec_ssb_disable(task); 1928 task_set_spec_ssb_force_disable(task); 1929 task_clear_spec_ssb_noexec(task); 1930 task_update_spec_tif(task); 1931 break; 1932 case PR_SPEC_DISABLE_NOEXEC: 1933 if (task_spec_ssb_force_disable(task)) 1934 return -EPERM; 1935 task_set_spec_ssb_disable(task); 1936 task_set_spec_ssb_noexec(task); 1937 task_update_spec_tif(task); 1938 break; 1939 default: 1940 return -ERANGE; 1941 } 1942 return 0; 1943 } 1944 1945 static bool is_spec_ib_user_controlled(void) 1946 { 1947 return spectre_v2_user_ibpb == SPECTRE_V2_USER_PRCTL || 1948 spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP || 1949 spectre_v2_user_stibp == SPECTRE_V2_USER_PRCTL || 1950 spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP; 1951 } 1952 1953 static int ib_prctl_set(struct task_struct *task, unsigned long ctrl) 1954 { 1955 switch (ctrl) { 1956 case PR_SPEC_ENABLE: 1957 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && 1958 spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) 1959 return 0; 1960 1961 /* 1962 * With strict mode for both IBPB and STIBP, the instruction 1963 * code paths avoid checking this task flag and instead, 1964 * unconditionally run the instruction. However, STIBP and IBPB 1965 * are independent and either can be set to conditionally 1966 * enabled regardless of the mode of the other. 1967 * 1968 * If either is set to conditional, allow the task flag to be 1969 * updated, unless it was force-disabled by a previous prctl 1970 * call. Currently, this is possible on an AMD CPU which has the 1971 * feature X86_FEATURE_AMD_STIBP_ALWAYS_ON. In this case, if the 1972 * kernel is booted with 'spectre_v2_user=seccomp', then 1973 * spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP and 1974 * spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED. 1975 */ 1976 if (!is_spec_ib_user_controlled() || 1977 task_spec_ib_force_disable(task)) 1978 return -EPERM; 1979 1980 task_clear_spec_ib_disable(task); 1981 task_update_spec_tif(task); 1982 break; 1983 case PR_SPEC_DISABLE: 1984 case PR_SPEC_FORCE_DISABLE: 1985 /* 1986 * Indirect branch speculation is always allowed when 1987 * mitigation is force disabled. 1988 */ 1989 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && 1990 spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) 1991 return -EPERM; 1992 1993 if (!is_spec_ib_user_controlled()) 1994 return 0; 1995 1996 task_set_spec_ib_disable(task); 1997 if (ctrl == PR_SPEC_FORCE_DISABLE) 1998 task_set_spec_ib_force_disable(task); 1999 task_update_spec_tif(task); 2000 if (task == current) 2001 indirect_branch_prediction_barrier(); 2002 break; 2003 default: 2004 return -ERANGE; 2005 } 2006 return 0; 2007 } 2008 2009 int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which, 2010 unsigned long ctrl) 2011 { 2012 switch (which) { 2013 case PR_SPEC_STORE_BYPASS: 2014 return ssb_prctl_set(task, ctrl); 2015 case PR_SPEC_INDIRECT_BRANCH: 2016 return ib_prctl_set(task, ctrl); 2017 case PR_SPEC_L1D_FLUSH: 2018 return l1d_flush_prctl_set(task, ctrl); 2019 default: 2020 return -ENODEV; 2021 } 2022 } 2023 2024 #ifdef CONFIG_SECCOMP 2025 void arch_seccomp_spec_mitigate(struct task_struct *task) 2026 { 2027 if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP) 2028 ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE); 2029 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP || 2030 spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP) 2031 ib_prctl_set(task, PR_SPEC_FORCE_DISABLE); 2032 } 2033 #endif 2034 2035 static int l1d_flush_prctl_get(struct task_struct *task) 2036 { 2037 if (!static_branch_unlikely(&switch_mm_cond_l1d_flush)) 2038 return PR_SPEC_FORCE_DISABLE; 2039 2040 if (test_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH)) 2041 return PR_SPEC_PRCTL | PR_SPEC_ENABLE; 2042 else 2043 return PR_SPEC_PRCTL | PR_SPEC_DISABLE; 2044 } 2045 2046 static int ssb_prctl_get(struct task_struct *task) 2047 { 2048 switch (ssb_mode) { 2049 case SPEC_STORE_BYPASS_DISABLE: 2050 return PR_SPEC_DISABLE; 2051 case SPEC_STORE_BYPASS_SECCOMP: 2052 case SPEC_STORE_BYPASS_PRCTL: 2053 if (task_spec_ssb_force_disable(task)) 2054 return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE; 2055 if (task_spec_ssb_noexec(task)) 2056 return PR_SPEC_PRCTL | PR_SPEC_DISABLE_NOEXEC; 2057 if (task_spec_ssb_disable(task)) 2058 return PR_SPEC_PRCTL | PR_SPEC_DISABLE; 2059 return PR_SPEC_PRCTL | PR_SPEC_ENABLE; 2060 default: 2061 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS)) 2062 return PR_SPEC_ENABLE; 2063 return PR_SPEC_NOT_AFFECTED; 2064 } 2065 } 2066 2067 static int ib_prctl_get(struct task_struct *task) 2068 { 2069 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2)) 2070 return PR_SPEC_NOT_AFFECTED; 2071 2072 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && 2073 spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) 2074 return PR_SPEC_ENABLE; 2075 else if (is_spec_ib_user_controlled()) { 2076 if (task_spec_ib_force_disable(task)) 2077 return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE; 2078 if (task_spec_ib_disable(task)) 2079 return PR_SPEC_PRCTL | PR_SPEC_DISABLE; 2080 return PR_SPEC_PRCTL | PR_SPEC_ENABLE; 2081 } else if (spectre_v2_user_ibpb == SPECTRE_V2_USER_STRICT || 2082 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT || 2083 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED) 2084 return PR_SPEC_DISABLE; 2085 else 2086 return PR_SPEC_NOT_AFFECTED; 2087 } 2088 2089 int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which) 2090 { 2091 switch (which) { 2092 case PR_SPEC_STORE_BYPASS: 2093 return ssb_prctl_get(task); 2094 case PR_SPEC_INDIRECT_BRANCH: 2095 return ib_prctl_get(task); 2096 case PR_SPEC_L1D_FLUSH: 2097 return l1d_flush_prctl_get(task); 2098 default: 2099 return -ENODEV; 2100 } 2101 } 2102 2103 void x86_spec_ctrl_setup_ap(void) 2104 { 2105 if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) 2106 update_spec_ctrl(x86_spec_ctrl_base); 2107 2108 if (ssb_mode == SPEC_STORE_BYPASS_DISABLE) 2109 x86_amd_ssb_disable(); 2110 } 2111 2112 bool itlb_multihit_kvm_mitigation; 2113 EXPORT_SYMBOL_GPL(itlb_multihit_kvm_mitigation); 2114 2115 #undef pr_fmt 2116 #define pr_fmt(fmt) "L1TF: " fmt 2117 2118 /* Default mitigation for L1TF-affected CPUs */ 2119 enum l1tf_mitigations l1tf_mitigation __ro_after_init = L1TF_MITIGATION_FLUSH; 2120 #if IS_ENABLED(CONFIG_KVM_INTEL) 2121 EXPORT_SYMBOL_GPL(l1tf_mitigation); 2122 #endif 2123 enum vmx_l1d_flush_state l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO; 2124 EXPORT_SYMBOL_GPL(l1tf_vmx_mitigation); 2125 2126 /* 2127 * These CPUs all support 44bits physical address space internally in the 2128 * cache but CPUID can report a smaller number of physical address bits. 2129 * 2130 * The L1TF mitigation uses the top most address bit for the inversion of 2131 * non present PTEs. When the installed memory reaches into the top most 2132 * address bit due to memory holes, which has been observed on machines 2133 * which report 36bits physical address bits and have 32G RAM installed, 2134 * then the mitigation range check in l1tf_select_mitigation() triggers. 2135 * This is a false positive because the mitigation is still possible due to 2136 * the fact that the cache uses 44bit internally. Use the cache bits 2137 * instead of the reported physical bits and adjust them on the affected 2138 * machines to 44bit if the reported bits are less than 44. 2139 */ 2140 static void override_cache_bits(struct cpuinfo_x86 *c) 2141 { 2142 if (c->x86 != 6) 2143 return; 2144 2145 switch (c->x86_model) { 2146 case INTEL_FAM6_NEHALEM: 2147 case INTEL_FAM6_WESTMERE: 2148 case INTEL_FAM6_SANDYBRIDGE: 2149 case INTEL_FAM6_IVYBRIDGE: 2150 case INTEL_FAM6_HASWELL: 2151 case INTEL_FAM6_HASWELL_L: 2152 case INTEL_FAM6_HASWELL_G: 2153 case INTEL_FAM6_BROADWELL: 2154 case INTEL_FAM6_BROADWELL_G: 2155 case INTEL_FAM6_SKYLAKE_L: 2156 case INTEL_FAM6_SKYLAKE: 2157 case INTEL_FAM6_KABYLAKE_L: 2158 case INTEL_FAM6_KABYLAKE: 2159 if (c->x86_cache_bits < 44) 2160 c->x86_cache_bits = 44; 2161 break; 2162 } 2163 } 2164 2165 static void __init l1tf_select_mitigation(void) 2166 { 2167 u64 half_pa; 2168 2169 if (!boot_cpu_has_bug(X86_BUG_L1TF)) 2170 return; 2171 2172 if (cpu_mitigations_off()) 2173 l1tf_mitigation = L1TF_MITIGATION_OFF; 2174 else if (cpu_mitigations_auto_nosmt()) 2175 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT; 2176 2177 override_cache_bits(&boot_cpu_data); 2178 2179 switch (l1tf_mitigation) { 2180 case L1TF_MITIGATION_OFF: 2181 case L1TF_MITIGATION_FLUSH_NOWARN: 2182 case L1TF_MITIGATION_FLUSH: 2183 break; 2184 case L1TF_MITIGATION_FLUSH_NOSMT: 2185 case L1TF_MITIGATION_FULL: 2186 cpu_smt_disable(false); 2187 break; 2188 case L1TF_MITIGATION_FULL_FORCE: 2189 cpu_smt_disable(true); 2190 break; 2191 } 2192 2193 #if CONFIG_PGTABLE_LEVELS == 2 2194 pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n"); 2195 return; 2196 #endif 2197 2198 half_pa = (u64)l1tf_pfn_limit() << PAGE_SHIFT; 2199 if (l1tf_mitigation != L1TF_MITIGATION_OFF && 2200 e820__mapped_any(half_pa, ULLONG_MAX - half_pa, E820_TYPE_RAM)) { 2201 pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n"); 2202 pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n", 2203 half_pa); 2204 pr_info("However, doing so will make a part of your RAM unusable.\n"); 2205 pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html might help you decide.\n"); 2206 return; 2207 } 2208 2209 setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV); 2210 } 2211 2212 static int __init l1tf_cmdline(char *str) 2213 { 2214 if (!boot_cpu_has_bug(X86_BUG_L1TF)) 2215 return 0; 2216 2217 if (!str) 2218 return -EINVAL; 2219 2220 if (!strcmp(str, "off")) 2221 l1tf_mitigation = L1TF_MITIGATION_OFF; 2222 else if (!strcmp(str, "flush,nowarn")) 2223 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOWARN; 2224 else if (!strcmp(str, "flush")) 2225 l1tf_mitigation = L1TF_MITIGATION_FLUSH; 2226 else if (!strcmp(str, "flush,nosmt")) 2227 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT; 2228 else if (!strcmp(str, "full")) 2229 l1tf_mitigation = L1TF_MITIGATION_FULL; 2230 else if (!strcmp(str, "full,force")) 2231 l1tf_mitigation = L1TF_MITIGATION_FULL_FORCE; 2232 2233 return 0; 2234 } 2235 early_param("l1tf", l1tf_cmdline); 2236 2237 #undef pr_fmt 2238 #define pr_fmt(fmt) fmt 2239 2240 #ifdef CONFIG_SYSFS 2241 2242 #define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion" 2243 2244 #if IS_ENABLED(CONFIG_KVM_INTEL) 2245 static const char * const l1tf_vmx_states[] = { 2246 [VMENTER_L1D_FLUSH_AUTO] = "auto", 2247 [VMENTER_L1D_FLUSH_NEVER] = "vulnerable", 2248 [VMENTER_L1D_FLUSH_COND] = "conditional cache flushes", 2249 [VMENTER_L1D_FLUSH_ALWAYS] = "cache flushes", 2250 [VMENTER_L1D_FLUSH_EPT_DISABLED] = "EPT disabled", 2251 [VMENTER_L1D_FLUSH_NOT_REQUIRED] = "flush not necessary" 2252 }; 2253 2254 static ssize_t l1tf_show_state(char *buf) 2255 { 2256 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO) 2257 return sysfs_emit(buf, "%s\n", L1TF_DEFAULT_MSG); 2258 2259 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED || 2260 (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER && 2261 sched_smt_active())) { 2262 return sysfs_emit(buf, "%s; VMX: %s\n", L1TF_DEFAULT_MSG, 2263 l1tf_vmx_states[l1tf_vmx_mitigation]); 2264 } 2265 2266 return sysfs_emit(buf, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG, 2267 l1tf_vmx_states[l1tf_vmx_mitigation], 2268 sched_smt_active() ? "vulnerable" : "disabled"); 2269 } 2270 2271 static ssize_t itlb_multihit_show_state(char *buf) 2272 { 2273 if (!boot_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) || 2274 !boot_cpu_has(X86_FEATURE_VMX)) 2275 return sysfs_emit(buf, "KVM: Mitigation: VMX unsupported\n"); 2276 else if (!(cr4_read_shadow() & X86_CR4_VMXE)) 2277 return sysfs_emit(buf, "KVM: Mitigation: VMX disabled\n"); 2278 else if (itlb_multihit_kvm_mitigation) 2279 return sysfs_emit(buf, "KVM: Mitigation: Split huge pages\n"); 2280 else 2281 return sysfs_emit(buf, "KVM: Vulnerable\n"); 2282 } 2283 #else 2284 static ssize_t l1tf_show_state(char *buf) 2285 { 2286 return sysfs_emit(buf, "%s\n", L1TF_DEFAULT_MSG); 2287 } 2288 2289 static ssize_t itlb_multihit_show_state(char *buf) 2290 { 2291 return sysfs_emit(buf, "Processor vulnerable\n"); 2292 } 2293 #endif 2294 2295 static ssize_t mds_show_state(char *buf) 2296 { 2297 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 2298 return sysfs_emit(buf, "%s; SMT Host state unknown\n", 2299 mds_strings[mds_mitigation]); 2300 } 2301 2302 if (boot_cpu_has(X86_BUG_MSBDS_ONLY)) { 2303 return sysfs_emit(buf, "%s; SMT %s\n", mds_strings[mds_mitigation], 2304 (mds_mitigation == MDS_MITIGATION_OFF ? "vulnerable" : 2305 sched_smt_active() ? "mitigated" : "disabled")); 2306 } 2307 2308 return sysfs_emit(buf, "%s; SMT %s\n", mds_strings[mds_mitigation], 2309 sched_smt_active() ? "vulnerable" : "disabled"); 2310 } 2311 2312 static ssize_t tsx_async_abort_show_state(char *buf) 2313 { 2314 if ((taa_mitigation == TAA_MITIGATION_TSX_DISABLED) || 2315 (taa_mitigation == TAA_MITIGATION_OFF)) 2316 return sysfs_emit(buf, "%s\n", taa_strings[taa_mitigation]); 2317 2318 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 2319 return sysfs_emit(buf, "%s; SMT Host state unknown\n", 2320 taa_strings[taa_mitigation]); 2321 } 2322 2323 return sysfs_emit(buf, "%s; SMT %s\n", taa_strings[taa_mitigation], 2324 sched_smt_active() ? "vulnerable" : "disabled"); 2325 } 2326 2327 static ssize_t mmio_stale_data_show_state(char *buf) 2328 { 2329 if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN)) 2330 return sysfs_emit(buf, "Unknown: No mitigations\n"); 2331 2332 if (mmio_mitigation == MMIO_MITIGATION_OFF) 2333 return sysfs_emit(buf, "%s\n", mmio_strings[mmio_mitigation]); 2334 2335 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 2336 return sysfs_emit(buf, "%s; SMT Host state unknown\n", 2337 mmio_strings[mmio_mitigation]); 2338 } 2339 2340 return sysfs_emit(buf, "%s; SMT %s\n", mmio_strings[mmio_mitigation], 2341 sched_smt_active() ? "vulnerable" : "disabled"); 2342 } 2343 2344 static char *stibp_state(void) 2345 { 2346 if (spectre_v2_in_eibrs_mode(spectre_v2_enabled)) 2347 return ""; 2348 2349 switch (spectre_v2_user_stibp) { 2350 case SPECTRE_V2_USER_NONE: 2351 return ", STIBP: disabled"; 2352 case SPECTRE_V2_USER_STRICT: 2353 return ", STIBP: forced"; 2354 case SPECTRE_V2_USER_STRICT_PREFERRED: 2355 return ", STIBP: always-on"; 2356 case SPECTRE_V2_USER_PRCTL: 2357 case SPECTRE_V2_USER_SECCOMP: 2358 if (static_key_enabled(&switch_to_cond_stibp)) 2359 return ", STIBP: conditional"; 2360 } 2361 return ""; 2362 } 2363 2364 static char *ibpb_state(void) 2365 { 2366 if (boot_cpu_has(X86_FEATURE_IBPB)) { 2367 if (static_key_enabled(&switch_mm_always_ibpb)) 2368 return ", IBPB: always-on"; 2369 if (static_key_enabled(&switch_mm_cond_ibpb)) 2370 return ", IBPB: conditional"; 2371 return ", IBPB: disabled"; 2372 } 2373 return ""; 2374 } 2375 2376 static char *pbrsb_eibrs_state(void) 2377 { 2378 if (boot_cpu_has_bug(X86_BUG_EIBRS_PBRSB)) { 2379 if (boot_cpu_has(X86_FEATURE_RSB_VMEXIT_LITE) || 2380 boot_cpu_has(X86_FEATURE_RSB_VMEXIT)) 2381 return ", PBRSB-eIBRS: SW sequence"; 2382 else 2383 return ", PBRSB-eIBRS: Vulnerable"; 2384 } else { 2385 return ", PBRSB-eIBRS: Not affected"; 2386 } 2387 } 2388 2389 static ssize_t spectre_v2_show_state(char *buf) 2390 { 2391 if (spectre_v2_enabled == SPECTRE_V2_LFENCE) 2392 return sysfs_emit(buf, "Vulnerable: LFENCE\n"); 2393 2394 if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled()) 2395 return sysfs_emit(buf, "Vulnerable: eIBRS with unprivileged eBPF\n"); 2396 2397 if (sched_smt_active() && unprivileged_ebpf_enabled() && 2398 spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE) 2399 return sysfs_emit(buf, "Vulnerable: eIBRS+LFENCE with unprivileged eBPF and SMT\n"); 2400 2401 return sysfs_emit(buf, "%s%s%s%s%s%s%s\n", 2402 spectre_v2_strings[spectre_v2_enabled], 2403 ibpb_state(), 2404 boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "", 2405 stibp_state(), 2406 boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "", 2407 pbrsb_eibrs_state(), 2408 spectre_v2_module_string()); 2409 } 2410 2411 static ssize_t srbds_show_state(char *buf) 2412 { 2413 return sysfs_emit(buf, "%s\n", srbds_strings[srbds_mitigation]); 2414 } 2415 2416 static ssize_t retbleed_show_state(char *buf) 2417 { 2418 if (retbleed_mitigation == RETBLEED_MITIGATION_UNRET || 2419 retbleed_mitigation == RETBLEED_MITIGATION_IBPB) { 2420 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && 2421 boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) 2422 return sysfs_emit(buf, "Vulnerable: untrained return thunk / IBPB on non-AMD based uarch\n"); 2423 2424 return sysfs_emit(buf, "%s; SMT %s\n", retbleed_strings[retbleed_mitigation], 2425 !sched_smt_active() ? "disabled" : 2426 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT || 2427 spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED ? 2428 "enabled with STIBP protection" : "vulnerable"); 2429 } 2430 2431 return sysfs_emit(buf, "%s\n", retbleed_strings[retbleed_mitigation]); 2432 } 2433 2434 static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr, 2435 char *buf, unsigned int bug) 2436 { 2437 if (!boot_cpu_has_bug(bug)) 2438 return sysfs_emit(buf, "Not affected\n"); 2439 2440 switch (bug) { 2441 case X86_BUG_CPU_MELTDOWN: 2442 if (boot_cpu_has(X86_FEATURE_PTI)) 2443 return sysfs_emit(buf, "Mitigation: PTI\n"); 2444 2445 if (hypervisor_is_type(X86_HYPER_XEN_PV)) 2446 return sysfs_emit(buf, "Unknown (XEN PV detected, hypervisor mitigation required)\n"); 2447 2448 break; 2449 2450 case X86_BUG_SPECTRE_V1: 2451 return sysfs_emit(buf, "%s\n", spectre_v1_strings[spectre_v1_mitigation]); 2452 2453 case X86_BUG_SPECTRE_V2: 2454 return spectre_v2_show_state(buf); 2455 2456 case X86_BUG_SPEC_STORE_BYPASS: 2457 return sysfs_emit(buf, "%s\n", ssb_strings[ssb_mode]); 2458 2459 case X86_BUG_L1TF: 2460 if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV)) 2461 return l1tf_show_state(buf); 2462 break; 2463 2464 case X86_BUG_MDS: 2465 return mds_show_state(buf); 2466 2467 case X86_BUG_TAA: 2468 return tsx_async_abort_show_state(buf); 2469 2470 case X86_BUG_ITLB_MULTIHIT: 2471 return itlb_multihit_show_state(buf); 2472 2473 case X86_BUG_SRBDS: 2474 return srbds_show_state(buf); 2475 2476 case X86_BUG_MMIO_STALE_DATA: 2477 case X86_BUG_MMIO_UNKNOWN: 2478 return mmio_stale_data_show_state(buf); 2479 2480 case X86_BUG_RETBLEED: 2481 return retbleed_show_state(buf); 2482 2483 default: 2484 break; 2485 } 2486 2487 return sysfs_emit(buf, "Vulnerable\n"); 2488 } 2489 2490 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf) 2491 { 2492 return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN); 2493 } 2494 2495 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf) 2496 { 2497 return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1); 2498 } 2499 2500 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf) 2501 { 2502 return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2); 2503 } 2504 2505 ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf) 2506 { 2507 return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS); 2508 } 2509 2510 ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf) 2511 { 2512 return cpu_show_common(dev, attr, buf, X86_BUG_L1TF); 2513 } 2514 2515 ssize_t cpu_show_mds(struct device *dev, struct device_attribute *attr, char *buf) 2516 { 2517 return cpu_show_common(dev, attr, buf, X86_BUG_MDS); 2518 } 2519 2520 ssize_t cpu_show_tsx_async_abort(struct device *dev, struct device_attribute *attr, char *buf) 2521 { 2522 return cpu_show_common(dev, attr, buf, X86_BUG_TAA); 2523 } 2524 2525 ssize_t cpu_show_itlb_multihit(struct device *dev, struct device_attribute *attr, char *buf) 2526 { 2527 return cpu_show_common(dev, attr, buf, X86_BUG_ITLB_MULTIHIT); 2528 } 2529 2530 ssize_t cpu_show_srbds(struct device *dev, struct device_attribute *attr, char *buf) 2531 { 2532 return cpu_show_common(dev, attr, buf, X86_BUG_SRBDS); 2533 } 2534 2535 ssize_t cpu_show_mmio_stale_data(struct device *dev, struct device_attribute *attr, char *buf) 2536 { 2537 if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN)) 2538 return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_UNKNOWN); 2539 else 2540 return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_STALE_DATA); 2541 } 2542 2543 ssize_t cpu_show_retbleed(struct device *dev, struct device_attribute *attr, char *buf) 2544 { 2545 return cpu_show_common(dev, attr, buf, X86_BUG_RETBLEED); 2546 } 2547 #endif 2548