1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * CPU Microcode Update Driver for Linux 4 * 5 * Copyright (C) 2000-2006 Tigran Aivazian <aivazian.tigran@gmail.com> 6 * 2006 Shaohua Li <shaohua.li@intel.com> 7 * 2013-2016 Borislav Petkov <bp@alien8.de> 8 * 9 * X86 CPU microcode early update for Linux: 10 * 11 * Copyright (C) 2012 Fenghua Yu <fenghua.yu@intel.com> 12 * H Peter Anvin" <hpa@zytor.com> 13 * (C) 2015 Borislav Petkov <bp@alien8.de> 14 * 15 * This driver allows to upgrade microcode on x86 processors. 16 */ 17 18 #define pr_fmt(fmt) "microcode: " fmt 19 20 #include <linux/platform_device.h> 21 #include <linux/stop_machine.h> 22 #include <linux/syscore_ops.h> 23 #include <linux/miscdevice.h> 24 #include <linux/capability.h> 25 #include <linux/firmware.h> 26 #include <linux/kernel.h> 27 #include <linux/delay.h> 28 #include <linux/mutex.h> 29 #include <linux/cpu.h> 30 #include <linux/nmi.h> 31 #include <linux/fs.h> 32 #include <linux/mm.h> 33 34 #include <asm/microcode_intel.h> 35 #include <asm/cpu_device_id.h> 36 #include <asm/microcode_amd.h> 37 #include <asm/perf_event.h> 38 #include <asm/microcode.h> 39 #include <asm/processor.h> 40 #include <asm/cmdline.h> 41 #include <asm/setup.h> 42 43 #define DRIVER_VERSION "2.2" 44 45 static struct microcode_ops *microcode_ops; 46 static bool dis_ucode_ldr = true; 47 48 bool initrd_gone; 49 50 LIST_HEAD(microcode_cache); 51 52 /* 53 * Synchronization. 54 * 55 * All non cpu-hotplug-callback call sites use: 56 * 57 * - cpus_read_lock/unlock() to synchronize with 58 * the cpu-hotplug-callback call sites. 59 * 60 * We guarantee that only a single cpu is being 61 * updated at any particular moment of time. 62 */ 63 struct ucode_cpu_info ucode_cpu_info[NR_CPUS]; 64 65 struct cpu_info_ctx { 66 struct cpu_signature *cpu_sig; 67 int err; 68 }; 69 70 /* 71 * Those patch levels cannot be updated to newer ones and thus should be final. 72 */ 73 static u32 final_levels[] = { 74 0x01000098, 75 0x0100009f, 76 0x010000af, 77 0, /* T-101 terminator */ 78 }; 79 80 /* 81 * Check the current patch level on this CPU. 82 * 83 * Returns: 84 * - true: if update should stop 85 * - false: otherwise 86 */ 87 static bool amd_check_current_patch_level(void) 88 { 89 u32 lvl, dummy, i; 90 u32 *levels; 91 92 native_rdmsr(MSR_AMD64_PATCH_LEVEL, lvl, dummy); 93 94 if (IS_ENABLED(CONFIG_X86_32)) 95 levels = (u32 *)__pa_nodebug(&final_levels); 96 else 97 levels = final_levels; 98 99 for (i = 0; levels[i]; i++) { 100 if (lvl == levels[i]) 101 return true; 102 } 103 return false; 104 } 105 106 static bool __init check_loader_disabled_bsp(void) 107 { 108 static const char *__dis_opt_str = "dis_ucode_ldr"; 109 110 #ifdef CONFIG_X86_32 111 const char *cmdline = (const char *)__pa_nodebug(boot_command_line); 112 const char *option = (const char *)__pa_nodebug(__dis_opt_str); 113 bool *res = (bool *)__pa_nodebug(&dis_ucode_ldr); 114 115 #else /* CONFIG_X86_64 */ 116 const char *cmdline = boot_command_line; 117 const char *option = __dis_opt_str; 118 bool *res = &dis_ucode_ldr; 119 #endif 120 121 /* 122 * CPUID(1).ECX[31]: reserved for hypervisor use. This is still not 123 * completely accurate as xen pv guests don't see that CPUID bit set but 124 * that's good enough as they don't land on the BSP path anyway. 125 */ 126 if (native_cpuid_ecx(1) & BIT(31)) 127 return *res; 128 129 if (x86_cpuid_vendor() == X86_VENDOR_AMD) { 130 if (amd_check_current_patch_level()) 131 return *res; 132 } 133 134 if (cmdline_find_option_bool(cmdline, option) <= 0) 135 *res = false; 136 137 return *res; 138 } 139 140 void __init load_ucode_bsp(void) 141 { 142 unsigned int cpuid_1_eax; 143 bool intel = true; 144 145 if (!have_cpuid_p()) 146 return; 147 148 cpuid_1_eax = native_cpuid_eax(1); 149 150 switch (x86_cpuid_vendor()) { 151 case X86_VENDOR_INTEL: 152 if (x86_family(cpuid_1_eax) < 6) 153 return; 154 break; 155 156 case X86_VENDOR_AMD: 157 if (x86_family(cpuid_1_eax) < 0x10) 158 return; 159 intel = false; 160 break; 161 162 default: 163 return; 164 } 165 166 if (check_loader_disabled_bsp()) 167 return; 168 169 if (intel) 170 load_ucode_intel_bsp(); 171 else 172 load_ucode_amd_early(cpuid_1_eax); 173 } 174 175 static bool check_loader_disabled_ap(void) 176 { 177 #ifdef CONFIG_X86_32 178 return *((bool *)__pa_nodebug(&dis_ucode_ldr)); 179 #else 180 return dis_ucode_ldr; 181 #endif 182 } 183 184 void load_ucode_ap(void) 185 { 186 unsigned int cpuid_1_eax; 187 188 if (check_loader_disabled_ap()) 189 return; 190 191 cpuid_1_eax = native_cpuid_eax(1); 192 193 switch (x86_cpuid_vendor()) { 194 case X86_VENDOR_INTEL: 195 if (x86_family(cpuid_1_eax) >= 6) 196 load_ucode_intel_ap(); 197 break; 198 case X86_VENDOR_AMD: 199 if (x86_family(cpuid_1_eax) >= 0x10) 200 load_ucode_amd_early(cpuid_1_eax); 201 break; 202 default: 203 break; 204 } 205 } 206 207 static int __init save_microcode_in_initrd(void) 208 { 209 struct cpuinfo_x86 *c = &boot_cpu_data; 210 int ret = -EINVAL; 211 212 switch (c->x86_vendor) { 213 case X86_VENDOR_INTEL: 214 if (c->x86 >= 6) 215 ret = save_microcode_in_initrd_intel(); 216 break; 217 case X86_VENDOR_AMD: 218 if (c->x86 >= 0x10) 219 ret = save_microcode_in_initrd_amd(cpuid_eax(1)); 220 break; 221 default: 222 break; 223 } 224 225 initrd_gone = true; 226 227 return ret; 228 } 229 230 struct cpio_data find_microcode_in_initrd(const char *path, bool use_pa) 231 { 232 #ifdef CONFIG_BLK_DEV_INITRD 233 unsigned long start = 0; 234 size_t size; 235 236 #ifdef CONFIG_X86_32 237 struct boot_params *params; 238 239 if (use_pa) 240 params = (struct boot_params *)__pa_nodebug(&boot_params); 241 else 242 params = &boot_params; 243 244 size = params->hdr.ramdisk_size; 245 246 /* 247 * Set start only if we have an initrd image. We cannot use initrd_start 248 * because it is not set that early yet. 249 */ 250 if (size) 251 start = params->hdr.ramdisk_image; 252 253 # else /* CONFIG_X86_64 */ 254 size = (unsigned long)boot_params.ext_ramdisk_size << 32; 255 size |= boot_params.hdr.ramdisk_size; 256 257 if (size) { 258 start = (unsigned long)boot_params.ext_ramdisk_image << 32; 259 start |= boot_params.hdr.ramdisk_image; 260 261 start += PAGE_OFFSET; 262 } 263 # endif 264 265 /* 266 * Fixup the start address: after reserve_initrd() runs, initrd_start 267 * has the virtual address of the beginning of the initrd. It also 268 * possibly relocates the ramdisk. In either case, initrd_start contains 269 * the updated address so use that instead. 270 * 271 * initrd_gone is for the hotplug case where we've thrown out initrd 272 * already. 273 */ 274 if (!use_pa) { 275 if (initrd_gone) 276 return (struct cpio_data){ NULL, 0, "" }; 277 if (initrd_start) 278 start = initrd_start; 279 } else { 280 /* 281 * The picture with physical addresses is a bit different: we 282 * need to get the *physical* address to which the ramdisk was 283 * relocated, i.e., relocated_ramdisk (not initrd_start) and 284 * since we're running from physical addresses, we need to access 285 * relocated_ramdisk through its *physical* address too. 286 */ 287 u64 *rr = (u64 *)__pa_nodebug(&relocated_ramdisk); 288 if (*rr) 289 start = *rr; 290 } 291 292 return find_cpio_data(path, (void *)start, size, NULL); 293 #else /* !CONFIG_BLK_DEV_INITRD */ 294 return (struct cpio_data){ NULL, 0, "" }; 295 #endif 296 } 297 298 void reload_early_microcode(unsigned int cpu) 299 { 300 int vendor, family; 301 302 vendor = x86_cpuid_vendor(); 303 family = x86_cpuid_family(); 304 305 switch (vendor) { 306 case X86_VENDOR_INTEL: 307 if (family >= 6) 308 reload_ucode_intel(); 309 break; 310 case X86_VENDOR_AMD: 311 if (family >= 0x10) 312 reload_ucode_amd(cpu); 313 break; 314 default: 315 break; 316 } 317 } 318 319 /* fake device for request_firmware */ 320 static struct platform_device *microcode_pdev; 321 322 #ifdef CONFIG_MICROCODE_LATE_LOADING 323 /* 324 * Late loading dance. Why the heavy-handed stomp_machine effort? 325 * 326 * - HT siblings must be idle and not execute other code while the other sibling 327 * is loading microcode in order to avoid any negative interactions caused by 328 * the loading. 329 * 330 * - In addition, microcode update on the cores must be serialized until this 331 * requirement can be relaxed in the future. Right now, this is conservative 332 * and good. 333 */ 334 #define SPINUNIT 100 /* 100 nsec */ 335 336 static int check_online_cpus(void) 337 { 338 unsigned int cpu; 339 340 /* 341 * Make sure all CPUs are online. It's fine for SMT to be disabled if 342 * all the primary threads are still online. 343 */ 344 for_each_present_cpu(cpu) { 345 if (topology_is_primary_thread(cpu) && !cpu_online(cpu)) { 346 pr_err("Not all CPUs online, aborting microcode update.\n"); 347 return -EINVAL; 348 } 349 } 350 351 return 0; 352 } 353 354 static atomic_t late_cpus_in; 355 static atomic_t late_cpus_out; 356 357 static int __wait_for_cpus(atomic_t *t, long long timeout) 358 { 359 int all_cpus = num_online_cpus(); 360 361 atomic_inc(t); 362 363 while (atomic_read(t) < all_cpus) { 364 if (timeout < SPINUNIT) { 365 pr_err("Timeout while waiting for CPUs rendezvous, remaining: %d\n", 366 all_cpus - atomic_read(t)); 367 return 1; 368 } 369 370 ndelay(SPINUNIT); 371 timeout -= SPINUNIT; 372 373 touch_nmi_watchdog(); 374 } 375 return 0; 376 } 377 378 /* 379 * Returns: 380 * < 0 - on error 381 * 0 - success (no update done or microcode was updated) 382 */ 383 static int __reload_late(void *info) 384 { 385 int cpu = smp_processor_id(); 386 enum ucode_state err; 387 int ret = 0; 388 389 /* 390 * Wait for all CPUs to arrive. A load will not be attempted unless all 391 * CPUs show up. 392 * */ 393 if (__wait_for_cpus(&late_cpus_in, NSEC_PER_SEC)) 394 return -1; 395 396 /* 397 * On an SMT system, it suffices to load the microcode on one sibling of 398 * the core because the microcode engine is shared between the threads. 399 * Synchronization still needs to take place so that no concurrent 400 * loading attempts happen on multiple threads of an SMT core. See 401 * below. 402 */ 403 if (cpumask_first(topology_sibling_cpumask(cpu)) == cpu) 404 err = microcode_ops->apply_microcode(cpu); 405 else 406 goto wait_for_siblings; 407 408 if (err >= UCODE_NFOUND) { 409 if (err == UCODE_ERROR) { 410 pr_warn("Error reloading microcode on CPU %d\n", cpu); 411 ret = -1; 412 } 413 } 414 415 wait_for_siblings: 416 if (__wait_for_cpus(&late_cpus_out, NSEC_PER_SEC)) 417 panic("Timeout during microcode update!\n"); 418 419 /* 420 * At least one thread has completed update on each core. 421 * For others, simply call the update to make sure the 422 * per-cpu cpuinfo can be updated with right microcode 423 * revision. 424 */ 425 if (cpumask_first(topology_sibling_cpumask(cpu)) != cpu) 426 err = microcode_ops->apply_microcode(cpu); 427 428 return ret; 429 } 430 431 /* 432 * Reload microcode late on all CPUs. Wait for a sec until they 433 * all gather together. 434 */ 435 static int microcode_reload_late(void) 436 { 437 int old = boot_cpu_data.microcode, ret; 438 struct cpuinfo_x86 prev_info; 439 440 pr_err("Attempting late microcode loading - it is dangerous and taints the kernel.\n"); 441 pr_err("You should switch to early loading, if possible.\n"); 442 443 atomic_set(&late_cpus_in, 0); 444 atomic_set(&late_cpus_out, 0); 445 446 /* 447 * Take a snapshot before the microcode update in order to compare and 448 * check whether any bits changed after an update. 449 */ 450 store_cpu_caps(&prev_info); 451 452 ret = stop_machine_cpuslocked(__reload_late, NULL, cpu_online_mask); 453 if (!ret) { 454 pr_info("Reload succeeded, microcode revision: 0x%x -> 0x%x\n", 455 old, boot_cpu_data.microcode); 456 microcode_check(&prev_info); 457 } else { 458 pr_info("Reload failed, current microcode revision: 0x%x\n", 459 boot_cpu_data.microcode); 460 } 461 462 return ret; 463 } 464 465 static ssize_t reload_store(struct device *dev, 466 struct device_attribute *attr, 467 const char *buf, size_t size) 468 { 469 enum ucode_state tmp_ret = UCODE_OK; 470 int bsp = boot_cpu_data.cpu_index; 471 unsigned long val; 472 ssize_t ret = 0; 473 474 ret = kstrtoul(buf, 0, &val); 475 if (ret || val != 1) 476 return -EINVAL; 477 478 cpus_read_lock(); 479 480 ret = check_online_cpus(); 481 if (ret) 482 goto put; 483 484 tmp_ret = microcode_ops->request_microcode_fw(bsp, µcode_pdev->dev); 485 if (tmp_ret != UCODE_NEW) 486 goto put; 487 488 ret = microcode_reload_late(); 489 put: 490 cpus_read_unlock(); 491 492 if (ret == 0) 493 ret = size; 494 495 add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK); 496 497 return ret; 498 } 499 500 static DEVICE_ATTR_WO(reload); 501 #endif 502 503 static ssize_t version_show(struct device *dev, 504 struct device_attribute *attr, char *buf) 505 { 506 struct ucode_cpu_info *uci = ucode_cpu_info + dev->id; 507 508 return sprintf(buf, "0x%x\n", uci->cpu_sig.rev); 509 } 510 511 static ssize_t processor_flags_show(struct device *dev, 512 struct device_attribute *attr, char *buf) 513 { 514 struct ucode_cpu_info *uci = ucode_cpu_info + dev->id; 515 516 return sprintf(buf, "0x%x\n", uci->cpu_sig.pf); 517 } 518 519 static DEVICE_ATTR_RO(version); 520 static DEVICE_ATTR_RO(processor_flags); 521 522 static struct attribute *mc_default_attrs[] = { 523 &dev_attr_version.attr, 524 &dev_attr_processor_flags.attr, 525 NULL 526 }; 527 528 static const struct attribute_group mc_attr_group = { 529 .attrs = mc_default_attrs, 530 .name = "microcode", 531 }; 532 533 static void microcode_fini_cpu(int cpu) 534 { 535 if (microcode_ops->microcode_fini_cpu) 536 microcode_ops->microcode_fini_cpu(cpu); 537 } 538 539 static enum ucode_state microcode_init_cpu(int cpu) 540 { 541 struct ucode_cpu_info *uci = ucode_cpu_info + cpu; 542 543 memset(uci, 0, sizeof(*uci)); 544 545 microcode_ops->collect_cpu_info(cpu, &uci->cpu_sig); 546 547 return microcode_ops->apply_microcode(cpu); 548 } 549 550 /** 551 * microcode_bsp_resume - Update boot CPU microcode during resume. 552 */ 553 void microcode_bsp_resume(void) 554 { 555 int cpu = smp_processor_id(); 556 struct ucode_cpu_info *uci = ucode_cpu_info + cpu; 557 558 if (uci->mc) 559 microcode_ops->apply_microcode(cpu); 560 else 561 reload_early_microcode(cpu); 562 } 563 564 static struct syscore_ops mc_syscore_ops = { 565 .resume = microcode_bsp_resume, 566 }; 567 568 static int mc_cpu_starting(unsigned int cpu) 569 { 570 enum ucode_state err = microcode_ops->apply_microcode(cpu); 571 572 pr_debug("%s: CPU%d, err: %d\n", __func__, cpu, err); 573 574 return err == UCODE_ERROR; 575 } 576 577 static int mc_cpu_online(unsigned int cpu) 578 { 579 struct device *dev = get_cpu_device(cpu); 580 581 if (sysfs_create_group(&dev->kobj, &mc_attr_group)) 582 pr_err("Failed to create group for CPU%d\n", cpu); 583 return 0; 584 } 585 586 static int mc_cpu_down_prep(unsigned int cpu) 587 { 588 struct device *dev; 589 590 dev = get_cpu_device(cpu); 591 592 microcode_fini_cpu(cpu); 593 594 /* Suspend is in progress, only remove the interface */ 595 sysfs_remove_group(&dev->kobj, &mc_attr_group); 596 pr_debug("%s: CPU%d\n", __func__, cpu); 597 598 return 0; 599 } 600 601 static void setup_online_cpu(struct work_struct *work) 602 { 603 int cpu = smp_processor_id(); 604 enum ucode_state err; 605 606 err = microcode_init_cpu(cpu); 607 if (err == UCODE_ERROR) { 608 pr_err("Error applying microcode on CPU%d\n", cpu); 609 return; 610 } 611 612 mc_cpu_online(cpu); 613 } 614 615 static struct attribute *cpu_root_microcode_attrs[] = { 616 #ifdef CONFIG_MICROCODE_LATE_LOADING 617 &dev_attr_reload.attr, 618 #endif 619 NULL 620 }; 621 622 static const struct attribute_group cpu_root_microcode_group = { 623 .name = "microcode", 624 .attrs = cpu_root_microcode_attrs, 625 }; 626 627 static int __init microcode_init(void) 628 { 629 struct device *dev_root; 630 struct cpuinfo_x86 *c = &boot_cpu_data; 631 int error; 632 633 if (dis_ucode_ldr) 634 return -EINVAL; 635 636 if (c->x86_vendor == X86_VENDOR_INTEL) 637 microcode_ops = init_intel_microcode(); 638 else if (c->x86_vendor == X86_VENDOR_AMD) 639 microcode_ops = init_amd_microcode(); 640 else 641 pr_err("no support for this CPU vendor\n"); 642 643 if (!microcode_ops) 644 return -ENODEV; 645 646 microcode_pdev = platform_device_register_simple("microcode", -1, NULL, 0); 647 if (IS_ERR(microcode_pdev)) 648 return PTR_ERR(microcode_pdev); 649 650 dev_root = bus_get_dev_root(&cpu_subsys); 651 if (dev_root) { 652 error = sysfs_create_group(&dev_root->kobj, &cpu_root_microcode_group); 653 put_device(dev_root); 654 if (error) { 655 pr_err("Error creating microcode group!\n"); 656 goto out_pdev; 657 } 658 } 659 660 /* Do per-CPU setup */ 661 schedule_on_each_cpu(setup_online_cpu); 662 663 register_syscore_ops(&mc_syscore_ops); 664 cpuhp_setup_state_nocalls(CPUHP_AP_MICROCODE_LOADER, "x86/microcode:starting", 665 mc_cpu_starting, NULL); 666 cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "x86/microcode:online", 667 mc_cpu_online, mc_cpu_down_prep); 668 669 pr_info("Microcode Update Driver: v%s.", DRIVER_VERSION); 670 671 return 0; 672 673 out_pdev: 674 platform_device_unregister(microcode_pdev); 675 return error; 676 677 } 678 fs_initcall(save_microcode_in_initrd); 679 late_initcall(microcode_init); 680