1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * CPU Microcode Update Driver for Linux
4 *
5 * Copyright (C) 2000-2006 Tigran Aivazian <aivazian.tigran@gmail.com>
6 * 2006 Shaohua Li <shaohua.li@intel.com>
7 * 2013-2016 Borislav Petkov <bp@alien8.de>
8 *
9 * X86 CPU microcode early update for Linux:
10 *
11 * Copyright (C) 2012 Fenghua Yu <fenghua.yu@intel.com>
12 * H Peter Anvin" <hpa@zytor.com>
13 * (C) 2015 Borislav Petkov <bp@alien8.de>
14 *
15 * This driver allows to upgrade microcode on x86 processors.
16 */
17
18 #define pr_fmt(fmt) "microcode: " fmt
19
20 #include <linux/platform_device.h>
21 #include <linux/stop_machine.h>
22 #include <linux/syscore_ops.h>
23 #include <linux/miscdevice.h>
24 #include <linux/capability.h>
25 #include <linux/firmware.h>
26 #include <linux/kernel.h>
27 #include <linux/delay.h>
28 #include <linux/mutex.h>
29 #include <linux/cpu.h>
30 #include <linux/nmi.h>
31 #include <linux/fs.h>
32 #include <linux/mm.h>
33
34 #include <asm/cpu_device_id.h>
35 #include <asm/perf_event.h>
36 #include <asm/processor.h>
37 #include <asm/cmdline.h>
38 #include <asm/setup.h>
39
40 #include "internal.h"
41
42 #define DRIVER_VERSION "2.2"
43
44 static struct microcode_ops *microcode_ops;
45 static bool dis_ucode_ldr = true;
46
47 bool initrd_gone;
48
49 LIST_HEAD(microcode_cache);
50
51 /*
52 * Synchronization.
53 *
54 * All non cpu-hotplug-callback call sites use:
55 *
56 * - cpus_read_lock/unlock() to synchronize with
57 * the cpu-hotplug-callback call sites.
58 *
59 * We guarantee that only a single cpu is being
60 * updated at any particular moment of time.
61 */
62 struct ucode_cpu_info ucode_cpu_info[NR_CPUS];
63
64 struct cpu_info_ctx {
65 struct cpu_signature *cpu_sig;
66 int err;
67 };
68
69 /*
70 * Those patch levels cannot be updated to newer ones and thus should be final.
71 */
72 static u32 final_levels[] = {
73 0x01000098,
74 0x0100009f,
75 0x010000af,
76 0, /* T-101 terminator */
77 };
78
79 /*
80 * Check the current patch level on this CPU.
81 *
82 * Returns:
83 * - true: if update should stop
84 * - false: otherwise
85 */
amd_check_current_patch_level(void)86 static bool amd_check_current_patch_level(void)
87 {
88 u32 lvl, dummy, i;
89 u32 *levels;
90
91 native_rdmsr(MSR_AMD64_PATCH_LEVEL, lvl, dummy);
92
93 if (IS_ENABLED(CONFIG_X86_32))
94 levels = (u32 *)__pa_nodebug(&final_levels);
95 else
96 levels = final_levels;
97
98 for (i = 0; levels[i]; i++) {
99 if (lvl == levels[i])
100 return true;
101 }
102 return false;
103 }
104
check_loader_disabled_bsp(void)105 static bool __init check_loader_disabled_bsp(void)
106 {
107 static const char *__dis_opt_str = "dis_ucode_ldr";
108
109 #ifdef CONFIG_X86_32
110 const char *cmdline = (const char *)__pa_nodebug(boot_command_line);
111 const char *option = (const char *)__pa_nodebug(__dis_opt_str);
112 bool *res = (bool *)__pa_nodebug(&dis_ucode_ldr);
113
114 #else /* CONFIG_X86_64 */
115 const char *cmdline = boot_command_line;
116 const char *option = __dis_opt_str;
117 bool *res = &dis_ucode_ldr;
118 #endif
119
120 /*
121 * CPUID(1).ECX[31]: reserved for hypervisor use. This is still not
122 * completely accurate as xen pv guests don't see that CPUID bit set but
123 * that's good enough as they don't land on the BSP path anyway.
124 */
125 if (native_cpuid_ecx(1) & BIT(31))
126 return *res;
127
128 if (x86_cpuid_vendor() == X86_VENDOR_AMD) {
129 if (amd_check_current_patch_level())
130 return *res;
131 }
132
133 if (cmdline_find_option_bool(cmdline, option) <= 0)
134 *res = false;
135
136 return *res;
137 }
138
load_ucode_bsp(void)139 void __init load_ucode_bsp(void)
140 {
141 unsigned int cpuid_1_eax;
142 bool intel = true;
143
144 if (!have_cpuid_p())
145 return;
146
147 cpuid_1_eax = native_cpuid_eax(1);
148
149 switch (x86_cpuid_vendor()) {
150 case X86_VENDOR_INTEL:
151 if (x86_family(cpuid_1_eax) < 6)
152 return;
153 break;
154
155 case X86_VENDOR_AMD:
156 if (x86_family(cpuid_1_eax) < 0x10)
157 return;
158 intel = false;
159 break;
160
161 default:
162 return;
163 }
164
165 if (check_loader_disabled_bsp())
166 return;
167
168 if (intel)
169 load_ucode_intel_bsp();
170 else
171 load_ucode_amd_early(cpuid_1_eax);
172 }
173
check_loader_disabled_ap(void)174 static bool check_loader_disabled_ap(void)
175 {
176 #ifdef CONFIG_X86_32
177 return *((bool *)__pa_nodebug(&dis_ucode_ldr));
178 #else
179 return dis_ucode_ldr;
180 #endif
181 }
182
load_ucode_ap(void)183 void load_ucode_ap(void)
184 {
185 unsigned int cpuid_1_eax;
186
187 if (check_loader_disabled_ap())
188 return;
189
190 cpuid_1_eax = native_cpuid_eax(1);
191
192 switch (x86_cpuid_vendor()) {
193 case X86_VENDOR_INTEL:
194 if (x86_family(cpuid_1_eax) >= 6)
195 load_ucode_intel_ap();
196 break;
197 case X86_VENDOR_AMD:
198 if (x86_family(cpuid_1_eax) >= 0x10)
199 load_ucode_amd_early(cpuid_1_eax);
200 break;
201 default:
202 break;
203 }
204 }
205
save_microcode_in_initrd(void)206 static int __init save_microcode_in_initrd(void)
207 {
208 struct cpuinfo_x86 *c = &boot_cpu_data;
209 int ret = -EINVAL;
210
211 if (dis_ucode_ldr) {
212 ret = 0;
213 goto out;
214 }
215
216 switch (c->x86_vendor) {
217 case X86_VENDOR_INTEL:
218 if (c->x86 >= 6)
219 ret = save_microcode_in_initrd_intel();
220 break;
221 case X86_VENDOR_AMD:
222 if (c->x86 >= 0x10)
223 ret = save_microcode_in_initrd_amd(cpuid_eax(1));
224 break;
225 default:
226 break;
227 }
228
229 out:
230 initrd_gone = true;
231
232 return ret;
233 }
234
find_microcode_in_initrd(const char * path,bool use_pa)235 struct cpio_data find_microcode_in_initrd(const char *path, bool use_pa)
236 {
237 #ifdef CONFIG_BLK_DEV_INITRD
238 unsigned long start = 0;
239 size_t size;
240
241 #ifdef CONFIG_X86_32
242 struct boot_params *params;
243
244 if (use_pa)
245 params = (struct boot_params *)__pa_nodebug(&boot_params);
246 else
247 params = &boot_params;
248
249 size = params->hdr.ramdisk_size;
250
251 /*
252 * Set start only if we have an initrd image. We cannot use initrd_start
253 * because it is not set that early yet.
254 */
255 if (size)
256 start = params->hdr.ramdisk_image;
257
258 # else /* CONFIG_X86_64 */
259 size = (unsigned long)boot_params.ext_ramdisk_size << 32;
260 size |= boot_params.hdr.ramdisk_size;
261
262 if (size) {
263 start = (unsigned long)boot_params.ext_ramdisk_image << 32;
264 start |= boot_params.hdr.ramdisk_image;
265
266 start += PAGE_OFFSET;
267 }
268 # endif
269
270 /*
271 * Fixup the start address: after reserve_initrd() runs, initrd_start
272 * has the virtual address of the beginning of the initrd. It also
273 * possibly relocates the ramdisk. In either case, initrd_start contains
274 * the updated address so use that instead.
275 *
276 * initrd_gone is for the hotplug case where we've thrown out initrd
277 * already.
278 */
279 if (!use_pa) {
280 if (initrd_gone)
281 return (struct cpio_data){ NULL, 0, "" };
282 if (initrd_start)
283 start = initrd_start;
284 } else {
285 /*
286 * The picture with physical addresses is a bit different: we
287 * need to get the *physical* address to which the ramdisk was
288 * relocated, i.e., relocated_ramdisk (not initrd_start) and
289 * since we're running from physical addresses, we need to access
290 * relocated_ramdisk through its *physical* address too.
291 */
292 u64 *rr = (u64 *)__pa_nodebug(&relocated_ramdisk);
293 if (*rr)
294 start = *rr;
295 }
296
297 return find_cpio_data(path, (void *)start, size, NULL);
298 #else /* !CONFIG_BLK_DEV_INITRD */
299 return (struct cpio_data){ NULL, 0, "" };
300 #endif
301 }
302
reload_early_microcode(unsigned int cpu)303 static void reload_early_microcode(unsigned int cpu)
304 {
305 int vendor, family;
306
307 vendor = x86_cpuid_vendor();
308 family = x86_cpuid_family();
309
310 switch (vendor) {
311 case X86_VENDOR_INTEL:
312 if (family >= 6)
313 reload_ucode_intel();
314 break;
315 case X86_VENDOR_AMD:
316 if (family >= 0x10)
317 reload_ucode_amd(cpu);
318 break;
319 default:
320 break;
321 }
322 }
323
324 /* fake device for request_firmware */
325 static struct platform_device *microcode_pdev;
326
327 #ifdef CONFIG_MICROCODE_LATE_LOADING
328 /*
329 * Late loading dance. Why the heavy-handed stomp_machine effort?
330 *
331 * - HT siblings must be idle and not execute other code while the other sibling
332 * is loading microcode in order to avoid any negative interactions caused by
333 * the loading.
334 *
335 * - In addition, microcode update on the cores must be serialized until this
336 * requirement can be relaxed in the future. Right now, this is conservative
337 * and good.
338 */
339 #define SPINUNIT 100 /* 100 nsec */
340
check_online_cpus(void)341 static int check_online_cpus(void)
342 {
343 unsigned int cpu;
344
345 /*
346 * Make sure all CPUs are online. It's fine for SMT to be disabled if
347 * all the primary threads are still online.
348 */
349 for_each_present_cpu(cpu) {
350 if (topology_is_primary_thread(cpu) && !cpu_online(cpu)) {
351 pr_err("Not all CPUs online, aborting microcode update.\n");
352 return -EINVAL;
353 }
354 }
355
356 return 0;
357 }
358
359 static atomic_t late_cpus_in;
360 static atomic_t late_cpus_out;
361
__wait_for_cpus(atomic_t * t,long long timeout)362 static int __wait_for_cpus(atomic_t *t, long long timeout)
363 {
364 int all_cpus = num_online_cpus();
365
366 atomic_inc(t);
367
368 while (atomic_read(t) < all_cpus) {
369 if (timeout < SPINUNIT) {
370 pr_err("Timeout while waiting for CPUs rendezvous, remaining: %d\n",
371 all_cpus - atomic_read(t));
372 return 1;
373 }
374
375 ndelay(SPINUNIT);
376 timeout -= SPINUNIT;
377
378 touch_nmi_watchdog();
379 }
380 return 0;
381 }
382
383 /*
384 * Returns:
385 * < 0 - on error
386 * 0 - success (no update done or microcode was updated)
387 */
__reload_late(void * info)388 static int __reload_late(void *info)
389 {
390 int cpu = smp_processor_id();
391 enum ucode_state err;
392 int ret = 0;
393
394 /*
395 * Wait for all CPUs to arrive. A load will not be attempted unless all
396 * CPUs show up.
397 * */
398 if (__wait_for_cpus(&late_cpus_in, NSEC_PER_SEC))
399 return -1;
400
401 /*
402 * On an SMT system, it suffices to load the microcode on one sibling of
403 * the core because the microcode engine is shared between the threads.
404 * Synchronization still needs to take place so that no concurrent
405 * loading attempts happen on multiple threads of an SMT core. See
406 * below.
407 */
408 if (cpumask_first(topology_sibling_cpumask(cpu)) == cpu)
409 err = microcode_ops->apply_microcode(cpu);
410 else
411 goto wait_for_siblings;
412
413 if (err >= UCODE_NFOUND) {
414 if (err == UCODE_ERROR) {
415 pr_warn("Error reloading microcode on CPU %d\n", cpu);
416 ret = -1;
417 }
418 }
419
420 wait_for_siblings:
421 if (__wait_for_cpus(&late_cpus_out, NSEC_PER_SEC))
422 panic("Timeout during microcode update!\n");
423
424 /*
425 * At least one thread has completed update on each core.
426 * For others, simply call the update to make sure the
427 * per-cpu cpuinfo can be updated with right microcode
428 * revision.
429 */
430 if (cpumask_first(topology_sibling_cpumask(cpu)) != cpu)
431 err = microcode_ops->apply_microcode(cpu);
432
433 return ret;
434 }
435
436 /*
437 * Reload microcode late on all CPUs. Wait for a sec until they
438 * all gather together.
439 */
microcode_reload_late(void)440 static int microcode_reload_late(void)
441 {
442 int old = boot_cpu_data.microcode, ret;
443 struct cpuinfo_x86 prev_info;
444
445 pr_err("Attempting late microcode loading - it is dangerous and taints the kernel.\n");
446 pr_err("You should switch to early loading, if possible.\n");
447
448 atomic_set(&late_cpus_in, 0);
449 atomic_set(&late_cpus_out, 0);
450
451 /*
452 * Take a snapshot before the microcode update in order to compare and
453 * check whether any bits changed after an update.
454 */
455 store_cpu_caps(&prev_info);
456
457 ret = stop_machine_cpuslocked(__reload_late, NULL, cpu_online_mask);
458 if (!ret) {
459 pr_info("Reload succeeded, microcode revision: 0x%x -> 0x%x\n",
460 old, boot_cpu_data.microcode);
461 microcode_check(&prev_info);
462 } else {
463 pr_info("Reload failed, current microcode revision: 0x%x\n",
464 boot_cpu_data.microcode);
465 }
466
467 return ret;
468 }
469
reload_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t size)470 static ssize_t reload_store(struct device *dev,
471 struct device_attribute *attr,
472 const char *buf, size_t size)
473 {
474 enum ucode_state tmp_ret = UCODE_OK;
475 int bsp = boot_cpu_data.cpu_index;
476 unsigned long val;
477 ssize_t ret = 0;
478
479 ret = kstrtoul(buf, 0, &val);
480 if (ret || val != 1)
481 return -EINVAL;
482
483 cpus_read_lock();
484
485 ret = check_online_cpus();
486 if (ret)
487 goto put;
488
489 tmp_ret = microcode_ops->request_microcode_fw(bsp, µcode_pdev->dev);
490 if (tmp_ret != UCODE_NEW)
491 goto put;
492
493 ret = microcode_reload_late();
494 put:
495 cpus_read_unlock();
496
497 if (ret == 0)
498 ret = size;
499
500 add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK);
501
502 return ret;
503 }
504
505 static DEVICE_ATTR_WO(reload);
506 #endif
507
version_show(struct device * dev,struct device_attribute * attr,char * buf)508 static ssize_t version_show(struct device *dev,
509 struct device_attribute *attr, char *buf)
510 {
511 struct ucode_cpu_info *uci = ucode_cpu_info + dev->id;
512
513 return sprintf(buf, "0x%x\n", uci->cpu_sig.rev);
514 }
515
processor_flags_show(struct device * dev,struct device_attribute * attr,char * buf)516 static ssize_t processor_flags_show(struct device *dev,
517 struct device_attribute *attr, char *buf)
518 {
519 struct ucode_cpu_info *uci = ucode_cpu_info + dev->id;
520
521 return sprintf(buf, "0x%x\n", uci->cpu_sig.pf);
522 }
523
524 static DEVICE_ATTR_RO(version);
525 static DEVICE_ATTR_RO(processor_flags);
526
527 static struct attribute *mc_default_attrs[] = {
528 &dev_attr_version.attr,
529 &dev_attr_processor_flags.attr,
530 NULL
531 };
532
533 static const struct attribute_group mc_attr_group = {
534 .attrs = mc_default_attrs,
535 .name = "microcode",
536 };
537
microcode_fini_cpu(int cpu)538 static void microcode_fini_cpu(int cpu)
539 {
540 if (microcode_ops->microcode_fini_cpu)
541 microcode_ops->microcode_fini_cpu(cpu);
542 }
543
microcode_init_cpu(int cpu)544 static enum ucode_state microcode_init_cpu(int cpu)
545 {
546 struct ucode_cpu_info *uci = ucode_cpu_info + cpu;
547
548 memset(uci, 0, sizeof(*uci));
549
550 microcode_ops->collect_cpu_info(cpu, &uci->cpu_sig);
551
552 return microcode_ops->apply_microcode(cpu);
553 }
554
555 /**
556 * microcode_bsp_resume - Update boot CPU microcode during resume.
557 */
microcode_bsp_resume(void)558 void microcode_bsp_resume(void)
559 {
560 int cpu = smp_processor_id();
561 struct ucode_cpu_info *uci = ucode_cpu_info + cpu;
562
563 if (uci->mc)
564 microcode_ops->apply_microcode(cpu);
565 else
566 reload_early_microcode(cpu);
567 }
568
569 static struct syscore_ops mc_syscore_ops = {
570 .resume = microcode_bsp_resume,
571 };
572
mc_cpu_starting(unsigned int cpu)573 static int mc_cpu_starting(unsigned int cpu)
574 {
575 enum ucode_state err = microcode_ops->apply_microcode(cpu);
576
577 pr_debug("%s: CPU%d, err: %d\n", __func__, cpu, err);
578
579 return err == UCODE_ERROR;
580 }
581
mc_cpu_online(unsigned int cpu)582 static int mc_cpu_online(unsigned int cpu)
583 {
584 struct device *dev = get_cpu_device(cpu);
585
586 if (sysfs_create_group(&dev->kobj, &mc_attr_group))
587 pr_err("Failed to create group for CPU%d\n", cpu);
588 return 0;
589 }
590
mc_cpu_down_prep(unsigned int cpu)591 static int mc_cpu_down_prep(unsigned int cpu)
592 {
593 struct device *dev;
594
595 dev = get_cpu_device(cpu);
596
597 microcode_fini_cpu(cpu);
598
599 /* Suspend is in progress, only remove the interface */
600 sysfs_remove_group(&dev->kobj, &mc_attr_group);
601 pr_debug("%s: CPU%d\n", __func__, cpu);
602
603 return 0;
604 }
605
setup_online_cpu(struct work_struct * work)606 static void setup_online_cpu(struct work_struct *work)
607 {
608 int cpu = smp_processor_id();
609 enum ucode_state err;
610
611 err = microcode_init_cpu(cpu);
612 if (err == UCODE_ERROR) {
613 pr_err("Error applying microcode on CPU%d\n", cpu);
614 return;
615 }
616
617 mc_cpu_online(cpu);
618 }
619
620 static struct attribute *cpu_root_microcode_attrs[] = {
621 #ifdef CONFIG_MICROCODE_LATE_LOADING
622 &dev_attr_reload.attr,
623 #endif
624 NULL
625 };
626
627 static const struct attribute_group cpu_root_microcode_group = {
628 .name = "microcode",
629 .attrs = cpu_root_microcode_attrs,
630 };
631
microcode_init(void)632 static int __init microcode_init(void)
633 {
634 struct device *dev_root;
635 struct cpuinfo_x86 *c = &boot_cpu_data;
636 int error;
637
638 if (dis_ucode_ldr)
639 return -EINVAL;
640
641 if (c->x86_vendor == X86_VENDOR_INTEL)
642 microcode_ops = init_intel_microcode();
643 else if (c->x86_vendor == X86_VENDOR_AMD)
644 microcode_ops = init_amd_microcode();
645 else
646 pr_err("no support for this CPU vendor\n");
647
648 if (!microcode_ops)
649 return -ENODEV;
650
651 microcode_pdev = platform_device_register_simple("microcode", -1, NULL, 0);
652 if (IS_ERR(microcode_pdev))
653 return PTR_ERR(microcode_pdev);
654
655 dev_root = bus_get_dev_root(&cpu_subsys);
656 if (dev_root) {
657 error = sysfs_create_group(&dev_root->kobj, &cpu_root_microcode_group);
658 put_device(dev_root);
659 if (error) {
660 pr_err("Error creating microcode group!\n");
661 goto out_pdev;
662 }
663 }
664
665 /* Do per-CPU setup */
666 schedule_on_each_cpu(setup_online_cpu);
667
668 register_syscore_ops(&mc_syscore_ops);
669 cpuhp_setup_state_nocalls(CPUHP_AP_MICROCODE_LOADER, "x86/microcode:starting",
670 mc_cpu_starting, NULL);
671 cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "x86/microcode:online",
672 mc_cpu_online, mc_cpu_down_prep);
673
674 pr_info("Microcode Update Driver: v%s.", DRIVER_VERSION);
675
676 return 0;
677
678 out_pdev:
679 platform_device_unregister(microcode_pdev);
680 return error;
681
682 }
683 fs_initcall(save_microcode_in_initrd);
684 late_initcall(microcode_init);
685