1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 221afaf18SBorislav Petkov /* 321afaf18SBorislav Petkov * Machine check handler. 421afaf18SBorislav Petkov * 521afaf18SBorislav Petkov * K8 parts Copyright 2002,2003 Andi Kleen, SuSE Labs. 621afaf18SBorislav Petkov * Rest from unknown author(s). 721afaf18SBorislav Petkov * 2004 Andi Kleen. Rewrote most of it. 821afaf18SBorislav Petkov * Copyright 2008 Intel Corporation 921afaf18SBorislav Petkov * Author: Andi Kleen 1021afaf18SBorislav Petkov */ 1121afaf18SBorislav Petkov 1221afaf18SBorislav Petkov #include <linux/thread_info.h> 1321afaf18SBorislav Petkov #include <linux/capability.h> 1421afaf18SBorislav Petkov #include <linux/miscdevice.h> 1521afaf18SBorislav Petkov #include <linux/ratelimit.h> 1621afaf18SBorislav Petkov #include <linux/rcupdate.h> 1721afaf18SBorislav Petkov #include <linux/kobject.h> 1821afaf18SBorislav Petkov #include <linux/uaccess.h> 1921afaf18SBorislav Petkov #include <linux/kdebug.h> 2021afaf18SBorislav Petkov #include <linux/kernel.h> 2121afaf18SBorislav Petkov #include <linux/percpu.h> 2221afaf18SBorislav Petkov #include <linux/string.h> 2321afaf18SBorislav Petkov #include <linux/device.h> 2421afaf18SBorislav Petkov #include <linux/syscore_ops.h> 2521afaf18SBorislav Petkov #include <linux/delay.h> 2621afaf18SBorislav Petkov #include <linux/ctype.h> 2721afaf18SBorislav Petkov #include <linux/sched.h> 2821afaf18SBorislav Petkov #include <linux/sysfs.h> 2921afaf18SBorislav Petkov #include <linux/types.h> 3021afaf18SBorislav Petkov #include <linux/slab.h> 3121afaf18SBorislav Petkov #include <linux/init.h> 3221afaf18SBorislav Petkov #include <linux/kmod.h> 3321afaf18SBorislav Petkov #include <linux/poll.h> 3421afaf18SBorislav Petkov #include <linux/nmi.h> 3521afaf18SBorislav Petkov #include <linux/cpu.h> 3621afaf18SBorislav Petkov #include <linux/ras.h> 3721afaf18SBorislav Petkov #include <linux/smp.h> 3821afaf18SBorislav Petkov #include <linux/fs.h> 3921afaf18SBorislav Petkov #include <linux/mm.h> 4021afaf18SBorislav Petkov #include <linux/debugfs.h> 4121afaf18SBorislav Petkov #include <linux/irq_work.h> 4221afaf18SBorislav Petkov #include <linux/export.h> 4321afaf18SBorislav Petkov #include <linux/set_memory.h> 449998a983SRicardo Neri #include <linux/sync_core.h> 455567d11cSPeter Zijlstra #include <linux/task_work.h> 460d00449cSPeter Zijlstra #include <linux/hardirq.h> 47*04c6948dSZhiquan Li #include <linux/kexec.h> 4821afaf18SBorislav Petkov 4921afaf18SBorislav Petkov #include <asm/intel-family.h> 5021afaf18SBorislav Petkov #include <asm/processor.h> 5121afaf18SBorislav Petkov #include <asm/traps.h> 5221afaf18SBorislav Petkov #include <asm/tlbflush.h> 5321afaf18SBorislav Petkov #include <asm/mce.h> 5421afaf18SBorislav Petkov #include <asm/msr.h> 5521afaf18SBorislav Petkov #include <asm/reboot.h> 5621afaf18SBorislav Petkov 5721afaf18SBorislav Petkov #include "internal.h" 5821afaf18SBorislav Petkov 5921afaf18SBorislav Petkov /* sysfs synchronization */ 6021afaf18SBorislav Petkov static DEFINE_MUTEX(mce_sysfs_mutex); 6121afaf18SBorislav Petkov 6221afaf18SBorislav Petkov #define CREATE_TRACE_POINTS 6321afaf18SBorislav Petkov #include <trace/events/mce.h> 6421afaf18SBorislav Petkov 6521afaf18SBorislav Petkov #define SPINUNIT 100 /* 100ns */ 6621afaf18SBorislav Petkov 6721afaf18SBorislav Petkov DEFINE_PER_CPU(unsigned, mce_exception_count); 6821afaf18SBorislav Petkov 69c7d314f3SYazen Ghannam DEFINE_PER_CPU_READ_MOSTLY(unsigned int, mce_num_banks); 70c7d314f3SYazen Ghannam 71fcd343a2SSmita Koralahalli DEFINE_PER_CPU_READ_MOSTLY(struct mce_bank[MAX_NR_BANKS], mce_banks_array); 72b4914508SYazen Ghannam 73b4914508SYazen Ghannam #define ATTR_LEN 16 74b4914508SYazen Ghannam /* One object for each MCE bank, shared by all CPUs */ 75b4914508SYazen Ghannam struct mce_bank_dev { 7695fdce6bSYazen Ghannam struct device_attribute attr; /* device attribute */ 7795fdce6bSYazen Ghannam char attrname[ATTR_LEN]; /* attribute name */ 78b4914508SYazen Ghannam u8 bank; /* bank number */ 7995fdce6bSYazen Ghannam }; 80b4914508SYazen Ghannam static struct mce_bank_dev mce_bank_devs[MAX_NR_BANKS]; 8195fdce6bSYazen Ghannam 8221afaf18SBorislav Petkov struct mce_vendor_flags mce_flags __read_mostly; 8321afaf18SBorislav Petkov 8421afaf18SBorislav Petkov struct mca_config mca_cfg __read_mostly = { 8521afaf18SBorislav Petkov .bootlog = -1, 8621afaf18SBorislav Petkov .monarch_timeout = -1 8721afaf18SBorislav Petkov }; 8821afaf18SBorislav Petkov 8921afaf18SBorislav Petkov static DEFINE_PER_CPU(struct mce, mces_seen); 9021afaf18SBorislav Petkov static unsigned long mce_need_notify; 9121afaf18SBorislav Petkov 9221afaf18SBorislav Petkov /* 9321afaf18SBorislav Petkov * MCA banks polled by the period polling timer for corrected events. 9421afaf18SBorislav Petkov * With Intel CMCI, this only has MCA banks which do not support CMCI (if any). 9521afaf18SBorislav Petkov */ 9621afaf18SBorislav Petkov DEFINE_PER_CPU(mce_banks_t, mce_poll_banks) = { 9721afaf18SBorislav Petkov [0 ... BITS_TO_LONGS(MAX_NR_BANKS)-1] = ~0UL 9821afaf18SBorislav Petkov }; 9921afaf18SBorislav Petkov 10021afaf18SBorislav Petkov /* 10121afaf18SBorislav Petkov * MCA banks controlled through firmware first for corrected errors. 10221afaf18SBorislav Petkov * This is a global list of banks for which we won't enable CMCI and we 10321afaf18SBorislav Petkov * won't poll. Firmware controls these banks and is responsible for 10421afaf18SBorislav Petkov * reporting corrected errors through GHES. Uncorrected/recoverable 10521afaf18SBorislav Petkov * errors are still notified through a machine check. 10621afaf18SBorislav Petkov */ 10721afaf18SBorislav Petkov mce_banks_t mce_banks_ce_disabled; 10821afaf18SBorislav Petkov 10921afaf18SBorislav Petkov static struct work_struct mce_work; 11021afaf18SBorislav Petkov static struct irq_work mce_irq_work; 11121afaf18SBorislav Petkov 11221afaf18SBorislav Petkov /* 11321afaf18SBorislav Petkov * CPU/chipset specific EDAC code can register a notifier call here to print 11421afaf18SBorislav Petkov * MCE errors in a human-readable form. 11521afaf18SBorislav Petkov */ 11621afaf18SBorislav Petkov BLOCKING_NOTIFIER_HEAD(x86_mce_decoder_chain); 11721afaf18SBorislav Petkov 11821afaf18SBorislav Petkov /* Do initial initialization of a struct mce */ 119487d654dSBorislav Petkov void mce_setup(struct mce *m) 12021afaf18SBorislav Petkov { 12121afaf18SBorislav Petkov memset(m, 0, sizeof(struct mce)); 12221afaf18SBorislav Petkov m->cpu = m->extcpu = smp_processor_id(); 12321afaf18SBorislav Petkov /* need the internal __ version to avoid deadlocks */ 12421afaf18SBorislav Petkov m->time = __ktime_get_real_seconds(); 12521afaf18SBorislav Petkov m->cpuvendor = boot_cpu_data.x86_vendor; 12621afaf18SBorislav Petkov m->cpuid = cpuid_eax(1); 12721afaf18SBorislav Petkov m->socketid = cpu_data(m->extcpu).phys_proc_id; 12821afaf18SBorislav Petkov m->apicid = cpu_data(m->extcpu).initial_apicid; 129865d3a9aSThomas Gleixner m->mcgcap = __rdmsr(MSR_IA32_MCG_CAP); 130822ccfadSTony Luck m->ppin = cpu_data(m->extcpu).ppin; 13121afaf18SBorislav Petkov m->microcode = boot_cpu_data.microcode; 13221afaf18SBorislav Petkov } 13321afaf18SBorislav Petkov 13421afaf18SBorislav Petkov DEFINE_PER_CPU(struct mce, injectm); 13521afaf18SBorislav Petkov EXPORT_PER_CPU_SYMBOL_GPL(injectm); 13621afaf18SBorislav Petkov 13721afaf18SBorislav Petkov void mce_log(struct mce *m) 13821afaf18SBorislav Petkov { 13921afaf18SBorislav Petkov if (!mce_gen_pool_add(m)) 14021afaf18SBorislav Petkov irq_work_queue(&mce_irq_work); 14121afaf18SBorislav Petkov } 14281736abdSJan H. Schönherr EXPORT_SYMBOL_GPL(mce_log); 14321afaf18SBorislav Petkov 14421afaf18SBorislav Petkov void mce_register_decode_chain(struct notifier_block *nb) 14521afaf18SBorislav Petkov { 14615af3659SZhen Lei if (WARN_ON(nb->priority < MCE_PRIO_LOWEST || 14715af3659SZhen Lei nb->priority > MCE_PRIO_HIGHEST)) 14821afaf18SBorislav Petkov return; 14921afaf18SBorislav Petkov 15021afaf18SBorislav Petkov blocking_notifier_chain_register(&x86_mce_decoder_chain, nb); 15121afaf18SBorislav Petkov } 15221afaf18SBorislav Petkov EXPORT_SYMBOL_GPL(mce_register_decode_chain); 15321afaf18SBorislav Petkov 15421afaf18SBorislav Petkov void mce_unregister_decode_chain(struct notifier_block *nb) 15521afaf18SBorislav Petkov { 15621afaf18SBorislav Petkov blocking_notifier_chain_unregister(&x86_mce_decoder_chain, nb); 15721afaf18SBorislav Petkov } 15821afaf18SBorislav Petkov EXPORT_SYMBOL_GPL(mce_unregister_decode_chain); 15921afaf18SBorislav Petkov 16021afaf18SBorislav Petkov static void __print_mce(struct mce *m) 16121afaf18SBorislav Petkov { 16221afaf18SBorislav Petkov pr_emerg(HW_ERR "CPU %d: Machine Check%s: %Lx Bank %d: %016Lx\n", 16321afaf18SBorislav Petkov m->extcpu, 16421afaf18SBorislav Petkov (m->mcgstatus & MCG_STATUS_MCIP ? " Exception" : ""), 16521afaf18SBorislav Petkov m->mcgstatus, m->bank, m->status); 16621afaf18SBorislav Petkov 16721afaf18SBorislav Petkov if (m->ip) { 16821afaf18SBorislav Petkov pr_emerg(HW_ERR "RIP%s %02x:<%016Lx> ", 16921afaf18SBorislav Petkov !(m->mcgstatus & MCG_STATUS_EIPV) ? " !INEXACT!" : "", 17021afaf18SBorislav Petkov m->cs, m->ip); 17121afaf18SBorislav Petkov 17221afaf18SBorislav Petkov if (m->cs == __KERNEL_CS) 17321afaf18SBorislav Petkov pr_cont("{%pS}", (void *)(unsigned long)m->ip); 17421afaf18SBorislav Petkov pr_cont("\n"); 17521afaf18SBorislav Petkov } 17621afaf18SBorislav Petkov 17721afaf18SBorislav Petkov pr_emerg(HW_ERR "TSC %llx ", m->tsc); 17821afaf18SBorislav Petkov if (m->addr) 17921afaf18SBorislav Petkov pr_cont("ADDR %llx ", m->addr); 18021afaf18SBorislav Petkov if (m->misc) 18121afaf18SBorislav Petkov pr_cont("MISC %llx ", m->misc); 182bb2de0adSSmita Koralahalli if (m->ppin) 183bb2de0adSSmita Koralahalli pr_cont("PPIN %llx ", m->ppin); 18421afaf18SBorislav Petkov 18521afaf18SBorislav Petkov if (mce_flags.smca) { 18621afaf18SBorislav Petkov if (m->synd) 18721afaf18SBorislav Petkov pr_cont("SYND %llx ", m->synd); 18821afaf18SBorislav Petkov if (m->ipid) 18921afaf18SBorislav Petkov pr_cont("IPID %llx ", m->ipid); 19021afaf18SBorislav Petkov } 19121afaf18SBorislav Petkov 19221afaf18SBorislav Petkov pr_cont("\n"); 193925946cfSTony Luck 19421afaf18SBorislav Petkov /* 19521afaf18SBorislav Petkov * Note this output is parsed by external tools and old fields 19621afaf18SBorislav Petkov * should not be changed. 19721afaf18SBorislav Petkov */ 19821afaf18SBorislav Petkov pr_emerg(HW_ERR "PROCESSOR %u:%x TIME %llu SOCKET %u APIC %x microcode %x\n", 19921afaf18SBorislav Petkov m->cpuvendor, m->cpuid, m->time, m->socketid, m->apicid, 20021afaf18SBorislav Petkov m->microcode); 20121afaf18SBorislav Petkov } 20221afaf18SBorislav Petkov 20321afaf18SBorislav Petkov static void print_mce(struct mce *m) 20421afaf18SBorislav Petkov { 20521afaf18SBorislav Petkov __print_mce(m); 20621afaf18SBorislav Petkov 20721afaf18SBorislav Petkov if (m->cpuvendor != X86_VENDOR_AMD && m->cpuvendor != X86_VENDOR_HYGON) 20821afaf18SBorislav Petkov pr_emerg_ratelimited(HW_ERR "Run the above through 'mcelog --ascii'\n"); 20921afaf18SBorislav Petkov } 21021afaf18SBorislav Petkov 21121afaf18SBorislav Petkov #define PANIC_TIMEOUT 5 /* 5 seconds */ 21221afaf18SBorislav Petkov 21321afaf18SBorislav Petkov static atomic_t mce_panicked; 21421afaf18SBorislav Petkov 21521afaf18SBorislav Petkov static int fake_panic; 21621afaf18SBorislav Petkov static atomic_t mce_fake_panicked; 21721afaf18SBorislav Petkov 21821afaf18SBorislav Petkov /* Panic in progress. Enable interrupts and wait for final IPI */ 21921afaf18SBorislav Petkov static void wait_for_panic(void) 22021afaf18SBorislav Petkov { 22121afaf18SBorislav Petkov long timeout = PANIC_TIMEOUT*USEC_PER_SEC; 22221afaf18SBorislav Petkov 22321afaf18SBorislav Petkov preempt_disable(); 22421afaf18SBorislav Petkov local_irq_enable(); 22521afaf18SBorislav Petkov while (timeout-- > 0) 22621afaf18SBorislav Petkov udelay(1); 22721afaf18SBorislav Petkov if (panic_timeout == 0) 22821afaf18SBorislav Petkov panic_timeout = mca_cfg.panic_timeout; 22921afaf18SBorislav Petkov panic("Panicing machine check CPU died"); 23021afaf18SBorislav Petkov } 23121afaf18SBorislav Petkov 2323c7ce80aSBorislav Petkov static noinstr void mce_panic(const char *msg, struct mce *final, char *exp) 23321afaf18SBorislav Petkov { 23421afaf18SBorislav Petkov struct llist_node *pending; 23521afaf18SBorislav Petkov struct mce_evt_llist *l; 2363c7ce80aSBorislav Petkov int apei_err = 0; 237*04c6948dSZhiquan Li struct page *p; 2383c7ce80aSBorislav Petkov 2393c7ce80aSBorislav Petkov /* 2403c7ce80aSBorislav Petkov * Allow instrumentation around external facilities usage. Not that it 2413c7ce80aSBorislav Petkov * matters a whole lot since the machine is going to panic anyway. 2423c7ce80aSBorislav Petkov */ 2433c7ce80aSBorislav Petkov instrumentation_begin(); 24421afaf18SBorislav Petkov 24521afaf18SBorislav Petkov if (!fake_panic) { 24621afaf18SBorislav Petkov /* 24721afaf18SBorislav Petkov * Make sure only one CPU runs in machine check panic 24821afaf18SBorislav Petkov */ 24921afaf18SBorislav Petkov if (atomic_inc_return(&mce_panicked) > 1) 25021afaf18SBorislav Petkov wait_for_panic(); 25121afaf18SBorislav Petkov barrier(); 25221afaf18SBorislav Petkov 25321afaf18SBorislav Petkov bust_spinlocks(1); 25421afaf18SBorislav Petkov console_verbose(); 25521afaf18SBorislav Petkov } else { 25621afaf18SBorislav Petkov /* Don't log too much for fake panic */ 25721afaf18SBorislav Petkov if (atomic_inc_return(&mce_fake_panicked) > 1) 2583c7ce80aSBorislav Petkov goto out; 25921afaf18SBorislav Petkov } 26021afaf18SBorislav Petkov pending = mce_gen_pool_prepare_records(); 26121afaf18SBorislav Petkov /* First print corrected ones that are still unlogged */ 26221afaf18SBorislav Petkov llist_for_each_entry(l, pending, llnode) { 26321afaf18SBorislav Petkov struct mce *m = &l->mce; 26421afaf18SBorislav Petkov if (!(m->status & MCI_STATUS_UC)) { 26521afaf18SBorislav Petkov print_mce(m); 26621afaf18SBorislav Petkov if (!apei_err) 26721afaf18SBorislav Petkov apei_err = apei_write_mce(m); 26821afaf18SBorislav Petkov } 26921afaf18SBorislav Petkov } 27021afaf18SBorislav Petkov /* Now print uncorrected but with the final one last */ 27121afaf18SBorislav Petkov llist_for_each_entry(l, pending, llnode) { 27221afaf18SBorislav Petkov struct mce *m = &l->mce; 27321afaf18SBorislav Petkov if (!(m->status & MCI_STATUS_UC)) 27421afaf18SBorislav Petkov continue; 27521afaf18SBorislav Petkov if (!final || mce_cmp(m, final)) { 27621afaf18SBorislav Petkov print_mce(m); 27721afaf18SBorislav Petkov if (!apei_err) 27821afaf18SBorislav Petkov apei_err = apei_write_mce(m); 27921afaf18SBorislav Petkov } 28021afaf18SBorislav Petkov } 28121afaf18SBorislav Petkov if (final) { 28221afaf18SBorislav Petkov print_mce(final); 28321afaf18SBorislav Petkov if (!apei_err) 28421afaf18SBorislav Petkov apei_err = apei_write_mce(final); 28521afaf18SBorislav Petkov } 28621afaf18SBorislav Petkov if (exp) 28721afaf18SBorislav Petkov pr_emerg(HW_ERR "Machine check: %s\n", exp); 28821afaf18SBorislav Petkov if (!fake_panic) { 28921afaf18SBorislav Petkov if (panic_timeout == 0) 29021afaf18SBorislav Petkov panic_timeout = mca_cfg.panic_timeout; 291*04c6948dSZhiquan Li 292*04c6948dSZhiquan Li /* 293*04c6948dSZhiquan Li * Kdump skips the poisoned page in order to avoid 294*04c6948dSZhiquan Li * touching the error bits again. Poison the page even 295*04c6948dSZhiquan Li * if the error is fatal and the machine is about to 296*04c6948dSZhiquan Li * panic. 297*04c6948dSZhiquan Li */ 298*04c6948dSZhiquan Li if (kexec_crash_loaded()) { 299*04c6948dSZhiquan Li if (final && (final->status & MCI_STATUS_ADDRV)) { 300*04c6948dSZhiquan Li p = pfn_to_online_page(final->addr >> PAGE_SHIFT); 301*04c6948dSZhiquan Li if (p) 302*04c6948dSZhiquan Li SetPageHWPoison(p); 303*04c6948dSZhiquan Li } 304*04c6948dSZhiquan Li } 30521afaf18SBorislav Petkov panic(msg); 30621afaf18SBorislav Petkov } else 30721afaf18SBorislav Petkov pr_emerg(HW_ERR "Fake kernel panic: %s\n", msg); 3083c7ce80aSBorislav Petkov 3093c7ce80aSBorislav Petkov out: 3103c7ce80aSBorislav Petkov instrumentation_end(); 31121afaf18SBorislav Petkov } 31221afaf18SBorislav Petkov 31321afaf18SBorislav Petkov /* Support code for software error injection */ 31421afaf18SBorislav Petkov 31521afaf18SBorislav Petkov static int msr_to_offset(u32 msr) 31621afaf18SBorislav Petkov { 31721afaf18SBorislav Petkov unsigned bank = __this_cpu_read(injectm.bank); 31821afaf18SBorislav Petkov 31921afaf18SBorislav Petkov if (msr == mca_cfg.rip_msr) 32021afaf18SBorislav Petkov return offsetof(struct mce, ip); 3218121b8f9SBorislav Petkov if (msr == mca_msr_reg(bank, MCA_STATUS)) 32221afaf18SBorislav Petkov return offsetof(struct mce, status); 3238121b8f9SBorislav Petkov if (msr == mca_msr_reg(bank, MCA_ADDR)) 32421afaf18SBorislav Petkov return offsetof(struct mce, addr); 3258121b8f9SBorislav Petkov if (msr == mca_msr_reg(bank, MCA_MISC)) 32621afaf18SBorislav Petkov return offsetof(struct mce, misc); 32721afaf18SBorislav Petkov if (msr == MSR_IA32_MCG_STATUS) 32821afaf18SBorislav Petkov return offsetof(struct mce, mcgstatus); 32921afaf18SBorislav Petkov return -1; 33021afaf18SBorislav Petkov } 33121afaf18SBorislav Petkov 33246d28947SThomas Gleixner void ex_handler_msr_mce(struct pt_regs *regs, bool wrmsr) 333e2def7d4SBorislav Petkov { 334e42404afSThomas Gleixner if (wrmsr) { 335e42404afSThomas Gleixner pr_emerg("MSR access error: WRMSR to 0x%x (tried to write 0x%08x%08x) at rIP: 0x%lx (%pS)\n", 336e42404afSThomas Gleixner (unsigned int)regs->cx, (unsigned int)regs->dx, (unsigned int)regs->ax, 337e42404afSThomas Gleixner regs->ip, (void *)regs->ip); 338e42404afSThomas Gleixner } else { 339e2def7d4SBorislav Petkov pr_emerg("MSR access error: RDMSR from 0x%x at rIP: 0x%lx (%pS)\n", 340e2def7d4SBorislav Petkov (unsigned int)regs->cx, regs->ip, (void *)regs->ip); 341e42404afSThomas Gleixner } 342e2def7d4SBorislav Petkov 343e2def7d4SBorislav Petkov show_stack_regs(regs); 344e2def7d4SBorislav Petkov 345e2def7d4SBorislav Petkov panic("MCA architectural violation!\n"); 346e2def7d4SBorislav Petkov 347e2def7d4SBorislav Petkov while (true) 348e2def7d4SBorislav Petkov cpu_relax(); 349e42404afSThomas Gleixner } 350e2def7d4SBorislav Petkov 35121afaf18SBorislav Petkov /* MSR access wrappers used for error injection */ 35288f66a42SBorislav Petkov noinstr u64 mce_rdmsrl(u32 msr) 35321afaf18SBorislav Petkov { 354e2def7d4SBorislav Petkov DECLARE_ARGS(val, low, high); 35521afaf18SBorislav Petkov 35621afaf18SBorislav Petkov if (__this_cpu_read(injectm.finished)) { 357e1007770SBorislav Petkov int offset; 358e1007770SBorislav Petkov u64 ret; 35921afaf18SBorislav Petkov 360e1007770SBorislav Petkov instrumentation_begin(); 361e1007770SBorislav Petkov 362e1007770SBorislav Petkov offset = msr_to_offset(msr); 36321afaf18SBorislav Petkov if (offset < 0) 364e1007770SBorislav Petkov ret = 0; 365e1007770SBorislav Petkov else 366e1007770SBorislav Petkov ret = *(u64 *)((char *)this_cpu_ptr(&injectm) + offset); 367e1007770SBorislav Petkov 368e1007770SBorislav Petkov instrumentation_end(); 369e1007770SBorislav Petkov 370e1007770SBorislav Petkov return ret; 37121afaf18SBorislav Petkov } 37221afaf18SBorislav Petkov 37321afaf18SBorislav Petkov /* 374e2def7d4SBorislav Petkov * RDMSR on MCA MSRs should not fault. If they do, this is very much an 375e2def7d4SBorislav Petkov * architectural violation and needs to be reported to hw vendor. Panic 376e2def7d4SBorislav Petkov * the box to not allow any further progress. 37721afaf18SBorislav Petkov */ 378e2def7d4SBorislav Petkov asm volatile("1: rdmsr\n" 379e2def7d4SBorislav Petkov "2:\n" 38046d28947SThomas Gleixner _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_RDMSR_IN_MCE) 381e2def7d4SBorislav Petkov : EAX_EDX_RET(val, low, high) : "c" (msr)); 382e2def7d4SBorislav Petkov 383e2def7d4SBorislav Petkov 384e2def7d4SBorislav Petkov return EAX_EDX_VAL(val, low, high); 38521afaf18SBorislav Petkov } 38621afaf18SBorislav Petkov 387e1007770SBorislav Petkov static noinstr void mce_wrmsrl(u32 msr, u64 v) 38821afaf18SBorislav Petkov { 389e2def7d4SBorislav Petkov u32 low, high; 390e2def7d4SBorislav Petkov 39121afaf18SBorislav Petkov if (__this_cpu_read(injectm.finished)) { 392e1007770SBorislav Petkov int offset; 39321afaf18SBorislav Petkov 394e1007770SBorislav Petkov instrumentation_begin(); 395e1007770SBorislav Petkov 396e1007770SBorislav Petkov offset = msr_to_offset(msr); 39721afaf18SBorislav Petkov if (offset >= 0) 39821afaf18SBorislav Petkov *(u64 *)((char *)this_cpu_ptr(&injectm) + offset) = v; 399e1007770SBorislav Petkov 400e1007770SBorislav Petkov instrumentation_end(); 401e1007770SBorislav Petkov 40221afaf18SBorislav Petkov return; 40321afaf18SBorislav Petkov } 404e2def7d4SBorislav Petkov 405e2def7d4SBorislav Petkov low = (u32)v; 406e2def7d4SBorislav Petkov high = (u32)(v >> 32); 407e2def7d4SBorislav Petkov 408e2def7d4SBorislav Petkov /* See comment in mce_rdmsrl() */ 409e2def7d4SBorislav Petkov asm volatile("1: wrmsr\n" 410e2def7d4SBorislav Petkov "2:\n" 41146d28947SThomas Gleixner _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_WRMSR_IN_MCE) 412e2def7d4SBorislav Petkov : : "c" (msr), "a"(low), "d" (high) : "memory"); 41321afaf18SBorislav Petkov } 41421afaf18SBorislav Petkov 41521afaf18SBorislav Petkov /* 41621afaf18SBorislav Petkov * Collect all global (w.r.t. this processor) status about this machine 41721afaf18SBorislav Petkov * check into our "mce" struct so that we can use it later to assess 41821afaf18SBorislav Petkov * the severity of the problem as we read per-bank specific details. 41921afaf18SBorislav Petkov */ 420487d654dSBorislav Petkov static noinstr void mce_gather_info(struct mce *m, struct pt_regs *regs) 42121afaf18SBorislav Petkov { 422487d654dSBorislav Petkov /* 423487d654dSBorislav Petkov * Enable instrumentation around mce_setup() which calls external 424487d654dSBorislav Petkov * facilities. 425487d654dSBorislav Petkov */ 426487d654dSBorislav Petkov instrumentation_begin(); 42721afaf18SBorislav Petkov mce_setup(m); 428487d654dSBorislav Petkov instrumentation_end(); 42921afaf18SBorislav Petkov 43021afaf18SBorislav Petkov m->mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS); 43121afaf18SBorislav Petkov if (regs) { 43221afaf18SBorislav Petkov /* 43321afaf18SBorislav Petkov * Get the address of the instruction at the time of 43421afaf18SBorislav Petkov * the machine check error. 43521afaf18SBorislav Petkov */ 43621afaf18SBorislav Petkov if (m->mcgstatus & (MCG_STATUS_RIPV|MCG_STATUS_EIPV)) { 43721afaf18SBorislav Petkov m->ip = regs->ip; 43821afaf18SBorislav Petkov m->cs = regs->cs; 43921afaf18SBorislav Petkov 44021afaf18SBorislav Petkov /* 44121afaf18SBorislav Petkov * When in VM86 mode make the cs look like ring 3 44221afaf18SBorislav Petkov * always. This is a lie, but it's better than passing 44321afaf18SBorislav Petkov * the additional vm86 bit around everywhere. 44421afaf18SBorislav Petkov */ 44521afaf18SBorislav Petkov if (v8086_mode(regs)) 44621afaf18SBorislav Petkov m->cs |= 3; 44721afaf18SBorislav Petkov } 44821afaf18SBorislav Petkov /* Use accurate RIP reporting if available. */ 44921afaf18SBorislav Petkov if (mca_cfg.rip_msr) 45021afaf18SBorislav Petkov m->ip = mce_rdmsrl(mca_cfg.rip_msr); 45121afaf18SBorislav Petkov } 45221afaf18SBorislav Petkov } 45321afaf18SBorislav Petkov 45421afaf18SBorislav Petkov int mce_available(struct cpuinfo_x86 *c) 45521afaf18SBorislav Petkov { 45621afaf18SBorislav Petkov if (mca_cfg.disabled) 45721afaf18SBorislav Petkov return 0; 45821afaf18SBorislav Petkov return cpu_has(c, X86_FEATURE_MCE) && cpu_has(c, X86_FEATURE_MCA); 45921afaf18SBorislav Petkov } 46021afaf18SBorislav Petkov 46121afaf18SBorislav Petkov static void mce_schedule_work(void) 46221afaf18SBorislav Petkov { 46321afaf18SBorislav Petkov if (!mce_gen_pool_empty()) 46421afaf18SBorislav Petkov schedule_work(&mce_work); 46521afaf18SBorislav Petkov } 46621afaf18SBorislav Petkov 46721afaf18SBorislav Petkov static void mce_irq_work_cb(struct irq_work *entry) 46821afaf18SBorislav Petkov { 46921afaf18SBorislav Petkov mce_schedule_work(); 47021afaf18SBorislav Petkov } 47121afaf18SBorislav Petkov 47221afaf18SBorislav Petkov /* 47321afaf18SBorislav Petkov * Check if the address reported by the CPU is in a format we can parse. 47421afaf18SBorislav Petkov * It would be possible to add code for most other cases, but all would 47521afaf18SBorislav Petkov * be somewhat complicated (e.g. segment offset would require an instruction 476d9f6e12fSIngo Molnar * parser). So only support physical addresses up to page granularity for now. 47721afaf18SBorislav Petkov */ 47821afaf18SBorislav Petkov int mce_usable_address(struct mce *m) 47921afaf18SBorislav Petkov { 48021afaf18SBorislav Petkov if (!(m->status & MCI_STATUS_ADDRV)) 48121afaf18SBorislav Petkov return 0; 48221afaf18SBorislav Petkov 4836e898d2bSTony W Wang-oc /* Checks after this one are Intel/Zhaoxin-specific: */ 4846e898d2bSTony W Wang-oc if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL && 4856e898d2bSTony W Wang-oc boot_cpu_data.x86_vendor != X86_VENDOR_ZHAOXIN) 48621afaf18SBorislav Petkov return 1; 48721afaf18SBorislav Petkov 48821afaf18SBorislav Petkov if (!(m->status & MCI_STATUS_MISCV)) 48921afaf18SBorislav Petkov return 0; 49021afaf18SBorislav Petkov 49121afaf18SBorislav Petkov if (MCI_MISC_ADDR_LSB(m->misc) > PAGE_SHIFT) 49221afaf18SBorislav Petkov return 0; 49321afaf18SBorislav Petkov 49421afaf18SBorislav Petkov if (MCI_MISC_ADDR_MODE(m->misc) != MCI_MISC_ADDR_PHYS) 49521afaf18SBorislav Petkov return 0; 49621afaf18SBorislav Petkov 49721afaf18SBorislav Petkov return 1; 49821afaf18SBorislav Petkov } 49921afaf18SBorislav Petkov EXPORT_SYMBOL_GPL(mce_usable_address); 50021afaf18SBorislav Petkov 50121afaf18SBorislav Petkov bool mce_is_memory_error(struct mce *m) 50221afaf18SBorislav Petkov { 5036e898d2bSTony W Wang-oc switch (m->cpuvendor) { 5046e898d2bSTony W Wang-oc case X86_VENDOR_AMD: 5056e898d2bSTony W Wang-oc case X86_VENDOR_HYGON: 50621afaf18SBorislav Petkov return amd_mce_is_memory_error(m); 5076e898d2bSTony W Wang-oc 5086e898d2bSTony W Wang-oc case X86_VENDOR_INTEL: 5096e898d2bSTony W Wang-oc case X86_VENDOR_ZHAOXIN: 51021afaf18SBorislav Petkov /* 51121afaf18SBorislav Petkov * Intel SDM Volume 3B - 15.9.2 Compound Error Codes 51221afaf18SBorislav Petkov * 51321afaf18SBorislav Petkov * Bit 7 of the MCACOD field of IA32_MCi_STATUS is used for 51421afaf18SBorislav Petkov * indicating a memory error. Bit 8 is used for indicating a 51521afaf18SBorislav Petkov * cache hierarchy error. The combination of bit 2 and bit 3 51621afaf18SBorislav Petkov * is used for indicating a `generic' cache hierarchy error 51721afaf18SBorislav Petkov * But we can't just blindly check the above bits, because if 51821afaf18SBorislav Petkov * bit 11 is set, then it is a bus/interconnect error - and 51921afaf18SBorislav Petkov * either way the above bits just gives more detail on what 52021afaf18SBorislav Petkov * bus/interconnect error happened. Note that bit 12 can be 52121afaf18SBorislav Petkov * ignored, as it's the "filter" bit. 52221afaf18SBorislav Petkov */ 52321afaf18SBorislav Petkov return (m->status & 0xef80) == BIT(7) || 52421afaf18SBorislav Petkov (m->status & 0xef00) == BIT(8) || 52521afaf18SBorislav Petkov (m->status & 0xeffc) == 0xc; 52621afaf18SBorislav Petkov 5276e898d2bSTony W Wang-oc default: 52821afaf18SBorislav Petkov return false; 52921afaf18SBorislav Petkov } 5306e898d2bSTony W Wang-oc } 53121afaf18SBorislav Petkov EXPORT_SYMBOL_GPL(mce_is_memory_error); 53221afaf18SBorislav Petkov 53317fae129STony Luck static bool whole_page(struct mce *m) 53417fae129STony Luck { 53517fae129STony Luck if (!mca_cfg.ser || !(m->status & MCI_STATUS_MISCV)) 53617fae129STony Luck return true; 53717fae129STony Luck 53817fae129STony Luck return MCI_MISC_ADDR_LSB(m->misc) >= PAGE_SHIFT; 53917fae129STony Luck } 54017fae129STony Luck 54121afaf18SBorislav Petkov bool mce_is_correctable(struct mce *m) 54221afaf18SBorislav Petkov { 54321afaf18SBorislav Petkov if (m->cpuvendor == X86_VENDOR_AMD && m->status & MCI_STATUS_DEFERRED) 54421afaf18SBorislav Petkov return false; 54521afaf18SBorislav Petkov 54621afaf18SBorislav Petkov if (m->cpuvendor == X86_VENDOR_HYGON && m->status & MCI_STATUS_DEFERRED) 54721afaf18SBorislav Petkov return false; 54821afaf18SBorislav Petkov 54921afaf18SBorislav Petkov if (m->status & MCI_STATUS_UC) 55021afaf18SBorislav Petkov return false; 55121afaf18SBorislav Petkov 55221afaf18SBorislav Petkov return true; 55321afaf18SBorislav Petkov } 55421afaf18SBorislav Petkov EXPORT_SYMBOL_GPL(mce_is_correctable); 55521afaf18SBorislav Petkov 556c9c6d216STony Luck static int mce_early_notifier(struct notifier_block *nb, unsigned long val, 55721afaf18SBorislav Petkov void *data) 55821afaf18SBorislav Petkov { 55921afaf18SBorislav Petkov struct mce *m = (struct mce *)data; 56021afaf18SBorislav Petkov 56121afaf18SBorislav Petkov if (!m) 56221afaf18SBorislav Petkov return NOTIFY_DONE; 56321afaf18SBorislav Petkov 56421afaf18SBorislav Petkov /* Emit the trace record: */ 56521afaf18SBorislav Petkov trace_mce_record(m); 56621afaf18SBorislav Petkov 56721afaf18SBorislav Petkov set_bit(0, &mce_need_notify); 56821afaf18SBorislav Petkov 56921afaf18SBorislav Petkov mce_notify_irq(); 57021afaf18SBorislav Petkov 57121afaf18SBorislav Petkov return NOTIFY_DONE; 57221afaf18SBorislav Petkov } 57321afaf18SBorislav Petkov 574c9c6d216STony Luck static struct notifier_block early_nb = { 575c9c6d216STony Luck .notifier_call = mce_early_notifier, 576c9c6d216STony Luck .priority = MCE_PRIO_EARLY, 57721afaf18SBorislav Petkov }; 57821afaf18SBorislav Petkov 5798438b84aSJan H. Schönherr static int uc_decode_notifier(struct notifier_block *nb, unsigned long val, 58021afaf18SBorislav Petkov void *data) 58121afaf18SBorislav Petkov { 58221afaf18SBorislav Petkov struct mce *mce = (struct mce *)data; 58321afaf18SBorislav Petkov unsigned long pfn; 58421afaf18SBorislav Petkov 5858438b84aSJan H. Schönherr if (!mce || !mce_usable_address(mce)) 58621afaf18SBorislav Petkov return NOTIFY_DONE; 58721afaf18SBorislav Petkov 5888438b84aSJan H. Schönherr if (mce->severity != MCE_AO_SEVERITY && 5898438b84aSJan H. Schönherr mce->severity != MCE_DEFERRED_SEVERITY) 5908438b84aSJan H. Schönherr return NOTIFY_DONE; 5918438b84aSJan H. Schönherr 5928a01ec97STony Luck pfn = (mce->addr & MCI_ADDR_PHYSADDR) >> PAGE_SHIFT; 59323ba710aSTony Luck if (!memory_failure(pfn, 0)) { 5945898b43aSJane Chu set_mce_nospec(pfn); 59523ba710aSTony Luck mce->kflags |= MCE_HANDLED_UC; 59623ba710aSTony Luck } 59721afaf18SBorislav Petkov 59821afaf18SBorislav Petkov return NOTIFY_OK; 59921afaf18SBorislav Petkov } 6008438b84aSJan H. Schönherr 6018438b84aSJan H. Schönherr static struct notifier_block mce_uc_nb = { 6028438b84aSJan H. Schönherr .notifier_call = uc_decode_notifier, 6038438b84aSJan H. Schönherr .priority = MCE_PRIO_UC, 60421afaf18SBorislav Petkov }; 60521afaf18SBorislav Petkov 60621afaf18SBorislav Petkov static int mce_default_notifier(struct notifier_block *nb, unsigned long val, 60721afaf18SBorislav Petkov void *data) 60821afaf18SBorislav Petkov { 60921afaf18SBorislav Petkov struct mce *m = (struct mce *)data; 61021afaf18SBorislav Petkov 61121afaf18SBorislav Petkov if (!m) 61221afaf18SBorislav Petkov return NOTIFY_DONE; 61321afaf18SBorislav Petkov 61443505646STony Luck if (mca_cfg.print_all || !m->kflags) 61521afaf18SBorislav Petkov __print_mce(m); 61621afaf18SBorislav Petkov 61721afaf18SBorislav Petkov return NOTIFY_DONE; 61821afaf18SBorislav Petkov } 61921afaf18SBorislav Petkov 62021afaf18SBorislav Petkov static struct notifier_block mce_default_nb = { 62121afaf18SBorislav Petkov .notifier_call = mce_default_notifier, 62221afaf18SBorislav Petkov /* lowest prio, we want it to run last. */ 62321afaf18SBorislav Petkov .priority = MCE_PRIO_LOWEST, 62421afaf18SBorislav Petkov }; 62521afaf18SBorislav Petkov 62621afaf18SBorislav Petkov /* 62721afaf18SBorislav Petkov * Read ADDR and MISC registers. 62821afaf18SBorislav Petkov */ 629db6c996dSBorislav Petkov static noinstr void mce_read_aux(struct mce *m, int i) 63021afaf18SBorislav Petkov { 63121afaf18SBorislav Petkov if (m->status & MCI_STATUS_MISCV) 6328121b8f9SBorislav Petkov m->misc = mce_rdmsrl(mca_msr_reg(i, MCA_MISC)); 63321afaf18SBorislav Petkov 63421afaf18SBorislav Petkov if (m->status & MCI_STATUS_ADDRV) { 6358121b8f9SBorislav Petkov m->addr = mce_rdmsrl(mca_msr_reg(i, MCA_ADDR)); 63621afaf18SBorislav Petkov 63721afaf18SBorislav Petkov /* 63821afaf18SBorislav Petkov * Mask the reported address by the reported granularity. 63921afaf18SBorislav Petkov */ 64021afaf18SBorislav Petkov if (mca_cfg.ser && (m->status & MCI_STATUS_MISCV)) { 64121afaf18SBorislav Petkov u8 shift = MCI_MISC_ADDR_LSB(m->misc); 64221afaf18SBorislav Petkov m->addr >>= shift; 64321afaf18SBorislav Petkov m->addr <<= shift; 64421afaf18SBorislav Petkov } 64521afaf18SBorislav Petkov 6462117654eSSmita Koralahalli smca_extract_err_addr(m); 64721afaf18SBorislav Petkov } 64821afaf18SBorislav Petkov 64921afaf18SBorislav Petkov if (mce_flags.smca) { 65021afaf18SBorislav Petkov m->ipid = mce_rdmsrl(MSR_AMD64_SMCA_MCx_IPID(i)); 65121afaf18SBorislav Petkov 65221afaf18SBorislav Petkov if (m->status & MCI_STATUS_SYNDV) 65321afaf18SBorislav Petkov m->synd = mce_rdmsrl(MSR_AMD64_SMCA_MCx_SYND(i)); 65421afaf18SBorislav Petkov } 65521afaf18SBorislav Petkov } 65621afaf18SBorislav Petkov 65721afaf18SBorislav Petkov DEFINE_PER_CPU(unsigned, mce_poll_count); 65821afaf18SBorislav Petkov 65921afaf18SBorislav Petkov /* 66021afaf18SBorislav Petkov * Poll for corrected events or events that happened before reset. 66121afaf18SBorislav Petkov * Those are just logged through /dev/mcelog. 66221afaf18SBorislav Petkov * 66321afaf18SBorislav Petkov * This is executed in standard interrupt context. 66421afaf18SBorislav Petkov * 66521afaf18SBorislav Petkov * Note: spec recommends to panic for fatal unsignalled 66621afaf18SBorislav Petkov * errors here. However this would be quite problematic -- 66721afaf18SBorislav Petkov * we would need to reimplement the Monarch handling and 66821afaf18SBorislav Petkov * it would mess up the exclusion between exception handler 669312a4661SLinus Torvalds * and poll handler -- * so we skip this for now. 67021afaf18SBorislav Petkov * These cases should not happen anyways, or only when the CPU 67121afaf18SBorislav Petkov * is already totally * confused. In this case it's likely it will 67221afaf18SBorislav Petkov * not fully execute the machine check handler either. 67321afaf18SBorislav Petkov */ 67421afaf18SBorislav Petkov bool machine_check_poll(enum mcp_flags flags, mce_banks_t *b) 67521afaf18SBorislav Petkov { 676b4914508SYazen Ghannam struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 67721afaf18SBorislav Petkov bool error_seen = false; 67821afaf18SBorislav Petkov struct mce m; 67921afaf18SBorislav Petkov int i; 68021afaf18SBorislav Petkov 68121afaf18SBorislav Petkov this_cpu_inc(mce_poll_count); 68221afaf18SBorislav Petkov 68321afaf18SBorislav Petkov mce_gather_info(&m, NULL); 68421afaf18SBorislav Petkov 68521afaf18SBorislav Petkov if (flags & MCP_TIMESTAMP) 68621afaf18SBorislav Petkov m.tsc = rdtsc(); 68721afaf18SBorislav Petkov 688c7d314f3SYazen Ghannam for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 68921afaf18SBorislav Petkov if (!mce_banks[i].ctl || !test_bit(i, *b)) 69021afaf18SBorislav Petkov continue; 69121afaf18SBorislav Petkov 69221afaf18SBorislav Petkov m.misc = 0; 69321afaf18SBorislav Petkov m.addr = 0; 69421afaf18SBorislav Petkov m.bank = i; 69521afaf18SBorislav Petkov 69621afaf18SBorislav Petkov barrier(); 6978121b8f9SBorislav Petkov m.status = mce_rdmsrl(mca_msr_reg(i, MCA_STATUS)); 698f19501aaSTony Luck 699f19501aaSTony Luck /* If this entry is not valid, ignore it */ 70021afaf18SBorislav Petkov if (!(m.status & MCI_STATUS_VAL)) 70121afaf18SBorislav Petkov continue; 70221afaf18SBorislav Petkov 70321afaf18SBorislav Petkov /* 704f19501aaSTony Luck * If we are logging everything (at CPU online) or this 705f19501aaSTony Luck * is a corrected error, then we must log it. 70621afaf18SBorislav Petkov */ 707f19501aaSTony Luck if ((flags & MCP_UC) || !(m.status & MCI_STATUS_UC)) 708f19501aaSTony Luck goto log_it; 709f19501aaSTony Luck 710f19501aaSTony Luck /* 711f19501aaSTony Luck * Newer Intel systems that support software error 712f19501aaSTony Luck * recovery need to make additional checks. Other 713f19501aaSTony Luck * CPUs should skip over uncorrected errors, but log 714f19501aaSTony Luck * everything else. 715f19501aaSTony Luck */ 716f19501aaSTony Luck if (!mca_cfg.ser) { 717f19501aaSTony Luck if (m.status & MCI_STATUS_UC) 718f19501aaSTony Luck continue; 719f19501aaSTony Luck goto log_it; 720f19501aaSTony Luck } 721f19501aaSTony Luck 722f19501aaSTony Luck /* Log "not enabled" (speculative) errors */ 723f19501aaSTony Luck if (!(m.status & MCI_STATUS_EN)) 724f19501aaSTony Luck goto log_it; 725f19501aaSTony Luck 726f19501aaSTony Luck /* 727f19501aaSTony Luck * Log UCNA (SDM: 15.6.3 "UCR Error Classification") 728f19501aaSTony Luck * UC == 1 && PCC == 0 && S == 0 729f19501aaSTony Luck */ 730f19501aaSTony Luck if (!(m.status & MCI_STATUS_PCC) && !(m.status & MCI_STATUS_S)) 731f19501aaSTony Luck goto log_it; 732f19501aaSTony Luck 733f19501aaSTony Luck /* 734f19501aaSTony Luck * Skip anything else. Presumption is that our read of this 735f19501aaSTony Luck * bank is racing with a machine check. Leave the log alone 736f19501aaSTony Luck * for do_machine_check() to deal with it. 737f19501aaSTony Luck */ 73821afaf18SBorislav Petkov continue; 73921afaf18SBorislav Petkov 740f19501aaSTony Luck log_it: 74121afaf18SBorislav Petkov error_seen = true; 74221afaf18SBorislav Petkov 74390454e49SJan H. Schönherr if (flags & MCP_DONTLOG) 74490454e49SJan H. Schönherr goto clear_it; 74590454e49SJan H. Schönherr 74621afaf18SBorislav Petkov mce_read_aux(&m, i); 7477f1b8e0dSBorislav Petkov m.severity = mce_severity(&m, NULL, NULL, false); 74821afaf18SBorislav Petkov /* 74921afaf18SBorislav Petkov * Don't get the IP here because it's unlikely to 75021afaf18SBorislav Petkov * have anything to do with the actual error location. 75121afaf18SBorislav Petkov */ 75221afaf18SBorislav Petkov 75390454e49SJan H. Schönherr if (mca_cfg.dont_log_ce && !mce_usable_address(&m)) 75490454e49SJan H. Schönherr goto clear_it; 75590454e49SJan H. Schönherr 7563bff147bSBorislav Petkov if (flags & MCP_QUEUE_LOG) 7573bff147bSBorislav Petkov mce_gen_pool_add(&m); 7583bff147bSBorislav Petkov else 75990454e49SJan H. Schönherr mce_log(&m); 76090454e49SJan H. Schönherr 76190454e49SJan H. Schönherr clear_it: 76221afaf18SBorislav Petkov /* 76321afaf18SBorislav Petkov * Clear state for this bank. 76421afaf18SBorislav Petkov */ 7658121b8f9SBorislav Petkov mce_wrmsrl(mca_msr_reg(i, MCA_STATUS), 0); 76621afaf18SBorislav Petkov } 76721afaf18SBorislav Petkov 76821afaf18SBorislav Petkov /* 76921afaf18SBorislav Petkov * Don't clear MCG_STATUS here because it's only defined for 77021afaf18SBorislav Petkov * exceptions. 77121afaf18SBorislav Petkov */ 77221afaf18SBorislav Petkov 77321afaf18SBorislav Petkov sync_core(); 77421afaf18SBorislav Petkov 77521afaf18SBorislav Petkov return error_seen; 77621afaf18SBorislav Petkov } 77721afaf18SBorislav Petkov EXPORT_SYMBOL_GPL(machine_check_poll); 77821afaf18SBorislav Petkov 77921afaf18SBorislav Petkov /* 780cc466666SBorislav Petkov * During IFU recovery Sandy Bridge -EP4S processors set the RIPV and 781cc466666SBorislav Petkov * EIPV bits in MCG_STATUS to zero on the affected logical processor (SDM 782cc466666SBorislav Petkov * Vol 3B Table 15-20). But this confuses both the code that determines 783cc466666SBorislav Petkov * whether the machine check occurred in kernel or user mode, and also 784cc466666SBorislav Petkov * the severity assessment code. Pretend that EIPV was set, and take the 785cc466666SBorislav Petkov * ip/cs values from the pt_regs that mce_gather_info() ignored earlier. 786cc466666SBorislav Petkov */ 787f11445baSBorislav Petkov static __always_inline void 788f11445baSBorislav Petkov quirk_sandybridge_ifu(int bank, struct mce *m, struct pt_regs *regs) 789cc466666SBorislav Petkov { 790cc466666SBorislav Petkov if (bank != 0) 791cc466666SBorislav Petkov return; 792cc466666SBorislav Petkov if ((m->mcgstatus & (MCG_STATUS_EIPV|MCG_STATUS_RIPV)) != 0) 793cc466666SBorislav Petkov return; 794cc466666SBorislav Petkov if ((m->status & (MCI_STATUS_OVER|MCI_STATUS_UC| 795cc466666SBorislav Petkov MCI_STATUS_EN|MCI_STATUS_MISCV|MCI_STATUS_ADDRV| 796cc466666SBorislav Petkov MCI_STATUS_PCC|MCI_STATUS_S|MCI_STATUS_AR| 797cc466666SBorislav Petkov MCACOD)) != 798cc466666SBorislav Petkov (MCI_STATUS_UC|MCI_STATUS_EN| 799cc466666SBorislav Petkov MCI_STATUS_MISCV|MCI_STATUS_ADDRV|MCI_STATUS_S| 800cc466666SBorislav Petkov MCI_STATUS_AR|MCACOD_INSTR)) 801cc466666SBorislav Petkov return; 802cc466666SBorislav Petkov 803cc466666SBorislav Petkov m->mcgstatus |= MCG_STATUS_EIPV; 804cc466666SBorislav Petkov m->ip = regs->ip; 805cc466666SBorislav Petkov m->cs = regs->cs; 806cc466666SBorislav Petkov } 807cc466666SBorislav Petkov 808cc466666SBorislav Petkov /* 8098ca97812SJue Wang * Disable fast string copy and return from the MCE handler upon the first SRAR 8108ca97812SJue Wang * MCE on bank 1 due to a CPU erratum on Intel Skylake/Cascade Lake/Cooper Lake 8118ca97812SJue Wang * CPUs. 8128ca97812SJue Wang * The fast string copy instructions ("REP; MOVS*") could consume an 8138ca97812SJue Wang * uncorrectable memory error in the cache line _right after_ the desired region 8148ca97812SJue Wang * to copy and raise an MCE with RIP pointing to the instruction _after_ the 8158ca97812SJue Wang * "REP; MOVS*". 8168ca97812SJue Wang * This mitigation addresses the issue completely with the caveat of performance 8178ca97812SJue Wang * degradation on the CPU affected. This is still better than the OS crashing on 8188ca97812SJue Wang * MCEs raised on an irrelevant process due to "REP; MOVS*" accesses from a 8198ca97812SJue Wang * kernel context (e.g., copy_page). 8208ca97812SJue Wang * 8218ca97812SJue Wang * Returns true when fast string copy on CPU has been disabled. 8228ca97812SJue Wang */ 8238ca97812SJue Wang static noinstr bool quirk_skylake_repmov(void) 8248ca97812SJue Wang { 8258ca97812SJue Wang u64 mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS); 8268ca97812SJue Wang u64 misc_enable = mce_rdmsrl(MSR_IA32_MISC_ENABLE); 8278ca97812SJue Wang u64 mc1_status; 8288ca97812SJue Wang 8298ca97812SJue Wang /* 8308ca97812SJue Wang * Apply the quirk only to local machine checks, i.e., no broadcast 8318ca97812SJue Wang * sync is needed. 8328ca97812SJue Wang */ 8338ca97812SJue Wang if (!(mcgstatus & MCG_STATUS_LMCES) || 8348ca97812SJue Wang !(misc_enable & MSR_IA32_MISC_ENABLE_FAST_STRING)) 8358ca97812SJue Wang return false; 8368ca97812SJue Wang 8378ca97812SJue Wang mc1_status = mce_rdmsrl(MSR_IA32_MCx_STATUS(1)); 8388ca97812SJue Wang 8398ca97812SJue Wang /* Check for a software-recoverable data fetch error. */ 8408ca97812SJue Wang if ((mc1_status & 8418ca97812SJue Wang (MCI_STATUS_VAL | MCI_STATUS_OVER | MCI_STATUS_UC | MCI_STATUS_EN | 8428ca97812SJue Wang MCI_STATUS_ADDRV | MCI_STATUS_MISCV | MCI_STATUS_PCC | 8438ca97812SJue Wang MCI_STATUS_AR | MCI_STATUS_S)) == 8448ca97812SJue Wang (MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN | 8458ca97812SJue Wang MCI_STATUS_ADDRV | MCI_STATUS_MISCV | 8468ca97812SJue Wang MCI_STATUS_AR | MCI_STATUS_S)) { 8478ca97812SJue Wang misc_enable &= ~MSR_IA32_MISC_ENABLE_FAST_STRING; 8488ca97812SJue Wang mce_wrmsrl(MSR_IA32_MISC_ENABLE, misc_enable); 8498ca97812SJue Wang mce_wrmsrl(MSR_IA32_MCx_STATUS(1), 0); 8508ca97812SJue Wang 8518ca97812SJue Wang instrumentation_begin(); 8528ca97812SJue Wang pr_err_once("Erratum detected, disable fast string copy instructions.\n"); 8538ca97812SJue Wang instrumentation_end(); 8548ca97812SJue Wang 8558ca97812SJue Wang return true; 8568ca97812SJue Wang } 8578ca97812SJue Wang 8588ca97812SJue Wang return false; 8598ca97812SJue Wang } 8608ca97812SJue Wang 8618ca97812SJue Wang /* 8624240e2ebSYazen Ghannam * Some Zen-based Instruction Fetch Units set EIPV=RIPV=0 on poison consumption 8634240e2ebSYazen Ghannam * errors. This means mce_gather_info() will not save the "ip" and "cs" registers. 8644240e2ebSYazen Ghannam * 8654240e2ebSYazen Ghannam * However, the context is still valid, so save the "cs" register for later use. 8664240e2ebSYazen Ghannam * 8674240e2ebSYazen Ghannam * The "ip" register is truly unknown, so don't save it or fixup EIPV/RIPV. 8684240e2ebSYazen Ghannam * 8694240e2ebSYazen Ghannam * The Instruction Fetch Unit is at MCA bank 1 for all affected systems. 8704240e2ebSYazen Ghannam */ 8714240e2ebSYazen Ghannam static __always_inline void quirk_zen_ifu(int bank, struct mce *m, struct pt_regs *regs) 8724240e2ebSYazen Ghannam { 8734240e2ebSYazen Ghannam if (bank != 1) 8744240e2ebSYazen Ghannam return; 8754240e2ebSYazen Ghannam if (!(m->status & MCI_STATUS_POISON)) 8764240e2ebSYazen Ghannam return; 8774240e2ebSYazen Ghannam 8784240e2ebSYazen Ghannam m->cs = regs->cs; 8794240e2ebSYazen Ghannam } 8804240e2ebSYazen Ghannam 8814240e2ebSYazen Ghannam /* 88221afaf18SBorislav Petkov * Do a quick check if any of the events requires a panic. 88321afaf18SBorislav Petkov * This decides if we keep the events around or clear them. 88421afaf18SBorislav Petkov */ 885f11445baSBorislav Petkov static __always_inline int mce_no_way_out(struct mce *m, char **msg, unsigned long *validp, 88621afaf18SBorislav Petkov struct pt_regs *regs) 88721afaf18SBorislav Petkov { 8887a8bc2b0SJan H. Schönherr char *tmp = *msg; 88921afaf18SBorislav Petkov int i; 89021afaf18SBorislav Petkov 891c7d314f3SYazen Ghannam for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 8928121b8f9SBorislav Petkov m->status = mce_rdmsrl(mca_msr_reg(i, MCA_STATUS)); 89321afaf18SBorislav Petkov if (!(m->status & MCI_STATUS_VAL)) 89421afaf18SBorislav Petkov continue; 89521afaf18SBorislav Petkov 896f11445baSBorislav Petkov arch___set_bit(i, validp); 897cc466666SBorislav Petkov if (mce_flags.snb_ifu_quirk) 898cc466666SBorislav Petkov quirk_sandybridge_ifu(i, m, regs); 89921afaf18SBorislav Petkov 9004240e2ebSYazen Ghannam if (mce_flags.zen_ifu_quirk) 9014240e2ebSYazen Ghannam quirk_zen_ifu(i, m, regs); 9024240e2ebSYazen Ghannam 903d28af26fSTony Luck m->bank = i; 9047f1b8e0dSBorislav Petkov if (mce_severity(m, regs, &tmp, true) >= MCE_PANIC_SEVERITY) { 90521afaf18SBorislav Petkov mce_read_aux(m, i); 90621afaf18SBorislav Petkov *msg = tmp; 90721afaf18SBorislav Petkov return 1; 90821afaf18SBorislav Petkov } 90921afaf18SBorislav Petkov } 91021afaf18SBorislav Petkov return 0; 91121afaf18SBorislav Petkov } 91221afaf18SBorislav Petkov 91321afaf18SBorislav Petkov /* 91421afaf18SBorislav Petkov * Variable to establish order between CPUs while scanning. 91521afaf18SBorislav Petkov * Each CPU spins initially until executing is equal its number. 91621afaf18SBorislav Petkov */ 91721afaf18SBorislav Petkov static atomic_t mce_executing; 91821afaf18SBorislav Petkov 91921afaf18SBorislav Petkov /* 92021afaf18SBorislav Petkov * Defines order of CPUs on entry. First CPU becomes Monarch. 92121afaf18SBorislav Petkov */ 92221afaf18SBorislav Petkov static atomic_t mce_callin; 92321afaf18SBorislav Petkov 92421afaf18SBorislav Petkov /* 9257bb39313SPaul E. McKenney * Track which CPUs entered the MCA broadcast synchronization and which not in 9267bb39313SPaul E. McKenney * order to print holdouts. 9277bb39313SPaul E. McKenney */ 9287bb39313SPaul E. McKenney static cpumask_t mce_missing_cpus = CPU_MASK_ALL; 9297bb39313SPaul E. McKenney 9307bb39313SPaul E. McKenney /* 93121afaf18SBorislav Petkov * Check if a timeout waiting for other CPUs happened. 93221afaf18SBorislav Petkov */ 933edb3d07eSBorislav Petkov static noinstr int mce_timed_out(u64 *t, const char *msg) 93421afaf18SBorislav Petkov { 935edb3d07eSBorislav Petkov int ret = 0; 936edb3d07eSBorislav Petkov 937edb3d07eSBorislav Petkov /* Enable instrumentation around calls to external facilities */ 938edb3d07eSBorislav Petkov instrumentation_begin(); 939edb3d07eSBorislav Petkov 94021afaf18SBorislav Petkov /* 94121afaf18SBorislav Petkov * The others already did panic for some reason. 94221afaf18SBorislav Petkov * Bail out like in a timeout. 94321afaf18SBorislav Petkov * rmb() to tell the compiler that system_state 94421afaf18SBorislav Petkov * might have been modified by someone else. 94521afaf18SBorislav Petkov */ 94621afaf18SBorislav Petkov rmb(); 94721afaf18SBorislav Petkov if (atomic_read(&mce_panicked)) 94821afaf18SBorislav Petkov wait_for_panic(); 94921afaf18SBorislav Petkov if (!mca_cfg.monarch_timeout) 95021afaf18SBorislav Petkov goto out; 95121afaf18SBorislav Petkov if ((s64)*t < SPINUNIT) { 9527bb39313SPaul E. McKenney if (cpumask_and(&mce_missing_cpus, cpu_online_mask, &mce_missing_cpus)) 9537bb39313SPaul E. McKenney pr_emerg("CPUs not responding to MCE broadcast (may include false positives): %*pbl\n", 9547bb39313SPaul E. McKenney cpumask_pr_args(&mce_missing_cpus)); 95521afaf18SBorislav Petkov mce_panic(msg, NULL, NULL); 9567f1b8e0dSBorislav Petkov 957edb3d07eSBorislav Petkov ret = 1; 958edb3d07eSBorislav Petkov goto out; 95921afaf18SBorislav Petkov } 96021afaf18SBorislav Petkov *t -= SPINUNIT; 961edb3d07eSBorislav Petkov 96221afaf18SBorislav Petkov out: 96321afaf18SBorislav Petkov touch_nmi_watchdog(); 964edb3d07eSBorislav Petkov 965edb3d07eSBorislav Petkov instrumentation_end(); 966edb3d07eSBorislav Petkov 967edb3d07eSBorislav Petkov return ret; 96821afaf18SBorislav Petkov } 96921afaf18SBorislav Petkov 97021afaf18SBorislav Petkov /* 97121afaf18SBorislav Petkov * The Monarch's reign. The Monarch is the CPU who entered 97221afaf18SBorislav Petkov * the machine check handler first. It waits for the others to 97321afaf18SBorislav Petkov * raise the exception too and then grades them. When any 97421afaf18SBorislav Petkov * error is fatal panic. Only then let the others continue. 97521afaf18SBorislav Petkov * 97621afaf18SBorislav Petkov * The other CPUs entering the MCE handler will be controlled by the 97721afaf18SBorislav Petkov * Monarch. They are called Subjects. 97821afaf18SBorislav Petkov * 97921afaf18SBorislav Petkov * This way we prevent any potential data corruption in a unrecoverable case 98021afaf18SBorislav Petkov * and also makes sure always all CPU's errors are examined. 98121afaf18SBorislav Petkov * 98221afaf18SBorislav Petkov * Also this detects the case of a machine check event coming from outer 98321afaf18SBorislav Petkov * space (not detected by any CPUs) In this case some external agent wants 98421afaf18SBorislav Petkov * us to shut down, so panic too. 98521afaf18SBorislav Petkov * 98621afaf18SBorislav Petkov * The other CPUs might still decide to panic if the handler happens 98721afaf18SBorislav Petkov * in a unrecoverable place, but in this case the system is in a semi-stable 98821afaf18SBorislav Petkov * state and won't corrupt anything by itself. It's ok to let the others 98921afaf18SBorislav Petkov * continue for a bit first. 99021afaf18SBorislav Petkov * 99121afaf18SBorislav Petkov * All the spin loops have timeouts; when a timeout happens a CPU 99221afaf18SBorislav Petkov * typically elects itself to be Monarch. 99321afaf18SBorislav Petkov */ 99421afaf18SBorislav Petkov static void mce_reign(void) 99521afaf18SBorislav Petkov { 99621afaf18SBorislav Petkov int cpu; 99721afaf18SBorislav Petkov struct mce *m = NULL; 99821afaf18SBorislav Petkov int global_worst = 0; 99921afaf18SBorislav Petkov char *msg = NULL; 100021afaf18SBorislav Petkov 100121afaf18SBorislav Petkov /* 100221afaf18SBorislav Petkov * This CPU is the Monarch and the other CPUs have run 100321afaf18SBorislav Petkov * through their handlers. 100421afaf18SBorislav Petkov * Grade the severity of the errors of all the CPUs. 100521afaf18SBorislav Petkov */ 100621afaf18SBorislav Petkov for_each_possible_cpu(cpu) { 100713c877f4STony Luck struct mce *mtmp = &per_cpu(mces_seen, cpu); 100813c877f4STony Luck 100913c877f4STony Luck if (mtmp->severity > global_worst) { 101013c877f4STony Luck global_worst = mtmp->severity; 101121afaf18SBorislav Petkov m = &per_cpu(mces_seen, cpu); 101221afaf18SBorislav Petkov } 101321afaf18SBorislav Petkov } 101421afaf18SBorislav Petkov 101521afaf18SBorislav Petkov /* 101621afaf18SBorislav Petkov * Cannot recover? Panic here then. 101721afaf18SBorislav Petkov * This dumps all the mces in the log buffer and stops the 101821afaf18SBorislav Petkov * other CPUs. 101921afaf18SBorislav Petkov */ 10207f1b8e0dSBorislav Petkov if (m && global_worst >= MCE_PANIC_SEVERITY) { 102113c877f4STony Luck /* call mce_severity() to get "msg" for panic */ 10227f1b8e0dSBorislav Petkov mce_severity(m, NULL, &msg, true); 102321afaf18SBorislav Petkov mce_panic("Fatal machine check", m, msg); 102413c877f4STony Luck } 102521afaf18SBorislav Petkov 102621afaf18SBorislav Petkov /* 102721afaf18SBorislav Petkov * For UC somewhere we let the CPU who detects it handle it. 102821afaf18SBorislav Petkov * Also must let continue the others, otherwise the handling 102921afaf18SBorislav Petkov * CPU could deadlock on a lock. 103021afaf18SBorislav Petkov */ 103121afaf18SBorislav Petkov 103221afaf18SBorislav Petkov /* 103321afaf18SBorislav Petkov * No machine check event found. Must be some external 103421afaf18SBorislav Petkov * source or one CPU is hung. Panic. 103521afaf18SBorislav Petkov */ 10367f1b8e0dSBorislav Petkov if (global_worst <= MCE_KEEP_SEVERITY) 103721afaf18SBorislav Petkov mce_panic("Fatal machine check from unknown source", NULL, NULL); 103821afaf18SBorislav Petkov 103921afaf18SBorislav Petkov /* 104021afaf18SBorislav Petkov * Now clear all the mces_seen so that they don't reappear on 104121afaf18SBorislav Petkov * the next mce. 104221afaf18SBorislav Petkov */ 104321afaf18SBorislav Petkov for_each_possible_cpu(cpu) 104421afaf18SBorislav Petkov memset(&per_cpu(mces_seen, cpu), 0, sizeof(struct mce)); 104521afaf18SBorislav Petkov } 104621afaf18SBorislav Petkov 104721afaf18SBorislav Petkov static atomic_t global_nwo; 104821afaf18SBorislav Petkov 104921afaf18SBorislav Petkov /* 105021afaf18SBorislav Petkov * Start of Monarch synchronization. This waits until all CPUs have 105121afaf18SBorislav Petkov * entered the exception handler and then determines if any of them 105221afaf18SBorislav Petkov * saw a fatal event that requires panic. Then it executes them 105321afaf18SBorislav Petkov * in the entry order. 105421afaf18SBorislav Petkov * TBD double check parallel CPU hotunplug 105521afaf18SBorislav Petkov */ 1056e3d72e8eSBorislav Petkov static noinstr int mce_start(int *no_way_out) 105721afaf18SBorislav Petkov { 105821afaf18SBorislav Petkov u64 timeout = (u64)mca_cfg.monarch_timeout * NSEC_PER_USEC; 1059e3d72e8eSBorislav Petkov int order, ret = -1; 106021afaf18SBorislav Petkov 106121afaf18SBorislav Petkov if (!timeout) 1062e3d72e8eSBorislav Petkov return ret; 106321afaf18SBorislav Petkov 10640f613bfaSMark Rutland raw_atomic_add(*no_way_out, &global_nwo); 106521afaf18SBorislav Petkov /* 106621afaf18SBorislav Petkov * Rely on the implied barrier below, such that global_nwo 106721afaf18SBorislav Petkov * is updated before mce_callin. 106821afaf18SBorislav Petkov */ 10690f613bfaSMark Rutland order = raw_atomic_inc_return(&mce_callin); 1070f11445baSBorislav Petkov arch_cpumask_clear_cpu(smp_processor_id(), &mce_missing_cpus); 107121afaf18SBorislav Petkov 1072e3d72e8eSBorislav Petkov /* Enable instrumentation around calls to external facilities */ 1073e3d72e8eSBorislav Petkov instrumentation_begin(); 1074e3d72e8eSBorislav Petkov 107521afaf18SBorislav Petkov /* 107621afaf18SBorislav Petkov * Wait for everyone. 107721afaf18SBorislav Petkov */ 10780f613bfaSMark Rutland while (raw_atomic_read(&mce_callin) != num_online_cpus()) { 107921afaf18SBorislav Petkov if (mce_timed_out(&timeout, 108021afaf18SBorislav Petkov "Timeout: Not all CPUs entered broadcast exception handler")) { 10810f613bfaSMark Rutland raw_atomic_set(&global_nwo, 0); 1082e3d72e8eSBorislav Petkov goto out; 108321afaf18SBorislav Petkov } 108421afaf18SBorislav Petkov ndelay(SPINUNIT); 108521afaf18SBorislav Petkov } 108621afaf18SBorislav Petkov 108721afaf18SBorislav Petkov /* 108821afaf18SBorislav Petkov * mce_callin should be read before global_nwo 108921afaf18SBorislav Petkov */ 109021afaf18SBorislav Petkov smp_rmb(); 109121afaf18SBorislav Petkov 109221afaf18SBorislav Petkov if (order == 1) { 109321afaf18SBorislav Petkov /* 109421afaf18SBorislav Petkov * Monarch: Starts executing now, the others wait. 109521afaf18SBorislav Petkov */ 10960f613bfaSMark Rutland raw_atomic_set(&mce_executing, 1); 109721afaf18SBorislav Petkov } else { 109821afaf18SBorislav Petkov /* 109921afaf18SBorislav Petkov * Subject: Now start the scanning loop one by one in 110021afaf18SBorislav Petkov * the original callin order. 110121afaf18SBorislav Petkov * This way when there are any shared banks it will be 110221afaf18SBorislav Petkov * only seen by one CPU before cleared, avoiding duplicates. 110321afaf18SBorislav Petkov */ 11040f613bfaSMark Rutland while (raw_atomic_read(&mce_executing) < order) { 110521afaf18SBorislav Petkov if (mce_timed_out(&timeout, 110621afaf18SBorislav Petkov "Timeout: Subject CPUs unable to finish machine check processing")) { 11070f613bfaSMark Rutland raw_atomic_set(&global_nwo, 0); 1108e3d72e8eSBorislav Petkov goto out; 110921afaf18SBorislav Petkov } 111021afaf18SBorislav Petkov ndelay(SPINUNIT); 111121afaf18SBorislav Petkov } 111221afaf18SBorislav Petkov } 111321afaf18SBorislav Petkov 111421afaf18SBorislav Petkov /* 111521afaf18SBorislav Petkov * Cache the global no_way_out state. 111621afaf18SBorislav Petkov */ 11170f613bfaSMark Rutland *no_way_out = raw_atomic_read(&global_nwo); 111821afaf18SBorislav Petkov 1119e3d72e8eSBorislav Petkov ret = order; 1120e3d72e8eSBorislav Petkov 1121e3d72e8eSBorislav Petkov out: 1122e3d72e8eSBorislav Petkov instrumentation_end(); 1123e3d72e8eSBorislav Petkov 1124e3d72e8eSBorislav Petkov return ret; 112521afaf18SBorislav Petkov } 112621afaf18SBorislav Petkov 112721afaf18SBorislav Petkov /* 112821afaf18SBorislav Petkov * Synchronize between CPUs after main scanning loop. 112921afaf18SBorislav Petkov * This invokes the bulk of the Monarch processing. 113021afaf18SBorislav Petkov */ 1131b4813539SBorislav Petkov static noinstr int mce_end(int order) 113221afaf18SBorislav Petkov { 113321afaf18SBorislav Petkov u64 timeout = (u64)mca_cfg.monarch_timeout * NSEC_PER_USEC; 1134b4813539SBorislav Petkov int ret = -1; 1135b4813539SBorislav Petkov 1136b4813539SBorislav Petkov /* Allow instrumentation around external facilities. */ 1137b4813539SBorislav Petkov instrumentation_begin(); 113821afaf18SBorislav Petkov 113921afaf18SBorislav Petkov if (!timeout) 114021afaf18SBorislav Petkov goto reset; 114121afaf18SBorislav Petkov if (order < 0) 114221afaf18SBorislav Petkov goto reset; 114321afaf18SBorislav Petkov 114421afaf18SBorislav Petkov /* 114521afaf18SBorislav Petkov * Allow others to run. 114621afaf18SBorislav Petkov */ 114721afaf18SBorislav Petkov atomic_inc(&mce_executing); 114821afaf18SBorislav Petkov 114921afaf18SBorislav Petkov if (order == 1) { 115021afaf18SBorislav Petkov /* 115121afaf18SBorislav Petkov * Monarch: Wait for everyone to go through their scanning 115221afaf18SBorislav Petkov * loops. 115321afaf18SBorislav Petkov */ 1154ad669ec1SBorislav Petkov while (atomic_read(&mce_executing) <= num_online_cpus()) { 115521afaf18SBorislav Petkov if (mce_timed_out(&timeout, 115621afaf18SBorislav Petkov "Timeout: Monarch CPU unable to finish machine check processing")) 115721afaf18SBorislav Petkov goto reset; 115821afaf18SBorislav Petkov ndelay(SPINUNIT); 115921afaf18SBorislav Petkov } 116021afaf18SBorislav Petkov 116121afaf18SBorislav Petkov mce_reign(); 116221afaf18SBorislav Petkov barrier(); 116321afaf18SBorislav Petkov ret = 0; 116421afaf18SBorislav Petkov } else { 116521afaf18SBorislav Petkov /* 116621afaf18SBorislav Petkov * Subject: Wait for Monarch to finish. 116721afaf18SBorislav Petkov */ 116821afaf18SBorislav Petkov while (atomic_read(&mce_executing) != 0) { 116921afaf18SBorislav Petkov if (mce_timed_out(&timeout, 117021afaf18SBorislav Petkov "Timeout: Monarch CPU did not finish machine check processing")) 117121afaf18SBorislav Petkov goto reset; 117221afaf18SBorislav Petkov ndelay(SPINUNIT); 117321afaf18SBorislav Petkov } 117421afaf18SBorislav Petkov 117521afaf18SBorislav Petkov /* 117621afaf18SBorislav Petkov * Don't reset anything. That's done by the Monarch. 117721afaf18SBorislav Petkov */ 1178b4813539SBorislav Petkov ret = 0; 1179b4813539SBorislav Petkov goto out; 118021afaf18SBorislav Petkov } 118121afaf18SBorislav Petkov 118221afaf18SBorislav Petkov /* 118321afaf18SBorislav Petkov * Reset all global state. 118421afaf18SBorislav Petkov */ 118521afaf18SBorislav Petkov reset: 118621afaf18SBorislav Petkov atomic_set(&global_nwo, 0); 118721afaf18SBorislav Petkov atomic_set(&mce_callin, 0); 11887bb39313SPaul E. McKenney cpumask_setall(&mce_missing_cpus); 118921afaf18SBorislav Petkov barrier(); 119021afaf18SBorislav Petkov 119121afaf18SBorislav Petkov /* 119221afaf18SBorislav Petkov * Let others run again. 119321afaf18SBorislav Petkov */ 119421afaf18SBorislav Petkov atomic_set(&mce_executing, 0); 1195b4813539SBorislav Petkov 1196b4813539SBorislav Petkov out: 1197b4813539SBorislav Petkov instrumentation_end(); 1198b4813539SBorislav Petkov 119921afaf18SBorislav Petkov return ret; 120021afaf18SBorislav Petkov } 120121afaf18SBorislav Petkov 1202f11445baSBorislav Petkov static __always_inline void mce_clear_state(unsigned long *toclear) 120321afaf18SBorislav Petkov { 120421afaf18SBorislav Petkov int i; 120521afaf18SBorislav Petkov 1206c7d314f3SYazen Ghannam for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 1207f11445baSBorislav Petkov if (arch_test_bit(i, toclear)) 12088121b8f9SBorislav Petkov mce_wrmsrl(mca_msr_reg(i, MCA_STATUS), 0); 120921afaf18SBorislav Petkov } 121021afaf18SBorislav Petkov } 121121afaf18SBorislav Petkov 121221afaf18SBorislav Petkov /* 121321afaf18SBorislav Petkov * Cases where we avoid rendezvous handler timeout: 121421afaf18SBorislav Petkov * 1) If this CPU is offline. 121521afaf18SBorislav Petkov * 121621afaf18SBorislav Petkov * 2) If crashing_cpu was set, e.g. we're entering kdump and we need to 121721afaf18SBorislav Petkov * skip those CPUs which remain looping in the 1st kernel - see 121821afaf18SBorislav Petkov * crash_nmi_callback(). 121921afaf18SBorislav Petkov * 122021afaf18SBorislav Petkov * Note: there still is a small window between kexec-ing and the new, 122121afaf18SBorislav Petkov * kdump kernel establishing a new #MC handler where a broadcasted MCE 122221afaf18SBorislav Petkov * might not get handled properly. 122321afaf18SBorislav Petkov */ 122494a46d31SThomas Gleixner static noinstr bool mce_check_crashing_cpu(void) 122521afaf18SBorislav Petkov { 122694a46d31SThomas Gleixner unsigned int cpu = smp_processor_id(); 122794a46d31SThomas Gleixner 122814d3b376SPeter Zijlstra if (arch_cpu_is_offline(cpu) || 122921afaf18SBorislav Petkov (crashing_cpu != -1 && crashing_cpu != cpu)) { 123021afaf18SBorislav Petkov u64 mcgstatus; 123121afaf18SBorislav Petkov 1232aedbdeabSThomas Gleixner mcgstatus = __rdmsr(MSR_IA32_MCG_STATUS); 123370f0c230STony W Wang-oc 123470f0c230STony W Wang-oc if (boot_cpu_data.x86_vendor == X86_VENDOR_ZHAOXIN) { 123570f0c230STony W Wang-oc if (mcgstatus & MCG_STATUS_LMCES) 123670f0c230STony W Wang-oc return false; 123770f0c230STony W Wang-oc } 123870f0c230STony W Wang-oc 123921afaf18SBorislav Petkov if (mcgstatus & MCG_STATUS_RIPV) { 1240aedbdeabSThomas Gleixner __wrmsr(MSR_IA32_MCG_STATUS, 0, 0); 124121afaf18SBorislav Petkov return true; 124221afaf18SBorislav Petkov } 124321afaf18SBorislav Petkov } 124421afaf18SBorislav Petkov return false; 124521afaf18SBorislav Petkov } 124621afaf18SBorislav Petkov 124775581a20SBorislav Petkov static __always_inline int 124875581a20SBorislav Petkov __mc_scan_banks(struct mce *m, struct pt_regs *regs, struct mce *final, 124975581a20SBorislav Petkov unsigned long *toclear, unsigned long *valid_banks, int no_way_out, 125075581a20SBorislav Petkov int *worst) 125121afaf18SBorislav Petkov { 1252b4914508SYazen Ghannam struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 125321afaf18SBorislav Petkov struct mca_config *cfg = &mca_cfg; 125475581a20SBorislav Petkov int severity, i, taint = 0; 125521afaf18SBorislav Petkov 1256c7d314f3SYazen Ghannam for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 1257f11445baSBorislav Petkov arch___clear_bit(i, toclear); 1258f11445baSBorislav Petkov if (!arch_test_bit(i, valid_banks)) 125921afaf18SBorislav Petkov continue; 126021afaf18SBorislav Petkov 126121afaf18SBorislav Petkov if (!mce_banks[i].ctl) 126221afaf18SBorislav Petkov continue; 126321afaf18SBorislav Petkov 126421afaf18SBorislav Petkov m->misc = 0; 126521afaf18SBorislav Petkov m->addr = 0; 126621afaf18SBorislav Petkov m->bank = i; 126721afaf18SBorislav Petkov 12688121b8f9SBorislav Petkov m->status = mce_rdmsrl(mca_msr_reg(i, MCA_STATUS)); 126921afaf18SBorislav Petkov if (!(m->status & MCI_STATUS_VAL)) 127021afaf18SBorislav Petkov continue; 127121afaf18SBorislav Petkov 127221afaf18SBorislav Petkov /* 127321afaf18SBorislav Petkov * Corrected or non-signaled errors are handled by 127421afaf18SBorislav Petkov * machine_check_poll(). Leave them alone, unless this panics. 127521afaf18SBorislav Petkov */ 127621afaf18SBorislav Petkov if (!(m->status & (cfg->ser ? MCI_STATUS_S : MCI_STATUS_UC)) && 127721afaf18SBorislav Petkov !no_way_out) 127821afaf18SBorislav Petkov continue; 127921afaf18SBorislav Petkov 128021afaf18SBorislav Petkov /* Set taint even when machine check was not enabled. */ 128175581a20SBorislav Petkov taint++; 128221afaf18SBorislav Petkov 12837f1b8e0dSBorislav Petkov severity = mce_severity(m, regs, NULL, true); 128421afaf18SBorislav Petkov 128521afaf18SBorislav Petkov /* 128621afaf18SBorislav Petkov * When machine check was for corrected/deferred handler don't 128721afaf18SBorislav Petkov * touch, unless we're panicking. 128821afaf18SBorislav Petkov */ 128921afaf18SBorislav Petkov if ((severity == MCE_KEEP_SEVERITY || 129021afaf18SBorislav Petkov severity == MCE_UCNA_SEVERITY) && !no_way_out) 129121afaf18SBorislav Petkov continue; 129221afaf18SBorislav Petkov 1293f11445baSBorislav Petkov arch___set_bit(i, toclear); 129421afaf18SBorislav Petkov 129521afaf18SBorislav Petkov /* Machine check event was not enabled. Clear, but ignore. */ 129621afaf18SBorislav Petkov if (severity == MCE_NO_SEVERITY) 129721afaf18SBorislav Petkov continue; 129821afaf18SBorislav Petkov 129921afaf18SBorislav Petkov mce_read_aux(m, i); 130021afaf18SBorislav Petkov 130121afaf18SBorislav Petkov /* assuming valid severity level != 0 */ 130221afaf18SBorislav Petkov m->severity = severity; 130321afaf18SBorislav Petkov 130475581a20SBorislav Petkov /* 130575581a20SBorislav Petkov * Enable instrumentation around the mce_log() call which is 130675581a20SBorislav Petkov * done in #MC context, where instrumentation is disabled. 130775581a20SBorislav Petkov */ 130875581a20SBorislav Petkov instrumentation_begin(); 130921afaf18SBorislav Petkov mce_log(m); 131075581a20SBorislav Petkov instrumentation_end(); 131121afaf18SBorislav Petkov 131221afaf18SBorislav Petkov if (severity > *worst) { 131321afaf18SBorislav Petkov *final = *m; 131421afaf18SBorislav Petkov *worst = severity; 131521afaf18SBorislav Petkov } 131621afaf18SBorislav Petkov } 131721afaf18SBorislav Petkov 131821afaf18SBorislav Petkov /* mce_clear_state will clear *final, save locally for use later */ 131921afaf18SBorislav Petkov *m = *final; 132075581a20SBorislav Petkov 132175581a20SBorislav Petkov return taint; 132221afaf18SBorislav Petkov } 132321afaf18SBorislav Petkov 13245567d11cSPeter Zijlstra static void kill_me_now(struct callback_head *ch) 13255567d11cSPeter Zijlstra { 132681065b35STony Luck struct task_struct *p = container_of(ch, struct task_struct, mce_kill_me); 132781065b35STony Luck 132881065b35STony Luck p->mce_count = 0; 13295567d11cSPeter Zijlstra force_sig(SIGBUS); 13305567d11cSPeter Zijlstra } 13315567d11cSPeter Zijlstra 13325567d11cSPeter Zijlstra static void kill_me_maybe(struct callback_head *cb) 13335567d11cSPeter Zijlstra { 13345567d11cSPeter Zijlstra struct task_struct *p = container_of(cb, struct task_struct, mce_kill_me); 13355567d11cSPeter Zijlstra int flags = MF_ACTION_REQUIRED; 13368a01ec97STony Luck unsigned long pfn; 1337a3f5d80eSNaoya Horiguchi int ret; 13385567d11cSPeter Zijlstra 133981065b35STony Luck p->mce_count = 0; 13405567d11cSPeter Zijlstra pr_err("Uncorrected hardware memory error in user-access at %llx", p->mce_addr); 134117fae129STony Luck 134217fae129STony Luck if (!p->mce_ripv) 13435567d11cSPeter Zijlstra flags |= MF_MUST_KILL; 13445567d11cSPeter Zijlstra 13458a01ec97STony Luck pfn = (p->mce_addr & MCI_ADDR_PHYSADDR) >> PAGE_SHIFT; 13468a01ec97STony Luck ret = memory_failure(pfn, flags); 1347a6e3cf70STony Luck if (!ret) { 13488a01ec97STony Luck set_mce_nospec(pfn); 13491e36d9c6STony Luck sync_core(); 13505567d11cSPeter Zijlstra return; 13515567d11cSPeter Zijlstra } 13525567d11cSPeter Zijlstra 1353a3f5d80eSNaoya Horiguchi /* 1354a3f5d80eSNaoya Horiguchi * -EHWPOISON from memory_failure() means that it already sent SIGBUS 1355d1fe111fSluofei * to the current process with the proper error info, 1356d1fe111fSluofei * -EOPNOTSUPP means hwpoison_filter() filtered the error event, 1357d1fe111fSluofei * 1358d1fe111fSluofei * In both cases, no further processing is required. 1359a3f5d80eSNaoya Horiguchi */ 1360d1fe111fSluofei if (ret == -EHWPOISON || ret == -EOPNOTSUPP) 1361a3f5d80eSNaoya Horiguchi return; 1362a3f5d80eSNaoya Horiguchi 13635567d11cSPeter Zijlstra pr_err("Memory error not recovered"); 13645567d11cSPeter Zijlstra kill_me_now(cb); 13655567d11cSPeter Zijlstra } 1366a6e3cf70STony Luck 1367a6e3cf70STony Luck static void kill_me_never(struct callback_head *cb) 1368a6e3cf70STony Luck { 1369a6e3cf70STony Luck struct task_struct *p = container_of(cb, struct task_struct, mce_kill_me); 13708a01ec97STony Luck unsigned long pfn; 1371a6e3cf70STony Luck 1372a6e3cf70STony Luck p->mce_count = 0; 1373a6e3cf70STony Luck pr_err("Kernel accessed poison in user space at %llx\n", p->mce_addr); 13748a01ec97STony Luck pfn = (p->mce_addr & MCI_ADDR_PHYSADDR) >> PAGE_SHIFT; 13758a01ec97STony Luck if (!memory_failure(pfn, 0)) 13768a01ec97STony Luck set_mce_nospec(pfn); 137730063810STony Luck } 13785567d11cSPeter Zijlstra 1379a6e3cf70STony Luck static void queue_task_work(struct mce *m, char *msg, void (*func)(struct callback_head *)) 1380c0ab7ffcSTony Luck { 138181065b35STony Luck int count = ++current->mce_count; 138281065b35STony Luck 138381065b35STony Luck /* First call, save all the details */ 138481065b35STony Luck if (count == 1) { 1385c0ab7ffcSTony Luck current->mce_addr = m->addr; 1386c0ab7ffcSTony Luck current->mce_kflags = m->kflags; 1387c0ab7ffcSTony Luck current->mce_ripv = !!(m->mcgstatus & MCG_STATUS_RIPV); 1388c0ab7ffcSTony Luck current->mce_whole_page = whole_page(m); 1389a6e3cf70STony Luck current->mce_kill_me.func = func; 139081065b35STony Luck } 139181065b35STony Luck 139281065b35STony Luck /* Ten is likely overkill. Don't expect more than two faults before task_work() */ 139381065b35STony Luck if (count > 10) 139481065b35STony Luck mce_panic("Too many consecutive machine checks while accessing user data", m, msg); 139581065b35STony Luck 139681065b35STony Luck /* Second or later call, make sure page address matches the one from first call */ 139781065b35STony Luck if (count > 1 && (current->mce_addr >> PAGE_SHIFT) != (m->addr >> PAGE_SHIFT)) 139881065b35STony Luck mce_panic("Consecutive machine checks to different user pages", m, msg); 139981065b35STony Luck 140081065b35STony Luck /* Do not call task_work_add() more than once */ 140181065b35STony Luck if (count > 1) 140281065b35STony Luck return; 1403c0ab7ffcSTony Luck 140491989c70SJens Axboe task_work_add(current, ¤t->mce_kill_me, TWA_RESUME); 1405c0ab7ffcSTony Luck } 140621afaf18SBorislav Petkov 1407cbe1de16SBorislav Petkov /* Handle unconfigured int18 (should never happen) */ 1408cbe1de16SBorislav Petkov static noinstr void unexpected_machine_check(struct pt_regs *regs) 1409cbe1de16SBorislav Petkov { 1410cbe1de16SBorislav Petkov instrumentation_begin(); 1411cbe1de16SBorislav Petkov pr_err("CPU#%d: Unexpected int18 (Machine Check)\n", 1412cbe1de16SBorislav Petkov smp_processor_id()); 1413cbe1de16SBorislav Petkov instrumentation_end(); 1414cbe1de16SBorislav Petkov } 1415cbe1de16SBorislav Petkov 141621afaf18SBorislav Petkov /* 1417487d654dSBorislav Petkov * The actual machine check handler. This only handles real exceptions when 1418487d654dSBorislav Petkov * something got corrupted coming in through int 18. 141921afaf18SBorislav Petkov * 1420487d654dSBorislav Petkov * This is executed in #MC context not subject to normal locking rules. 1421487d654dSBorislav Petkov * This implies that most kernel services cannot be safely used. Don't even 142221afaf18SBorislav Petkov * think about putting a printk in there! 142321afaf18SBorislav Petkov * 142421afaf18SBorislav Petkov * On Intel systems this is entered on all CPUs in parallel through 142521afaf18SBorislav Petkov * MCE broadcast. However some CPUs might be broken beyond repair, 142621afaf18SBorislav Petkov * so be always careful when synchronizing with others. 142755ba18d6SAndy Lutomirski * 142855ba18d6SAndy Lutomirski * Tracing and kprobes are disabled: if we interrupted a kernel context 142955ba18d6SAndy Lutomirski * with IF=1, we need to minimize stack usage. There are also recursion 143055ba18d6SAndy Lutomirski * issues: if the machine check was due to a failure of the memory 143155ba18d6SAndy Lutomirski * backing the user stack, tracing that reads the user stack will cause 143255ba18d6SAndy Lutomirski * potentially infinite recursion. 1433487d654dSBorislav Petkov * 1434487d654dSBorislav Petkov * Currently, the #MC handler calls out to a number of external facilities 1435487d654dSBorislav Petkov * and, therefore, allows instrumentation around them. The optimal thing to 1436487d654dSBorislav Petkov * have would be to do the absolutely minimal work required in #MC context 1437487d654dSBorislav Petkov * and have instrumentation disabled only around that. Further processing can 1438487d654dSBorislav Petkov * then happen in process context where instrumentation is allowed. Achieving 1439487d654dSBorislav Petkov * that requires careful auditing and modifications. Until then, the code 1440487d654dSBorislav Petkov * allows instrumentation temporarily, where required. * 144121afaf18SBorislav Petkov */ 14427f6fa101SIra Weiny noinstr void do_machine_check(struct pt_regs *regs) 144321afaf18SBorislav Petkov { 144475581a20SBorislav Petkov int worst = 0, order, no_way_out, kill_current_task, lmce, taint = 0; 1445cd5e0d1fSBorislav Petkov DECLARE_BITMAP(valid_banks, MAX_NR_BANKS) = { 0 }; 1446cd5e0d1fSBorislav Petkov DECLARE_BITMAP(toclear, MAX_NR_BANKS) = { 0 }; 144721afaf18SBorislav Petkov struct mce m, *final; 14487a8bc2b0SJan H. Schönherr char *msg = NULL; 1449cbe1de16SBorislav Petkov 1450cbe1de16SBorislav Petkov if (unlikely(mce_flags.p5)) 1451cbe1de16SBorislav Petkov return pentium_machine_check(regs); 1452cbe1de16SBorislav Petkov else if (unlikely(mce_flags.winchip)) 1453cbe1de16SBorislav Petkov return winchip_machine_check(regs); 1454cbe1de16SBorislav Petkov else if (unlikely(!mca_cfg.initialized)) 1455cbe1de16SBorislav Petkov return unexpected_machine_check(regs); 145621afaf18SBorislav Petkov 14578ca97812SJue Wang if (mce_flags.skx_repmov_quirk && quirk_skylake_repmov()) 14588ca97812SJue Wang goto clear; 14598ca97812SJue Wang 146021afaf18SBorislav Petkov /* 146121afaf18SBorislav Petkov * Establish sequential order between the CPUs entering the machine 146221afaf18SBorislav Petkov * check handler. 146321afaf18SBorislav Petkov */ 1464cbe1de16SBorislav Petkov order = -1; 146521afaf18SBorislav Petkov 146621afaf18SBorislav Petkov /* 146721afaf18SBorislav Petkov * If no_way_out gets set, there is no safe way to recover from this 14687f1b8e0dSBorislav Petkov * MCE. 146921afaf18SBorislav Petkov */ 1470cbe1de16SBorislav Petkov no_way_out = 0; 147121afaf18SBorislav Petkov 147221afaf18SBorislav Petkov /* 1473e1c06d23SGabriele Paoloni * If kill_current_task is not set, there might be a way to recover from this 147421afaf18SBorislav Petkov * error. 147521afaf18SBorislav Petkov */ 1476cbe1de16SBorislav Petkov kill_current_task = 0; 147721afaf18SBorislav Petkov 147821afaf18SBorislav Petkov /* 147921afaf18SBorislav Petkov * MCEs are always local on AMD. Same is determined by MCG_STATUS_LMCES 148021afaf18SBorislav Petkov * on Intel. 148121afaf18SBorislav Petkov */ 1482cbe1de16SBorislav Petkov lmce = 1; 148321afaf18SBorislav Petkov 148421afaf18SBorislav Petkov this_cpu_inc(mce_exception_count); 148521afaf18SBorislav Petkov 148621afaf18SBorislav Petkov mce_gather_info(&m, regs); 148721afaf18SBorislav Petkov m.tsc = rdtsc(); 148821afaf18SBorislav Petkov 148921afaf18SBorislav Petkov final = this_cpu_ptr(&mces_seen); 149021afaf18SBorislav Petkov *final = m; 149121afaf18SBorislav Petkov 149221afaf18SBorislav Petkov no_way_out = mce_no_way_out(&m, &msg, valid_banks, regs); 149321afaf18SBorislav Petkov 149421afaf18SBorislav Petkov barrier(); 149521afaf18SBorislav Petkov 149621afaf18SBorislav Petkov /* 149721afaf18SBorislav Petkov * When no restart IP might need to kill or panic. 149821afaf18SBorislav Petkov * Assume the worst for now, but if we find the 149921afaf18SBorislav Petkov * severity is MCE_AR_SEVERITY we have other options. 150021afaf18SBorislav Petkov */ 150121afaf18SBorislav Petkov if (!(m.mcgstatus & MCG_STATUS_RIPV)) 15027f1b8e0dSBorislav Petkov kill_current_task = 1; 150321afaf18SBorislav Petkov /* 150421afaf18SBorislav Petkov * Check if this MCE is signaled to only this logical processor, 150570f0c230STony W Wang-oc * on Intel, Zhaoxin only. 150621afaf18SBorislav Petkov */ 150770f0c230STony W Wang-oc if (m.cpuvendor == X86_VENDOR_INTEL || 150870f0c230STony W Wang-oc m.cpuvendor == X86_VENDOR_ZHAOXIN) 150921afaf18SBorislav Petkov lmce = m.mcgstatus & MCG_STATUS_LMCES; 151021afaf18SBorislav Petkov 151121afaf18SBorislav Petkov /* 151221afaf18SBorislav Petkov * Local machine check may already know that we have to panic. 151321afaf18SBorislav Petkov * Broadcast machine check begins rendezvous in mce_start() 151421afaf18SBorislav Petkov * Go through all banks in exclusion of the other CPUs. This way we 151521afaf18SBorislav Petkov * don't report duplicated events on shared banks because the first one 151621afaf18SBorislav Petkov * to see it will clear it. 151721afaf18SBorislav Petkov */ 151821afaf18SBorislav Petkov if (lmce) { 15197f1b8e0dSBorislav Petkov if (no_way_out) 152021afaf18SBorislav Petkov mce_panic("Fatal local machine check", &m, msg); 152121afaf18SBorislav Petkov } else { 152221afaf18SBorislav Petkov order = mce_start(&no_way_out); 152321afaf18SBorislav Petkov } 152421afaf18SBorislav Petkov 152575581a20SBorislav Petkov taint = __mc_scan_banks(&m, regs, final, toclear, valid_banks, no_way_out, &worst); 152621afaf18SBorislav Petkov 152721afaf18SBorislav Petkov if (!no_way_out) 152821afaf18SBorislav Petkov mce_clear_state(toclear); 152921afaf18SBorislav Petkov 153021afaf18SBorislav Petkov /* 153121afaf18SBorislav Petkov * Do most of the synchronization with other CPUs. 153221afaf18SBorislav Petkov * When there's any problem use only local no_way_out state. 153321afaf18SBorislav Petkov */ 153421afaf18SBorislav Petkov if (!lmce) { 153525bc65d8SGabriele Paoloni if (mce_end(order) < 0) { 153625bc65d8SGabriele Paoloni if (!no_way_out) 153721afaf18SBorislav Petkov no_way_out = worst >= MCE_PANIC_SEVERITY; 1538e273e6e1SGabriele Paoloni 15397f1b8e0dSBorislav Petkov if (no_way_out) 1540e273e6e1SGabriele Paoloni mce_panic("Fatal machine check on current CPU", &m, msg); 154125bc65d8SGabriele Paoloni } 154221afaf18SBorislav Petkov } else { 154321afaf18SBorislav Petkov /* 154421afaf18SBorislav Petkov * If there was a fatal machine check we should have 154521afaf18SBorislav Petkov * already called mce_panic earlier in this function. 154621afaf18SBorislav Petkov * Since we re-read the banks, we might have found 154721afaf18SBorislav Petkov * something new. Check again to see if we found a 154821afaf18SBorislav Petkov * fatal error. We call "mce_severity()" again to 154921afaf18SBorislav Petkov * make sure we have the right "msg". 155021afaf18SBorislav Petkov */ 15517f1b8e0dSBorislav Petkov if (worst >= MCE_PANIC_SEVERITY) { 15527f1b8e0dSBorislav Petkov mce_severity(&m, regs, &msg, true); 155321afaf18SBorislav Petkov mce_panic("Local fatal machine check!", &m, msg); 155421afaf18SBorislav Petkov } 155521afaf18SBorislav Petkov } 155621afaf18SBorislav Petkov 15574fbce464SBorislav Petkov /* 155875581a20SBorislav Petkov * Enable instrumentation around the external facilities like task_work_add() 155975581a20SBorislav Petkov * (via queue_task_work()), fixup_exception() etc. For now, that is. Fixing this 156075581a20SBorislav Petkov * properly would need a lot more involved reorganization. 15614fbce464SBorislav Petkov */ 15624fbce464SBorislav Petkov instrumentation_begin(); 15634fbce464SBorislav Petkov 156475581a20SBorislav Petkov if (taint) 156575581a20SBorislav Petkov add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE); 156675581a20SBorislav Petkov 156775581a20SBorislav Petkov if (worst != MCE_AR_SEVERITY && !kill_current_task) 156875581a20SBorislav Petkov goto out; 156975581a20SBorislav Petkov 157021afaf18SBorislav Petkov /* Fault was in user mode and we need to take some action */ 157121afaf18SBorislav Petkov if ((m.cs & 3) == 3) { 1572b052df3dSThomas Gleixner /* If this triggers there is no way to recover. Die hard. */ 1573b052df3dSThomas Gleixner BUG_ON(!on_thread_stack() || !user_mode(regs)); 157421afaf18SBorislav Petkov 1575e40879b6SYazen Ghannam if (!mce_usable_address(&m)) 1576a6e3cf70STony Luck queue_task_work(&m, msg, kill_me_now); 1577a6e3cf70STony Luck else 1578a6e3cf70STony Luck queue_task_work(&m, msg, kill_me_maybe); 1579c0ab7ffcSTony Luck 158021afaf18SBorislav Petkov } else { 15811df73b21SBorislav Petkov /* 15821df73b21SBorislav Petkov * Handle an MCE which has happened in kernel space but from 15831df73b21SBorislav Petkov * which the kernel can recover: ex_has_fault_handler() has 15841df73b21SBorislav Petkov * already verified that the rIP at which the error happened is 15851df73b21SBorislav Petkov * a rIP from which the kernel can recover (by jumping to 15861df73b21SBorislav Petkov * recovery code specified in _ASM_EXTABLE_FAULT()) and the 15871df73b21SBorislav Petkov * corresponding exception handler which would do that is the 15881df73b21SBorislav Petkov * proper one. 15891df73b21SBorislav Petkov */ 15901df73b21SBorislav Petkov if (m.kflags & MCE_IN_KERNEL_RECOV) { 15918cd501c1SThomas Gleixner if (!fixup_exception(regs, X86_TRAP_MC, 0, 0)) 15922d806d07SJan H. Schönherr mce_panic("Failed kernel mode recovery", &m, msg); 159321afaf18SBorislav Petkov } 1594c0ab7ffcSTony Luck 1595c0ab7ffcSTony Luck if (m.kflags & MCE_IN_KERNEL_COPYIN) 1596a6e3cf70STony Luck queue_task_work(&m, msg, kill_me_never); 15971df73b21SBorislav Petkov } 15984fbce464SBorislav Petkov 159975581a20SBorislav Petkov out: 16004fbce464SBorislav Petkov instrumentation_end(); 16014fbce464SBorislav Petkov 16028ca97812SJue Wang clear: 16031e36d9c6STony Luck mce_wrmsrl(MSR_IA32_MCG_STATUS, 0); 160421afaf18SBorislav Petkov } 160521afaf18SBorislav Petkov EXPORT_SYMBOL_GPL(do_machine_check); 160621afaf18SBorislav Petkov 160721afaf18SBorislav Petkov #ifndef CONFIG_MEMORY_FAILURE 160821afaf18SBorislav Petkov int memory_failure(unsigned long pfn, int flags) 160921afaf18SBorislav Petkov { 161021afaf18SBorislav Petkov /* mce_severity() should not hand us an ACTION_REQUIRED error */ 161121afaf18SBorislav Petkov BUG_ON(flags & MF_ACTION_REQUIRED); 161221afaf18SBorislav Petkov pr_err("Uncorrected memory error in page 0x%lx ignored\n" 161321afaf18SBorislav Petkov "Rebuild kernel with CONFIG_MEMORY_FAILURE=y for smarter handling\n", 161421afaf18SBorislav Petkov pfn); 161521afaf18SBorislav Petkov 161621afaf18SBorislav Petkov return 0; 161721afaf18SBorislav Petkov } 161821afaf18SBorislav Petkov #endif 161921afaf18SBorislav Petkov 162021afaf18SBorislav Petkov /* 162121afaf18SBorislav Petkov * Periodic polling timer for "silent" machine check errors. If the 162221afaf18SBorislav Petkov * poller finds an MCE, poll 2x faster. When the poller finds no more 162321afaf18SBorislav Petkov * errors, poll 2x slower (up to check_interval seconds). 162421afaf18SBorislav Petkov */ 162521afaf18SBorislav Petkov static unsigned long check_interval = INITIAL_CHECK_INTERVAL; 162621afaf18SBorislav Petkov 162721afaf18SBorislav Petkov static DEFINE_PER_CPU(unsigned long, mce_next_interval); /* in jiffies */ 162821afaf18SBorislav Petkov static DEFINE_PER_CPU(struct timer_list, mce_timer); 162921afaf18SBorislav Petkov 163021afaf18SBorislav Petkov static unsigned long mce_adjust_timer_default(unsigned long interval) 163121afaf18SBorislav Petkov { 163221afaf18SBorislav Petkov return interval; 163321afaf18SBorislav Petkov } 163421afaf18SBorislav Petkov 163521afaf18SBorislav Petkov static unsigned long (*mce_adjust_timer)(unsigned long interval) = mce_adjust_timer_default; 163621afaf18SBorislav Petkov 163721afaf18SBorislav Petkov static void __start_timer(struct timer_list *t, unsigned long interval) 163821afaf18SBorislav Petkov { 163921afaf18SBorislav Petkov unsigned long when = jiffies + interval; 164021afaf18SBorislav Petkov unsigned long flags; 164121afaf18SBorislav Petkov 164221afaf18SBorislav Petkov local_irq_save(flags); 164321afaf18SBorislav Petkov 164421afaf18SBorislav Petkov if (!timer_pending(t) || time_before(when, t->expires)) 164521afaf18SBorislav Petkov mod_timer(t, round_jiffies(when)); 164621afaf18SBorislav Petkov 164721afaf18SBorislav Petkov local_irq_restore(flags); 164821afaf18SBorislav Petkov } 164921afaf18SBorislav Petkov 1650c3629dd7SBorislav Petkov (AMD) static void mc_poll_banks_default(void) 1651c3629dd7SBorislav Petkov (AMD) { 1652c3629dd7SBorislav Petkov (AMD) machine_check_poll(0, this_cpu_ptr(&mce_poll_banks)); 1653c3629dd7SBorislav Petkov (AMD) } 1654c3629dd7SBorislav Petkov (AMD) 1655c3629dd7SBorislav Petkov (AMD) void (*mc_poll_banks)(void) = mc_poll_banks_default; 1656c3629dd7SBorislav Petkov (AMD) 165721afaf18SBorislav Petkov static void mce_timer_fn(struct timer_list *t) 165821afaf18SBorislav Petkov { 165921afaf18SBorislav Petkov struct timer_list *cpu_t = this_cpu_ptr(&mce_timer); 166021afaf18SBorislav Petkov unsigned long iv; 166121afaf18SBorislav Petkov 166221afaf18SBorislav Petkov WARN_ON(cpu_t != t); 166321afaf18SBorislav Petkov 166421afaf18SBorislav Petkov iv = __this_cpu_read(mce_next_interval); 166521afaf18SBorislav Petkov 166621afaf18SBorislav Petkov if (mce_available(this_cpu_ptr(&cpu_info))) { 1667c3629dd7SBorislav Petkov (AMD) mc_poll_banks(); 166821afaf18SBorislav Petkov 166921afaf18SBorislav Petkov if (mce_intel_cmci_poll()) { 167021afaf18SBorislav Petkov iv = mce_adjust_timer(iv); 167121afaf18SBorislav Petkov goto done; 167221afaf18SBorislav Petkov } 167321afaf18SBorislav Petkov } 167421afaf18SBorislav Petkov 167521afaf18SBorislav Petkov /* 167621afaf18SBorislav Petkov * Alert userspace if needed. If we logged an MCE, reduce the polling 167721afaf18SBorislav Petkov * interval, otherwise increase the polling interval. 167821afaf18SBorislav Petkov */ 167921afaf18SBorislav Petkov if (mce_notify_irq()) 168021afaf18SBorislav Petkov iv = max(iv / 2, (unsigned long) HZ/100); 168121afaf18SBorislav Petkov else 168221afaf18SBorislav Petkov iv = min(iv * 2, round_jiffies_relative(check_interval * HZ)); 168321afaf18SBorislav Petkov 168421afaf18SBorislav Petkov done: 168521afaf18SBorislav Petkov __this_cpu_write(mce_next_interval, iv); 168621afaf18SBorislav Petkov __start_timer(t, iv); 168721afaf18SBorislav Petkov } 168821afaf18SBorislav Petkov 168921afaf18SBorislav Petkov /* 169021afaf18SBorislav Petkov * Ensure that the timer is firing in @interval from now. 169121afaf18SBorislav Petkov */ 169221afaf18SBorislav Petkov void mce_timer_kick(unsigned long interval) 169321afaf18SBorislav Petkov { 169421afaf18SBorislav Petkov struct timer_list *t = this_cpu_ptr(&mce_timer); 169521afaf18SBorislav Petkov unsigned long iv = __this_cpu_read(mce_next_interval); 169621afaf18SBorislav Petkov 169721afaf18SBorislav Petkov __start_timer(t, interval); 169821afaf18SBorislav Petkov 169921afaf18SBorislav Petkov if (interval < iv) 170021afaf18SBorislav Petkov __this_cpu_write(mce_next_interval, interval); 170121afaf18SBorislav Petkov } 170221afaf18SBorislav Petkov 170321afaf18SBorislav Petkov /* Must not be called in IRQ context where del_timer_sync() can deadlock */ 170421afaf18SBorislav Petkov static void mce_timer_delete_all(void) 170521afaf18SBorislav Petkov { 170621afaf18SBorislav Petkov int cpu; 170721afaf18SBorislav Petkov 170821afaf18SBorislav Petkov for_each_online_cpu(cpu) 170921afaf18SBorislav Petkov del_timer_sync(&per_cpu(mce_timer, cpu)); 171021afaf18SBorislav Petkov } 171121afaf18SBorislav Petkov 171221afaf18SBorislav Petkov /* 171321afaf18SBorislav Petkov * Notify the user(s) about new machine check events. 171421afaf18SBorislav Petkov * Can be called from interrupt context, but not from machine check/NMI 171521afaf18SBorislav Petkov * context. 171621afaf18SBorislav Petkov */ 171721afaf18SBorislav Petkov int mce_notify_irq(void) 171821afaf18SBorislav Petkov { 171921afaf18SBorislav Petkov /* Not more than two messages every minute */ 172021afaf18SBorislav Petkov static DEFINE_RATELIMIT_STATE(ratelimit, 60*HZ, 2); 172121afaf18SBorislav Petkov 172221afaf18SBorislav Petkov if (test_and_clear_bit(0, &mce_need_notify)) { 172321afaf18SBorislav Petkov mce_work_trigger(); 172421afaf18SBorislav Petkov 172521afaf18SBorislav Petkov if (__ratelimit(&ratelimit)) 172621afaf18SBorislav Petkov pr_info(HW_ERR "Machine check events logged\n"); 172721afaf18SBorislav Petkov 172821afaf18SBorislav Petkov return 1; 172921afaf18SBorislav Petkov } 173021afaf18SBorislav Petkov return 0; 173121afaf18SBorislav Petkov } 173221afaf18SBorislav Petkov EXPORT_SYMBOL_GPL(mce_notify_irq); 173321afaf18SBorislav Petkov 1734b4914508SYazen Ghannam static void __mcheck_cpu_mce_banks_init(void) 173521afaf18SBorislav Petkov { 1736b4914508SYazen Ghannam struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 1737c7d314f3SYazen Ghannam u8 n_banks = this_cpu_read(mce_num_banks); 173821afaf18SBorislav Petkov int i; 173921afaf18SBorislav Petkov 1740c7d314f3SYazen Ghannam for (i = 0; i < n_banks; i++) { 174121afaf18SBorislav Petkov struct mce_bank *b = &mce_banks[i]; 174221afaf18SBorislav Petkov 1743068b053dSYazen Ghannam /* 1744068b053dSYazen Ghannam * Init them all, __mcheck_cpu_apply_quirks() is going to apply 1745068b053dSYazen Ghannam * the required vendor quirks before 1746068b053dSYazen Ghannam * __mcheck_cpu_init_clear_banks() does the final bank setup. 1747068b053dSYazen Ghannam */ 174821afaf18SBorislav Petkov b->ctl = -1ULL; 174977080929SKaixu Xia b->init = true; 175021afaf18SBorislav Petkov } 175121afaf18SBorislav Petkov } 175221afaf18SBorislav Petkov 175321afaf18SBorislav Petkov /* 175421afaf18SBorislav Petkov * Initialize Machine Checks for a CPU. 175521afaf18SBorislav Petkov */ 1756b4914508SYazen Ghannam static void __mcheck_cpu_cap_init(void) 175721afaf18SBorislav Petkov { 175821afaf18SBorislav Petkov u64 cap; 1759006c0770SYazen Ghannam u8 b; 176021afaf18SBorislav Petkov 176121afaf18SBorislav Petkov rdmsrl(MSR_IA32_MCG_CAP, cap); 176221afaf18SBorislav Petkov 176321afaf18SBorislav Petkov b = cap & MCG_BANKCNT_MASK; 176421afaf18SBorislav Petkov 1765c7d314f3SYazen Ghannam if (b > MAX_NR_BANKS) { 1766c7d314f3SYazen Ghannam pr_warn("CPU%d: Using only %u machine check banks out of %u\n", 1767c7d314f3SYazen Ghannam smp_processor_id(), MAX_NR_BANKS, b); 1768c7d314f3SYazen Ghannam b = MAX_NR_BANKS; 1769c7d314f3SYazen Ghannam } 1770c7d314f3SYazen Ghannam 1771c7d314f3SYazen Ghannam this_cpu_write(mce_num_banks, b); 177221afaf18SBorislav Petkov 1773b4914508SYazen Ghannam __mcheck_cpu_mce_banks_init(); 177421afaf18SBorislav Petkov 177521afaf18SBorislav Petkov /* Use accurate RIP reporting if available. */ 177621afaf18SBorislav Petkov if ((cap & MCG_EXT_P) && MCG_EXT_CNT(cap) >= 9) 177721afaf18SBorislav Petkov mca_cfg.rip_msr = MSR_IA32_MCG_EIP; 177821afaf18SBorislav Petkov 177921afaf18SBorislav Petkov if (cap & MCG_SER_P) 178021afaf18SBorislav Petkov mca_cfg.ser = 1; 178121afaf18SBorislav Petkov } 178221afaf18SBorislav Petkov 178321afaf18SBorislav Petkov static void __mcheck_cpu_init_generic(void) 178421afaf18SBorislav Petkov { 178521afaf18SBorislav Petkov enum mcp_flags m_fl = 0; 178621afaf18SBorislav Petkov mce_banks_t all_banks; 178721afaf18SBorislav Petkov u64 cap; 178821afaf18SBorislav Petkov 178921afaf18SBorislav Petkov if (!mca_cfg.bootlog) 179021afaf18SBorislav Petkov m_fl = MCP_DONTLOG; 179121afaf18SBorislav Petkov 179221afaf18SBorislav Petkov /* 17933bff147bSBorislav Petkov * Log the machine checks left over from the previous reset. Log them 17943bff147bSBorislav Petkov * only, do not start processing them. That will happen in mcheck_late_init() 17953bff147bSBorislav Petkov * when all consumers have been registered on the notifier chain. 179621afaf18SBorislav Petkov */ 179721afaf18SBorislav Petkov bitmap_fill(all_banks, MAX_NR_BANKS); 17983bff147bSBorislav Petkov machine_check_poll(MCP_UC | MCP_QUEUE_LOG | m_fl, &all_banks); 179921afaf18SBorislav Petkov 180021afaf18SBorislav Petkov cr4_set_bits(X86_CR4_MCE); 180121afaf18SBorislav Petkov 180221afaf18SBorislav Petkov rdmsrl(MSR_IA32_MCG_CAP, cap); 180321afaf18SBorislav Petkov if (cap & MCG_CTL_P) 180421afaf18SBorislav Petkov wrmsr(MSR_IA32_MCG_CTL, 0xffffffff, 0xffffffff); 180521afaf18SBorislav Petkov } 180621afaf18SBorislav Petkov 180721afaf18SBorislav Petkov static void __mcheck_cpu_init_clear_banks(void) 180821afaf18SBorislav Petkov { 1809b4914508SYazen Ghannam struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 181021afaf18SBorislav Petkov int i; 181121afaf18SBorislav Petkov 1812c7d314f3SYazen Ghannam for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 181321afaf18SBorislav Petkov struct mce_bank *b = &mce_banks[i]; 181421afaf18SBorislav Petkov 181521afaf18SBorislav Petkov if (!b->init) 181621afaf18SBorislav Petkov continue; 18178121b8f9SBorislav Petkov wrmsrl(mca_msr_reg(i, MCA_CTL), b->ctl); 18188121b8f9SBorislav Petkov wrmsrl(mca_msr_reg(i, MCA_STATUS), 0); 181921afaf18SBorislav Petkov } 182021afaf18SBorislav Petkov } 182121afaf18SBorislav Petkov 182221afaf18SBorislav Petkov /* 1823068b053dSYazen Ghannam * Do a final check to see if there are any unused/RAZ banks. 1824068b053dSYazen Ghannam * 1825068b053dSYazen Ghannam * This must be done after the banks have been initialized and any quirks have 1826068b053dSYazen Ghannam * been applied. 1827068b053dSYazen Ghannam * 1828068b053dSYazen Ghannam * Do not call this from any user-initiated flows, e.g. CPU hotplug or sysfs. 1829068b053dSYazen Ghannam * Otherwise, a user who disables a bank will not be able to re-enable it 1830068b053dSYazen Ghannam * without a system reboot. 1831068b053dSYazen Ghannam */ 1832068b053dSYazen Ghannam static void __mcheck_cpu_check_banks(void) 1833068b053dSYazen Ghannam { 1834068b053dSYazen Ghannam struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 1835068b053dSYazen Ghannam u64 msrval; 1836068b053dSYazen Ghannam int i; 1837068b053dSYazen Ghannam 1838068b053dSYazen Ghannam for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 1839068b053dSYazen Ghannam struct mce_bank *b = &mce_banks[i]; 1840068b053dSYazen Ghannam 1841068b053dSYazen Ghannam if (!b->init) 1842068b053dSYazen Ghannam continue; 1843068b053dSYazen Ghannam 18448121b8f9SBorislav Petkov rdmsrl(mca_msr_reg(i, MCA_CTL), msrval); 1845068b053dSYazen Ghannam b->init = !!msrval; 1846068b053dSYazen Ghannam } 1847068b053dSYazen Ghannam } 1848068b053dSYazen Ghannam 184921afaf18SBorislav Petkov /* Add per CPU specific workarounds here */ 185021afaf18SBorislav Petkov static int __mcheck_cpu_apply_quirks(struct cpuinfo_x86 *c) 185121afaf18SBorislav Petkov { 1852b4914508SYazen Ghannam struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 185321afaf18SBorislav Petkov struct mca_config *cfg = &mca_cfg; 185421afaf18SBorislav Petkov 185521afaf18SBorislav Petkov if (c->x86_vendor == X86_VENDOR_UNKNOWN) { 185621afaf18SBorislav Petkov pr_info("unknown CPU type - not enabling MCE support\n"); 185721afaf18SBorislav Petkov return -EOPNOTSUPP; 185821afaf18SBorislav Petkov } 185921afaf18SBorislav Petkov 186021afaf18SBorislav Petkov /* This should be disabled by the BIOS, but isn't always */ 186121afaf18SBorislav Petkov if (c->x86_vendor == X86_VENDOR_AMD) { 1862c7d314f3SYazen Ghannam if (c->x86 == 15 && this_cpu_read(mce_num_banks) > 4) { 186321afaf18SBorislav Petkov /* 186421afaf18SBorislav Petkov * disable GART TBL walk error reporting, which 186521afaf18SBorislav Petkov * trips off incorrectly with the IOMMU & 3ware 186621afaf18SBorislav Petkov * & Cerberus: 186721afaf18SBorislav Petkov */ 186821afaf18SBorislav Petkov clear_bit(10, (unsigned long *)&mce_banks[4].ctl); 186921afaf18SBorislav Petkov } 187021afaf18SBorislav Petkov if (c->x86 < 0x11 && cfg->bootlog < 0) { 187121afaf18SBorislav Petkov /* 187221afaf18SBorislav Petkov * Lots of broken BIOS around that don't clear them 187321afaf18SBorislav Petkov * by default and leave crap in there. Don't log: 187421afaf18SBorislav Petkov */ 187521afaf18SBorislav Petkov cfg->bootlog = 0; 187621afaf18SBorislav Petkov } 187721afaf18SBorislav Petkov /* 187821afaf18SBorislav Petkov * Various K7s with broken bank 0 around. Always disable 187921afaf18SBorislav Petkov * by default. 188021afaf18SBorislav Petkov */ 1881c7d314f3SYazen Ghannam if (c->x86 == 6 && this_cpu_read(mce_num_banks) > 0) 188221afaf18SBorislav Petkov mce_banks[0].ctl = 0; 188321afaf18SBorislav Petkov 188421afaf18SBorislav Petkov /* 188521afaf18SBorislav Petkov * overflow_recov is supported for F15h Models 00h-0fh 188621afaf18SBorislav Petkov * even though we don't have a CPUID bit for it. 188721afaf18SBorislav Petkov */ 188821afaf18SBorislav Petkov if (c->x86 == 0x15 && c->x86_model <= 0xf) 188921afaf18SBorislav Petkov mce_flags.overflow_recov = 1; 189021afaf18SBorislav Petkov 18914240e2ebSYazen Ghannam if (c->x86 >= 0x17 && c->x86 <= 0x1A) 18924240e2ebSYazen Ghannam mce_flags.zen_ifu_quirk = 1; 18934240e2ebSYazen Ghannam 189421afaf18SBorislav Petkov } 189521afaf18SBorislav Petkov 189621afaf18SBorislav Petkov if (c->x86_vendor == X86_VENDOR_INTEL) { 189721afaf18SBorislav Petkov /* 189821afaf18SBorislav Petkov * SDM documents that on family 6 bank 0 should not be written 189921afaf18SBorislav Petkov * because it aliases to another special BIOS controlled 190021afaf18SBorislav Petkov * register. 190121afaf18SBorislav Petkov * But it's not aliased anymore on model 0x1a+ 190221afaf18SBorislav Petkov * Don't ignore bank 0 completely because there could be a 190321afaf18SBorislav Petkov * valid event later, merely don't write CTL0. 190421afaf18SBorislav Petkov */ 190521afaf18SBorislav Petkov 1906c7d314f3SYazen Ghannam if (c->x86 == 6 && c->x86_model < 0x1A && this_cpu_read(mce_num_banks) > 0) 190777080929SKaixu Xia mce_banks[0].init = false; 190821afaf18SBorislav Petkov 190921afaf18SBorislav Petkov /* 191021afaf18SBorislav Petkov * All newer Intel systems support MCE broadcasting. Enable 191121afaf18SBorislav Petkov * synchronization with a one second timeout. 191221afaf18SBorislav Petkov */ 191321afaf18SBorislav Petkov if ((c->x86 > 6 || (c->x86 == 6 && c->x86_model >= 0xe)) && 191421afaf18SBorislav Petkov cfg->monarch_timeout < 0) 191521afaf18SBorislav Petkov cfg->monarch_timeout = USEC_PER_SEC; 191621afaf18SBorislav Petkov 191721afaf18SBorislav Petkov /* 191821afaf18SBorislav Petkov * There are also broken BIOSes on some Pentium M and 191921afaf18SBorislav Petkov * earlier systems: 192021afaf18SBorislav Petkov */ 192121afaf18SBorislav Petkov if (c->x86 == 6 && c->x86_model <= 13 && cfg->bootlog < 0) 192221afaf18SBorislav Petkov cfg->bootlog = 0; 192321afaf18SBorislav Petkov 192421afaf18SBorislav Petkov if (c->x86 == 6 && c->x86_model == 45) 1925cc466666SBorislav Petkov mce_flags.snb_ifu_quirk = 1; 19268ca97812SJue Wang 19278ca97812SJue Wang /* 19288ca97812SJue Wang * Skylake, Cascacde Lake and Cooper Lake require a quirk on 19298ca97812SJue Wang * rep movs. 19308ca97812SJue Wang */ 19318ca97812SJue Wang if (c->x86 == 6 && c->x86_model == INTEL_FAM6_SKYLAKE_X) 19328ca97812SJue Wang mce_flags.skx_repmov_quirk = 1; 193321afaf18SBorislav Petkov } 19346e898d2bSTony W Wang-oc 19356e898d2bSTony W Wang-oc if (c->x86_vendor == X86_VENDOR_ZHAOXIN) { 19366e898d2bSTony W Wang-oc /* 19376e898d2bSTony W Wang-oc * All newer Zhaoxin CPUs support MCE broadcasting. Enable 19386e898d2bSTony W Wang-oc * synchronization with a one second timeout. 19396e898d2bSTony W Wang-oc */ 19406e898d2bSTony W Wang-oc if (c->x86 > 6 || (c->x86_model == 0x19 || c->x86_model == 0x1f)) { 19416e898d2bSTony W Wang-oc if (cfg->monarch_timeout < 0) 19426e898d2bSTony W Wang-oc cfg->monarch_timeout = USEC_PER_SEC; 19436e898d2bSTony W Wang-oc } 19446e898d2bSTony W Wang-oc } 19456e898d2bSTony W Wang-oc 194621afaf18SBorislav Petkov if (cfg->monarch_timeout < 0) 194721afaf18SBorislav Petkov cfg->monarch_timeout = 0; 194821afaf18SBorislav Petkov if (cfg->bootlog != 0) 194921afaf18SBorislav Petkov cfg->panic_timeout = 30; 195021afaf18SBorislav Petkov 195121afaf18SBorislav Petkov return 0; 195221afaf18SBorislav Petkov } 195321afaf18SBorislav Petkov 195421afaf18SBorislav Petkov static int __mcheck_cpu_ancient_init(struct cpuinfo_x86 *c) 195521afaf18SBorislav Petkov { 195621afaf18SBorislav Petkov if (c->x86 != 5) 195721afaf18SBorislav Petkov return 0; 195821afaf18SBorislav Petkov 195921afaf18SBorislav Petkov switch (c->x86_vendor) { 196021afaf18SBorislav Petkov case X86_VENDOR_INTEL: 196121afaf18SBorislav Petkov intel_p5_mcheck_init(c); 1962cbe1de16SBorislav Petkov mce_flags.p5 = 1; 196321afaf18SBorislav Petkov return 1; 196421afaf18SBorislav Petkov case X86_VENDOR_CENTAUR: 196521afaf18SBorislav Petkov winchip_mcheck_init(c); 1966cbe1de16SBorislav Petkov mce_flags.winchip = 1; 196721afaf18SBorislav Petkov return 1; 196821afaf18SBorislav Petkov default: 196921afaf18SBorislav Petkov return 0; 197021afaf18SBorislav Petkov } 197121afaf18SBorislav Petkov 197221afaf18SBorislav Petkov return 0; 197321afaf18SBorislav Petkov } 197421afaf18SBorislav Petkov 197521afaf18SBorislav Petkov /* 197621afaf18SBorislav Petkov * Init basic CPU features needed for early decoding of MCEs. 197721afaf18SBorislav Petkov */ 197821afaf18SBorislav Petkov static void __mcheck_cpu_init_early(struct cpuinfo_x86 *c) 197921afaf18SBorislav Petkov { 198021afaf18SBorislav Petkov if (c->x86_vendor == X86_VENDOR_AMD || c->x86_vendor == X86_VENDOR_HYGON) { 198121afaf18SBorislav Petkov mce_flags.overflow_recov = !!cpu_has(c, X86_FEATURE_OVERFLOW_RECOV); 198221afaf18SBorislav Petkov mce_flags.succor = !!cpu_has(c, X86_FEATURE_SUCCOR); 198321afaf18SBorislav Petkov mce_flags.smca = !!cpu_has(c, X86_FEATURE_SMCA); 1984c9bf318fSThomas Gleixner mce_flags.amd_threshold = 1; 198521afaf18SBorislav Petkov } 198621afaf18SBorislav Petkov } 198721afaf18SBorislav Petkov 198821afaf18SBorislav Petkov static void mce_centaur_feature_init(struct cpuinfo_x86 *c) 198921afaf18SBorislav Petkov { 199021afaf18SBorislav Petkov struct mca_config *cfg = &mca_cfg; 199121afaf18SBorislav Petkov 199221afaf18SBorislav Petkov /* 199321afaf18SBorislav Petkov * All newer Centaur CPUs support MCE broadcasting. Enable 199421afaf18SBorislav Petkov * synchronization with a one second timeout. 199521afaf18SBorislav Petkov */ 199621afaf18SBorislav Petkov if ((c->x86 == 6 && c->x86_model == 0xf && c->x86_stepping >= 0xe) || 199721afaf18SBorislav Petkov c->x86 > 6) { 199821afaf18SBorislav Petkov if (cfg->monarch_timeout < 0) 199921afaf18SBorislav Petkov cfg->monarch_timeout = USEC_PER_SEC; 200021afaf18SBorislav Petkov } 200121afaf18SBorislav Petkov } 200221afaf18SBorislav Petkov 20035a3d56a0STony W Wang-oc static void mce_zhaoxin_feature_init(struct cpuinfo_x86 *c) 20045a3d56a0STony W Wang-oc { 20055a3d56a0STony W Wang-oc struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 20065a3d56a0STony W Wang-oc 20075a3d56a0STony W Wang-oc /* 20085a3d56a0STony W Wang-oc * These CPUs have MCA bank 8 which reports only one error type called 20095a3d56a0STony W Wang-oc * SVAD (System View Address Decoder). The reporting of that error is 20105a3d56a0STony W Wang-oc * controlled by IA32_MC8.CTL.0. 20115a3d56a0STony W Wang-oc * 20125a3d56a0STony W Wang-oc * If enabled, prefetching on these CPUs will cause SVAD MCE when 20135a3d56a0STony W Wang-oc * virtual machines start and result in a system panic. Always disable 20145a3d56a0STony W Wang-oc * bank 8 SVAD error by default. 20155a3d56a0STony W Wang-oc */ 20165a3d56a0STony W Wang-oc if ((c->x86 == 7 && c->x86_model == 0x1b) || 20175a3d56a0STony W Wang-oc (c->x86_model == 0x19 || c->x86_model == 0x1f)) { 20185a3d56a0STony W Wang-oc if (this_cpu_read(mce_num_banks) > 8) 20195a3d56a0STony W Wang-oc mce_banks[8].ctl = 0; 20205a3d56a0STony W Wang-oc } 20215a3d56a0STony W Wang-oc 20225a3d56a0STony W Wang-oc intel_init_cmci(); 202370f0c230STony W Wang-oc intel_init_lmce(); 20245a3d56a0STony W Wang-oc mce_adjust_timer = cmci_intel_adjust_timer; 20255a3d56a0STony W Wang-oc } 20265a3d56a0STony W Wang-oc 202770f0c230STony W Wang-oc static void mce_zhaoxin_feature_clear(struct cpuinfo_x86 *c) 202870f0c230STony W Wang-oc { 202970f0c230STony W Wang-oc intel_clear_lmce(); 203070f0c230STony W Wang-oc } 203170f0c230STony W Wang-oc 203221afaf18SBorislav Petkov static void __mcheck_cpu_init_vendor(struct cpuinfo_x86 *c) 203321afaf18SBorislav Petkov { 203421afaf18SBorislav Petkov switch (c->x86_vendor) { 203521afaf18SBorislav Petkov case X86_VENDOR_INTEL: 203621afaf18SBorislav Petkov mce_intel_feature_init(c); 203721afaf18SBorislav Petkov mce_adjust_timer = cmci_intel_adjust_timer; 203821afaf18SBorislav Petkov break; 203921afaf18SBorislav Petkov 204021afaf18SBorislav Petkov case X86_VENDOR_AMD: { 204121afaf18SBorislav Petkov mce_amd_feature_init(c); 204221afaf18SBorislav Petkov break; 204321afaf18SBorislav Petkov } 204421afaf18SBorislav Petkov 204521afaf18SBorislav Petkov case X86_VENDOR_HYGON: 204621afaf18SBorislav Petkov mce_hygon_feature_init(c); 204721afaf18SBorislav Petkov break; 204821afaf18SBorislav Petkov 204921afaf18SBorislav Petkov case X86_VENDOR_CENTAUR: 205021afaf18SBorislav Petkov mce_centaur_feature_init(c); 205121afaf18SBorislav Petkov break; 205221afaf18SBorislav Petkov 20535a3d56a0STony W Wang-oc case X86_VENDOR_ZHAOXIN: 20545a3d56a0STony W Wang-oc mce_zhaoxin_feature_init(c); 20555a3d56a0STony W Wang-oc break; 20565a3d56a0STony W Wang-oc 205721afaf18SBorislav Petkov default: 205821afaf18SBorislav Petkov break; 205921afaf18SBorislav Petkov } 206021afaf18SBorislav Petkov } 206121afaf18SBorislav Petkov 206221afaf18SBorislav Petkov static void __mcheck_cpu_clear_vendor(struct cpuinfo_x86 *c) 206321afaf18SBorislav Petkov { 206421afaf18SBorislav Petkov switch (c->x86_vendor) { 206521afaf18SBorislav Petkov case X86_VENDOR_INTEL: 206621afaf18SBorislav Petkov mce_intel_feature_clear(c); 206721afaf18SBorislav Petkov break; 206870f0c230STony W Wang-oc 206970f0c230STony W Wang-oc case X86_VENDOR_ZHAOXIN: 207070f0c230STony W Wang-oc mce_zhaoxin_feature_clear(c); 207170f0c230STony W Wang-oc break; 207270f0c230STony W Wang-oc 207321afaf18SBorislav Petkov default: 207421afaf18SBorislav Petkov break; 207521afaf18SBorislav Petkov } 207621afaf18SBorislav Petkov } 207721afaf18SBorislav Petkov 207821afaf18SBorislav Petkov static void mce_start_timer(struct timer_list *t) 207921afaf18SBorislav Petkov { 208021afaf18SBorislav Petkov unsigned long iv = check_interval * HZ; 208121afaf18SBorislav Petkov 208221afaf18SBorislav Petkov if (mca_cfg.ignore_ce || !iv) 208321afaf18SBorislav Petkov return; 208421afaf18SBorislav Petkov 208521afaf18SBorislav Petkov this_cpu_write(mce_next_interval, iv); 208621afaf18SBorislav Petkov __start_timer(t, iv); 208721afaf18SBorislav Petkov } 208821afaf18SBorislav Petkov 208921afaf18SBorislav Petkov static void __mcheck_cpu_setup_timer(void) 209021afaf18SBorislav Petkov { 209121afaf18SBorislav Petkov struct timer_list *t = this_cpu_ptr(&mce_timer); 209221afaf18SBorislav Petkov 209321afaf18SBorislav Petkov timer_setup(t, mce_timer_fn, TIMER_PINNED); 209421afaf18SBorislav Petkov } 209521afaf18SBorislav Petkov 209621afaf18SBorislav Petkov static void __mcheck_cpu_init_timer(void) 209721afaf18SBorislav Petkov { 209821afaf18SBorislav Petkov struct timer_list *t = this_cpu_ptr(&mce_timer); 209921afaf18SBorislav Petkov 210021afaf18SBorislav Petkov timer_setup(t, mce_timer_fn, TIMER_PINNED); 210121afaf18SBorislav Petkov mce_start_timer(t); 210221afaf18SBorislav Petkov } 210321afaf18SBorislav Petkov 210445d4b7b9SYazen Ghannam bool filter_mce(struct mce *m) 210545d4b7b9SYazen Ghannam { 210671a84402SYazen Ghannam if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) 210771a84402SYazen Ghannam return amd_filter_mce(m); 21082976908eSPrarit Bhargava if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) 21092976908eSPrarit Bhargava return intel_filter_mce(m); 211071a84402SYazen Ghannam 211145d4b7b9SYazen Ghannam return false; 211245d4b7b9SYazen Ghannam } 211345d4b7b9SYazen Ghannam 21144c0dcd83SThomas Gleixner static __always_inline void exc_machine_check_kernel(struct pt_regs *regs) 211521afaf18SBorislav Petkov { 2116b6be002bSThomas Gleixner irqentry_state_t irq_state; 2117bc21a291SThomas Gleixner 211813cbc0cdSAndy Lutomirski WARN_ON_ONCE(user_mode(regs)); 211913cbc0cdSAndy Lutomirski 21204c0dcd83SThomas Gleixner /* 21214c0dcd83SThomas Gleixner * Only required when from kernel mode. See 21224c0dcd83SThomas Gleixner * mce_check_crashing_cpu() for details. 21234c0dcd83SThomas Gleixner */ 2124cbe1de16SBorislav Petkov if (mca_cfg.initialized && mce_check_crashing_cpu()) 212594a46d31SThomas Gleixner return; 212694a46d31SThomas Gleixner 2127b6be002bSThomas Gleixner irq_state = irqentry_nmi_enter(regs); 212873749536SPeter Zijlstra 2129cbe1de16SBorislav Petkov do_machine_check(regs); 213073749536SPeter Zijlstra 2131b6be002bSThomas Gleixner irqentry_nmi_exit(regs, irq_state); 213221afaf18SBorislav Petkov } 213321afaf18SBorislav Petkov 21344c0dcd83SThomas Gleixner static __always_inline void exc_machine_check_user(struct pt_regs *regs) 21354c0dcd83SThomas Gleixner { 2136517e4992SThomas Gleixner irqentry_enter_from_user_mode(regs); 213773749536SPeter Zijlstra 2138cbe1de16SBorislav Petkov do_machine_check(regs); 213973749536SPeter Zijlstra 2140517e4992SThomas Gleixner irqentry_exit_to_user_mode(regs); 21414c0dcd83SThomas Gleixner } 21424c0dcd83SThomas Gleixner 21434c0dcd83SThomas Gleixner #ifdef CONFIG_X86_64 21444c0dcd83SThomas Gleixner /* MCE hit kernel mode */ 21454c0dcd83SThomas Gleixner DEFINE_IDTENTRY_MCE(exc_machine_check) 21464c0dcd83SThomas Gleixner { 2147cd840e42SPeter Zijlstra unsigned long dr7; 2148cd840e42SPeter Zijlstra 2149cd840e42SPeter Zijlstra dr7 = local_db_save(); 21504c0dcd83SThomas Gleixner exc_machine_check_kernel(regs); 2151cd840e42SPeter Zijlstra local_db_restore(dr7); 21524c0dcd83SThomas Gleixner } 21534c0dcd83SThomas Gleixner 21544c0dcd83SThomas Gleixner /* The user mode variant. */ 21554c0dcd83SThomas Gleixner DEFINE_IDTENTRY_MCE_USER(exc_machine_check) 21564c0dcd83SThomas Gleixner { 2157cd840e42SPeter Zijlstra unsigned long dr7; 2158cd840e42SPeter Zijlstra 2159cd840e42SPeter Zijlstra dr7 = local_db_save(); 21604c0dcd83SThomas Gleixner exc_machine_check_user(regs); 2161cd840e42SPeter Zijlstra local_db_restore(dr7); 21624c0dcd83SThomas Gleixner } 21634c0dcd83SThomas Gleixner #else 21644c0dcd83SThomas Gleixner /* 32bit unified entry point */ 216513cbc0cdSAndy Lutomirski DEFINE_IDTENTRY_RAW(exc_machine_check) 21664c0dcd83SThomas Gleixner { 2167cd840e42SPeter Zijlstra unsigned long dr7; 2168cd840e42SPeter Zijlstra 2169cd840e42SPeter Zijlstra dr7 = local_db_save(); 21704c0dcd83SThomas Gleixner if (user_mode(regs)) 21714c0dcd83SThomas Gleixner exc_machine_check_user(regs); 21724c0dcd83SThomas Gleixner else 21734c0dcd83SThomas Gleixner exc_machine_check_kernel(regs); 2174cd840e42SPeter Zijlstra local_db_restore(dr7); 21754c0dcd83SThomas Gleixner } 21764c0dcd83SThomas Gleixner #endif 217721afaf18SBorislav Petkov 217821afaf18SBorislav Petkov /* 217921afaf18SBorislav Petkov * Called for each booted CPU to set up machine checks. 218021afaf18SBorislav Petkov * Must be called with preempt off: 218121afaf18SBorislav Petkov */ 218221afaf18SBorislav Petkov void mcheck_cpu_init(struct cpuinfo_x86 *c) 218321afaf18SBorislav Petkov { 218421afaf18SBorislav Petkov if (mca_cfg.disabled) 218521afaf18SBorislav Petkov return; 218621afaf18SBorislav Petkov 218721afaf18SBorislav Petkov if (__mcheck_cpu_ancient_init(c)) 218821afaf18SBorislav Petkov return; 218921afaf18SBorislav Petkov 219021afaf18SBorislav Petkov if (!mce_available(c)) 219121afaf18SBorislav Petkov return; 219221afaf18SBorislav Petkov 2193b4914508SYazen Ghannam __mcheck_cpu_cap_init(); 2194b4914508SYazen Ghannam 2195b4914508SYazen Ghannam if (__mcheck_cpu_apply_quirks(c) < 0) { 219621afaf18SBorislav Petkov mca_cfg.disabled = 1; 219721afaf18SBorislav Petkov return; 219821afaf18SBorislav Petkov } 219921afaf18SBorislav Petkov 220021afaf18SBorislav Petkov if (mce_gen_pool_init()) { 220121afaf18SBorislav Petkov mca_cfg.disabled = 1; 220221afaf18SBorislav Petkov pr_emerg("Couldn't allocate MCE records pool!\n"); 220321afaf18SBorislav Petkov return; 220421afaf18SBorislav Petkov } 220521afaf18SBorislav Petkov 2206cbe1de16SBorislav Petkov mca_cfg.initialized = 1; 220721afaf18SBorislav Petkov 220821afaf18SBorislav Petkov __mcheck_cpu_init_early(c); 220921afaf18SBorislav Petkov __mcheck_cpu_init_generic(); 221021afaf18SBorislav Petkov __mcheck_cpu_init_vendor(c); 221121afaf18SBorislav Petkov __mcheck_cpu_init_clear_banks(); 2212068b053dSYazen Ghannam __mcheck_cpu_check_banks(); 221321afaf18SBorislav Petkov __mcheck_cpu_setup_timer(); 221421afaf18SBorislav Petkov } 221521afaf18SBorislav Petkov 221621afaf18SBorislav Petkov /* 221721afaf18SBorislav Petkov * Called for each booted CPU to clear some machine checks opt-ins 221821afaf18SBorislav Petkov */ 221921afaf18SBorislav Petkov void mcheck_cpu_clear(struct cpuinfo_x86 *c) 222021afaf18SBorislav Petkov { 222121afaf18SBorislav Petkov if (mca_cfg.disabled) 222221afaf18SBorislav Petkov return; 222321afaf18SBorislav Petkov 222421afaf18SBorislav Petkov if (!mce_available(c)) 222521afaf18SBorislav Petkov return; 222621afaf18SBorislav Petkov 222721afaf18SBorislav Petkov /* 222821afaf18SBorislav Petkov * Possibly to clear general settings generic to x86 222921afaf18SBorislav Petkov * __mcheck_cpu_clear_generic(c); 223021afaf18SBorislav Petkov */ 223121afaf18SBorislav Petkov __mcheck_cpu_clear_vendor(c); 223221afaf18SBorislav Petkov 223321afaf18SBorislav Petkov } 223421afaf18SBorislav Petkov 223521afaf18SBorislav Petkov static void __mce_disable_bank(void *arg) 223621afaf18SBorislav Petkov { 223721afaf18SBorislav Petkov int bank = *((int *)arg); 223821afaf18SBorislav Petkov __clear_bit(bank, this_cpu_ptr(mce_poll_banks)); 223921afaf18SBorislav Petkov cmci_disable_bank(bank); 224021afaf18SBorislav Petkov } 224121afaf18SBorislav Petkov 224221afaf18SBorislav Petkov void mce_disable_bank(int bank) 224321afaf18SBorislav Petkov { 2244c7d314f3SYazen Ghannam if (bank >= this_cpu_read(mce_num_banks)) { 224521afaf18SBorislav Petkov pr_warn(FW_BUG 224621afaf18SBorislav Petkov "Ignoring request to disable invalid MCA bank %d.\n", 224721afaf18SBorislav Petkov bank); 224821afaf18SBorislav Petkov return; 224921afaf18SBorislav Petkov } 225021afaf18SBorislav Petkov set_bit(bank, mce_banks_ce_disabled); 225121afaf18SBorislav Petkov on_each_cpu(__mce_disable_bank, &bank, 1); 225221afaf18SBorislav Petkov } 225321afaf18SBorislav Petkov 225421afaf18SBorislav Petkov /* 225521afaf18SBorislav Petkov * mce=off Disables machine check 225621afaf18SBorislav Petkov * mce=no_cmci Disables CMCI 225721afaf18SBorislav Petkov * mce=no_lmce Disables LMCE 225821afaf18SBorislav Petkov * mce=dont_log_ce Clears corrected events silently, no log created for CEs. 225943505646STony Luck * mce=print_all Print all machine check logs to console 226021afaf18SBorislav Petkov * mce=ignore_ce Disables polling and CMCI, corrected events are not cleared. 226121afaf18SBorislav Petkov * mce=TOLERANCELEVEL[,monarchtimeout] (number, see above) 226221afaf18SBorislav Petkov * monarchtimeout is how long to wait for other CPUs on machine 226321afaf18SBorislav Petkov * check, or 0 to not wait 226421afaf18SBorislav Petkov * mce=bootlog Log MCEs from before booting. Disabled by default on AMD Fam10h 226521afaf18SBorislav Petkov and older. 226621afaf18SBorislav Petkov * mce=nobootlog Don't log MCEs from before booting. 226721afaf18SBorislav Petkov * mce=bios_cmci_threshold Don't program the CMCI threshold 2268ec6347bbSDan Williams * mce=recovery force enable copy_mc_fragile() 226921afaf18SBorislav Petkov */ 227021afaf18SBorislav Petkov static int __init mcheck_enable(char *str) 227121afaf18SBorislav Petkov { 227221afaf18SBorislav Petkov struct mca_config *cfg = &mca_cfg; 227321afaf18SBorislav Petkov 227421afaf18SBorislav Petkov if (*str == 0) { 227521afaf18SBorislav Petkov enable_p5_mce(); 227621afaf18SBorislav Petkov return 1; 227721afaf18SBorislav Petkov } 227821afaf18SBorislav Petkov if (*str == '=') 227921afaf18SBorislav Petkov str++; 228021afaf18SBorislav Petkov if (!strcmp(str, "off")) 228121afaf18SBorislav Petkov cfg->disabled = 1; 228221afaf18SBorislav Petkov else if (!strcmp(str, "no_cmci")) 228321afaf18SBorislav Petkov cfg->cmci_disabled = true; 228421afaf18SBorislav Petkov else if (!strcmp(str, "no_lmce")) 228521afaf18SBorislav Petkov cfg->lmce_disabled = 1; 228621afaf18SBorislav Petkov else if (!strcmp(str, "dont_log_ce")) 228721afaf18SBorislav Petkov cfg->dont_log_ce = true; 228843505646STony Luck else if (!strcmp(str, "print_all")) 228943505646STony Luck cfg->print_all = true; 229021afaf18SBorislav Petkov else if (!strcmp(str, "ignore_ce")) 229121afaf18SBorislav Petkov cfg->ignore_ce = true; 229221afaf18SBorislav Petkov else if (!strcmp(str, "bootlog") || !strcmp(str, "nobootlog")) 229321afaf18SBorislav Petkov cfg->bootlog = (str[0] == 'b'); 229421afaf18SBorislav Petkov else if (!strcmp(str, "bios_cmci_threshold")) 229521afaf18SBorislav Petkov cfg->bios_cmci_threshold = 1; 229621afaf18SBorislav Petkov else if (!strcmp(str, "recovery")) 229721afaf18SBorislav Petkov cfg->recovery = 1; 22987f1b8e0dSBorislav Petkov else if (isdigit(str[0])) 229921afaf18SBorislav Petkov get_option(&str, &(cfg->monarch_timeout)); 23007f1b8e0dSBorislav Petkov else { 230121afaf18SBorislav Petkov pr_info("mce argument %s ignored. Please use /sys\n", str); 230221afaf18SBorislav Petkov return 0; 230321afaf18SBorislav Petkov } 230421afaf18SBorislav Petkov return 1; 230521afaf18SBorislav Petkov } 230621afaf18SBorislav Petkov __setup("mce", mcheck_enable); 230721afaf18SBorislav Petkov 230821afaf18SBorislav Petkov int __init mcheck_init(void) 230921afaf18SBorislav Petkov { 2310c9c6d216STony Luck mce_register_decode_chain(&early_nb); 23118438b84aSJan H. Schönherr mce_register_decode_chain(&mce_uc_nb); 231221afaf18SBorislav Petkov mce_register_decode_chain(&mce_default_nb); 231321afaf18SBorislav Petkov 231421afaf18SBorislav Petkov INIT_WORK(&mce_work, mce_gen_pool_process); 231521afaf18SBorislav Petkov init_irq_work(&mce_irq_work, mce_irq_work_cb); 231621afaf18SBorislav Petkov 231721afaf18SBorislav Petkov return 0; 231821afaf18SBorislav Petkov } 231921afaf18SBorislav Petkov 232021afaf18SBorislav Petkov /* 232121afaf18SBorislav Petkov * mce_syscore: PM support 232221afaf18SBorislav Petkov */ 232321afaf18SBorislav Petkov 232421afaf18SBorislav Petkov /* 232521afaf18SBorislav Petkov * Disable machine checks on suspend and shutdown. We can't really handle 232621afaf18SBorislav Petkov * them later. 232721afaf18SBorislav Petkov */ 232821afaf18SBorislav Petkov static void mce_disable_error_reporting(void) 232921afaf18SBorislav Petkov { 2330b4914508SYazen Ghannam struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 233121afaf18SBorislav Petkov int i; 233221afaf18SBorislav Petkov 2333c7d314f3SYazen Ghannam for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 233421afaf18SBorislav Petkov struct mce_bank *b = &mce_banks[i]; 233521afaf18SBorislav Petkov 233621afaf18SBorislav Petkov if (b->init) 23378121b8f9SBorislav Petkov wrmsrl(mca_msr_reg(i, MCA_CTL), 0); 233821afaf18SBorislav Petkov } 233921afaf18SBorislav Petkov return; 234021afaf18SBorislav Petkov } 234121afaf18SBorislav Petkov 234221afaf18SBorislav Petkov static void vendor_disable_error_reporting(void) 234321afaf18SBorislav Petkov { 234421afaf18SBorislav Petkov /* 23456e898d2bSTony W Wang-oc * Don't clear on Intel or AMD or Hygon or Zhaoxin CPUs. Some of these 23466e898d2bSTony W Wang-oc * MSRs are socket-wide. Disabling them for just a single offlined CPU 23476e898d2bSTony W Wang-oc * is bad, since it will inhibit reporting for all shared resources on 23486e898d2bSTony W Wang-oc * the socket like the last level cache (LLC), the integrated memory 23496e898d2bSTony W Wang-oc * controller (iMC), etc. 235021afaf18SBorislav Petkov */ 235121afaf18SBorislav Petkov if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL || 235221afaf18SBorislav Petkov boot_cpu_data.x86_vendor == X86_VENDOR_HYGON || 23536e898d2bSTony W Wang-oc boot_cpu_data.x86_vendor == X86_VENDOR_AMD || 23546e898d2bSTony W Wang-oc boot_cpu_data.x86_vendor == X86_VENDOR_ZHAOXIN) 235521afaf18SBorislav Petkov return; 235621afaf18SBorislav Petkov 235721afaf18SBorislav Petkov mce_disable_error_reporting(); 235821afaf18SBorislav Petkov } 235921afaf18SBorislav Petkov 236021afaf18SBorislav Petkov static int mce_syscore_suspend(void) 236121afaf18SBorislav Petkov { 236221afaf18SBorislav Petkov vendor_disable_error_reporting(); 236321afaf18SBorislav Petkov return 0; 236421afaf18SBorislav Petkov } 236521afaf18SBorislav Petkov 236621afaf18SBorislav Petkov static void mce_syscore_shutdown(void) 236721afaf18SBorislav Petkov { 236821afaf18SBorislav Petkov vendor_disable_error_reporting(); 236921afaf18SBorislav Petkov } 237021afaf18SBorislav Petkov 237121afaf18SBorislav Petkov /* 237221afaf18SBorislav Petkov * On resume clear all MCE state. Don't want to see leftovers from the BIOS. 237321afaf18SBorislav Petkov * Only one CPU is active at this time, the others get re-added later using 237421afaf18SBorislav Petkov * CPU hotplug: 237521afaf18SBorislav Petkov */ 237621afaf18SBorislav Petkov static void mce_syscore_resume(void) 237721afaf18SBorislav Petkov { 237821afaf18SBorislav Petkov __mcheck_cpu_init_generic(); 237921afaf18SBorislav Petkov __mcheck_cpu_init_vendor(raw_cpu_ptr(&cpu_info)); 238021afaf18SBorislav Petkov __mcheck_cpu_init_clear_banks(); 238121afaf18SBorislav Petkov } 238221afaf18SBorislav Petkov 238321afaf18SBorislav Petkov static struct syscore_ops mce_syscore_ops = { 238421afaf18SBorislav Petkov .suspend = mce_syscore_suspend, 238521afaf18SBorislav Petkov .shutdown = mce_syscore_shutdown, 238621afaf18SBorislav Petkov .resume = mce_syscore_resume, 238721afaf18SBorislav Petkov }; 238821afaf18SBorislav Petkov 238921afaf18SBorislav Petkov /* 239021afaf18SBorislav Petkov * mce_device: Sysfs support 239121afaf18SBorislav Petkov */ 239221afaf18SBorislav Petkov 239321afaf18SBorislav Petkov static void mce_cpu_restart(void *data) 239421afaf18SBorislav Petkov { 239521afaf18SBorislav Petkov if (!mce_available(raw_cpu_ptr(&cpu_info))) 239621afaf18SBorislav Petkov return; 239721afaf18SBorislav Petkov __mcheck_cpu_init_generic(); 239821afaf18SBorislav Petkov __mcheck_cpu_init_clear_banks(); 239921afaf18SBorislav Petkov __mcheck_cpu_init_timer(); 240021afaf18SBorislav Petkov } 240121afaf18SBorislav Petkov 240221afaf18SBorislav Petkov /* Reinit MCEs after user configuration changes */ 240321afaf18SBorislav Petkov static void mce_restart(void) 240421afaf18SBorislav Petkov { 240521afaf18SBorislav Petkov mce_timer_delete_all(); 240621afaf18SBorislav Petkov on_each_cpu(mce_cpu_restart, NULL, 1); 24074783b9cbSYazen Ghannam mce_schedule_work(); 240821afaf18SBorislav Petkov } 240921afaf18SBorislav Petkov 241021afaf18SBorislav Petkov /* Toggle features for corrected errors */ 241121afaf18SBorislav Petkov static void mce_disable_cmci(void *data) 241221afaf18SBorislav Petkov { 241321afaf18SBorislav Petkov if (!mce_available(raw_cpu_ptr(&cpu_info))) 241421afaf18SBorislav Petkov return; 241521afaf18SBorislav Petkov cmci_clear(); 241621afaf18SBorislav Petkov } 241721afaf18SBorislav Petkov 241821afaf18SBorislav Petkov static void mce_enable_ce(void *all) 241921afaf18SBorislav Petkov { 242021afaf18SBorislav Petkov if (!mce_available(raw_cpu_ptr(&cpu_info))) 242121afaf18SBorislav Petkov return; 242221afaf18SBorislav Petkov cmci_reenable(); 242321afaf18SBorislav Petkov cmci_recheck(); 242421afaf18SBorislav Petkov if (all) 242521afaf18SBorislav Petkov __mcheck_cpu_init_timer(); 242621afaf18SBorislav Petkov } 242721afaf18SBorislav Petkov 242821afaf18SBorislav Petkov static struct bus_type mce_subsys = { 242921afaf18SBorislav Petkov .name = "machinecheck", 243021afaf18SBorislav Petkov .dev_name = "machinecheck", 243121afaf18SBorislav Petkov }; 243221afaf18SBorislav Petkov 243321afaf18SBorislav Petkov DEFINE_PER_CPU(struct device *, mce_device); 243421afaf18SBorislav Petkov 2435b4914508SYazen Ghannam static inline struct mce_bank_dev *attr_to_bank(struct device_attribute *attr) 243621afaf18SBorislav Petkov { 2437b4914508SYazen Ghannam return container_of(attr, struct mce_bank_dev, attr); 243821afaf18SBorislav Petkov } 243921afaf18SBorislav Petkov 244021afaf18SBorislav Petkov static ssize_t show_bank(struct device *s, struct device_attribute *attr, 244121afaf18SBorislav Petkov char *buf) 244221afaf18SBorislav Petkov { 2443b4914508SYazen Ghannam u8 bank = attr_to_bank(attr)->bank; 2444b4914508SYazen Ghannam struct mce_bank *b; 2445b4914508SYazen Ghannam 2446c7d314f3SYazen Ghannam if (bank >= per_cpu(mce_num_banks, s->id)) 2447b4914508SYazen Ghannam return -EINVAL; 2448b4914508SYazen Ghannam 2449b4914508SYazen Ghannam b = &per_cpu(mce_banks_array, s->id)[bank]; 2450b4914508SYazen Ghannam 2451068b053dSYazen Ghannam if (!b->init) 2452068b053dSYazen Ghannam return -ENODEV; 2453068b053dSYazen Ghannam 2454b4914508SYazen Ghannam return sprintf(buf, "%llx\n", b->ctl); 245521afaf18SBorislav Petkov } 245621afaf18SBorislav Petkov 245721afaf18SBorislav Petkov static ssize_t set_bank(struct device *s, struct device_attribute *attr, 245821afaf18SBorislav Petkov const char *buf, size_t size) 245921afaf18SBorislav Petkov { 2460b4914508SYazen Ghannam u8 bank = attr_to_bank(attr)->bank; 2461b4914508SYazen Ghannam struct mce_bank *b; 246221afaf18SBorislav Petkov u64 new; 246321afaf18SBorislav Petkov 246421afaf18SBorislav Petkov if (kstrtou64(buf, 0, &new) < 0) 246521afaf18SBorislav Petkov return -EINVAL; 246621afaf18SBorislav Petkov 2467c7d314f3SYazen Ghannam if (bank >= per_cpu(mce_num_banks, s->id)) 2468b4914508SYazen Ghannam return -EINVAL; 2469b4914508SYazen Ghannam 2470b4914508SYazen Ghannam b = &per_cpu(mce_banks_array, s->id)[bank]; 2471b4914508SYazen Ghannam 2472068b053dSYazen Ghannam if (!b->init) 2473068b053dSYazen Ghannam return -ENODEV; 2474068b053dSYazen Ghannam 2475b4914508SYazen Ghannam b->ctl = new; 247621afaf18SBorislav Petkov mce_restart(); 247721afaf18SBorislav Petkov 247821afaf18SBorislav Petkov return size; 247921afaf18SBorislav Petkov } 248021afaf18SBorislav Petkov 248121afaf18SBorislav Petkov static ssize_t set_ignore_ce(struct device *s, 248221afaf18SBorislav Petkov struct device_attribute *attr, 248321afaf18SBorislav Petkov const char *buf, size_t size) 248421afaf18SBorislav Petkov { 248521afaf18SBorislav Petkov u64 new; 248621afaf18SBorislav Petkov 248721afaf18SBorislav Petkov if (kstrtou64(buf, 0, &new) < 0) 248821afaf18SBorislav Petkov return -EINVAL; 248921afaf18SBorislav Petkov 249021afaf18SBorislav Petkov mutex_lock(&mce_sysfs_mutex); 249121afaf18SBorislav Petkov if (mca_cfg.ignore_ce ^ !!new) { 249221afaf18SBorislav Petkov if (new) { 249321afaf18SBorislav Petkov /* disable ce features */ 249421afaf18SBorislav Petkov mce_timer_delete_all(); 249521afaf18SBorislav Petkov on_each_cpu(mce_disable_cmci, NULL, 1); 249621afaf18SBorislav Petkov mca_cfg.ignore_ce = true; 249721afaf18SBorislav Petkov } else { 249821afaf18SBorislav Petkov /* enable ce features */ 249921afaf18SBorislav Petkov mca_cfg.ignore_ce = false; 250021afaf18SBorislav Petkov on_each_cpu(mce_enable_ce, (void *)1, 1); 250121afaf18SBorislav Petkov } 250221afaf18SBorislav Petkov } 250321afaf18SBorislav Petkov mutex_unlock(&mce_sysfs_mutex); 250421afaf18SBorislav Petkov 250521afaf18SBorislav Petkov return size; 250621afaf18SBorislav Petkov } 250721afaf18SBorislav Petkov 250821afaf18SBorislav Petkov static ssize_t set_cmci_disabled(struct device *s, 250921afaf18SBorislav Petkov struct device_attribute *attr, 251021afaf18SBorislav Petkov const char *buf, size_t size) 251121afaf18SBorislav Petkov { 251221afaf18SBorislav Petkov u64 new; 251321afaf18SBorislav Petkov 251421afaf18SBorislav Petkov if (kstrtou64(buf, 0, &new) < 0) 251521afaf18SBorislav Petkov return -EINVAL; 251621afaf18SBorislav Petkov 251721afaf18SBorislav Petkov mutex_lock(&mce_sysfs_mutex); 251821afaf18SBorislav Petkov if (mca_cfg.cmci_disabled ^ !!new) { 251921afaf18SBorislav Petkov if (new) { 252021afaf18SBorislav Petkov /* disable cmci */ 252121afaf18SBorislav Petkov on_each_cpu(mce_disable_cmci, NULL, 1); 252221afaf18SBorislav Petkov mca_cfg.cmci_disabled = true; 252321afaf18SBorislav Petkov } else { 252421afaf18SBorislav Petkov /* enable cmci */ 252521afaf18SBorislav Petkov mca_cfg.cmci_disabled = false; 252621afaf18SBorislav Petkov on_each_cpu(mce_enable_ce, NULL, 1); 252721afaf18SBorislav Petkov } 252821afaf18SBorislav Petkov } 252921afaf18SBorislav Petkov mutex_unlock(&mce_sysfs_mutex); 253021afaf18SBorislav Petkov 253121afaf18SBorislav Petkov return size; 253221afaf18SBorislav Petkov } 253321afaf18SBorislav Petkov 253421afaf18SBorislav Petkov static ssize_t store_int_with_restart(struct device *s, 253521afaf18SBorislav Petkov struct device_attribute *attr, 253621afaf18SBorislav Petkov const char *buf, size_t size) 253721afaf18SBorislav Petkov { 253821afaf18SBorislav Petkov unsigned long old_check_interval = check_interval; 253921afaf18SBorislav Petkov ssize_t ret = device_store_ulong(s, attr, buf, size); 254021afaf18SBorislav Petkov 254121afaf18SBorislav Petkov if (check_interval == old_check_interval) 254221afaf18SBorislav Petkov return ret; 254321afaf18SBorislav Petkov 254421afaf18SBorislav Petkov mutex_lock(&mce_sysfs_mutex); 254521afaf18SBorislav Petkov mce_restart(); 254621afaf18SBorislav Petkov mutex_unlock(&mce_sysfs_mutex); 254721afaf18SBorislav Petkov 254821afaf18SBorislav Petkov return ret; 254921afaf18SBorislav Petkov } 255021afaf18SBorislav Petkov 255121afaf18SBorislav Petkov static DEVICE_INT_ATTR(monarch_timeout, 0644, mca_cfg.monarch_timeout); 255221afaf18SBorislav Petkov static DEVICE_BOOL_ATTR(dont_log_ce, 0644, mca_cfg.dont_log_ce); 255343505646STony Luck static DEVICE_BOOL_ATTR(print_all, 0644, mca_cfg.print_all); 255421afaf18SBorislav Petkov 255521afaf18SBorislav Petkov static struct dev_ext_attribute dev_attr_check_interval = { 255621afaf18SBorislav Petkov __ATTR(check_interval, 0644, device_show_int, store_int_with_restart), 255721afaf18SBorislav Petkov &check_interval 255821afaf18SBorislav Petkov }; 255921afaf18SBorislav Petkov 256021afaf18SBorislav Petkov static struct dev_ext_attribute dev_attr_ignore_ce = { 256121afaf18SBorislav Petkov __ATTR(ignore_ce, 0644, device_show_bool, set_ignore_ce), 256221afaf18SBorislav Petkov &mca_cfg.ignore_ce 256321afaf18SBorislav Petkov }; 256421afaf18SBorislav Petkov 256521afaf18SBorislav Petkov static struct dev_ext_attribute dev_attr_cmci_disabled = { 256621afaf18SBorislav Petkov __ATTR(cmci_disabled, 0644, device_show_bool, set_cmci_disabled), 256721afaf18SBorislav Petkov &mca_cfg.cmci_disabled 256821afaf18SBorislav Petkov }; 256921afaf18SBorislav Petkov 257021afaf18SBorislav Petkov static struct device_attribute *mce_device_attrs[] = { 257121afaf18SBorislav Petkov &dev_attr_check_interval.attr, 257221afaf18SBorislav Petkov #ifdef CONFIG_X86_MCELOG_LEGACY 257321afaf18SBorislav Petkov &dev_attr_trigger, 257421afaf18SBorislav Petkov #endif 257521afaf18SBorislav Petkov &dev_attr_monarch_timeout.attr, 257621afaf18SBorislav Petkov &dev_attr_dont_log_ce.attr, 257743505646STony Luck &dev_attr_print_all.attr, 257821afaf18SBorislav Petkov &dev_attr_ignore_ce.attr, 257921afaf18SBorislav Petkov &dev_attr_cmci_disabled.attr, 258021afaf18SBorislav Petkov NULL 258121afaf18SBorislav Petkov }; 258221afaf18SBorislav Petkov 258321afaf18SBorislav Petkov static cpumask_var_t mce_device_initialized; 258421afaf18SBorislav Petkov 258521afaf18SBorislav Petkov static void mce_device_release(struct device *dev) 258621afaf18SBorislav Petkov { 258721afaf18SBorislav Petkov kfree(dev); 258821afaf18SBorislav Petkov } 258921afaf18SBorislav Petkov 2590b4914508SYazen Ghannam /* Per CPU device init. All of the CPUs still share the same bank device: */ 259121afaf18SBorislav Petkov static int mce_device_create(unsigned int cpu) 259221afaf18SBorislav Petkov { 259321afaf18SBorislav Petkov struct device *dev; 259421afaf18SBorislav Petkov int err; 259521afaf18SBorislav Petkov int i, j; 259621afaf18SBorislav Petkov 259721afaf18SBorislav Petkov if (!mce_available(&boot_cpu_data)) 259821afaf18SBorislav Petkov return -EIO; 259921afaf18SBorislav Petkov 260021afaf18SBorislav Petkov dev = per_cpu(mce_device, cpu); 260121afaf18SBorislav Petkov if (dev) 260221afaf18SBorislav Petkov return 0; 260321afaf18SBorislav Petkov 260421afaf18SBorislav Petkov dev = kzalloc(sizeof(*dev), GFP_KERNEL); 260521afaf18SBorislav Petkov if (!dev) 260621afaf18SBorislav Petkov return -ENOMEM; 260721afaf18SBorislav Petkov dev->id = cpu; 260821afaf18SBorislav Petkov dev->bus = &mce_subsys; 260921afaf18SBorislav Petkov dev->release = &mce_device_release; 261021afaf18SBorislav Petkov 261121afaf18SBorislav Petkov err = device_register(dev); 261221afaf18SBorislav Petkov if (err) { 261321afaf18SBorislav Petkov put_device(dev); 261421afaf18SBorislav Petkov return err; 261521afaf18SBorislav Petkov } 261621afaf18SBorislav Petkov 261721afaf18SBorislav Petkov for (i = 0; mce_device_attrs[i]; i++) { 261821afaf18SBorislav Petkov err = device_create_file(dev, mce_device_attrs[i]); 261921afaf18SBorislav Petkov if (err) 262021afaf18SBorislav Petkov goto error; 262121afaf18SBorislav Petkov } 2622c7d314f3SYazen Ghannam for (j = 0; j < per_cpu(mce_num_banks, cpu); j++) { 2623b4914508SYazen Ghannam err = device_create_file(dev, &mce_bank_devs[j].attr); 262421afaf18SBorislav Petkov if (err) 262521afaf18SBorislav Petkov goto error2; 262621afaf18SBorislav Petkov } 262721afaf18SBorislav Petkov cpumask_set_cpu(cpu, mce_device_initialized); 262821afaf18SBorislav Petkov per_cpu(mce_device, cpu) = dev; 262921afaf18SBorislav Petkov 263021afaf18SBorislav Petkov return 0; 263121afaf18SBorislav Petkov error2: 263221afaf18SBorislav Petkov while (--j >= 0) 2633b4914508SYazen Ghannam device_remove_file(dev, &mce_bank_devs[j].attr); 263421afaf18SBorislav Petkov error: 263521afaf18SBorislav Petkov while (--i >= 0) 263621afaf18SBorislav Petkov device_remove_file(dev, mce_device_attrs[i]); 263721afaf18SBorislav Petkov 263821afaf18SBorislav Petkov device_unregister(dev); 263921afaf18SBorislav Petkov 264021afaf18SBorislav Petkov return err; 264121afaf18SBorislav Petkov } 264221afaf18SBorislav Petkov 264321afaf18SBorislav Petkov static void mce_device_remove(unsigned int cpu) 264421afaf18SBorislav Petkov { 264521afaf18SBorislav Petkov struct device *dev = per_cpu(mce_device, cpu); 264621afaf18SBorislav Petkov int i; 264721afaf18SBorislav Petkov 264821afaf18SBorislav Petkov if (!cpumask_test_cpu(cpu, mce_device_initialized)) 264921afaf18SBorislav Petkov return; 265021afaf18SBorislav Petkov 265121afaf18SBorislav Petkov for (i = 0; mce_device_attrs[i]; i++) 265221afaf18SBorislav Petkov device_remove_file(dev, mce_device_attrs[i]); 265321afaf18SBorislav Petkov 2654c7d314f3SYazen Ghannam for (i = 0; i < per_cpu(mce_num_banks, cpu); i++) 2655b4914508SYazen Ghannam device_remove_file(dev, &mce_bank_devs[i].attr); 265621afaf18SBorislav Petkov 265721afaf18SBorislav Petkov device_unregister(dev); 265821afaf18SBorislav Petkov cpumask_clear_cpu(cpu, mce_device_initialized); 265921afaf18SBorislav Petkov per_cpu(mce_device, cpu) = NULL; 266021afaf18SBorislav Petkov } 266121afaf18SBorislav Petkov 266221afaf18SBorislav Petkov /* Make sure there are no machine checks on offlined CPUs. */ 266321afaf18SBorislav Petkov static void mce_disable_cpu(void) 266421afaf18SBorislav Petkov { 266521afaf18SBorislav Petkov if (!mce_available(raw_cpu_ptr(&cpu_info))) 266621afaf18SBorislav Petkov return; 266721afaf18SBorislav Petkov 266821afaf18SBorislav Petkov if (!cpuhp_tasks_frozen) 266921afaf18SBorislav Petkov cmci_clear(); 267021afaf18SBorislav Petkov 267121afaf18SBorislav Petkov vendor_disable_error_reporting(); 267221afaf18SBorislav Petkov } 267321afaf18SBorislav Petkov 267421afaf18SBorislav Petkov static void mce_reenable_cpu(void) 267521afaf18SBorislav Petkov { 2676b4914508SYazen Ghannam struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 267721afaf18SBorislav Petkov int i; 267821afaf18SBorislav Petkov 267921afaf18SBorislav Petkov if (!mce_available(raw_cpu_ptr(&cpu_info))) 268021afaf18SBorislav Petkov return; 268121afaf18SBorislav Petkov 268221afaf18SBorislav Petkov if (!cpuhp_tasks_frozen) 268321afaf18SBorislav Petkov cmci_reenable(); 2684c7d314f3SYazen Ghannam for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 268521afaf18SBorislav Petkov struct mce_bank *b = &mce_banks[i]; 268621afaf18SBorislav Petkov 268721afaf18SBorislav Petkov if (b->init) 26888121b8f9SBorislav Petkov wrmsrl(mca_msr_reg(i, MCA_CTL), b->ctl); 268921afaf18SBorislav Petkov } 269021afaf18SBorislav Petkov } 269121afaf18SBorislav Petkov 269221afaf18SBorislav Petkov static int mce_cpu_dead(unsigned int cpu) 269321afaf18SBorislav Petkov { 269421afaf18SBorislav Petkov mce_intel_hcpu_update(cpu); 269521afaf18SBorislav Petkov 269621afaf18SBorislav Petkov /* intentionally ignoring frozen here */ 269721afaf18SBorislav Petkov if (!cpuhp_tasks_frozen) 269821afaf18SBorislav Petkov cmci_rediscover(); 269921afaf18SBorislav Petkov return 0; 270021afaf18SBorislav Petkov } 270121afaf18SBorislav Petkov 270221afaf18SBorislav Petkov static int mce_cpu_online(unsigned int cpu) 270321afaf18SBorislav Petkov { 270421afaf18SBorislav Petkov struct timer_list *t = this_cpu_ptr(&mce_timer); 270521afaf18SBorislav Petkov int ret; 270621afaf18SBorislav Petkov 270721afaf18SBorislav Petkov mce_device_create(cpu); 270821afaf18SBorislav Petkov 270921afaf18SBorislav Petkov ret = mce_threshold_create_device(cpu); 271021afaf18SBorislav Petkov if (ret) { 271121afaf18SBorislav Petkov mce_device_remove(cpu); 271221afaf18SBorislav Petkov return ret; 271321afaf18SBorislav Petkov } 271421afaf18SBorislav Petkov mce_reenable_cpu(); 271521afaf18SBorislav Petkov mce_start_timer(t); 271621afaf18SBorislav Petkov return 0; 271721afaf18SBorislav Petkov } 271821afaf18SBorislav Petkov 271921afaf18SBorislav Petkov static int mce_cpu_pre_down(unsigned int cpu) 272021afaf18SBorislav Petkov { 272121afaf18SBorislav Petkov struct timer_list *t = this_cpu_ptr(&mce_timer); 272221afaf18SBorislav Petkov 272321afaf18SBorislav Petkov mce_disable_cpu(); 272421afaf18SBorislav Petkov del_timer_sync(t); 272521afaf18SBorislav Petkov mce_threshold_remove_device(cpu); 272621afaf18SBorislav Petkov mce_device_remove(cpu); 272721afaf18SBorislav Petkov return 0; 272821afaf18SBorislav Petkov } 272921afaf18SBorislav Petkov 273021afaf18SBorislav Petkov static __init void mce_init_banks(void) 273121afaf18SBorislav Petkov { 273221afaf18SBorislav Petkov int i; 273321afaf18SBorislav Petkov 2734b4914508SYazen Ghannam for (i = 0; i < MAX_NR_BANKS; i++) { 2735b4914508SYazen Ghannam struct mce_bank_dev *b = &mce_bank_devs[i]; 273621afaf18SBorislav Petkov struct device_attribute *a = &b->attr; 273721afaf18SBorislav Petkov 2738b4914508SYazen Ghannam b->bank = i; 2739b4914508SYazen Ghannam 274021afaf18SBorislav Petkov sysfs_attr_init(&a->attr); 274121afaf18SBorislav Petkov a->attr.name = b->attrname; 274221afaf18SBorislav Petkov snprintf(b->attrname, ATTR_LEN, "bank%d", i); 274321afaf18SBorislav Petkov 274421afaf18SBorislav Petkov a->attr.mode = 0644; 274521afaf18SBorislav Petkov a->show = show_bank; 274621afaf18SBorislav Petkov a->store = set_bank; 274721afaf18SBorislav Petkov } 274821afaf18SBorislav Petkov } 274921afaf18SBorislav Petkov 27506e7a41c6SThomas Gleixner /* 27516e7a41c6SThomas Gleixner * When running on XEN, this initcall is ordered against the XEN mcelog 27526e7a41c6SThomas Gleixner * initcall: 27536e7a41c6SThomas Gleixner * 27546e7a41c6SThomas Gleixner * device_initcall(xen_late_init_mcelog); 27556e7a41c6SThomas Gleixner * device_initcall_sync(mcheck_init_device); 27566e7a41c6SThomas Gleixner */ 275721afaf18SBorislav Petkov static __init int mcheck_init_device(void) 275821afaf18SBorislav Petkov { 275921afaf18SBorislav Petkov int err; 276021afaf18SBorislav Petkov 276121afaf18SBorislav Petkov /* 276221afaf18SBorislav Petkov * Check if we have a spare virtual bit. This will only become 276321afaf18SBorislav Petkov * a problem if/when we move beyond 5-level page tables. 276421afaf18SBorislav Petkov */ 276521afaf18SBorislav Petkov MAYBE_BUILD_BUG_ON(__VIRTUAL_MASK_SHIFT >= 63); 276621afaf18SBorislav Petkov 276721afaf18SBorislav Petkov if (!mce_available(&boot_cpu_data)) { 276821afaf18SBorislav Petkov err = -EIO; 276921afaf18SBorislav Petkov goto err_out; 277021afaf18SBorislav Petkov } 277121afaf18SBorislav Petkov 277221afaf18SBorislav Petkov if (!zalloc_cpumask_var(&mce_device_initialized, GFP_KERNEL)) { 277321afaf18SBorislav Petkov err = -ENOMEM; 277421afaf18SBorislav Petkov goto err_out; 277521afaf18SBorislav Petkov } 277621afaf18SBorislav Petkov 277721afaf18SBorislav Petkov mce_init_banks(); 277821afaf18SBorislav Petkov 277921afaf18SBorislav Petkov err = subsys_system_register(&mce_subsys, NULL); 278021afaf18SBorislav Petkov if (err) 278121afaf18SBorislav Petkov goto err_out_mem; 278221afaf18SBorislav Petkov 278321afaf18SBorislav Petkov err = cpuhp_setup_state(CPUHP_X86_MCE_DEAD, "x86/mce:dead", NULL, 278421afaf18SBorislav Petkov mce_cpu_dead); 278521afaf18SBorislav Petkov if (err) 278621afaf18SBorislav Petkov goto err_out_mem; 278721afaf18SBorislav Petkov 27886e7a41c6SThomas Gleixner /* 27896e7a41c6SThomas Gleixner * Invokes mce_cpu_online() on all CPUs which are online when 27906e7a41c6SThomas Gleixner * the state is installed. 27916e7a41c6SThomas Gleixner */ 279221afaf18SBorislav Petkov err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/mce:online", 279321afaf18SBorislav Petkov mce_cpu_online, mce_cpu_pre_down); 279421afaf18SBorislav Petkov if (err < 0) 279521afaf18SBorislav Petkov goto err_out_online; 279621afaf18SBorislav Petkov 279721afaf18SBorislav Petkov register_syscore_ops(&mce_syscore_ops); 279821afaf18SBorislav Petkov 279921afaf18SBorislav Petkov return 0; 280021afaf18SBorislav Petkov 280121afaf18SBorislav Petkov err_out_online: 280221afaf18SBorislav Petkov cpuhp_remove_state(CPUHP_X86_MCE_DEAD); 280321afaf18SBorislav Petkov 280421afaf18SBorislav Petkov err_out_mem: 280521afaf18SBorislav Petkov free_cpumask_var(mce_device_initialized); 280621afaf18SBorislav Petkov 280721afaf18SBorislav Petkov err_out: 280821afaf18SBorislav Petkov pr_err("Unable to init MCE device (rc: %d)\n", err); 280921afaf18SBorislav Petkov 281021afaf18SBorislav Petkov return err; 281121afaf18SBorislav Petkov } 281221afaf18SBorislav Petkov device_initcall_sync(mcheck_init_device); 281321afaf18SBorislav Petkov 281421afaf18SBorislav Petkov /* 281521afaf18SBorislav Petkov * Old style boot options parsing. Only for compatibility. 281621afaf18SBorislav Petkov */ 281721afaf18SBorislav Petkov static int __init mcheck_disable(char *str) 281821afaf18SBorislav Petkov { 281921afaf18SBorislav Petkov mca_cfg.disabled = 1; 282021afaf18SBorislav Petkov return 1; 282121afaf18SBorislav Petkov } 282221afaf18SBorislav Petkov __setup("nomce", mcheck_disable); 282321afaf18SBorislav Petkov 282421afaf18SBorislav Petkov #ifdef CONFIG_DEBUG_FS 282521afaf18SBorislav Petkov struct dentry *mce_get_debugfs_dir(void) 282621afaf18SBorislav Petkov { 282721afaf18SBorislav Petkov static struct dentry *dmce; 282821afaf18SBorislav Petkov 282921afaf18SBorislav Petkov if (!dmce) 283021afaf18SBorislav Petkov dmce = debugfs_create_dir("mce", NULL); 283121afaf18SBorislav Petkov 283221afaf18SBorislav Petkov return dmce; 283321afaf18SBorislav Petkov } 283421afaf18SBorislav Petkov 283521afaf18SBorislav Petkov static void mce_reset(void) 283621afaf18SBorislav Petkov { 283721afaf18SBorislav Petkov atomic_set(&mce_fake_panicked, 0); 283821afaf18SBorislav Petkov atomic_set(&mce_executing, 0); 283921afaf18SBorislav Petkov atomic_set(&mce_callin, 0); 284021afaf18SBorislav Petkov atomic_set(&global_nwo, 0); 28417bb39313SPaul E. McKenney cpumask_setall(&mce_missing_cpus); 284221afaf18SBorislav Petkov } 284321afaf18SBorislav Petkov 284421afaf18SBorislav Petkov static int fake_panic_get(void *data, u64 *val) 284521afaf18SBorislav Petkov { 284621afaf18SBorislav Petkov *val = fake_panic; 284721afaf18SBorislav Petkov return 0; 284821afaf18SBorislav Petkov } 284921afaf18SBorislav Petkov 285021afaf18SBorislav Petkov static int fake_panic_set(void *data, u64 val) 285121afaf18SBorislav Petkov { 285221afaf18SBorislav Petkov mce_reset(); 285321afaf18SBorislav Petkov fake_panic = val; 285421afaf18SBorislav Petkov return 0; 285521afaf18SBorislav Petkov } 285621afaf18SBorislav Petkov 285728156d76SYueHaibing DEFINE_DEBUGFS_ATTRIBUTE(fake_panic_fops, fake_panic_get, fake_panic_set, 285828156d76SYueHaibing "%llu\n"); 285921afaf18SBorislav Petkov 28606e4f929eSGreg Kroah-Hartman static void __init mcheck_debugfs_init(void) 286121afaf18SBorislav Petkov { 28626e4f929eSGreg Kroah-Hartman struct dentry *dmce; 286321afaf18SBorislav Petkov 286421afaf18SBorislav Petkov dmce = mce_get_debugfs_dir(); 28656e4f929eSGreg Kroah-Hartman debugfs_create_file_unsafe("fake_panic", 0444, dmce, NULL, 28666e4f929eSGreg Kroah-Hartman &fake_panic_fops); 286721afaf18SBorislav Petkov } 286821afaf18SBorislav Petkov #else 28696e4f929eSGreg Kroah-Hartman static void __init mcheck_debugfs_init(void) { } 287021afaf18SBorislav Petkov #endif 287121afaf18SBorislav Petkov 287221afaf18SBorislav Petkov static int __init mcheck_late_init(void) 287321afaf18SBorislav Petkov { 287421afaf18SBorislav Petkov if (mca_cfg.recovery) 2875ec6347bbSDan Williams enable_copy_mc_fragile(); 287621afaf18SBorislav Petkov 287721afaf18SBorislav Petkov mcheck_debugfs_init(); 287821afaf18SBorislav Petkov 287921afaf18SBorislav Petkov /* 288021afaf18SBorislav Petkov * Flush out everything that has been logged during early boot, now that 288121afaf18SBorislav Petkov * everything has been initialized (workqueues, decoders, ...). 288221afaf18SBorislav Petkov */ 288321afaf18SBorislav Petkov mce_schedule_work(); 288421afaf18SBorislav Petkov 288521afaf18SBorislav Petkov return 0; 288621afaf18SBorislav Petkov } 288721afaf18SBorislav Petkov late_initcall(mcheck_late_init); 2888