1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 221afaf18SBorislav Petkov /* 321afaf18SBorislav Petkov * Machine check handler. 421afaf18SBorislav Petkov * 521afaf18SBorislav Petkov * K8 parts Copyright 2002,2003 Andi Kleen, SuSE Labs. 621afaf18SBorislav Petkov * Rest from unknown author(s). 721afaf18SBorislav Petkov * 2004 Andi Kleen. Rewrote most of it. 821afaf18SBorislav Petkov * Copyright 2008 Intel Corporation 921afaf18SBorislav Petkov * Author: Andi Kleen 1021afaf18SBorislav Petkov */ 1121afaf18SBorislav Petkov 1221afaf18SBorislav Petkov #include <linux/thread_info.h> 1321afaf18SBorislav Petkov #include <linux/capability.h> 1421afaf18SBorislav Petkov #include <linux/miscdevice.h> 1521afaf18SBorislav Petkov #include <linux/ratelimit.h> 1621afaf18SBorislav Petkov #include <linux/rcupdate.h> 1721afaf18SBorislav Petkov #include <linux/kobject.h> 1821afaf18SBorislav Petkov #include <linux/uaccess.h> 1921afaf18SBorislav Petkov #include <linux/kdebug.h> 2021afaf18SBorislav Petkov #include <linux/kernel.h> 2121afaf18SBorislav Petkov #include <linux/percpu.h> 2221afaf18SBorislav Petkov #include <linux/string.h> 2321afaf18SBorislav Petkov #include <linux/device.h> 2421afaf18SBorislav Petkov #include <linux/syscore_ops.h> 2521afaf18SBorislav Petkov #include <linux/delay.h> 2621afaf18SBorislav Petkov #include <linux/ctype.h> 2721afaf18SBorislav Petkov #include <linux/sched.h> 2821afaf18SBorislav Petkov #include <linux/sysfs.h> 2921afaf18SBorislav Petkov #include <linux/types.h> 3021afaf18SBorislav Petkov #include <linux/slab.h> 3121afaf18SBorislav Petkov #include <linux/init.h> 3221afaf18SBorislav Petkov #include <linux/kmod.h> 3321afaf18SBorislav Petkov #include <linux/poll.h> 3421afaf18SBorislav Petkov #include <linux/nmi.h> 3521afaf18SBorislav Petkov #include <linux/cpu.h> 3621afaf18SBorislav Petkov #include <linux/ras.h> 3721afaf18SBorislav Petkov #include <linux/smp.h> 3821afaf18SBorislav Petkov #include <linux/fs.h> 3921afaf18SBorislav Petkov #include <linux/mm.h> 4021afaf18SBorislav Petkov #include <linux/debugfs.h> 4121afaf18SBorislav Petkov #include <linux/irq_work.h> 4221afaf18SBorislav Petkov #include <linux/export.h> 4321afaf18SBorislav Petkov #include <linux/set_memory.h> 449998a983SRicardo Neri #include <linux/sync_core.h> 455567d11cSPeter Zijlstra #include <linux/task_work.h> 460d00449cSPeter Zijlstra #include <linux/hardirq.h> 4721afaf18SBorislav Petkov 4821afaf18SBorislav Petkov #include <asm/intel-family.h> 4921afaf18SBorislav Petkov #include <asm/processor.h> 5021afaf18SBorislav Petkov #include <asm/traps.h> 5121afaf18SBorislav Petkov #include <asm/tlbflush.h> 5221afaf18SBorislav Petkov #include <asm/mce.h> 5321afaf18SBorislav Petkov #include <asm/msr.h> 5421afaf18SBorislav Petkov #include <asm/reboot.h> 5521afaf18SBorislav Petkov 5621afaf18SBorislav Petkov #include "internal.h" 5721afaf18SBorislav Petkov 5821afaf18SBorislav Petkov /* sysfs synchronization */ 5921afaf18SBorislav Petkov static DEFINE_MUTEX(mce_sysfs_mutex); 6021afaf18SBorislav Petkov 6121afaf18SBorislav Petkov #define CREATE_TRACE_POINTS 6221afaf18SBorislav Petkov #include <trace/events/mce.h> 6321afaf18SBorislav Petkov 6421afaf18SBorislav Petkov #define SPINUNIT 100 /* 100ns */ 6521afaf18SBorislav Petkov 6621afaf18SBorislav Petkov DEFINE_PER_CPU(unsigned, mce_exception_count); 6721afaf18SBorislav Petkov 68c7d314f3SYazen Ghannam DEFINE_PER_CPU_READ_MOSTLY(unsigned int, mce_num_banks); 69c7d314f3SYazen Ghannam 7095fdce6bSYazen Ghannam struct mce_bank { 7195fdce6bSYazen Ghannam u64 ctl; /* subevents to enable */ 7295fdce6bSYazen Ghannam bool init; /* initialise bank? */ 73b4914508SYazen Ghannam }; 74b4914508SYazen Ghannam static DEFINE_PER_CPU_READ_MOSTLY(struct mce_bank[MAX_NR_BANKS], mce_banks_array); 75b4914508SYazen Ghannam 76b4914508SYazen Ghannam #define ATTR_LEN 16 77b4914508SYazen Ghannam /* One object for each MCE bank, shared by all CPUs */ 78b4914508SYazen Ghannam struct mce_bank_dev { 7995fdce6bSYazen Ghannam struct device_attribute attr; /* device attribute */ 8095fdce6bSYazen Ghannam char attrname[ATTR_LEN]; /* attribute name */ 81b4914508SYazen Ghannam u8 bank; /* bank number */ 8295fdce6bSYazen Ghannam }; 83b4914508SYazen Ghannam static struct mce_bank_dev mce_bank_devs[MAX_NR_BANKS]; 8495fdce6bSYazen Ghannam 8521afaf18SBorislav Petkov struct mce_vendor_flags mce_flags __read_mostly; 8621afaf18SBorislav Petkov 8721afaf18SBorislav Petkov struct mca_config mca_cfg __read_mostly = { 8821afaf18SBorislav Petkov .bootlog = -1, 8921afaf18SBorislav Petkov /* 9021afaf18SBorislav Petkov * Tolerant levels: 9121afaf18SBorislav Petkov * 0: always panic on uncorrected errors, log corrected errors 9221afaf18SBorislav Petkov * 1: panic or SIGBUS on uncorrected errors, log corrected errors 9321afaf18SBorislav Petkov * 2: SIGBUS or log uncorrected errors (if possible), log corr. errors 9421afaf18SBorislav Petkov * 3: never panic or SIGBUS, log all errors (for testing only) 9521afaf18SBorislav Petkov */ 9621afaf18SBorislav Petkov .tolerant = 1, 9721afaf18SBorislav Petkov .monarch_timeout = -1 9821afaf18SBorislav Petkov }; 9921afaf18SBorislav Petkov 10021afaf18SBorislav Petkov static DEFINE_PER_CPU(struct mce, mces_seen); 10121afaf18SBorislav Petkov static unsigned long mce_need_notify; 10221afaf18SBorislav Petkov static int cpu_missing; 10321afaf18SBorislav Petkov 10421afaf18SBorislav Petkov /* 10521afaf18SBorislav Petkov * MCA banks polled by the period polling timer for corrected events. 10621afaf18SBorislav Petkov * With Intel CMCI, this only has MCA banks which do not support CMCI (if any). 10721afaf18SBorislav Petkov */ 10821afaf18SBorislav Petkov DEFINE_PER_CPU(mce_banks_t, mce_poll_banks) = { 10921afaf18SBorislav Petkov [0 ... BITS_TO_LONGS(MAX_NR_BANKS)-1] = ~0UL 11021afaf18SBorislav Petkov }; 11121afaf18SBorislav Petkov 11221afaf18SBorislav Petkov /* 11321afaf18SBorislav Petkov * MCA banks controlled through firmware first for corrected errors. 11421afaf18SBorislav Petkov * This is a global list of banks for which we won't enable CMCI and we 11521afaf18SBorislav Petkov * won't poll. Firmware controls these banks and is responsible for 11621afaf18SBorislav Petkov * reporting corrected errors through GHES. Uncorrected/recoverable 11721afaf18SBorislav Petkov * errors are still notified through a machine check. 11821afaf18SBorislav Petkov */ 11921afaf18SBorislav Petkov mce_banks_t mce_banks_ce_disabled; 12021afaf18SBorislav Petkov 12121afaf18SBorislav Petkov static struct work_struct mce_work; 12221afaf18SBorislav Petkov static struct irq_work mce_irq_work; 12321afaf18SBorislav Petkov 12421afaf18SBorislav Petkov static void (*quirk_no_way_out)(int bank, struct mce *m, struct pt_regs *regs); 12521afaf18SBorislav Petkov 12621afaf18SBorislav Petkov /* 12721afaf18SBorislav Petkov * CPU/chipset specific EDAC code can register a notifier call here to print 12821afaf18SBorislav Petkov * MCE errors in a human-readable form. 12921afaf18SBorislav Petkov */ 13021afaf18SBorislav Petkov BLOCKING_NOTIFIER_HEAD(x86_mce_decoder_chain); 13121afaf18SBorislav Petkov 13221afaf18SBorislav Petkov /* Do initial initialization of a struct mce */ 133865d3a9aSThomas Gleixner noinstr void mce_setup(struct mce *m) 13421afaf18SBorislav Petkov { 13521afaf18SBorislav Petkov memset(m, 0, sizeof(struct mce)); 13621afaf18SBorislav Petkov m->cpu = m->extcpu = smp_processor_id(); 13721afaf18SBorislav Petkov /* need the internal __ version to avoid deadlocks */ 13821afaf18SBorislav Petkov m->time = __ktime_get_real_seconds(); 13921afaf18SBorislav Petkov m->cpuvendor = boot_cpu_data.x86_vendor; 14021afaf18SBorislav Petkov m->cpuid = cpuid_eax(1); 14121afaf18SBorislav Petkov m->socketid = cpu_data(m->extcpu).phys_proc_id; 14221afaf18SBorislav Petkov m->apicid = cpu_data(m->extcpu).initial_apicid; 143865d3a9aSThomas Gleixner m->mcgcap = __rdmsr(MSR_IA32_MCG_CAP); 14421afaf18SBorislav Petkov 14521afaf18SBorislav Petkov if (this_cpu_has(X86_FEATURE_INTEL_PPIN)) 146865d3a9aSThomas Gleixner m->ppin = __rdmsr(MSR_PPIN); 147077168e2SWei Huang else if (this_cpu_has(X86_FEATURE_AMD_PPIN)) 148865d3a9aSThomas Gleixner m->ppin = __rdmsr(MSR_AMD_PPIN); 14921afaf18SBorislav Petkov 15021afaf18SBorislav Petkov m->microcode = boot_cpu_data.microcode; 15121afaf18SBorislav Petkov } 15221afaf18SBorislav Petkov 15321afaf18SBorislav Petkov DEFINE_PER_CPU(struct mce, injectm); 15421afaf18SBorislav Petkov EXPORT_PER_CPU_SYMBOL_GPL(injectm); 15521afaf18SBorislav Petkov 15621afaf18SBorislav Petkov void mce_log(struct mce *m) 15721afaf18SBorislav Petkov { 15821afaf18SBorislav Petkov if (!mce_gen_pool_add(m)) 15921afaf18SBorislav Petkov irq_work_queue(&mce_irq_work); 16021afaf18SBorislav Petkov } 16181736abdSJan H. Schönherr EXPORT_SYMBOL_GPL(mce_log); 16221afaf18SBorislav Petkov 16321afaf18SBorislav Petkov void mce_register_decode_chain(struct notifier_block *nb) 16421afaf18SBorislav Petkov { 16515af3659SZhen Lei if (WARN_ON(nb->priority < MCE_PRIO_LOWEST || 16615af3659SZhen Lei nb->priority > MCE_PRIO_HIGHEST)) 16721afaf18SBorislav Petkov return; 16821afaf18SBorislav Petkov 16921afaf18SBorislav Petkov blocking_notifier_chain_register(&x86_mce_decoder_chain, nb); 17021afaf18SBorislav Petkov } 17121afaf18SBorislav Petkov EXPORT_SYMBOL_GPL(mce_register_decode_chain); 17221afaf18SBorislav Petkov 17321afaf18SBorislav Petkov void mce_unregister_decode_chain(struct notifier_block *nb) 17421afaf18SBorislav Petkov { 17521afaf18SBorislav Petkov blocking_notifier_chain_unregister(&x86_mce_decoder_chain, nb); 17621afaf18SBorislav Petkov } 17721afaf18SBorislav Petkov EXPORT_SYMBOL_GPL(mce_unregister_decode_chain); 17821afaf18SBorislav Petkov 17921afaf18SBorislav Petkov static inline u32 ctl_reg(int bank) 18021afaf18SBorislav Petkov { 18121afaf18SBorislav Petkov return MSR_IA32_MCx_CTL(bank); 18221afaf18SBorislav Petkov } 18321afaf18SBorislav Petkov 18421afaf18SBorislav Petkov static inline u32 status_reg(int bank) 18521afaf18SBorislav Petkov { 18621afaf18SBorislav Petkov return MSR_IA32_MCx_STATUS(bank); 18721afaf18SBorislav Petkov } 18821afaf18SBorislav Petkov 18921afaf18SBorislav Petkov static inline u32 addr_reg(int bank) 19021afaf18SBorislav Petkov { 19121afaf18SBorislav Petkov return MSR_IA32_MCx_ADDR(bank); 19221afaf18SBorislav Petkov } 19321afaf18SBorislav Petkov 19421afaf18SBorislav Petkov static inline u32 misc_reg(int bank) 19521afaf18SBorislav Petkov { 19621afaf18SBorislav Petkov return MSR_IA32_MCx_MISC(bank); 19721afaf18SBorislav Petkov } 19821afaf18SBorislav Petkov 19921afaf18SBorislav Petkov static inline u32 smca_ctl_reg(int bank) 20021afaf18SBorislav Petkov { 20121afaf18SBorislav Petkov return MSR_AMD64_SMCA_MCx_CTL(bank); 20221afaf18SBorislav Petkov } 20321afaf18SBorislav Petkov 20421afaf18SBorislav Petkov static inline u32 smca_status_reg(int bank) 20521afaf18SBorislav Petkov { 20621afaf18SBorislav Petkov return MSR_AMD64_SMCA_MCx_STATUS(bank); 20721afaf18SBorislav Petkov } 20821afaf18SBorislav Petkov 20921afaf18SBorislav Petkov static inline u32 smca_addr_reg(int bank) 21021afaf18SBorislav Petkov { 21121afaf18SBorislav Petkov return MSR_AMD64_SMCA_MCx_ADDR(bank); 21221afaf18SBorislav Petkov } 21321afaf18SBorislav Petkov 21421afaf18SBorislav Petkov static inline u32 smca_misc_reg(int bank) 21521afaf18SBorislav Petkov { 21621afaf18SBorislav Petkov return MSR_AMD64_SMCA_MCx_MISC(bank); 21721afaf18SBorislav Petkov } 21821afaf18SBorislav Petkov 21921afaf18SBorislav Petkov struct mca_msr_regs msr_ops = { 22021afaf18SBorislav Petkov .ctl = ctl_reg, 22121afaf18SBorislav Petkov .status = status_reg, 22221afaf18SBorislav Petkov .addr = addr_reg, 22321afaf18SBorislav Petkov .misc = misc_reg 22421afaf18SBorislav Petkov }; 22521afaf18SBorislav Petkov 22621afaf18SBorislav Petkov static void __print_mce(struct mce *m) 22721afaf18SBorislav Petkov { 22821afaf18SBorislav Petkov pr_emerg(HW_ERR "CPU %d: Machine Check%s: %Lx Bank %d: %016Lx\n", 22921afaf18SBorislav Petkov m->extcpu, 23021afaf18SBorislav Petkov (m->mcgstatus & MCG_STATUS_MCIP ? " Exception" : ""), 23121afaf18SBorislav Petkov m->mcgstatus, m->bank, m->status); 23221afaf18SBorislav Petkov 23321afaf18SBorislav Petkov if (m->ip) { 23421afaf18SBorislav Petkov pr_emerg(HW_ERR "RIP%s %02x:<%016Lx> ", 23521afaf18SBorislav Petkov !(m->mcgstatus & MCG_STATUS_EIPV) ? " !INEXACT!" : "", 23621afaf18SBorislav Petkov m->cs, m->ip); 23721afaf18SBorislav Petkov 23821afaf18SBorislav Petkov if (m->cs == __KERNEL_CS) 23921afaf18SBorislav Petkov pr_cont("{%pS}", (void *)(unsigned long)m->ip); 24021afaf18SBorislav Petkov pr_cont("\n"); 24121afaf18SBorislav Petkov } 24221afaf18SBorislav Petkov 24321afaf18SBorislav Petkov pr_emerg(HW_ERR "TSC %llx ", m->tsc); 24421afaf18SBorislav Petkov if (m->addr) 24521afaf18SBorislav Petkov pr_cont("ADDR %llx ", m->addr); 24621afaf18SBorislav Petkov if (m->misc) 24721afaf18SBorislav Petkov pr_cont("MISC %llx ", m->misc); 248bb2de0adSSmita Koralahalli if (m->ppin) 249bb2de0adSSmita Koralahalli pr_cont("PPIN %llx ", m->ppin); 25021afaf18SBorislav Petkov 25121afaf18SBorislav Petkov if (mce_flags.smca) { 25221afaf18SBorislav Petkov if (m->synd) 25321afaf18SBorislav Petkov pr_cont("SYND %llx ", m->synd); 25421afaf18SBorislav Petkov if (m->ipid) 25521afaf18SBorislav Petkov pr_cont("IPID %llx ", m->ipid); 25621afaf18SBorislav Petkov } 25721afaf18SBorislav Petkov 25821afaf18SBorislav Petkov pr_cont("\n"); 259925946cfSTony Luck 26021afaf18SBorislav Petkov /* 26121afaf18SBorislav Petkov * Note this output is parsed by external tools and old fields 26221afaf18SBorislav Petkov * should not be changed. 26321afaf18SBorislav Petkov */ 26421afaf18SBorislav Petkov pr_emerg(HW_ERR "PROCESSOR %u:%x TIME %llu SOCKET %u APIC %x microcode %x\n", 26521afaf18SBorislav Petkov m->cpuvendor, m->cpuid, m->time, m->socketid, m->apicid, 26621afaf18SBorislav Petkov m->microcode); 26721afaf18SBorislav Petkov } 26821afaf18SBorislav Petkov 26921afaf18SBorislav Petkov static void print_mce(struct mce *m) 27021afaf18SBorislav Petkov { 27121afaf18SBorislav Petkov __print_mce(m); 27221afaf18SBorislav Petkov 27321afaf18SBorislav Petkov if (m->cpuvendor != X86_VENDOR_AMD && m->cpuvendor != X86_VENDOR_HYGON) 27421afaf18SBorislav Petkov pr_emerg_ratelimited(HW_ERR "Run the above through 'mcelog --ascii'\n"); 27521afaf18SBorislav Petkov } 27621afaf18SBorislav Petkov 27721afaf18SBorislav Petkov #define PANIC_TIMEOUT 5 /* 5 seconds */ 27821afaf18SBorislav Petkov 27921afaf18SBorislav Petkov static atomic_t mce_panicked; 28021afaf18SBorislav Petkov 28121afaf18SBorislav Petkov static int fake_panic; 28221afaf18SBorislav Petkov static atomic_t mce_fake_panicked; 28321afaf18SBorislav Petkov 28421afaf18SBorislav Petkov /* Panic in progress. Enable interrupts and wait for final IPI */ 28521afaf18SBorislav Petkov static void wait_for_panic(void) 28621afaf18SBorislav Petkov { 28721afaf18SBorislav Petkov long timeout = PANIC_TIMEOUT*USEC_PER_SEC; 28821afaf18SBorislav Petkov 28921afaf18SBorislav Petkov preempt_disable(); 29021afaf18SBorislav Petkov local_irq_enable(); 29121afaf18SBorislav Petkov while (timeout-- > 0) 29221afaf18SBorislav Petkov udelay(1); 29321afaf18SBorislav Petkov if (panic_timeout == 0) 29421afaf18SBorislav Petkov panic_timeout = mca_cfg.panic_timeout; 29521afaf18SBorislav Petkov panic("Panicing machine check CPU died"); 29621afaf18SBorislav Petkov } 29721afaf18SBorislav Petkov 29821afaf18SBorislav Petkov static void mce_panic(const char *msg, struct mce *final, char *exp) 29921afaf18SBorislav Petkov { 30021afaf18SBorislav Petkov int apei_err = 0; 30121afaf18SBorislav Petkov struct llist_node *pending; 30221afaf18SBorislav Petkov struct mce_evt_llist *l; 30321afaf18SBorislav Petkov 30421afaf18SBorislav Petkov if (!fake_panic) { 30521afaf18SBorislav Petkov /* 30621afaf18SBorislav Petkov * Make sure only one CPU runs in machine check panic 30721afaf18SBorislav Petkov */ 30821afaf18SBorislav Petkov if (atomic_inc_return(&mce_panicked) > 1) 30921afaf18SBorislav Petkov wait_for_panic(); 31021afaf18SBorislav Petkov barrier(); 31121afaf18SBorislav Petkov 31221afaf18SBorislav Petkov bust_spinlocks(1); 31321afaf18SBorislav Petkov console_verbose(); 31421afaf18SBorislav Petkov } else { 31521afaf18SBorislav Petkov /* Don't log too much for fake panic */ 31621afaf18SBorislav Petkov if (atomic_inc_return(&mce_fake_panicked) > 1) 31721afaf18SBorislav Petkov return; 31821afaf18SBorislav Petkov } 31921afaf18SBorislav Petkov pending = mce_gen_pool_prepare_records(); 32021afaf18SBorislav Petkov /* First print corrected ones that are still unlogged */ 32121afaf18SBorislav Petkov llist_for_each_entry(l, pending, llnode) { 32221afaf18SBorislav Petkov struct mce *m = &l->mce; 32321afaf18SBorislav Petkov if (!(m->status & MCI_STATUS_UC)) { 32421afaf18SBorislav Petkov print_mce(m); 32521afaf18SBorislav Petkov if (!apei_err) 32621afaf18SBorislav Petkov apei_err = apei_write_mce(m); 32721afaf18SBorislav Petkov } 32821afaf18SBorislav Petkov } 32921afaf18SBorislav Petkov /* Now print uncorrected but with the final one last */ 33021afaf18SBorislav Petkov llist_for_each_entry(l, pending, llnode) { 33121afaf18SBorislav Petkov struct mce *m = &l->mce; 33221afaf18SBorislav Petkov if (!(m->status & MCI_STATUS_UC)) 33321afaf18SBorislav Petkov continue; 33421afaf18SBorislav Petkov if (!final || mce_cmp(m, final)) { 33521afaf18SBorislav Petkov print_mce(m); 33621afaf18SBorislav Petkov if (!apei_err) 33721afaf18SBorislav Petkov apei_err = apei_write_mce(m); 33821afaf18SBorislav Petkov } 33921afaf18SBorislav Petkov } 34021afaf18SBorislav Petkov if (final) { 34121afaf18SBorislav Petkov print_mce(final); 34221afaf18SBorislav Petkov if (!apei_err) 34321afaf18SBorislav Petkov apei_err = apei_write_mce(final); 34421afaf18SBorislav Petkov } 34521afaf18SBorislav Petkov if (cpu_missing) 34621afaf18SBorislav Petkov pr_emerg(HW_ERR "Some CPUs didn't answer in synchronization\n"); 34721afaf18SBorislav Petkov if (exp) 34821afaf18SBorislav Petkov pr_emerg(HW_ERR "Machine check: %s\n", exp); 34921afaf18SBorislav Petkov if (!fake_panic) { 35021afaf18SBorislav Petkov if (panic_timeout == 0) 35121afaf18SBorislav Petkov panic_timeout = mca_cfg.panic_timeout; 35221afaf18SBorislav Petkov panic(msg); 35321afaf18SBorislav Petkov } else 35421afaf18SBorislav Petkov pr_emerg(HW_ERR "Fake kernel panic: %s\n", msg); 35521afaf18SBorislav Petkov } 35621afaf18SBorislav Petkov 35721afaf18SBorislav Petkov /* Support code for software error injection */ 35821afaf18SBorislav Petkov 35921afaf18SBorislav Petkov static int msr_to_offset(u32 msr) 36021afaf18SBorislav Petkov { 36121afaf18SBorislav Petkov unsigned bank = __this_cpu_read(injectm.bank); 36221afaf18SBorislav Petkov 36321afaf18SBorislav Petkov if (msr == mca_cfg.rip_msr) 36421afaf18SBorislav Petkov return offsetof(struct mce, ip); 36521afaf18SBorislav Petkov if (msr == msr_ops.status(bank)) 36621afaf18SBorislav Petkov return offsetof(struct mce, status); 36721afaf18SBorislav Petkov if (msr == msr_ops.addr(bank)) 36821afaf18SBorislav Petkov return offsetof(struct mce, addr); 36921afaf18SBorislav Petkov if (msr == msr_ops.misc(bank)) 37021afaf18SBorislav Petkov return offsetof(struct mce, misc); 37121afaf18SBorislav Petkov if (msr == MSR_IA32_MCG_STATUS) 37221afaf18SBorislav Petkov return offsetof(struct mce, mcgstatus); 37321afaf18SBorislav Petkov return -1; 37421afaf18SBorislav Petkov } 37521afaf18SBorislav Petkov 376e2def7d4SBorislav Petkov __visible bool ex_handler_rdmsr_fault(const struct exception_table_entry *fixup, 377e2def7d4SBorislav Petkov struct pt_regs *regs, int trapnr, 378e2def7d4SBorislav Petkov unsigned long error_code, 379e2def7d4SBorislav Petkov unsigned long fault_addr) 380e2def7d4SBorislav Petkov { 381e2def7d4SBorislav Petkov pr_emerg("MSR access error: RDMSR from 0x%x at rIP: 0x%lx (%pS)\n", 382e2def7d4SBorislav Petkov (unsigned int)regs->cx, regs->ip, (void *)regs->ip); 383e2def7d4SBorislav Petkov 384e2def7d4SBorislav Petkov show_stack_regs(regs); 385e2def7d4SBorislav Petkov 386e2def7d4SBorislav Petkov panic("MCA architectural violation!\n"); 387e2def7d4SBorislav Petkov 388e2def7d4SBorislav Petkov while (true) 389e2def7d4SBorislav Petkov cpu_relax(); 390e2def7d4SBorislav Petkov 391e2def7d4SBorislav Petkov return true; 392e2def7d4SBorislav Petkov } 393e2def7d4SBorislav Petkov 39421afaf18SBorislav Petkov /* MSR access wrappers used for error injection */ 395e1007770SBorislav Petkov static noinstr u64 mce_rdmsrl(u32 msr) 39621afaf18SBorislav Petkov { 397e2def7d4SBorislav Petkov DECLARE_ARGS(val, low, high); 39821afaf18SBorislav Petkov 39921afaf18SBorislav Petkov if (__this_cpu_read(injectm.finished)) { 400e1007770SBorislav Petkov int offset; 401e1007770SBorislav Petkov u64 ret; 40221afaf18SBorislav Petkov 403e1007770SBorislav Petkov instrumentation_begin(); 404e1007770SBorislav Petkov 405e1007770SBorislav Petkov offset = msr_to_offset(msr); 40621afaf18SBorislav Petkov if (offset < 0) 407e1007770SBorislav Petkov ret = 0; 408e1007770SBorislav Petkov else 409e1007770SBorislav Petkov ret = *(u64 *)((char *)this_cpu_ptr(&injectm) + offset); 410e1007770SBorislav Petkov 411e1007770SBorislav Petkov instrumentation_end(); 412e1007770SBorislav Petkov 413e1007770SBorislav Petkov return ret; 41421afaf18SBorislav Petkov } 41521afaf18SBorislav Petkov 41621afaf18SBorislav Petkov /* 417e2def7d4SBorislav Petkov * RDMSR on MCA MSRs should not fault. If they do, this is very much an 418e2def7d4SBorislav Petkov * architectural violation and needs to be reported to hw vendor. Panic 419e2def7d4SBorislav Petkov * the box to not allow any further progress. 42021afaf18SBorislav Petkov */ 421e2def7d4SBorislav Petkov asm volatile("1: rdmsr\n" 422e2def7d4SBorislav Petkov "2:\n" 423e2def7d4SBorislav Petkov _ASM_EXTABLE_HANDLE(1b, 2b, ex_handler_rdmsr_fault) 424e2def7d4SBorislav Petkov : EAX_EDX_RET(val, low, high) : "c" (msr)); 425e2def7d4SBorislav Petkov 426e2def7d4SBorislav Petkov 427e2def7d4SBorislav Petkov return EAX_EDX_VAL(val, low, high); 42821afaf18SBorislav Petkov } 42921afaf18SBorislav Petkov 430e2def7d4SBorislav Petkov __visible bool ex_handler_wrmsr_fault(const struct exception_table_entry *fixup, 431e2def7d4SBorislav Petkov struct pt_regs *regs, int trapnr, 432e2def7d4SBorislav Petkov unsigned long error_code, 433e2def7d4SBorislav Petkov unsigned long fault_addr) 43421afaf18SBorislav Petkov { 435e2def7d4SBorislav Petkov pr_emerg("MSR access error: WRMSR to 0x%x (tried to write 0x%08x%08x) at rIP: 0x%lx (%pS)\n", 436e2def7d4SBorislav Petkov (unsigned int)regs->cx, (unsigned int)regs->dx, (unsigned int)regs->ax, 437e2def7d4SBorislav Petkov regs->ip, (void *)regs->ip); 43821afaf18SBorislav Petkov 439e2def7d4SBorislav Petkov show_stack_regs(regs); 440e2def7d4SBorislav Petkov 441e2def7d4SBorislav Petkov panic("MCA architectural violation!\n"); 442e2def7d4SBorislav Petkov 443e2def7d4SBorislav Petkov while (true) 444e2def7d4SBorislav Petkov cpu_relax(); 445e2def7d4SBorislav Petkov 446e2def7d4SBorislav Petkov return true; 44721afaf18SBorislav Petkov } 44821afaf18SBorislav Petkov 449e1007770SBorislav Petkov static noinstr void mce_wrmsrl(u32 msr, u64 v) 45021afaf18SBorislav Petkov { 451e2def7d4SBorislav Petkov u32 low, high; 452e2def7d4SBorislav Petkov 45321afaf18SBorislav Petkov if (__this_cpu_read(injectm.finished)) { 454e1007770SBorislav Petkov int offset; 45521afaf18SBorislav Petkov 456e1007770SBorislav Petkov instrumentation_begin(); 457e1007770SBorislav Petkov 458e1007770SBorislav Petkov offset = msr_to_offset(msr); 45921afaf18SBorislav Petkov if (offset >= 0) 46021afaf18SBorislav Petkov *(u64 *)((char *)this_cpu_ptr(&injectm) + offset) = v; 461e1007770SBorislav Petkov 462e1007770SBorislav Petkov instrumentation_end(); 463e1007770SBorislav Petkov 46421afaf18SBorislav Petkov return; 46521afaf18SBorislav Petkov } 466e2def7d4SBorislav Petkov 467e2def7d4SBorislav Petkov low = (u32)v; 468e2def7d4SBorislav Petkov high = (u32)(v >> 32); 469e2def7d4SBorislav Petkov 470e2def7d4SBorislav Petkov /* See comment in mce_rdmsrl() */ 471e2def7d4SBorislav Petkov asm volatile("1: wrmsr\n" 472e2def7d4SBorislav Petkov "2:\n" 473e2def7d4SBorislav Petkov _ASM_EXTABLE_HANDLE(1b, 2b, ex_handler_wrmsr_fault) 474e2def7d4SBorislav Petkov : : "c" (msr), "a"(low), "d" (high) : "memory"); 47521afaf18SBorislav Petkov } 47621afaf18SBorislav Petkov 47721afaf18SBorislav Petkov /* 47821afaf18SBorislav Petkov * Collect all global (w.r.t. this processor) status about this machine 47921afaf18SBorislav Petkov * check into our "mce" struct so that we can use it later to assess 48021afaf18SBorislav Petkov * the severity of the problem as we read per-bank specific details. 48121afaf18SBorislav Petkov */ 48221afaf18SBorislav Petkov static inline void mce_gather_info(struct mce *m, struct pt_regs *regs) 48321afaf18SBorislav Petkov { 48421afaf18SBorislav Petkov mce_setup(m); 48521afaf18SBorislav Petkov 48621afaf18SBorislav Petkov m->mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS); 48721afaf18SBorislav Petkov if (regs) { 48821afaf18SBorislav Petkov /* 48921afaf18SBorislav Petkov * Get the address of the instruction at the time of 49021afaf18SBorislav Petkov * the machine check error. 49121afaf18SBorislav Petkov */ 49221afaf18SBorislav Petkov if (m->mcgstatus & (MCG_STATUS_RIPV|MCG_STATUS_EIPV)) { 49321afaf18SBorislav Petkov m->ip = regs->ip; 49421afaf18SBorislav Petkov m->cs = regs->cs; 49521afaf18SBorislav Petkov 49621afaf18SBorislav Petkov /* 49721afaf18SBorislav Petkov * When in VM86 mode make the cs look like ring 3 49821afaf18SBorislav Petkov * always. This is a lie, but it's better than passing 49921afaf18SBorislav Petkov * the additional vm86 bit around everywhere. 50021afaf18SBorislav Petkov */ 50121afaf18SBorislav Petkov if (v8086_mode(regs)) 50221afaf18SBorislav Petkov m->cs |= 3; 50321afaf18SBorislav Petkov } 50421afaf18SBorislav Petkov /* Use accurate RIP reporting if available. */ 50521afaf18SBorislav Petkov if (mca_cfg.rip_msr) 50621afaf18SBorislav Petkov m->ip = mce_rdmsrl(mca_cfg.rip_msr); 50721afaf18SBorislav Petkov } 50821afaf18SBorislav Petkov } 50921afaf18SBorislav Petkov 51021afaf18SBorislav Petkov int mce_available(struct cpuinfo_x86 *c) 51121afaf18SBorislav Petkov { 51221afaf18SBorislav Petkov if (mca_cfg.disabled) 51321afaf18SBorislav Petkov return 0; 51421afaf18SBorislav Petkov return cpu_has(c, X86_FEATURE_MCE) && cpu_has(c, X86_FEATURE_MCA); 51521afaf18SBorislav Petkov } 51621afaf18SBorislav Petkov 51721afaf18SBorislav Petkov static void mce_schedule_work(void) 51821afaf18SBorislav Petkov { 51921afaf18SBorislav Petkov if (!mce_gen_pool_empty()) 52021afaf18SBorislav Petkov schedule_work(&mce_work); 52121afaf18SBorislav Petkov } 52221afaf18SBorislav Petkov 52321afaf18SBorislav Petkov static void mce_irq_work_cb(struct irq_work *entry) 52421afaf18SBorislav Petkov { 52521afaf18SBorislav Petkov mce_schedule_work(); 52621afaf18SBorislav Petkov } 52721afaf18SBorislav Petkov 52821afaf18SBorislav Petkov /* 52921afaf18SBorislav Petkov * Check if the address reported by the CPU is in a format we can parse. 53021afaf18SBorislav Petkov * It would be possible to add code for most other cases, but all would 53121afaf18SBorislav Petkov * be somewhat complicated (e.g. segment offset would require an instruction 532d9f6e12fSIngo Molnar * parser). So only support physical addresses up to page granularity for now. 53321afaf18SBorislav Petkov */ 53421afaf18SBorislav Petkov int mce_usable_address(struct mce *m) 53521afaf18SBorislav Petkov { 53621afaf18SBorislav Petkov if (!(m->status & MCI_STATUS_ADDRV)) 53721afaf18SBorislav Petkov return 0; 53821afaf18SBorislav Petkov 5396e898d2bSTony W Wang-oc /* Checks after this one are Intel/Zhaoxin-specific: */ 5406e898d2bSTony W Wang-oc if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL && 5416e898d2bSTony W Wang-oc boot_cpu_data.x86_vendor != X86_VENDOR_ZHAOXIN) 54221afaf18SBorislav Petkov return 1; 54321afaf18SBorislav Petkov 54421afaf18SBorislav Petkov if (!(m->status & MCI_STATUS_MISCV)) 54521afaf18SBorislav Petkov return 0; 54621afaf18SBorislav Petkov 54721afaf18SBorislav Petkov if (MCI_MISC_ADDR_LSB(m->misc) > PAGE_SHIFT) 54821afaf18SBorislav Petkov return 0; 54921afaf18SBorislav Petkov 55021afaf18SBorislav Petkov if (MCI_MISC_ADDR_MODE(m->misc) != MCI_MISC_ADDR_PHYS) 55121afaf18SBorislav Petkov return 0; 55221afaf18SBorislav Petkov 55321afaf18SBorislav Petkov return 1; 55421afaf18SBorislav Petkov } 55521afaf18SBorislav Petkov EXPORT_SYMBOL_GPL(mce_usable_address); 55621afaf18SBorislav Petkov 55721afaf18SBorislav Petkov bool mce_is_memory_error(struct mce *m) 55821afaf18SBorislav Petkov { 5596e898d2bSTony W Wang-oc switch (m->cpuvendor) { 5606e898d2bSTony W Wang-oc case X86_VENDOR_AMD: 5616e898d2bSTony W Wang-oc case X86_VENDOR_HYGON: 56221afaf18SBorislav Petkov return amd_mce_is_memory_error(m); 5636e898d2bSTony W Wang-oc 5646e898d2bSTony W Wang-oc case X86_VENDOR_INTEL: 5656e898d2bSTony W Wang-oc case X86_VENDOR_ZHAOXIN: 56621afaf18SBorislav Petkov /* 56721afaf18SBorislav Petkov * Intel SDM Volume 3B - 15.9.2 Compound Error Codes 56821afaf18SBorislav Petkov * 56921afaf18SBorislav Petkov * Bit 7 of the MCACOD field of IA32_MCi_STATUS is used for 57021afaf18SBorislav Petkov * indicating a memory error. Bit 8 is used for indicating a 57121afaf18SBorislav Petkov * cache hierarchy error. The combination of bit 2 and bit 3 57221afaf18SBorislav Petkov * is used for indicating a `generic' cache hierarchy error 57321afaf18SBorislav Petkov * But we can't just blindly check the above bits, because if 57421afaf18SBorislav Petkov * bit 11 is set, then it is a bus/interconnect error - and 57521afaf18SBorislav Petkov * either way the above bits just gives more detail on what 57621afaf18SBorislav Petkov * bus/interconnect error happened. Note that bit 12 can be 57721afaf18SBorislav Petkov * ignored, as it's the "filter" bit. 57821afaf18SBorislav Petkov */ 57921afaf18SBorislav Petkov return (m->status & 0xef80) == BIT(7) || 58021afaf18SBorislav Petkov (m->status & 0xef00) == BIT(8) || 58121afaf18SBorislav Petkov (m->status & 0xeffc) == 0xc; 58221afaf18SBorislav Petkov 5836e898d2bSTony W Wang-oc default: 58421afaf18SBorislav Petkov return false; 58521afaf18SBorislav Petkov } 5866e898d2bSTony W Wang-oc } 58721afaf18SBorislav Petkov EXPORT_SYMBOL_GPL(mce_is_memory_error); 58821afaf18SBorislav Petkov 58917fae129STony Luck static bool whole_page(struct mce *m) 59017fae129STony Luck { 59117fae129STony Luck if (!mca_cfg.ser || !(m->status & MCI_STATUS_MISCV)) 59217fae129STony Luck return true; 59317fae129STony Luck 59417fae129STony Luck return MCI_MISC_ADDR_LSB(m->misc) >= PAGE_SHIFT; 59517fae129STony Luck } 59617fae129STony Luck 59721afaf18SBorislav Petkov bool mce_is_correctable(struct mce *m) 59821afaf18SBorislav Petkov { 59921afaf18SBorislav Petkov if (m->cpuvendor == X86_VENDOR_AMD && m->status & MCI_STATUS_DEFERRED) 60021afaf18SBorislav Petkov return false; 60121afaf18SBorislav Petkov 60221afaf18SBorislav Petkov if (m->cpuvendor == X86_VENDOR_HYGON && m->status & MCI_STATUS_DEFERRED) 60321afaf18SBorislav Petkov return false; 60421afaf18SBorislav Petkov 60521afaf18SBorislav Petkov if (m->status & MCI_STATUS_UC) 60621afaf18SBorislav Petkov return false; 60721afaf18SBorislav Petkov 60821afaf18SBorislav Petkov return true; 60921afaf18SBorislav Petkov } 61021afaf18SBorislav Petkov EXPORT_SYMBOL_GPL(mce_is_correctable); 61121afaf18SBorislav Petkov 612c9c6d216STony Luck static int mce_early_notifier(struct notifier_block *nb, unsigned long val, 61321afaf18SBorislav Petkov void *data) 61421afaf18SBorislav Petkov { 61521afaf18SBorislav Petkov struct mce *m = (struct mce *)data; 61621afaf18SBorislav Petkov 61721afaf18SBorislav Petkov if (!m) 61821afaf18SBorislav Petkov return NOTIFY_DONE; 61921afaf18SBorislav Petkov 62021afaf18SBorislav Petkov /* Emit the trace record: */ 62121afaf18SBorislav Petkov trace_mce_record(m); 62221afaf18SBorislav Petkov 62321afaf18SBorislav Petkov set_bit(0, &mce_need_notify); 62421afaf18SBorislav Petkov 62521afaf18SBorislav Petkov mce_notify_irq(); 62621afaf18SBorislav Petkov 62721afaf18SBorislav Petkov return NOTIFY_DONE; 62821afaf18SBorislav Petkov } 62921afaf18SBorislav Petkov 630c9c6d216STony Luck static struct notifier_block early_nb = { 631c9c6d216STony Luck .notifier_call = mce_early_notifier, 632c9c6d216STony Luck .priority = MCE_PRIO_EARLY, 63321afaf18SBorislav Petkov }; 63421afaf18SBorislav Petkov 6358438b84aSJan H. Schönherr static int uc_decode_notifier(struct notifier_block *nb, unsigned long val, 63621afaf18SBorislav Petkov void *data) 63721afaf18SBorislav Petkov { 63821afaf18SBorislav Petkov struct mce *mce = (struct mce *)data; 63921afaf18SBorislav Petkov unsigned long pfn; 64021afaf18SBorislav Petkov 6418438b84aSJan H. Schönherr if (!mce || !mce_usable_address(mce)) 64221afaf18SBorislav Petkov return NOTIFY_DONE; 64321afaf18SBorislav Petkov 6448438b84aSJan H. Schönherr if (mce->severity != MCE_AO_SEVERITY && 6458438b84aSJan H. Schönherr mce->severity != MCE_DEFERRED_SEVERITY) 6468438b84aSJan H. Schönherr return NOTIFY_DONE; 6478438b84aSJan H. Schönherr 64821afaf18SBorislav Petkov pfn = mce->addr >> PAGE_SHIFT; 64923ba710aSTony Luck if (!memory_failure(pfn, 0)) { 65017fae129STony Luck set_mce_nospec(pfn, whole_page(mce)); 65123ba710aSTony Luck mce->kflags |= MCE_HANDLED_UC; 65223ba710aSTony Luck } 65321afaf18SBorislav Petkov 65421afaf18SBorislav Petkov return NOTIFY_OK; 65521afaf18SBorislav Petkov } 6568438b84aSJan H. Schönherr 6578438b84aSJan H. Schönherr static struct notifier_block mce_uc_nb = { 6588438b84aSJan H. Schönherr .notifier_call = uc_decode_notifier, 6598438b84aSJan H. Schönherr .priority = MCE_PRIO_UC, 66021afaf18SBorislav Petkov }; 66121afaf18SBorislav Petkov 66221afaf18SBorislav Petkov static int mce_default_notifier(struct notifier_block *nb, unsigned long val, 66321afaf18SBorislav Petkov void *data) 66421afaf18SBorislav Petkov { 66521afaf18SBorislav Petkov struct mce *m = (struct mce *)data; 66621afaf18SBorislav Petkov 66721afaf18SBorislav Petkov if (!m) 66821afaf18SBorislav Petkov return NOTIFY_DONE; 66921afaf18SBorislav Petkov 67043505646STony Luck if (mca_cfg.print_all || !m->kflags) 67121afaf18SBorislav Petkov __print_mce(m); 67221afaf18SBorislav Petkov 67321afaf18SBorislav Petkov return NOTIFY_DONE; 67421afaf18SBorislav Petkov } 67521afaf18SBorislav Petkov 67621afaf18SBorislav Petkov static struct notifier_block mce_default_nb = { 67721afaf18SBorislav Petkov .notifier_call = mce_default_notifier, 67821afaf18SBorislav Petkov /* lowest prio, we want it to run last. */ 67921afaf18SBorislav Petkov .priority = MCE_PRIO_LOWEST, 68021afaf18SBorislav Petkov }; 68121afaf18SBorislav Petkov 68221afaf18SBorislav Petkov /* 68321afaf18SBorislav Petkov * Read ADDR and MISC registers. 68421afaf18SBorislav Petkov */ 68521afaf18SBorislav Petkov static void mce_read_aux(struct mce *m, int i) 68621afaf18SBorislav Petkov { 68721afaf18SBorislav Petkov if (m->status & MCI_STATUS_MISCV) 68821afaf18SBorislav Petkov m->misc = mce_rdmsrl(msr_ops.misc(i)); 68921afaf18SBorislav Petkov 69021afaf18SBorislav Petkov if (m->status & MCI_STATUS_ADDRV) { 69121afaf18SBorislav Petkov m->addr = mce_rdmsrl(msr_ops.addr(i)); 69221afaf18SBorislav Petkov 69321afaf18SBorislav Petkov /* 69421afaf18SBorislav Petkov * Mask the reported address by the reported granularity. 69521afaf18SBorislav Petkov */ 69621afaf18SBorislav Petkov if (mca_cfg.ser && (m->status & MCI_STATUS_MISCV)) { 69721afaf18SBorislav Petkov u8 shift = MCI_MISC_ADDR_LSB(m->misc); 69821afaf18SBorislav Petkov m->addr >>= shift; 69921afaf18SBorislav Petkov m->addr <<= shift; 70021afaf18SBorislav Petkov } 70121afaf18SBorislav Petkov 70221afaf18SBorislav Petkov /* 70321afaf18SBorislav Petkov * Extract [55:<lsb>] where lsb is the least significant 70421afaf18SBorislav Petkov * *valid* bit of the address bits. 70521afaf18SBorislav Petkov */ 70621afaf18SBorislav Petkov if (mce_flags.smca) { 70721afaf18SBorislav Petkov u8 lsb = (m->addr >> 56) & 0x3f; 70821afaf18SBorislav Petkov 70921afaf18SBorislav Petkov m->addr &= GENMASK_ULL(55, lsb); 71021afaf18SBorislav Petkov } 71121afaf18SBorislav Petkov } 71221afaf18SBorislav Petkov 71321afaf18SBorislav Petkov if (mce_flags.smca) { 71421afaf18SBorislav Petkov m->ipid = mce_rdmsrl(MSR_AMD64_SMCA_MCx_IPID(i)); 71521afaf18SBorislav Petkov 71621afaf18SBorislav Petkov if (m->status & MCI_STATUS_SYNDV) 71721afaf18SBorislav Petkov m->synd = mce_rdmsrl(MSR_AMD64_SMCA_MCx_SYND(i)); 71821afaf18SBorislav Petkov } 71921afaf18SBorislav Petkov } 72021afaf18SBorislav Petkov 72121afaf18SBorislav Petkov DEFINE_PER_CPU(unsigned, mce_poll_count); 72221afaf18SBorislav Petkov 72321afaf18SBorislav Petkov /* 72421afaf18SBorislav Petkov * Poll for corrected events or events that happened before reset. 72521afaf18SBorislav Petkov * Those are just logged through /dev/mcelog. 72621afaf18SBorislav Petkov * 72721afaf18SBorislav Petkov * This is executed in standard interrupt context. 72821afaf18SBorislav Petkov * 72921afaf18SBorislav Petkov * Note: spec recommends to panic for fatal unsignalled 73021afaf18SBorislav Petkov * errors here. However this would be quite problematic -- 73121afaf18SBorislav Petkov * we would need to reimplement the Monarch handling and 73221afaf18SBorislav Petkov * it would mess up the exclusion between exception handler 733312a4661SLinus Torvalds * and poll handler -- * so we skip this for now. 73421afaf18SBorislav Petkov * These cases should not happen anyways, or only when the CPU 73521afaf18SBorislav Petkov * is already totally * confused. In this case it's likely it will 73621afaf18SBorislav Petkov * not fully execute the machine check handler either. 73721afaf18SBorislav Petkov */ 73821afaf18SBorislav Petkov bool machine_check_poll(enum mcp_flags flags, mce_banks_t *b) 73921afaf18SBorislav Petkov { 740b4914508SYazen Ghannam struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 74121afaf18SBorislav Petkov bool error_seen = false; 74221afaf18SBorislav Petkov struct mce m; 74321afaf18SBorislav Petkov int i; 74421afaf18SBorislav Petkov 74521afaf18SBorislav Petkov this_cpu_inc(mce_poll_count); 74621afaf18SBorislav Petkov 74721afaf18SBorislav Petkov mce_gather_info(&m, NULL); 74821afaf18SBorislav Petkov 74921afaf18SBorislav Petkov if (flags & MCP_TIMESTAMP) 75021afaf18SBorislav Petkov m.tsc = rdtsc(); 75121afaf18SBorislav Petkov 752c7d314f3SYazen Ghannam for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 75321afaf18SBorislav Petkov if (!mce_banks[i].ctl || !test_bit(i, *b)) 75421afaf18SBorislav Petkov continue; 75521afaf18SBorislav Petkov 75621afaf18SBorislav Petkov m.misc = 0; 75721afaf18SBorislav Petkov m.addr = 0; 75821afaf18SBorislav Petkov m.bank = i; 75921afaf18SBorislav Petkov 76021afaf18SBorislav Petkov barrier(); 76121afaf18SBorislav Petkov m.status = mce_rdmsrl(msr_ops.status(i)); 762f19501aaSTony Luck 763f19501aaSTony Luck /* If this entry is not valid, ignore it */ 76421afaf18SBorislav Petkov if (!(m.status & MCI_STATUS_VAL)) 76521afaf18SBorislav Petkov continue; 76621afaf18SBorislav Petkov 76721afaf18SBorislav Petkov /* 768f19501aaSTony Luck * If we are logging everything (at CPU online) or this 769f19501aaSTony Luck * is a corrected error, then we must log it. 77021afaf18SBorislav Petkov */ 771f19501aaSTony Luck if ((flags & MCP_UC) || !(m.status & MCI_STATUS_UC)) 772f19501aaSTony Luck goto log_it; 773f19501aaSTony Luck 774f19501aaSTony Luck /* 775f19501aaSTony Luck * Newer Intel systems that support software error 776f19501aaSTony Luck * recovery need to make additional checks. Other 777f19501aaSTony Luck * CPUs should skip over uncorrected errors, but log 778f19501aaSTony Luck * everything else. 779f19501aaSTony Luck */ 780f19501aaSTony Luck if (!mca_cfg.ser) { 781f19501aaSTony Luck if (m.status & MCI_STATUS_UC) 782f19501aaSTony Luck continue; 783f19501aaSTony Luck goto log_it; 784f19501aaSTony Luck } 785f19501aaSTony Luck 786f19501aaSTony Luck /* Log "not enabled" (speculative) errors */ 787f19501aaSTony Luck if (!(m.status & MCI_STATUS_EN)) 788f19501aaSTony Luck goto log_it; 789f19501aaSTony Luck 790f19501aaSTony Luck /* 791f19501aaSTony Luck * Log UCNA (SDM: 15.6.3 "UCR Error Classification") 792f19501aaSTony Luck * UC == 1 && PCC == 0 && S == 0 793f19501aaSTony Luck */ 794f19501aaSTony Luck if (!(m.status & MCI_STATUS_PCC) && !(m.status & MCI_STATUS_S)) 795f19501aaSTony Luck goto log_it; 796f19501aaSTony Luck 797f19501aaSTony Luck /* 798f19501aaSTony Luck * Skip anything else. Presumption is that our read of this 799f19501aaSTony Luck * bank is racing with a machine check. Leave the log alone 800f19501aaSTony Luck * for do_machine_check() to deal with it. 801f19501aaSTony Luck */ 80221afaf18SBorislav Petkov continue; 80321afaf18SBorislav Petkov 804f19501aaSTony Luck log_it: 80521afaf18SBorislav Petkov error_seen = true; 80621afaf18SBorislav Petkov 80790454e49SJan H. Schönherr if (flags & MCP_DONTLOG) 80890454e49SJan H. Schönherr goto clear_it; 80990454e49SJan H. Schönherr 81021afaf18SBorislav Petkov mce_read_aux(&m, i); 81141ce0564SYouquan Song m.severity = mce_severity(&m, NULL, mca_cfg.tolerant, NULL, false); 81221afaf18SBorislav Petkov /* 81321afaf18SBorislav Petkov * Don't get the IP here because it's unlikely to 81421afaf18SBorislav Petkov * have anything to do with the actual error location. 81521afaf18SBorislav Petkov */ 81621afaf18SBorislav Petkov 81790454e49SJan H. Schönherr if (mca_cfg.dont_log_ce && !mce_usable_address(&m)) 81890454e49SJan H. Schönherr goto clear_it; 81990454e49SJan H. Schönherr 820*3bff147bSBorislav Petkov if (flags & MCP_QUEUE_LOG) 821*3bff147bSBorislav Petkov mce_gen_pool_add(&m); 822*3bff147bSBorislav Petkov else 82390454e49SJan H. Schönherr mce_log(&m); 82490454e49SJan H. Schönherr 82590454e49SJan H. Schönherr clear_it: 82621afaf18SBorislav Petkov /* 82721afaf18SBorislav Petkov * Clear state for this bank. 82821afaf18SBorislav Petkov */ 82921afaf18SBorislav Petkov mce_wrmsrl(msr_ops.status(i), 0); 83021afaf18SBorislav Petkov } 83121afaf18SBorislav Petkov 83221afaf18SBorislav Petkov /* 83321afaf18SBorislav Petkov * Don't clear MCG_STATUS here because it's only defined for 83421afaf18SBorislav Petkov * exceptions. 83521afaf18SBorislav Petkov */ 83621afaf18SBorislav Petkov 83721afaf18SBorislav Petkov sync_core(); 83821afaf18SBorislav Petkov 83921afaf18SBorislav Petkov return error_seen; 84021afaf18SBorislav Petkov } 84121afaf18SBorislav Petkov EXPORT_SYMBOL_GPL(machine_check_poll); 84221afaf18SBorislav Petkov 84321afaf18SBorislav Petkov /* 84421afaf18SBorislav Petkov * Do a quick check if any of the events requires a panic. 84521afaf18SBorislav Petkov * This decides if we keep the events around or clear them. 84621afaf18SBorislav Petkov */ 84721afaf18SBorislav Petkov static int mce_no_way_out(struct mce *m, char **msg, unsigned long *validp, 84821afaf18SBorislav Petkov struct pt_regs *regs) 84921afaf18SBorislav Petkov { 8507a8bc2b0SJan H. Schönherr char *tmp = *msg; 85121afaf18SBorislav Petkov int i; 85221afaf18SBorislav Petkov 853c7d314f3SYazen Ghannam for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 85421afaf18SBorislav Petkov m->status = mce_rdmsrl(msr_ops.status(i)); 85521afaf18SBorislav Petkov if (!(m->status & MCI_STATUS_VAL)) 85621afaf18SBorislav Petkov continue; 85721afaf18SBorislav Petkov 85821afaf18SBorislav Petkov __set_bit(i, validp); 85921afaf18SBorislav Petkov if (quirk_no_way_out) 86021afaf18SBorislav Petkov quirk_no_way_out(i, m, regs); 86121afaf18SBorislav Petkov 862d28af26fSTony Luck m->bank = i; 86341ce0564SYouquan Song if (mce_severity(m, regs, mca_cfg.tolerant, &tmp, true) >= MCE_PANIC_SEVERITY) { 86421afaf18SBorislav Petkov mce_read_aux(m, i); 86521afaf18SBorislav Petkov *msg = tmp; 86621afaf18SBorislav Petkov return 1; 86721afaf18SBorislav Petkov } 86821afaf18SBorislav Petkov } 86921afaf18SBorislav Petkov return 0; 87021afaf18SBorislav Petkov } 87121afaf18SBorislav Petkov 87221afaf18SBorislav Petkov /* 87321afaf18SBorislav Petkov * Variable to establish order between CPUs while scanning. 87421afaf18SBorislav Petkov * Each CPU spins initially until executing is equal its number. 87521afaf18SBorislav Petkov */ 87621afaf18SBorislav Petkov static atomic_t mce_executing; 87721afaf18SBorislav Petkov 87821afaf18SBorislav Petkov /* 87921afaf18SBorislav Petkov * Defines order of CPUs on entry. First CPU becomes Monarch. 88021afaf18SBorislav Petkov */ 88121afaf18SBorislav Petkov static atomic_t mce_callin; 88221afaf18SBorislav Petkov 88321afaf18SBorislav Petkov /* 8847bb39313SPaul E. McKenney * Track which CPUs entered the MCA broadcast synchronization and which not in 8857bb39313SPaul E. McKenney * order to print holdouts. 8867bb39313SPaul E. McKenney */ 8877bb39313SPaul E. McKenney static cpumask_t mce_missing_cpus = CPU_MASK_ALL; 8887bb39313SPaul E. McKenney 8897bb39313SPaul E. McKenney /* 89021afaf18SBorislav Petkov * Check if a timeout waiting for other CPUs happened. 89121afaf18SBorislav Petkov */ 89221afaf18SBorislav Petkov static int mce_timed_out(u64 *t, const char *msg) 89321afaf18SBorislav Petkov { 89421afaf18SBorislav Petkov /* 89521afaf18SBorislav Petkov * The others already did panic for some reason. 89621afaf18SBorislav Petkov * Bail out like in a timeout. 89721afaf18SBorislav Petkov * rmb() to tell the compiler that system_state 89821afaf18SBorislav Petkov * might have been modified by someone else. 89921afaf18SBorislav Petkov */ 90021afaf18SBorislav Petkov rmb(); 90121afaf18SBorislav Petkov if (atomic_read(&mce_panicked)) 90221afaf18SBorislav Petkov wait_for_panic(); 90321afaf18SBorislav Petkov if (!mca_cfg.monarch_timeout) 90421afaf18SBorislav Petkov goto out; 90521afaf18SBorislav Petkov if ((s64)*t < SPINUNIT) { 9067bb39313SPaul E. McKenney if (mca_cfg.tolerant <= 1) { 9077bb39313SPaul E. McKenney if (cpumask_and(&mce_missing_cpus, cpu_online_mask, &mce_missing_cpus)) 9087bb39313SPaul E. McKenney pr_emerg("CPUs not responding to MCE broadcast (may include false positives): %*pbl\n", 9097bb39313SPaul E. McKenney cpumask_pr_args(&mce_missing_cpus)); 91021afaf18SBorislav Petkov mce_panic(msg, NULL, NULL); 9117bb39313SPaul E. McKenney } 91221afaf18SBorislav Petkov cpu_missing = 1; 91321afaf18SBorislav Petkov return 1; 91421afaf18SBorislav Petkov } 91521afaf18SBorislav Petkov *t -= SPINUNIT; 91621afaf18SBorislav Petkov out: 91721afaf18SBorislav Petkov touch_nmi_watchdog(); 91821afaf18SBorislav Petkov return 0; 91921afaf18SBorislav Petkov } 92021afaf18SBorislav Petkov 92121afaf18SBorislav Petkov /* 92221afaf18SBorislav Petkov * The Monarch's reign. The Monarch is the CPU who entered 92321afaf18SBorislav Petkov * the machine check handler first. It waits for the others to 92421afaf18SBorislav Petkov * raise the exception too and then grades them. When any 92521afaf18SBorislav Petkov * error is fatal panic. Only then let the others continue. 92621afaf18SBorislav Petkov * 92721afaf18SBorislav Petkov * The other CPUs entering the MCE handler will be controlled by the 92821afaf18SBorislav Petkov * Monarch. They are called Subjects. 92921afaf18SBorislav Petkov * 93021afaf18SBorislav Petkov * This way we prevent any potential data corruption in a unrecoverable case 93121afaf18SBorislav Petkov * and also makes sure always all CPU's errors are examined. 93221afaf18SBorislav Petkov * 93321afaf18SBorislav Petkov * Also this detects the case of a machine check event coming from outer 93421afaf18SBorislav Petkov * space (not detected by any CPUs) In this case some external agent wants 93521afaf18SBorislav Petkov * us to shut down, so panic too. 93621afaf18SBorislav Petkov * 93721afaf18SBorislav Petkov * The other CPUs might still decide to panic if the handler happens 93821afaf18SBorislav Petkov * in a unrecoverable place, but in this case the system is in a semi-stable 93921afaf18SBorislav Petkov * state and won't corrupt anything by itself. It's ok to let the others 94021afaf18SBorislav Petkov * continue for a bit first. 94121afaf18SBorislav Petkov * 94221afaf18SBorislav Petkov * All the spin loops have timeouts; when a timeout happens a CPU 94321afaf18SBorislav Petkov * typically elects itself to be Monarch. 94421afaf18SBorislav Petkov */ 94521afaf18SBorislav Petkov static void mce_reign(void) 94621afaf18SBorislav Petkov { 94721afaf18SBorislav Petkov int cpu; 94821afaf18SBorislav Petkov struct mce *m = NULL; 94921afaf18SBorislav Petkov int global_worst = 0; 95021afaf18SBorislav Petkov char *msg = NULL; 95121afaf18SBorislav Petkov 95221afaf18SBorislav Petkov /* 95321afaf18SBorislav Petkov * This CPU is the Monarch and the other CPUs have run 95421afaf18SBorislav Petkov * through their handlers. 95521afaf18SBorislav Petkov * Grade the severity of the errors of all the CPUs. 95621afaf18SBorislav Petkov */ 95721afaf18SBorislav Petkov for_each_possible_cpu(cpu) { 95813c877f4STony Luck struct mce *mtmp = &per_cpu(mces_seen, cpu); 95913c877f4STony Luck 96013c877f4STony Luck if (mtmp->severity > global_worst) { 96113c877f4STony Luck global_worst = mtmp->severity; 96221afaf18SBorislav Petkov m = &per_cpu(mces_seen, cpu); 96321afaf18SBorislav Petkov } 96421afaf18SBorislav Petkov } 96521afaf18SBorislav Petkov 96621afaf18SBorislav Petkov /* 96721afaf18SBorislav Petkov * Cannot recover? Panic here then. 96821afaf18SBorislav Petkov * This dumps all the mces in the log buffer and stops the 96921afaf18SBorislav Petkov * other CPUs. 97021afaf18SBorislav Petkov */ 97113c877f4STony Luck if (m && global_worst >= MCE_PANIC_SEVERITY && mca_cfg.tolerant < 3) { 97213c877f4STony Luck /* call mce_severity() to get "msg" for panic */ 97341ce0564SYouquan Song mce_severity(m, NULL, mca_cfg.tolerant, &msg, true); 97421afaf18SBorislav Petkov mce_panic("Fatal machine check", m, msg); 97513c877f4STony Luck } 97621afaf18SBorislav Petkov 97721afaf18SBorislav Petkov /* 97821afaf18SBorislav Petkov * For UC somewhere we let the CPU who detects it handle it. 97921afaf18SBorislav Petkov * Also must let continue the others, otherwise the handling 98021afaf18SBorislav Petkov * CPU could deadlock on a lock. 98121afaf18SBorislav Petkov */ 98221afaf18SBorislav Petkov 98321afaf18SBorislav Petkov /* 98421afaf18SBorislav Petkov * No machine check event found. Must be some external 98521afaf18SBorislav Petkov * source or one CPU is hung. Panic. 98621afaf18SBorislav Petkov */ 98721afaf18SBorislav Petkov if (global_worst <= MCE_KEEP_SEVERITY && mca_cfg.tolerant < 3) 98821afaf18SBorislav Petkov mce_panic("Fatal machine check from unknown source", NULL, NULL); 98921afaf18SBorislav Petkov 99021afaf18SBorislav Petkov /* 99121afaf18SBorislav Petkov * Now clear all the mces_seen so that they don't reappear on 99221afaf18SBorislav Petkov * the next mce. 99321afaf18SBorislav Petkov */ 99421afaf18SBorislav Petkov for_each_possible_cpu(cpu) 99521afaf18SBorislav Petkov memset(&per_cpu(mces_seen, cpu), 0, sizeof(struct mce)); 99621afaf18SBorislav Petkov } 99721afaf18SBorislav Petkov 99821afaf18SBorislav Petkov static atomic_t global_nwo; 99921afaf18SBorislav Petkov 100021afaf18SBorislav Petkov /* 100121afaf18SBorislav Petkov * Start of Monarch synchronization. This waits until all CPUs have 100221afaf18SBorislav Petkov * entered the exception handler and then determines if any of them 100321afaf18SBorislav Petkov * saw a fatal event that requires panic. Then it executes them 100421afaf18SBorislav Petkov * in the entry order. 100521afaf18SBorislav Petkov * TBD double check parallel CPU hotunplug 100621afaf18SBorislav Petkov */ 100721afaf18SBorislav Petkov static int mce_start(int *no_way_out) 100821afaf18SBorislav Petkov { 100921afaf18SBorislav Petkov int order; 101021afaf18SBorislav Petkov int cpus = num_online_cpus(); 101121afaf18SBorislav Petkov u64 timeout = (u64)mca_cfg.monarch_timeout * NSEC_PER_USEC; 101221afaf18SBorislav Petkov 101321afaf18SBorislav Petkov if (!timeout) 101421afaf18SBorislav Petkov return -1; 101521afaf18SBorislav Petkov 101621afaf18SBorislav Petkov atomic_add(*no_way_out, &global_nwo); 101721afaf18SBorislav Petkov /* 101821afaf18SBorislav Petkov * Rely on the implied barrier below, such that global_nwo 101921afaf18SBorislav Petkov * is updated before mce_callin. 102021afaf18SBorislav Petkov */ 102121afaf18SBorislav Petkov order = atomic_inc_return(&mce_callin); 10227bb39313SPaul E. McKenney cpumask_clear_cpu(smp_processor_id(), &mce_missing_cpus); 102321afaf18SBorislav Petkov 102421afaf18SBorislav Petkov /* 102521afaf18SBorislav Petkov * Wait for everyone. 102621afaf18SBorislav Petkov */ 102721afaf18SBorislav Petkov while (atomic_read(&mce_callin) != cpus) { 102821afaf18SBorislav Petkov if (mce_timed_out(&timeout, 102921afaf18SBorislav Petkov "Timeout: Not all CPUs entered broadcast exception handler")) { 103021afaf18SBorislav Petkov atomic_set(&global_nwo, 0); 103121afaf18SBorislav Petkov return -1; 103221afaf18SBorislav Petkov } 103321afaf18SBorislav Petkov ndelay(SPINUNIT); 103421afaf18SBorislav Petkov } 103521afaf18SBorislav Petkov 103621afaf18SBorislav Petkov /* 103721afaf18SBorislav Petkov * mce_callin should be read before global_nwo 103821afaf18SBorislav Petkov */ 103921afaf18SBorislav Petkov smp_rmb(); 104021afaf18SBorislav Petkov 104121afaf18SBorislav Petkov if (order == 1) { 104221afaf18SBorislav Petkov /* 104321afaf18SBorislav Petkov * Monarch: Starts executing now, the others wait. 104421afaf18SBorislav Petkov */ 104521afaf18SBorislav Petkov atomic_set(&mce_executing, 1); 104621afaf18SBorislav Petkov } else { 104721afaf18SBorislav Petkov /* 104821afaf18SBorislav Petkov * Subject: Now start the scanning loop one by one in 104921afaf18SBorislav Petkov * the original callin order. 105021afaf18SBorislav Petkov * This way when there are any shared banks it will be 105121afaf18SBorislav Petkov * only seen by one CPU before cleared, avoiding duplicates. 105221afaf18SBorislav Petkov */ 105321afaf18SBorislav Petkov while (atomic_read(&mce_executing) < order) { 105421afaf18SBorislav Petkov if (mce_timed_out(&timeout, 105521afaf18SBorislav Petkov "Timeout: Subject CPUs unable to finish machine check processing")) { 105621afaf18SBorislav Petkov atomic_set(&global_nwo, 0); 105721afaf18SBorislav Petkov return -1; 105821afaf18SBorislav Petkov } 105921afaf18SBorislav Petkov ndelay(SPINUNIT); 106021afaf18SBorislav Petkov } 106121afaf18SBorislav Petkov } 106221afaf18SBorislav Petkov 106321afaf18SBorislav Petkov /* 106421afaf18SBorislav Petkov * Cache the global no_way_out state. 106521afaf18SBorislav Petkov */ 106621afaf18SBorislav Petkov *no_way_out = atomic_read(&global_nwo); 106721afaf18SBorislav Petkov 106821afaf18SBorislav Petkov return order; 106921afaf18SBorislav Petkov } 107021afaf18SBorislav Petkov 107121afaf18SBorislav Petkov /* 107221afaf18SBorislav Petkov * Synchronize between CPUs after main scanning loop. 107321afaf18SBorislav Petkov * This invokes the bulk of the Monarch processing. 107421afaf18SBorislav Petkov */ 107521afaf18SBorislav Petkov static int mce_end(int order) 107621afaf18SBorislav Petkov { 107721afaf18SBorislav Petkov int ret = -1; 107821afaf18SBorislav Petkov u64 timeout = (u64)mca_cfg.monarch_timeout * NSEC_PER_USEC; 107921afaf18SBorislav Petkov 108021afaf18SBorislav Petkov if (!timeout) 108121afaf18SBorislav Petkov goto reset; 108221afaf18SBorislav Petkov if (order < 0) 108321afaf18SBorislav Petkov goto reset; 108421afaf18SBorislav Petkov 108521afaf18SBorislav Petkov /* 108621afaf18SBorislav Petkov * Allow others to run. 108721afaf18SBorislav Petkov */ 108821afaf18SBorislav Petkov atomic_inc(&mce_executing); 108921afaf18SBorislav Petkov 109021afaf18SBorislav Petkov if (order == 1) { 109121afaf18SBorislav Petkov /* CHECKME: Can this race with a parallel hotplug? */ 109221afaf18SBorislav Petkov int cpus = num_online_cpus(); 109321afaf18SBorislav Petkov 109421afaf18SBorislav Petkov /* 109521afaf18SBorislav Petkov * Monarch: Wait for everyone to go through their scanning 109621afaf18SBorislav Petkov * loops. 109721afaf18SBorislav Petkov */ 109821afaf18SBorislav Petkov while (atomic_read(&mce_executing) <= cpus) { 109921afaf18SBorislav Petkov if (mce_timed_out(&timeout, 110021afaf18SBorislav Petkov "Timeout: Monarch CPU unable to finish machine check processing")) 110121afaf18SBorislav Petkov goto reset; 110221afaf18SBorislav Petkov ndelay(SPINUNIT); 110321afaf18SBorislav Petkov } 110421afaf18SBorislav Petkov 110521afaf18SBorislav Petkov mce_reign(); 110621afaf18SBorislav Petkov barrier(); 110721afaf18SBorislav Petkov ret = 0; 110821afaf18SBorislav Petkov } else { 110921afaf18SBorislav Petkov /* 111021afaf18SBorislav Petkov * Subject: Wait for Monarch to finish. 111121afaf18SBorislav Petkov */ 111221afaf18SBorislav Petkov while (atomic_read(&mce_executing) != 0) { 111321afaf18SBorislav Petkov if (mce_timed_out(&timeout, 111421afaf18SBorislav Petkov "Timeout: Monarch CPU did not finish machine check processing")) 111521afaf18SBorislav Petkov goto reset; 111621afaf18SBorislav Petkov ndelay(SPINUNIT); 111721afaf18SBorislav Petkov } 111821afaf18SBorislav Petkov 111921afaf18SBorislav Petkov /* 112021afaf18SBorislav Petkov * Don't reset anything. That's done by the Monarch. 112121afaf18SBorislav Petkov */ 112221afaf18SBorislav Petkov return 0; 112321afaf18SBorislav Petkov } 112421afaf18SBorislav Petkov 112521afaf18SBorislav Petkov /* 112621afaf18SBorislav Petkov * Reset all global state. 112721afaf18SBorislav Petkov */ 112821afaf18SBorislav Petkov reset: 112921afaf18SBorislav Petkov atomic_set(&global_nwo, 0); 113021afaf18SBorislav Petkov atomic_set(&mce_callin, 0); 11317bb39313SPaul E. McKenney cpumask_setall(&mce_missing_cpus); 113221afaf18SBorislav Petkov barrier(); 113321afaf18SBorislav Petkov 113421afaf18SBorislav Petkov /* 113521afaf18SBorislav Petkov * Let others run again. 113621afaf18SBorislav Petkov */ 113721afaf18SBorislav Petkov atomic_set(&mce_executing, 0); 113821afaf18SBorislav Petkov return ret; 113921afaf18SBorislav Petkov } 114021afaf18SBorislav Petkov 114121afaf18SBorislav Petkov static void mce_clear_state(unsigned long *toclear) 114221afaf18SBorislav Petkov { 114321afaf18SBorislav Petkov int i; 114421afaf18SBorislav Petkov 1145c7d314f3SYazen Ghannam for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 114621afaf18SBorislav Petkov if (test_bit(i, toclear)) 114721afaf18SBorislav Petkov mce_wrmsrl(msr_ops.status(i), 0); 114821afaf18SBorislav Petkov } 114921afaf18SBorislav Petkov } 115021afaf18SBorislav Petkov 115121afaf18SBorislav Petkov /* 115221afaf18SBorislav Petkov * Cases where we avoid rendezvous handler timeout: 115321afaf18SBorislav Petkov * 1) If this CPU is offline. 115421afaf18SBorislav Petkov * 115521afaf18SBorislav Petkov * 2) If crashing_cpu was set, e.g. we're entering kdump and we need to 115621afaf18SBorislav Petkov * skip those CPUs which remain looping in the 1st kernel - see 115721afaf18SBorislav Petkov * crash_nmi_callback(). 115821afaf18SBorislav Petkov * 115921afaf18SBorislav Petkov * Note: there still is a small window between kexec-ing and the new, 116021afaf18SBorislav Petkov * kdump kernel establishing a new #MC handler where a broadcasted MCE 116121afaf18SBorislav Petkov * might not get handled properly. 116221afaf18SBorislav Petkov */ 116394a46d31SThomas Gleixner static noinstr bool mce_check_crashing_cpu(void) 116421afaf18SBorislav Petkov { 116594a46d31SThomas Gleixner unsigned int cpu = smp_processor_id(); 116694a46d31SThomas Gleixner 116714d3b376SPeter Zijlstra if (arch_cpu_is_offline(cpu) || 116821afaf18SBorislav Petkov (crashing_cpu != -1 && crashing_cpu != cpu)) { 116921afaf18SBorislav Petkov u64 mcgstatus; 117021afaf18SBorislav Petkov 1171aedbdeabSThomas Gleixner mcgstatus = __rdmsr(MSR_IA32_MCG_STATUS); 117270f0c230STony W Wang-oc 117370f0c230STony W Wang-oc if (boot_cpu_data.x86_vendor == X86_VENDOR_ZHAOXIN) { 117470f0c230STony W Wang-oc if (mcgstatus & MCG_STATUS_LMCES) 117570f0c230STony W Wang-oc return false; 117670f0c230STony W Wang-oc } 117770f0c230STony W Wang-oc 117821afaf18SBorislav Petkov if (mcgstatus & MCG_STATUS_RIPV) { 1179aedbdeabSThomas Gleixner __wrmsr(MSR_IA32_MCG_STATUS, 0, 0); 118021afaf18SBorislav Petkov return true; 118121afaf18SBorislav Petkov } 118221afaf18SBorislav Petkov } 118321afaf18SBorislav Petkov return false; 118421afaf18SBorislav Petkov } 118521afaf18SBorislav Petkov 118641ce0564SYouquan Song static void __mc_scan_banks(struct mce *m, struct pt_regs *regs, struct mce *final, 118721afaf18SBorislav Petkov unsigned long *toclear, unsigned long *valid_banks, 118821afaf18SBorislav Petkov int no_way_out, int *worst) 118921afaf18SBorislav Petkov { 1190b4914508SYazen Ghannam struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 119121afaf18SBorislav Petkov struct mca_config *cfg = &mca_cfg; 119221afaf18SBorislav Petkov int severity, i; 119321afaf18SBorislav Petkov 1194c7d314f3SYazen Ghannam for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 119521afaf18SBorislav Petkov __clear_bit(i, toclear); 119621afaf18SBorislav Petkov if (!test_bit(i, valid_banks)) 119721afaf18SBorislav Petkov continue; 119821afaf18SBorislav Petkov 119921afaf18SBorislav Petkov if (!mce_banks[i].ctl) 120021afaf18SBorislav Petkov continue; 120121afaf18SBorislav Petkov 120221afaf18SBorislav Petkov m->misc = 0; 120321afaf18SBorislav Petkov m->addr = 0; 120421afaf18SBorislav Petkov m->bank = i; 120521afaf18SBorislav Petkov 120621afaf18SBorislav Petkov m->status = mce_rdmsrl(msr_ops.status(i)); 120721afaf18SBorislav Petkov if (!(m->status & MCI_STATUS_VAL)) 120821afaf18SBorislav Petkov continue; 120921afaf18SBorislav Petkov 121021afaf18SBorislav Petkov /* 121121afaf18SBorislav Petkov * Corrected or non-signaled errors are handled by 121221afaf18SBorislav Petkov * machine_check_poll(). Leave them alone, unless this panics. 121321afaf18SBorislav Petkov */ 121421afaf18SBorislav Petkov if (!(m->status & (cfg->ser ? MCI_STATUS_S : MCI_STATUS_UC)) && 121521afaf18SBorislav Petkov !no_way_out) 121621afaf18SBorislav Petkov continue; 121721afaf18SBorislav Petkov 121821afaf18SBorislav Petkov /* Set taint even when machine check was not enabled. */ 121921afaf18SBorislav Petkov add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE); 122021afaf18SBorislav Petkov 122141ce0564SYouquan Song severity = mce_severity(m, regs, cfg->tolerant, NULL, true); 122221afaf18SBorislav Petkov 122321afaf18SBorislav Petkov /* 122421afaf18SBorislav Petkov * When machine check was for corrected/deferred handler don't 122521afaf18SBorislav Petkov * touch, unless we're panicking. 122621afaf18SBorislav Petkov */ 122721afaf18SBorislav Petkov if ((severity == MCE_KEEP_SEVERITY || 122821afaf18SBorislav Petkov severity == MCE_UCNA_SEVERITY) && !no_way_out) 122921afaf18SBorislav Petkov continue; 123021afaf18SBorislav Petkov 123121afaf18SBorislav Petkov __set_bit(i, toclear); 123221afaf18SBorislav Petkov 123321afaf18SBorislav Petkov /* Machine check event was not enabled. Clear, but ignore. */ 123421afaf18SBorislav Petkov if (severity == MCE_NO_SEVERITY) 123521afaf18SBorislav Petkov continue; 123621afaf18SBorislav Petkov 123721afaf18SBorislav Petkov mce_read_aux(m, i); 123821afaf18SBorislav Petkov 123921afaf18SBorislav Petkov /* assuming valid severity level != 0 */ 124021afaf18SBorislav Petkov m->severity = severity; 124121afaf18SBorislav Petkov 124221afaf18SBorislav Petkov mce_log(m); 124321afaf18SBorislav Petkov 124421afaf18SBorislav Petkov if (severity > *worst) { 124521afaf18SBorislav Petkov *final = *m; 124621afaf18SBorislav Petkov *worst = severity; 124721afaf18SBorislav Petkov } 124821afaf18SBorislav Petkov } 124921afaf18SBorislav Petkov 125021afaf18SBorislav Petkov /* mce_clear_state will clear *final, save locally for use later */ 125121afaf18SBorislav Petkov *m = *final; 125221afaf18SBorislav Petkov } 125321afaf18SBorislav Petkov 12545567d11cSPeter Zijlstra static void kill_me_now(struct callback_head *ch) 12555567d11cSPeter Zijlstra { 12565567d11cSPeter Zijlstra force_sig(SIGBUS); 12575567d11cSPeter Zijlstra } 12585567d11cSPeter Zijlstra 12595567d11cSPeter Zijlstra static void kill_me_maybe(struct callback_head *cb) 12605567d11cSPeter Zijlstra { 12615567d11cSPeter Zijlstra struct task_struct *p = container_of(cb, struct task_struct, mce_kill_me); 12625567d11cSPeter Zijlstra int flags = MF_ACTION_REQUIRED; 1263a3f5d80eSNaoya Horiguchi int ret; 12645567d11cSPeter Zijlstra 12655567d11cSPeter Zijlstra pr_err("Uncorrected hardware memory error in user-access at %llx", p->mce_addr); 126617fae129STony Luck 126717fae129STony Luck if (!p->mce_ripv) 12685567d11cSPeter Zijlstra flags |= MF_MUST_KILL; 12695567d11cSPeter Zijlstra 1270a3f5d80eSNaoya Horiguchi ret = memory_failure(p->mce_addr >> PAGE_SHIFT, flags); 1271a3f5d80eSNaoya Horiguchi if (!ret && !(p->mce_kflags & MCE_IN_KERNEL_COPYIN)) { 127217fae129STony Luck set_mce_nospec(p->mce_addr >> PAGE_SHIFT, p->mce_whole_page); 12731e36d9c6STony Luck sync_core(); 12745567d11cSPeter Zijlstra return; 12755567d11cSPeter Zijlstra } 12765567d11cSPeter Zijlstra 1277a3f5d80eSNaoya Horiguchi /* 1278a3f5d80eSNaoya Horiguchi * -EHWPOISON from memory_failure() means that it already sent SIGBUS 1279a3f5d80eSNaoya Horiguchi * to the current process with the proper error info, so no need to 1280a3f5d80eSNaoya Horiguchi * send SIGBUS here again. 1281a3f5d80eSNaoya Horiguchi */ 1282a3f5d80eSNaoya Horiguchi if (ret == -EHWPOISON) 1283a3f5d80eSNaoya Horiguchi return; 1284a3f5d80eSNaoya Horiguchi 128530063810STony Luck if (p->mce_vaddr != (void __user *)-1l) { 128630063810STony Luck force_sig_mceerr(BUS_MCEERR_AR, p->mce_vaddr, PAGE_SHIFT); 128730063810STony Luck } else { 12885567d11cSPeter Zijlstra pr_err("Memory error not recovered"); 12895567d11cSPeter Zijlstra kill_me_now(cb); 12905567d11cSPeter Zijlstra } 129130063810STony Luck } 12925567d11cSPeter Zijlstra 1293e1c06d23SGabriele Paoloni static void queue_task_work(struct mce *m, int kill_current_task) 1294c0ab7ffcSTony Luck { 1295c0ab7ffcSTony Luck current->mce_addr = m->addr; 1296c0ab7ffcSTony Luck current->mce_kflags = m->kflags; 1297c0ab7ffcSTony Luck current->mce_ripv = !!(m->mcgstatus & MCG_STATUS_RIPV); 1298c0ab7ffcSTony Luck current->mce_whole_page = whole_page(m); 1299c0ab7ffcSTony Luck 1300e1c06d23SGabriele Paoloni if (kill_current_task) 1301c0ab7ffcSTony Luck current->mce_kill_me.func = kill_me_now; 1302c0ab7ffcSTony Luck else 1303c0ab7ffcSTony Luck current->mce_kill_me.func = kill_me_maybe; 1304c0ab7ffcSTony Luck 130591989c70SJens Axboe task_work_add(current, ¤t->mce_kill_me, TWA_RESUME); 1306c0ab7ffcSTony Luck } 130721afaf18SBorislav Petkov 130821afaf18SBorislav Petkov /* 130921afaf18SBorislav Petkov * The actual machine check handler. This only handles real 131021afaf18SBorislav Petkov * exceptions when something got corrupted coming in through int 18. 131121afaf18SBorislav Petkov * 131221afaf18SBorislav Petkov * This is executed in NMI context not subject to normal locking rules. This 131321afaf18SBorislav Petkov * implies that most kernel services cannot be safely used. Don't even 131421afaf18SBorislav Petkov * think about putting a printk in there! 131521afaf18SBorislav Petkov * 131621afaf18SBorislav Petkov * On Intel systems this is entered on all CPUs in parallel through 131721afaf18SBorislav Petkov * MCE broadcast. However some CPUs might be broken beyond repair, 131821afaf18SBorislav Petkov * so be always careful when synchronizing with others. 131955ba18d6SAndy Lutomirski * 132055ba18d6SAndy Lutomirski * Tracing and kprobes are disabled: if we interrupted a kernel context 132155ba18d6SAndy Lutomirski * with IF=1, we need to minimize stack usage. There are also recursion 132255ba18d6SAndy Lutomirski * issues: if the machine check was due to a failure of the memory 132355ba18d6SAndy Lutomirski * backing the user stack, tracing that reads the user stack will cause 132455ba18d6SAndy Lutomirski * potentially infinite recursion. 132521afaf18SBorislav Petkov */ 13267f6fa101SIra Weiny noinstr void do_machine_check(struct pt_regs *regs) 132721afaf18SBorislav Petkov { 132821afaf18SBorislav Petkov DECLARE_BITMAP(valid_banks, MAX_NR_BANKS); 132921afaf18SBorislav Petkov DECLARE_BITMAP(toclear, MAX_NR_BANKS); 133021afaf18SBorislav Petkov struct mca_config *cfg = &mca_cfg; 133121afaf18SBorislav Petkov struct mce m, *final; 13327a8bc2b0SJan H. Schönherr char *msg = NULL; 133321afaf18SBorislav Petkov int worst = 0; 133421afaf18SBorislav Petkov 133521afaf18SBorislav Petkov /* 133621afaf18SBorislav Petkov * Establish sequential order between the CPUs entering the machine 133721afaf18SBorislav Petkov * check handler. 133821afaf18SBorislav Petkov */ 133921afaf18SBorislav Petkov int order = -1; 134021afaf18SBorislav Petkov 134121afaf18SBorislav Petkov /* 134221afaf18SBorislav Petkov * If no_way_out gets set, there is no safe way to recover from this 134321afaf18SBorislav Petkov * MCE. If mca_cfg.tolerant is cranked up, we'll try anyway. 134421afaf18SBorislav Petkov */ 134521afaf18SBorislav Petkov int no_way_out = 0; 134621afaf18SBorislav Petkov 134721afaf18SBorislav Petkov /* 1348e1c06d23SGabriele Paoloni * If kill_current_task is not set, there might be a way to recover from this 134921afaf18SBorislav Petkov * error. 135021afaf18SBorislav Petkov */ 1351e1c06d23SGabriele Paoloni int kill_current_task = 0; 135221afaf18SBorislav Petkov 135321afaf18SBorislav Petkov /* 135421afaf18SBorislav Petkov * MCEs are always local on AMD. Same is determined by MCG_STATUS_LMCES 135521afaf18SBorislav Petkov * on Intel. 135621afaf18SBorislav Petkov */ 135721afaf18SBorislav Petkov int lmce = 1; 135821afaf18SBorislav Petkov 135921afaf18SBorislav Petkov this_cpu_inc(mce_exception_count); 136021afaf18SBorislav Petkov 136121afaf18SBorislav Petkov mce_gather_info(&m, regs); 136221afaf18SBorislav Petkov m.tsc = rdtsc(); 136321afaf18SBorislav Petkov 136421afaf18SBorislav Petkov final = this_cpu_ptr(&mces_seen); 136521afaf18SBorislav Petkov *final = m; 136621afaf18SBorislav Petkov 136721afaf18SBorislav Petkov memset(valid_banks, 0, sizeof(valid_banks)); 136821afaf18SBorislav Petkov no_way_out = mce_no_way_out(&m, &msg, valid_banks, regs); 136921afaf18SBorislav Petkov 137021afaf18SBorislav Petkov barrier(); 137121afaf18SBorislav Petkov 137221afaf18SBorislav Petkov /* 137321afaf18SBorislav Petkov * When no restart IP might need to kill or panic. 137421afaf18SBorislav Petkov * Assume the worst for now, but if we find the 137521afaf18SBorislav Petkov * severity is MCE_AR_SEVERITY we have other options. 137621afaf18SBorislav Petkov */ 137721afaf18SBorislav Petkov if (!(m.mcgstatus & MCG_STATUS_RIPV)) 1378e1c06d23SGabriele Paoloni kill_current_task = (cfg->tolerant == 3) ? 0 : 1; 137921afaf18SBorislav Petkov /* 138021afaf18SBorislav Petkov * Check if this MCE is signaled to only this logical processor, 138170f0c230STony W Wang-oc * on Intel, Zhaoxin only. 138221afaf18SBorislav Petkov */ 138370f0c230STony W Wang-oc if (m.cpuvendor == X86_VENDOR_INTEL || 138470f0c230STony W Wang-oc m.cpuvendor == X86_VENDOR_ZHAOXIN) 138521afaf18SBorislav Petkov lmce = m.mcgstatus & MCG_STATUS_LMCES; 138621afaf18SBorislav Petkov 138721afaf18SBorislav Petkov /* 138821afaf18SBorislav Petkov * Local machine check may already know that we have to panic. 138921afaf18SBorislav Petkov * Broadcast machine check begins rendezvous in mce_start() 139021afaf18SBorislav Petkov * Go through all banks in exclusion of the other CPUs. This way we 139121afaf18SBorislav Petkov * don't report duplicated events on shared banks because the first one 139221afaf18SBorislav Petkov * to see it will clear it. 139321afaf18SBorislav Petkov */ 139421afaf18SBorislav Petkov if (lmce) { 13953a866b16SGabriele Paoloni if (no_way_out && cfg->tolerant < 3) 139621afaf18SBorislav Petkov mce_panic("Fatal local machine check", &m, msg); 139721afaf18SBorislav Petkov } else { 139821afaf18SBorislav Petkov order = mce_start(&no_way_out); 139921afaf18SBorislav Petkov } 140021afaf18SBorislav Petkov 140141ce0564SYouquan Song __mc_scan_banks(&m, regs, final, toclear, valid_banks, no_way_out, &worst); 140221afaf18SBorislav Petkov 140321afaf18SBorislav Petkov if (!no_way_out) 140421afaf18SBorislav Petkov mce_clear_state(toclear); 140521afaf18SBorislav Petkov 140621afaf18SBorislav Petkov /* 140721afaf18SBorislav Petkov * Do most of the synchronization with other CPUs. 140821afaf18SBorislav Petkov * When there's any problem use only local no_way_out state. 140921afaf18SBorislav Petkov */ 141021afaf18SBorislav Petkov if (!lmce) { 141125bc65d8SGabriele Paoloni if (mce_end(order) < 0) { 141225bc65d8SGabriele Paoloni if (!no_way_out) 141321afaf18SBorislav Petkov no_way_out = worst >= MCE_PANIC_SEVERITY; 1414e273e6e1SGabriele Paoloni 1415e273e6e1SGabriele Paoloni if (no_way_out && cfg->tolerant < 3) 1416e273e6e1SGabriele Paoloni mce_panic("Fatal machine check on current CPU", &m, msg); 141725bc65d8SGabriele Paoloni } 141821afaf18SBorislav Petkov } else { 141921afaf18SBorislav Petkov /* 142021afaf18SBorislav Petkov * If there was a fatal machine check we should have 142121afaf18SBorislav Petkov * already called mce_panic earlier in this function. 142221afaf18SBorislav Petkov * Since we re-read the banks, we might have found 142321afaf18SBorislav Petkov * something new. Check again to see if we found a 142421afaf18SBorislav Petkov * fatal error. We call "mce_severity()" again to 142521afaf18SBorislav Petkov * make sure we have the right "msg". 142621afaf18SBorislav Petkov */ 142721afaf18SBorislav Petkov if (worst >= MCE_PANIC_SEVERITY && mca_cfg.tolerant < 3) { 142841ce0564SYouquan Song mce_severity(&m, regs, cfg->tolerant, &msg, true); 142921afaf18SBorislav Petkov mce_panic("Local fatal machine check!", &m, msg); 143021afaf18SBorislav Petkov } 143121afaf18SBorislav Petkov } 143221afaf18SBorislav Petkov 1433e1c06d23SGabriele Paoloni if (worst != MCE_AR_SEVERITY && !kill_current_task) 14341e36d9c6STony Luck goto out; 143521afaf18SBorislav Petkov 143621afaf18SBorislav Petkov /* Fault was in user mode and we need to take some action */ 143721afaf18SBorislav Petkov if ((m.cs & 3) == 3) { 1438b052df3dSThomas Gleixner /* If this triggers there is no way to recover. Die hard. */ 1439b052df3dSThomas Gleixner BUG_ON(!on_thread_stack() || !user_mode(regs)); 144021afaf18SBorislav Petkov 1441e1c06d23SGabriele Paoloni queue_task_work(&m, kill_current_task); 1442c0ab7ffcSTony Luck 144321afaf18SBorislav Petkov } else { 14441df73b21SBorislav Petkov /* 14451df73b21SBorislav Petkov * Handle an MCE which has happened in kernel space but from 14461df73b21SBorislav Petkov * which the kernel can recover: ex_has_fault_handler() has 14471df73b21SBorislav Petkov * already verified that the rIP at which the error happened is 14481df73b21SBorislav Petkov * a rIP from which the kernel can recover (by jumping to 14491df73b21SBorislav Petkov * recovery code specified in _ASM_EXTABLE_FAULT()) and the 14501df73b21SBorislav Petkov * corresponding exception handler which would do that is the 14511df73b21SBorislav Petkov * proper one. 14521df73b21SBorislav Petkov */ 14531df73b21SBorislav Petkov if (m.kflags & MCE_IN_KERNEL_RECOV) { 14548cd501c1SThomas Gleixner if (!fixup_exception(regs, X86_TRAP_MC, 0, 0)) 14552d806d07SJan H. Schönherr mce_panic("Failed kernel mode recovery", &m, msg); 145621afaf18SBorislav Petkov } 1457c0ab7ffcSTony Luck 1458c0ab7ffcSTony Luck if (m.kflags & MCE_IN_KERNEL_COPYIN) 1459e1c06d23SGabriele Paoloni queue_task_work(&m, kill_current_task); 14601df73b21SBorislav Petkov } 14611e36d9c6STony Luck out: 14621e36d9c6STony Luck mce_wrmsrl(MSR_IA32_MCG_STATUS, 0); 146321afaf18SBorislav Petkov } 146421afaf18SBorislav Petkov EXPORT_SYMBOL_GPL(do_machine_check); 146521afaf18SBorislav Petkov 146621afaf18SBorislav Petkov #ifndef CONFIG_MEMORY_FAILURE 146721afaf18SBorislav Petkov int memory_failure(unsigned long pfn, int flags) 146821afaf18SBorislav Petkov { 146921afaf18SBorislav Petkov /* mce_severity() should not hand us an ACTION_REQUIRED error */ 147021afaf18SBorislav Petkov BUG_ON(flags & MF_ACTION_REQUIRED); 147121afaf18SBorislav Petkov pr_err("Uncorrected memory error in page 0x%lx ignored\n" 147221afaf18SBorislav Petkov "Rebuild kernel with CONFIG_MEMORY_FAILURE=y for smarter handling\n", 147321afaf18SBorislav Petkov pfn); 147421afaf18SBorislav Petkov 147521afaf18SBorislav Petkov return 0; 147621afaf18SBorislav Petkov } 147721afaf18SBorislav Petkov #endif 147821afaf18SBorislav Petkov 147921afaf18SBorislav Petkov /* 148021afaf18SBorislav Petkov * Periodic polling timer for "silent" machine check errors. If the 148121afaf18SBorislav Petkov * poller finds an MCE, poll 2x faster. When the poller finds no more 148221afaf18SBorislav Petkov * errors, poll 2x slower (up to check_interval seconds). 148321afaf18SBorislav Petkov */ 148421afaf18SBorislav Petkov static unsigned long check_interval = INITIAL_CHECK_INTERVAL; 148521afaf18SBorislav Petkov 148621afaf18SBorislav Petkov static DEFINE_PER_CPU(unsigned long, mce_next_interval); /* in jiffies */ 148721afaf18SBorislav Petkov static DEFINE_PER_CPU(struct timer_list, mce_timer); 148821afaf18SBorislav Petkov 148921afaf18SBorislav Petkov static unsigned long mce_adjust_timer_default(unsigned long interval) 149021afaf18SBorislav Petkov { 149121afaf18SBorislav Petkov return interval; 149221afaf18SBorislav Petkov } 149321afaf18SBorislav Petkov 149421afaf18SBorislav Petkov static unsigned long (*mce_adjust_timer)(unsigned long interval) = mce_adjust_timer_default; 149521afaf18SBorislav Petkov 149621afaf18SBorislav Petkov static void __start_timer(struct timer_list *t, unsigned long interval) 149721afaf18SBorislav Petkov { 149821afaf18SBorislav Petkov unsigned long when = jiffies + interval; 149921afaf18SBorislav Petkov unsigned long flags; 150021afaf18SBorislav Petkov 150121afaf18SBorislav Petkov local_irq_save(flags); 150221afaf18SBorislav Petkov 150321afaf18SBorislav Petkov if (!timer_pending(t) || time_before(when, t->expires)) 150421afaf18SBorislav Petkov mod_timer(t, round_jiffies(when)); 150521afaf18SBorislav Petkov 150621afaf18SBorislav Petkov local_irq_restore(flags); 150721afaf18SBorislav Petkov } 150821afaf18SBorislav Petkov 150921afaf18SBorislav Petkov static void mce_timer_fn(struct timer_list *t) 151021afaf18SBorislav Petkov { 151121afaf18SBorislav Petkov struct timer_list *cpu_t = this_cpu_ptr(&mce_timer); 151221afaf18SBorislav Petkov unsigned long iv; 151321afaf18SBorislav Petkov 151421afaf18SBorislav Petkov WARN_ON(cpu_t != t); 151521afaf18SBorislav Petkov 151621afaf18SBorislav Petkov iv = __this_cpu_read(mce_next_interval); 151721afaf18SBorislav Petkov 151821afaf18SBorislav Petkov if (mce_available(this_cpu_ptr(&cpu_info))) { 151921afaf18SBorislav Petkov machine_check_poll(0, this_cpu_ptr(&mce_poll_banks)); 152021afaf18SBorislav Petkov 152121afaf18SBorislav Petkov if (mce_intel_cmci_poll()) { 152221afaf18SBorislav Petkov iv = mce_adjust_timer(iv); 152321afaf18SBorislav Petkov goto done; 152421afaf18SBorislav Petkov } 152521afaf18SBorislav Petkov } 152621afaf18SBorislav Petkov 152721afaf18SBorislav Petkov /* 152821afaf18SBorislav Petkov * Alert userspace if needed. If we logged an MCE, reduce the polling 152921afaf18SBorislav Petkov * interval, otherwise increase the polling interval. 153021afaf18SBorislav Petkov */ 153121afaf18SBorislav Petkov if (mce_notify_irq()) 153221afaf18SBorislav Petkov iv = max(iv / 2, (unsigned long) HZ/100); 153321afaf18SBorislav Petkov else 153421afaf18SBorislav Petkov iv = min(iv * 2, round_jiffies_relative(check_interval * HZ)); 153521afaf18SBorislav Petkov 153621afaf18SBorislav Petkov done: 153721afaf18SBorislav Petkov __this_cpu_write(mce_next_interval, iv); 153821afaf18SBorislav Petkov __start_timer(t, iv); 153921afaf18SBorislav Petkov } 154021afaf18SBorislav Petkov 154121afaf18SBorislav Petkov /* 154221afaf18SBorislav Petkov * Ensure that the timer is firing in @interval from now. 154321afaf18SBorislav Petkov */ 154421afaf18SBorislav Petkov void mce_timer_kick(unsigned long interval) 154521afaf18SBorislav Petkov { 154621afaf18SBorislav Petkov struct timer_list *t = this_cpu_ptr(&mce_timer); 154721afaf18SBorislav Petkov unsigned long iv = __this_cpu_read(mce_next_interval); 154821afaf18SBorislav Petkov 154921afaf18SBorislav Petkov __start_timer(t, interval); 155021afaf18SBorislav Petkov 155121afaf18SBorislav Petkov if (interval < iv) 155221afaf18SBorislav Petkov __this_cpu_write(mce_next_interval, interval); 155321afaf18SBorislav Petkov } 155421afaf18SBorislav Petkov 155521afaf18SBorislav Petkov /* Must not be called in IRQ context where del_timer_sync() can deadlock */ 155621afaf18SBorislav Petkov static void mce_timer_delete_all(void) 155721afaf18SBorislav Petkov { 155821afaf18SBorislav Petkov int cpu; 155921afaf18SBorislav Petkov 156021afaf18SBorislav Petkov for_each_online_cpu(cpu) 156121afaf18SBorislav Petkov del_timer_sync(&per_cpu(mce_timer, cpu)); 156221afaf18SBorislav Petkov } 156321afaf18SBorislav Petkov 156421afaf18SBorislav Petkov /* 156521afaf18SBorislav Petkov * Notify the user(s) about new machine check events. 156621afaf18SBorislav Petkov * Can be called from interrupt context, but not from machine check/NMI 156721afaf18SBorislav Petkov * context. 156821afaf18SBorislav Petkov */ 156921afaf18SBorislav Petkov int mce_notify_irq(void) 157021afaf18SBorislav Petkov { 157121afaf18SBorislav Petkov /* Not more than two messages every minute */ 157221afaf18SBorislav Petkov static DEFINE_RATELIMIT_STATE(ratelimit, 60*HZ, 2); 157321afaf18SBorislav Petkov 157421afaf18SBorislav Petkov if (test_and_clear_bit(0, &mce_need_notify)) { 157521afaf18SBorislav Petkov mce_work_trigger(); 157621afaf18SBorislav Petkov 157721afaf18SBorislav Petkov if (__ratelimit(&ratelimit)) 157821afaf18SBorislav Petkov pr_info(HW_ERR "Machine check events logged\n"); 157921afaf18SBorislav Petkov 158021afaf18SBorislav Petkov return 1; 158121afaf18SBorislav Petkov } 158221afaf18SBorislav Petkov return 0; 158321afaf18SBorislav Petkov } 158421afaf18SBorislav Petkov EXPORT_SYMBOL_GPL(mce_notify_irq); 158521afaf18SBorislav Petkov 1586b4914508SYazen Ghannam static void __mcheck_cpu_mce_banks_init(void) 158721afaf18SBorislav Petkov { 1588b4914508SYazen Ghannam struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 1589c7d314f3SYazen Ghannam u8 n_banks = this_cpu_read(mce_num_banks); 159021afaf18SBorislav Petkov int i; 159121afaf18SBorislav Petkov 1592c7d314f3SYazen Ghannam for (i = 0; i < n_banks; i++) { 159321afaf18SBorislav Petkov struct mce_bank *b = &mce_banks[i]; 159421afaf18SBorislav Petkov 1595068b053dSYazen Ghannam /* 1596068b053dSYazen Ghannam * Init them all, __mcheck_cpu_apply_quirks() is going to apply 1597068b053dSYazen Ghannam * the required vendor quirks before 1598068b053dSYazen Ghannam * __mcheck_cpu_init_clear_banks() does the final bank setup. 1599068b053dSYazen Ghannam */ 160021afaf18SBorislav Petkov b->ctl = -1ULL; 160177080929SKaixu Xia b->init = true; 160221afaf18SBorislav Petkov } 160321afaf18SBorislav Petkov } 160421afaf18SBorislav Petkov 160521afaf18SBorislav Petkov /* 160621afaf18SBorislav Petkov * Initialize Machine Checks for a CPU. 160721afaf18SBorislav Petkov */ 1608b4914508SYazen Ghannam static void __mcheck_cpu_cap_init(void) 160921afaf18SBorislav Petkov { 161021afaf18SBorislav Petkov u64 cap; 1611006c0770SYazen Ghannam u8 b; 161221afaf18SBorislav Petkov 161321afaf18SBorislav Petkov rdmsrl(MSR_IA32_MCG_CAP, cap); 161421afaf18SBorislav Petkov 161521afaf18SBorislav Petkov b = cap & MCG_BANKCNT_MASK; 161621afaf18SBorislav Petkov 1617c7d314f3SYazen Ghannam if (b > MAX_NR_BANKS) { 1618c7d314f3SYazen Ghannam pr_warn("CPU%d: Using only %u machine check banks out of %u\n", 1619c7d314f3SYazen Ghannam smp_processor_id(), MAX_NR_BANKS, b); 1620c7d314f3SYazen Ghannam b = MAX_NR_BANKS; 1621c7d314f3SYazen Ghannam } 1622c7d314f3SYazen Ghannam 1623c7d314f3SYazen Ghannam this_cpu_write(mce_num_banks, b); 162421afaf18SBorislav Petkov 1625b4914508SYazen Ghannam __mcheck_cpu_mce_banks_init(); 162621afaf18SBorislav Petkov 162721afaf18SBorislav Petkov /* Use accurate RIP reporting if available. */ 162821afaf18SBorislav Petkov if ((cap & MCG_EXT_P) && MCG_EXT_CNT(cap) >= 9) 162921afaf18SBorislav Petkov mca_cfg.rip_msr = MSR_IA32_MCG_EIP; 163021afaf18SBorislav Petkov 163121afaf18SBorislav Petkov if (cap & MCG_SER_P) 163221afaf18SBorislav Petkov mca_cfg.ser = 1; 163321afaf18SBorislav Petkov } 163421afaf18SBorislav Petkov 163521afaf18SBorislav Petkov static void __mcheck_cpu_init_generic(void) 163621afaf18SBorislav Petkov { 163721afaf18SBorislav Petkov enum mcp_flags m_fl = 0; 163821afaf18SBorislav Petkov mce_banks_t all_banks; 163921afaf18SBorislav Petkov u64 cap; 164021afaf18SBorislav Petkov 164121afaf18SBorislav Petkov if (!mca_cfg.bootlog) 164221afaf18SBorislav Petkov m_fl = MCP_DONTLOG; 164321afaf18SBorislav Petkov 164421afaf18SBorislav Petkov /* 1645*3bff147bSBorislav Petkov * Log the machine checks left over from the previous reset. Log them 1646*3bff147bSBorislav Petkov * only, do not start processing them. That will happen in mcheck_late_init() 1647*3bff147bSBorislav Petkov * when all consumers have been registered on the notifier chain. 164821afaf18SBorislav Petkov */ 164921afaf18SBorislav Petkov bitmap_fill(all_banks, MAX_NR_BANKS); 1650*3bff147bSBorislav Petkov machine_check_poll(MCP_UC | MCP_QUEUE_LOG | m_fl, &all_banks); 165121afaf18SBorislav Petkov 165221afaf18SBorislav Petkov cr4_set_bits(X86_CR4_MCE); 165321afaf18SBorislav Petkov 165421afaf18SBorislav Petkov rdmsrl(MSR_IA32_MCG_CAP, cap); 165521afaf18SBorislav Petkov if (cap & MCG_CTL_P) 165621afaf18SBorislav Petkov wrmsr(MSR_IA32_MCG_CTL, 0xffffffff, 0xffffffff); 165721afaf18SBorislav Petkov } 165821afaf18SBorislav Petkov 165921afaf18SBorislav Petkov static void __mcheck_cpu_init_clear_banks(void) 166021afaf18SBorislav Petkov { 1661b4914508SYazen Ghannam struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 166221afaf18SBorislav Petkov int i; 166321afaf18SBorislav Petkov 1664c7d314f3SYazen Ghannam for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 166521afaf18SBorislav Petkov struct mce_bank *b = &mce_banks[i]; 166621afaf18SBorislav Petkov 166721afaf18SBorislav Petkov if (!b->init) 166821afaf18SBorislav Petkov continue; 166921afaf18SBorislav Petkov wrmsrl(msr_ops.ctl(i), b->ctl); 167021afaf18SBorislav Petkov wrmsrl(msr_ops.status(i), 0); 167121afaf18SBorislav Petkov } 167221afaf18SBorislav Petkov } 167321afaf18SBorislav Petkov 167421afaf18SBorislav Petkov /* 1675068b053dSYazen Ghannam * Do a final check to see if there are any unused/RAZ banks. 1676068b053dSYazen Ghannam * 1677068b053dSYazen Ghannam * This must be done after the banks have been initialized and any quirks have 1678068b053dSYazen Ghannam * been applied. 1679068b053dSYazen Ghannam * 1680068b053dSYazen Ghannam * Do not call this from any user-initiated flows, e.g. CPU hotplug or sysfs. 1681068b053dSYazen Ghannam * Otherwise, a user who disables a bank will not be able to re-enable it 1682068b053dSYazen Ghannam * without a system reboot. 1683068b053dSYazen Ghannam */ 1684068b053dSYazen Ghannam static void __mcheck_cpu_check_banks(void) 1685068b053dSYazen Ghannam { 1686068b053dSYazen Ghannam struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 1687068b053dSYazen Ghannam u64 msrval; 1688068b053dSYazen Ghannam int i; 1689068b053dSYazen Ghannam 1690068b053dSYazen Ghannam for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 1691068b053dSYazen Ghannam struct mce_bank *b = &mce_banks[i]; 1692068b053dSYazen Ghannam 1693068b053dSYazen Ghannam if (!b->init) 1694068b053dSYazen Ghannam continue; 1695068b053dSYazen Ghannam 1696068b053dSYazen Ghannam rdmsrl(msr_ops.ctl(i), msrval); 1697068b053dSYazen Ghannam b->init = !!msrval; 1698068b053dSYazen Ghannam } 1699068b053dSYazen Ghannam } 1700068b053dSYazen Ghannam 1701068b053dSYazen Ghannam /* 170221afaf18SBorislav Petkov * During IFU recovery Sandy Bridge -EP4S processors set the RIPV and 170321afaf18SBorislav Petkov * EIPV bits in MCG_STATUS to zero on the affected logical processor (SDM 170421afaf18SBorislav Petkov * Vol 3B Table 15-20). But this confuses both the code that determines 170521afaf18SBorislav Petkov * whether the machine check occurred in kernel or user mode, and also 170621afaf18SBorislav Petkov * the severity assessment code. Pretend that EIPV was set, and take the 170721afaf18SBorislav Petkov * ip/cs values from the pt_regs that mce_gather_info() ignored earlier. 170821afaf18SBorislav Petkov */ 170921afaf18SBorislav Petkov static void quirk_sandybridge_ifu(int bank, struct mce *m, struct pt_regs *regs) 171021afaf18SBorislav Petkov { 171121afaf18SBorislav Petkov if (bank != 0) 171221afaf18SBorislav Petkov return; 171321afaf18SBorislav Petkov if ((m->mcgstatus & (MCG_STATUS_EIPV|MCG_STATUS_RIPV)) != 0) 171421afaf18SBorislav Petkov return; 171521afaf18SBorislav Petkov if ((m->status & (MCI_STATUS_OVER|MCI_STATUS_UC| 171621afaf18SBorislav Petkov MCI_STATUS_EN|MCI_STATUS_MISCV|MCI_STATUS_ADDRV| 171721afaf18SBorislav Petkov MCI_STATUS_PCC|MCI_STATUS_S|MCI_STATUS_AR| 171821afaf18SBorislav Petkov MCACOD)) != 171921afaf18SBorislav Petkov (MCI_STATUS_UC|MCI_STATUS_EN| 172021afaf18SBorislav Petkov MCI_STATUS_MISCV|MCI_STATUS_ADDRV|MCI_STATUS_S| 172121afaf18SBorislav Petkov MCI_STATUS_AR|MCACOD_INSTR)) 172221afaf18SBorislav Petkov return; 172321afaf18SBorislav Petkov 172421afaf18SBorislav Petkov m->mcgstatus |= MCG_STATUS_EIPV; 172521afaf18SBorislav Petkov m->ip = regs->ip; 172621afaf18SBorislav Petkov m->cs = regs->cs; 172721afaf18SBorislav Petkov } 172821afaf18SBorislav Petkov 172921afaf18SBorislav Petkov /* Add per CPU specific workarounds here */ 173021afaf18SBorislav Petkov static int __mcheck_cpu_apply_quirks(struct cpuinfo_x86 *c) 173121afaf18SBorislav Petkov { 1732b4914508SYazen Ghannam struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 173321afaf18SBorislav Petkov struct mca_config *cfg = &mca_cfg; 173421afaf18SBorislav Petkov 173521afaf18SBorislav Petkov if (c->x86_vendor == X86_VENDOR_UNKNOWN) { 173621afaf18SBorislav Petkov pr_info("unknown CPU type - not enabling MCE support\n"); 173721afaf18SBorislav Petkov return -EOPNOTSUPP; 173821afaf18SBorislav Petkov } 173921afaf18SBorislav Petkov 174021afaf18SBorislav Petkov /* This should be disabled by the BIOS, but isn't always */ 174121afaf18SBorislav Petkov if (c->x86_vendor == X86_VENDOR_AMD) { 1742c7d314f3SYazen Ghannam if (c->x86 == 15 && this_cpu_read(mce_num_banks) > 4) { 174321afaf18SBorislav Petkov /* 174421afaf18SBorislav Petkov * disable GART TBL walk error reporting, which 174521afaf18SBorislav Petkov * trips off incorrectly with the IOMMU & 3ware 174621afaf18SBorislav Petkov * & Cerberus: 174721afaf18SBorislav Petkov */ 174821afaf18SBorislav Petkov clear_bit(10, (unsigned long *)&mce_banks[4].ctl); 174921afaf18SBorislav Petkov } 175021afaf18SBorislav Petkov if (c->x86 < 0x11 && cfg->bootlog < 0) { 175121afaf18SBorislav Petkov /* 175221afaf18SBorislav Petkov * Lots of broken BIOS around that don't clear them 175321afaf18SBorislav Petkov * by default and leave crap in there. Don't log: 175421afaf18SBorislav Petkov */ 175521afaf18SBorislav Petkov cfg->bootlog = 0; 175621afaf18SBorislav Petkov } 175721afaf18SBorislav Petkov /* 175821afaf18SBorislav Petkov * Various K7s with broken bank 0 around. Always disable 175921afaf18SBorislav Petkov * by default. 176021afaf18SBorislav Petkov */ 1761c7d314f3SYazen Ghannam if (c->x86 == 6 && this_cpu_read(mce_num_banks) > 0) 176221afaf18SBorislav Petkov mce_banks[0].ctl = 0; 176321afaf18SBorislav Petkov 176421afaf18SBorislav Petkov /* 176521afaf18SBorislav Petkov * overflow_recov is supported for F15h Models 00h-0fh 176621afaf18SBorislav Petkov * even though we don't have a CPUID bit for it. 176721afaf18SBorislav Petkov */ 176821afaf18SBorislav Petkov if (c->x86 == 0x15 && c->x86_model <= 0xf) 176921afaf18SBorislav Petkov mce_flags.overflow_recov = 1; 177021afaf18SBorislav Petkov 177121afaf18SBorislav Petkov } 177221afaf18SBorislav Petkov 177321afaf18SBorislav Petkov if (c->x86_vendor == X86_VENDOR_INTEL) { 177421afaf18SBorislav Petkov /* 177521afaf18SBorislav Petkov * SDM documents that on family 6 bank 0 should not be written 177621afaf18SBorislav Petkov * because it aliases to another special BIOS controlled 177721afaf18SBorislav Petkov * register. 177821afaf18SBorislav Petkov * But it's not aliased anymore on model 0x1a+ 177921afaf18SBorislav Petkov * Don't ignore bank 0 completely because there could be a 178021afaf18SBorislav Petkov * valid event later, merely don't write CTL0. 178121afaf18SBorislav Petkov */ 178221afaf18SBorislav Petkov 1783c7d314f3SYazen Ghannam if (c->x86 == 6 && c->x86_model < 0x1A && this_cpu_read(mce_num_banks) > 0) 178477080929SKaixu Xia mce_banks[0].init = false; 178521afaf18SBorislav Petkov 178621afaf18SBorislav Petkov /* 178721afaf18SBorislav Petkov * All newer Intel systems support MCE broadcasting. Enable 178821afaf18SBorislav Petkov * synchronization with a one second timeout. 178921afaf18SBorislav Petkov */ 179021afaf18SBorislav Petkov if ((c->x86 > 6 || (c->x86 == 6 && c->x86_model >= 0xe)) && 179121afaf18SBorislav Petkov cfg->monarch_timeout < 0) 179221afaf18SBorislav Petkov cfg->monarch_timeout = USEC_PER_SEC; 179321afaf18SBorislav Petkov 179421afaf18SBorislav Petkov /* 179521afaf18SBorislav Petkov * There are also broken BIOSes on some Pentium M and 179621afaf18SBorislav Petkov * earlier systems: 179721afaf18SBorislav Petkov */ 179821afaf18SBorislav Petkov if (c->x86 == 6 && c->x86_model <= 13 && cfg->bootlog < 0) 179921afaf18SBorislav Petkov cfg->bootlog = 0; 180021afaf18SBorislav Petkov 180121afaf18SBorislav Petkov if (c->x86 == 6 && c->x86_model == 45) 180221afaf18SBorislav Petkov quirk_no_way_out = quirk_sandybridge_ifu; 180321afaf18SBorislav Petkov } 18046e898d2bSTony W Wang-oc 18056e898d2bSTony W Wang-oc if (c->x86_vendor == X86_VENDOR_ZHAOXIN) { 18066e898d2bSTony W Wang-oc /* 18076e898d2bSTony W Wang-oc * All newer Zhaoxin CPUs support MCE broadcasting. Enable 18086e898d2bSTony W Wang-oc * synchronization with a one second timeout. 18096e898d2bSTony W Wang-oc */ 18106e898d2bSTony W Wang-oc if (c->x86 > 6 || (c->x86_model == 0x19 || c->x86_model == 0x1f)) { 18116e898d2bSTony W Wang-oc if (cfg->monarch_timeout < 0) 18126e898d2bSTony W Wang-oc cfg->monarch_timeout = USEC_PER_SEC; 18136e898d2bSTony W Wang-oc } 18146e898d2bSTony W Wang-oc } 18156e898d2bSTony W Wang-oc 181621afaf18SBorislav Petkov if (cfg->monarch_timeout < 0) 181721afaf18SBorislav Petkov cfg->monarch_timeout = 0; 181821afaf18SBorislav Petkov if (cfg->bootlog != 0) 181921afaf18SBorislav Petkov cfg->panic_timeout = 30; 182021afaf18SBorislav Petkov 182121afaf18SBorislav Petkov return 0; 182221afaf18SBorislav Petkov } 182321afaf18SBorislav Petkov 182421afaf18SBorislav Petkov static int __mcheck_cpu_ancient_init(struct cpuinfo_x86 *c) 182521afaf18SBorislav Petkov { 182621afaf18SBorislav Petkov if (c->x86 != 5) 182721afaf18SBorislav Petkov return 0; 182821afaf18SBorislav Petkov 182921afaf18SBorislav Petkov switch (c->x86_vendor) { 183021afaf18SBorislav Petkov case X86_VENDOR_INTEL: 183121afaf18SBorislav Petkov intel_p5_mcheck_init(c); 183221afaf18SBorislav Petkov return 1; 183321afaf18SBorislav Petkov case X86_VENDOR_CENTAUR: 183421afaf18SBorislav Petkov winchip_mcheck_init(c); 183521afaf18SBorislav Petkov return 1; 183621afaf18SBorislav Petkov default: 183721afaf18SBorislav Petkov return 0; 183821afaf18SBorislav Petkov } 183921afaf18SBorislav Petkov 184021afaf18SBorislav Petkov return 0; 184121afaf18SBorislav Petkov } 184221afaf18SBorislav Petkov 184321afaf18SBorislav Petkov /* 184421afaf18SBorislav Petkov * Init basic CPU features needed for early decoding of MCEs. 184521afaf18SBorislav Petkov */ 184621afaf18SBorislav Petkov static void __mcheck_cpu_init_early(struct cpuinfo_x86 *c) 184721afaf18SBorislav Petkov { 184821afaf18SBorislav Petkov if (c->x86_vendor == X86_VENDOR_AMD || c->x86_vendor == X86_VENDOR_HYGON) { 184921afaf18SBorislav Petkov mce_flags.overflow_recov = !!cpu_has(c, X86_FEATURE_OVERFLOW_RECOV); 185021afaf18SBorislav Petkov mce_flags.succor = !!cpu_has(c, X86_FEATURE_SUCCOR); 185121afaf18SBorislav Petkov mce_flags.smca = !!cpu_has(c, X86_FEATURE_SMCA); 1852c9bf318fSThomas Gleixner mce_flags.amd_threshold = 1; 185321afaf18SBorislav Petkov 185421afaf18SBorislav Petkov if (mce_flags.smca) { 185521afaf18SBorislav Petkov msr_ops.ctl = smca_ctl_reg; 185621afaf18SBorislav Petkov msr_ops.status = smca_status_reg; 185721afaf18SBorislav Petkov msr_ops.addr = smca_addr_reg; 185821afaf18SBorislav Petkov msr_ops.misc = smca_misc_reg; 185921afaf18SBorislav Petkov } 186021afaf18SBorislav Petkov } 186121afaf18SBorislav Petkov } 186221afaf18SBorislav Petkov 186321afaf18SBorislav Petkov static void mce_centaur_feature_init(struct cpuinfo_x86 *c) 186421afaf18SBorislav Petkov { 186521afaf18SBorislav Petkov struct mca_config *cfg = &mca_cfg; 186621afaf18SBorislav Petkov 186721afaf18SBorislav Petkov /* 186821afaf18SBorislav Petkov * All newer Centaur CPUs support MCE broadcasting. Enable 186921afaf18SBorislav Petkov * synchronization with a one second timeout. 187021afaf18SBorislav Petkov */ 187121afaf18SBorislav Petkov if ((c->x86 == 6 && c->x86_model == 0xf && c->x86_stepping >= 0xe) || 187221afaf18SBorislav Petkov c->x86 > 6) { 187321afaf18SBorislav Petkov if (cfg->monarch_timeout < 0) 187421afaf18SBorislav Petkov cfg->monarch_timeout = USEC_PER_SEC; 187521afaf18SBorislav Petkov } 187621afaf18SBorislav Petkov } 187721afaf18SBorislav Petkov 18785a3d56a0STony W Wang-oc static void mce_zhaoxin_feature_init(struct cpuinfo_x86 *c) 18795a3d56a0STony W Wang-oc { 18805a3d56a0STony W Wang-oc struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 18815a3d56a0STony W Wang-oc 18825a3d56a0STony W Wang-oc /* 18835a3d56a0STony W Wang-oc * These CPUs have MCA bank 8 which reports only one error type called 18845a3d56a0STony W Wang-oc * SVAD (System View Address Decoder). The reporting of that error is 18855a3d56a0STony W Wang-oc * controlled by IA32_MC8.CTL.0. 18865a3d56a0STony W Wang-oc * 18875a3d56a0STony W Wang-oc * If enabled, prefetching on these CPUs will cause SVAD MCE when 18885a3d56a0STony W Wang-oc * virtual machines start and result in a system panic. Always disable 18895a3d56a0STony W Wang-oc * bank 8 SVAD error by default. 18905a3d56a0STony W Wang-oc */ 18915a3d56a0STony W Wang-oc if ((c->x86 == 7 && c->x86_model == 0x1b) || 18925a3d56a0STony W Wang-oc (c->x86_model == 0x19 || c->x86_model == 0x1f)) { 18935a3d56a0STony W Wang-oc if (this_cpu_read(mce_num_banks) > 8) 18945a3d56a0STony W Wang-oc mce_banks[8].ctl = 0; 18955a3d56a0STony W Wang-oc } 18965a3d56a0STony W Wang-oc 18975a3d56a0STony W Wang-oc intel_init_cmci(); 189870f0c230STony W Wang-oc intel_init_lmce(); 18995a3d56a0STony W Wang-oc mce_adjust_timer = cmci_intel_adjust_timer; 19005a3d56a0STony W Wang-oc } 19015a3d56a0STony W Wang-oc 190270f0c230STony W Wang-oc static void mce_zhaoxin_feature_clear(struct cpuinfo_x86 *c) 190370f0c230STony W Wang-oc { 190470f0c230STony W Wang-oc intel_clear_lmce(); 190570f0c230STony W Wang-oc } 190670f0c230STony W Wang-oc 190721afaf18SBorislav Petkov static void __mcheck_cpu_init_vendor(struct cpuinfo_x86 *c) 190821afaf18SBorislav Petkov { 190921afaf18SBorislav Petkov switch (c->x86_vendor) { 191021afaf18SBorislav Petkov case X86_VENDOR_INTEL: 191121afaf18SBorislav Petkov mce_intel_feature_init(c); 191221afaf18SBorislav Petkov mce_adjust_timer = cmci_intel_adjust_timer; 191321afaf18SBorislav Petkov break; 191421afaf18SBorislav Petkov 191521afaf18SBorislav Petkov case X86_VENDOR_AMD: { 191621afaf18SBorislav Petkov mce_amd_feature_init(c); 191721afaf18SBorislav Petkov break; 191821afaf18SBorislav Petkov } 191921afaf18SBorislav Petkov 192021afaf18SBorislav Petkov case X86_VENDOR_HYGON: 192121afaf18SBorislav Petkov mce_hygon_feature_init(c); 192221afaf18SBorislav Petkov break; 192321afaf18SBorislav Petkov 192421afaf18SBorislav Petkov case X86_VENDOR_CENTAUR: 192521afaf18SBorislav Petkov mce_centaur_feature_init(c); 192621afaf18SBorislav Petkov break; 192721afaf18SBorislav Petkov 19285a3d56a0STony W Wang-oc case X86_VENDOR_ZHAOXIN: 19295a3d56a0STony W Wang-oc mce_zhaoxin_feature_init(c); 19305a3d56a0STony W Wang-oc break; 19315a3d56a0STony W Wang-oc 193221afaf18SBorislav Petkov default: 193321afaf18SBorislav Petkov break; 193421afaf18SBorislav Petkov } 193521afaf18SBorislav Petkov } 193621afaf18SBorislav Petkov 193721afaf18SBorislav Petkov static void __mcheck_cpu_clear_vendor(struct cpuinfo_x86 *c) 193821afaf18SBorislav Petkov { 193921afaf18SBorislav Petkov switch (c->x86_vendor) { 194021afaf18SBorislav Petkov case X86_VENDOR_INTEL: 194121afaf18SBorislav Petkov mce_intel_feature_clear(c); 194221afaf18SBorislav Petkov break; 194370f0c230STony W Wang-oc 194470f0c230STony W Wang-oc case X86_VENDOR_ZHAOXIN: 194570f0c230STony W Wang-oc mce_zhaoxin_feature_clear(c); 194670f0c230STony W Wang-oc break; 194770f0c230STony W Wang-oc 194821afaf18SBorislav Petkov default: 194921afaf18SBorislav Petkov break; 195021afaf18SBorislav Petkov } 195121afaf18SBorislav Petkov } 195221afaf18SBorislav Petkov 195321afaf18SBorislav Petkov static void mce_start_timer(struct timer_list *t) 195421afaf18SBorislav Petkov { 195521afaf18SBorislav Petkov unsigned long iv = check_interval * HZ; 195621afaf18SBorislav Petkov 195721afaf18SBorislav Petkov if (mca_cfg.ignore_ce || !iv) 195821afaf18SBorislav Petkov return; 195921afaf18SBorislav Petkov 196021afaf18SBorislav Petkov this_cpu_write(mce_next_interval, iv); 196121afaf18SBorislav Petkov __start_timer(t, iv); 196221afaf18SBorislav Petkov } 196321afaf18SBorislav Petkov 196421afaf18SBorislav Petkov static void __mcheck_cpu_setup_timer(void) 196521afaf18SBorislav Petkov { 196621afaf18SBorislav Petkov struct timer_list *t = this_cpu_ptr(&mce_timer); 196721afaf18SBorislav Petkov 196821afaf18SBorislav Petkov timer_setup(t, mce_timer_fn, TIMER_PINNED); 196921afaf18SBorislav Petkov } 197021afaf18SBorislav Petkov 197121afaf18SBorislav Petkov static void __mcheck_cpu_init_timer(void) 197221afaf18SBorislav Petkov { 197321afaf18SBorislav Petkov struct timer_list *t = this_cpu_ptr(&mce_timer); 197421afaf18SBorislav Petkov 197521afaf18SBorislav Petkov timer_setup(t, mce_timer_fn, TIMER_PINNED); 197621afaf18SBorislav Petkov mce_start_timer(t); 197721afaf18SBorislav Petkov } 197821afaf18SBorislav Petkov 197945d4b7b9SYazen Ghannam bool filter_mce(struct mce *m) 198045d4b7b9SYazen Ghannam { 198171a84402SYazen Ghannam if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) 198271a84402SYazen Ghannam return amd_filter_mce(m); 19832976908eSPrarit Bhargava if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) 19842976908eSPrarit Bhargava return intel_filter_mce(m); 198571a84402SYazen Ghannam 198645d4b7b9SYazen Ghannam return false; 198745d4b7b9SYazen Ghannam } 198845d4b7b9SYazen Ghannam 198921afaf18SBorislav Petkov /* Handle unconfigured int18 (should never happen) */ 1990865d3a9aSThomas Gleixner static noinstr void unexpected_machine_check(struct pt_regs *regs) 199121afaf18SBorislav Petkov { 1992865d3a9aSThomas Gleixner instrumentation_begin(); 199321afaf18SBorislav Petkov pr_err("CPU#%d: Unexpected int18 (Machine Check)\n", 199421afaf18SBorislav Petkov smp_processor_id()); 1995865d3a9aSThomas Gleixner instrumentation_end(); 199621afaf18SBorislav Petkov } 199721afaf18SBorislav Petkov 199821afaf18SBorislav Petkov /* Call the installed machine check handler for this CPU setup. */ 19998cd501c1SThomas Gleixner void (*machine_check_vector)(struct pt_regs *) = unexpected_machine_check; 200021afaf18SBorislav Petkov 20014c0dcd83SThomas Gleixner static __always_inline void exc_machine_check_kernel(struct pt_regs *regs) 200221afaf18SBorislav Petkov { 2003b6be002bSThomas Gleixner irqentry_state_t irq_state; 2004bc21a291SThomas Gleixner 200513cbc0cdSAndy Lutomirski WARN_ON_ONCE(user_mode(regs)); 200613cbc0cdSAndy Lutomirski 20074c0dcd83SThomas Gleixner /* 20084c0dcd83SThomas Gleixner * Only required when from kernel mode. See 20094c0dcd83SThomas Gleixner * mce_check_crashing_cpu() for details. 20104c0dcd83SThomas Gleixner */ 201194a46d31SThomas Gleixner if (machine_check_vector == do_machine_check && 201294a46d31SThomas Gleixner mce_check_crashing_cpu()) 201394a46d31SThomas Gleixner return; 201494a46d31SThomas Gleixner 2015b6be002bSThomas Gleixner irq_state = irqentry_nmi_enter(regs); 2016865d3a9aSThomas Gleixner /* 2017865d3a9aSThomas Gleixner * The call targets are marked noinstr, but objtool can't figure 2018865d3a9aSThomas Gleixner * that out because it's an indirect call. Annotate it. 2019865d3a9aSThomas Gleixner */ 2020865d3a9aSThomas Gleixner instrumentation_begin(); 202173749536SPeter Zijlstra 20228cd501c1SThomas Gleixner machine_check_vector(regs); 202373749536SPeter Zijlstra 2024865d3a9aSThomas Gleixner instrumentation_end(); 2025b6be002bSThomas Gleixner irqentry_nmi_exit(regs, irq_state); 202621afaf18SBorislav Petkov } 202721afaf18SBorislav Petkov 20284c0dcd83SThomas Gleixner static __always_inline void exc_machine_check_user(struct pt_regs *regs) 20294c0dcd83SThomas Gleixner { 2030517e4992SThomas Gleixner irqentry_enter_from_user_mode(regs); 2031865d3a9aSThomas Gleixner instrumentation_begin(); 203273749536SPeter Zijlstra 20334c0dcd83SThomas Gleixner machine_check_vector(regs); 203473749536SPeter Zijlstra 2035865d3a9aSThomas Gleixner instrumentation_end(); 2036517e4992SThomas Gleixner irqentry_exit_to_user_mode(regs); 20374c0dcd83SThomas Gleixner } 20384c0dcd83SThomas Gleixner 20394c0dcd83SThomas Gleixner #ifdef CONFIG_X86_64 20404c0dcd83SThomas Gleixner /* MCE hit kernel mode */ 20414c0dcd83SThomas Gleixner DEFINE_IDTENTRY_MCE(exc_machine_check) 20424c0dcd83SThomas Gleixner { 2043cd840e42SPeter Zijlstra unsigned long dr7; 2044cd840e42SPeter Zijlstra 2045cd840e42SPeter Zijlstra dr7 = local_db_save(); 20464c0dcd83SThomas Gleixner exc_machine_check_kernel(regs); 2047cd840e42SPeter Zijlstra local_db_restore(dr7); 20484c0dcd83SThomas Gleixner } 20494c0dcd83SThomas Gleixner 20504c0dcd83SThomas Gleixner /* The user mode variant. */ 20514c0dcd83SThomas Gleixner DEFINE_IDTENTRY_MCE_USER(exc_machine_check) 20524c0dcd83SThomas Gleixner { 2053cd840e42SPeter Zijlstra unsigned long dr7; 2054cd840e42SPeter Zijlstra 2055cd840e42SPeter Zijlstra dr7 = local_db_save(); 20564c0dcd83SThomas Gleixner exc_machine_check_user(regs); 2057cd840e42SPeter Zijlstra local_db_restore(dr7); 20584c0dcd83SThomas Gleixner } 20594c0dcd83SThomas Gleixner #else 20604c0dcd83SThomas Gleixner /* 32bit unified entry point */ 206113cbc0cdSAndy Lutomirski DEFINE_IDTENTRY_RAW(exc_machine_check) 20624c0dcd83SThomas Gleixner { 2063cd840e42SPeter Zijlstra unsigned long dr7; 2064cd840e42SPeter Zijlstra 2065cd840e42SPeter Zijlstra dr7 = local_db_save(); 20664c0dcd83SThomas Gleixner if (user_mode(regs)) 20674c0dcd83SThomas Gleixner exc_machine_check_user(regs); 20684c0dcd83SThomas Gleixner else 20694c0dcd83SThomas Gleixner exc_machine_check_kernel(regs); 2070cd840e42SPeter Zijlstra local_db_restore(dr7); 20714c0dcd83SThomas Gleixner } 20724c0dcd83SThomas Gleixner #endif 207321afaf18SBorislav Petkov 207421afaf18SBorislav Petkov /* 207521afaf18SBorislav Petkov * Called for each booted CPU to set up machine checks. 207621afaf18SBorislav Petkov * Must be called with preempt off: 207721afaf18SBorislav Petkov */ 207821afaf18SBorislav Petkov void mcheck_cpu_init(struct cpuinfo_x86 *c) 207921afaf18SBorislav Petkov { 208021afaf18SBorislav Petkov if (mca_cfg.disabled) 208121afaf18SBorislav Petkov return; 208221afaf18SBorislav Petkov 208321afaf18SBorislav Petkov if (__mcheck_cpu_ancient_init(c)) 208421afaf18SBorislav Petkov return; 208521afaf18SBorislav Petkov 208621afaf18SBorislav Petkov if (!mce_available(c)) 208721afaf18SBorislav Petkov return; 208821afaf18SBorislav Petkov 2089b4914508SYazen Ghannam __mcheck_cpu_cap_init(); 2090b4914508SYazen Ghannam 2091b4914508SYazen Ghannam if (__mcheck_cpu_apply_quirks(c) < 0) { 209221afaf18SBorislav Petkov mca_cfg.disabled = 1; 209321afaf18SBorislav Petkov return; 209421afaf18SBorislav Petkov } 209521afaf18SBorislav Petkov 209621afaf18SBorislav Petkov if (mce_gen_pool_init()) { 209721afaf18SBorislav Petkov mca_cfg.disabled = 1; 209821afaf18SBorislav Petkov pr_emerg("Couldn't allocate MCE records pool!\n"); 209921afaf18SBorislav Petkov return; 210021afaf18SBorislav Petkov } 210121afaf18SBorislav Petkov 210221afaf18SBorislav Petkov machine_check_vector = do_machine_check; 210321afaf18SBorislav Petkov 210421afaf18SBorislav Petkov __mcheck_cpu_init_early(c); 210521afaf18SBorislav Petkov __mcheck_cpu_init_generic(); 210621afaf18SBorislav Petkov __mcheck_cpu_init_vendor(c); 210721afaf18SBorislav Petkov __mcheck_cpu_init_clear_banks(); 2108068b053dSYazen Ghannam __mcheck_cpu_check_banks(); 210921afaf18SBorislav Petkov __mcheck_cpu_setup_timer(); 211021afaf18SBorislav Petkov } 211121afaf18SBorislav Petkov 211221afaf18SBorislav Petkov /* 211321afaf18SBorislav Petkov * Called for each booted CPU to clear some machine checks opt-ins 211421afaf18SBorislav Petkov */ 211521afaf18SBorislav Petkov void mcheck_cpu_clear(struct cpuinfo_x86 *c) 211621afaf18SBorislav Petkov { 211721afaf18SBorislav Petkov if (mca_cfg.disabled) 211821afaf18SBorislav Petkov return; 211921afaf18SBorislav Petkov 212021afaf18SBorislav Petkov if (!mce_available(c)) 212121afaf18SBorislav Petkov return; 212221afaf18SBorislav Petkov 212321afaf18SBorislav Petkov /* 212421afaf18SBorislav Petkov * Possibly to clear general settings generic to x86 212521afaf18SBorislav Petkov * __mcheck_cpu_clear_generic(c); 212621afaf18SBorislav Petkov */ 212721afaf18SBorislav Petkov __mcheck_cpu_clear_vendor(c); 212821afaf18SBorislav Petkov 212921afaf18SBorislav Petkov } 213021afaf18SBorislav Petkov 213121afaf18SBorislav Petkov static void __mce_disable_bank(void *arg) 213221afaf18SBorislav Petkov { 213321afaf18SBorislav Petkov int bank = *((int *)arg); 213421afaf18SBorislav Petkov __clear_bit(bank, this_cpu_ptr(mce_poll_banks)); 213521afaf18SBorislav Petkov cmci_disable_bank(bank); 213621afaf18SBorislav Petkov } 213721afaf18SBorislav Petkov 213821afaf18SBorislav Petkov void mce_disable_bank(int bank) 213921afaf18SBorislav Petkov { 2140c7d314f3SYazen Ghannam if (bank >= this_cpu_read(mce_num_banks)) { 214121afaf18SBorislav Petkov pr_warn(FW_BUG 214221afaf18SBorislav Petkov "Ignoring request to disable invalid MCA bank %d.\n", 214321afaf18SBorislav Petkov bank); 214421afaf18SBorislav Petkov return; 214521afaf18SBorislav Petkov } 214621afaf18SBorislav Petkov set_bit(bank, mce_banks_ce_disabled); 214721afaf18SBorislav Petkov on_each_cpu(__mce_disable_bank, &bank, 1); 214821afaf18SBorislav Petkov } 214921afaf18SBorislav Petkov 215021afaf18SBorislav Petkov /* 215121afaf18SBorislav Petkov * mce=off Disables machine check 215221afaf18SBorislav Petkov * mce=no_cmci Disables CMCI 215321afaf18SBorislav Petkov * mce=no_lmce Disables LMCE 215421afaf18SBorislav Petkov * mce=dont_log_ce Clears corrected events silently, no log created for CEs. 215543505646STony Luck * mce=print_all Print all machine check logs to console 215621afaf18SBorislav Petkov * mce=ignore_ce Disables polling and CMCI, corrected events are not cleared. 215721afaf18SBorislav Petkov * mce=TOLERANCELEVEL[,monarchtimeout] (number, see above) 215821afaf18SBorislav Petkov * monarchtimeout is how long to wait for other CPUs on machine 215921afaf18SBorislav Petkov * check, or 0 to not wait 216021afaf18SBorislav Petkov * mce=bootlog Log MCEs from before booting. Disabled by default on AMD Fam10h 216121afaf18SBorislav Petkov and older. 216221afaf18SBorislav Petkov * mce=nobootlog Don't log MCEs from before booting. 216321afaf18SBorislav Petkov * mce=bios_cmci_threshold Don't program the CMCI threshold 2164ec6347bbSDan Williams * mce=recovery force enable copy_mc_fragile() 216521afaf18SBorislav Petkov */ 216621afaf18SBorislav Petkov static int __init mcheck_enable(char *str) 216721afaf18SBorislav Petkov { 216821afaf18SBorislav Petkov struct mca_config *cfg = &mca_cfg; 216921afaf18SBorislav Petkov 217021afaf18SBorislav Petkov if (*str == 0) { 217121afaf18SBorislav Petkov enable_p5_mce(); 217221afaf18SBorislav Petkov return 1; 217321afaf18SBorislav Petkov } 217421afaf18SBorislav Petkov if (*str == '=') 217521afaf18SBorislav Petkov str++; 217621afaf18SBorislav Petkov if (!strcmp(str, "off")) 217721afaf18SBorislav Petkov cfg->disabled = 1; 217821afaf18SBorislav Petkov else if (!strcmp(str, "no_cmci")) 217921afaf18SBorislav Petkov cfg->cmci_disabled = true; 218021afaf18SBorislav Petkov else if (!strcmp(str, "no_lmce")) 218121afaf18SBorislav Petkov cfg->lmce_disabled = 1; 218221afaf18SBorislav Petkov else if (!strcmp(str, "dont_log_ce")) 218321afaf18SBorislav Petkov cfg->dont_log_ce = true; 218443505646STony Luck else if (!strcmp(str, "print_all")) 218543505646STony Luck cfg->print_all = true; 218621afaf18SBorislav Petkov else if (!strcmp(str, "ignore_ce")) 218721afaf18SBorislav Petkov cfg->ignore_ce = true; 218821afaf18SBorislav Petkov else if (!strcmp(str, "bootlog") || !strcmp(str, "nobootlog")) 218921afaf18SBorislav Petkov cfg->bootlog = (str[0] == 'b'); 219021afaf18SBorislav Petkov else if (!strcmp(str, "bios_cmci_threshold")) 219121afaf18SBorislav Petkov cfg->bios_cmci_threshold = 1; 219221afaf18SBorislav Petkov else if (!strcmp(str, "recovery")) 219321afaf18SBorislav Petkov cfg->recovery = 1; 219421afaf18SBorislav Petkov else if (isdigit(str[0])) { 219521afaf18SBorislav Petkov if (get_option(&str, &cfg->tolerant) == 2) 219621afaf18SBorislav Petkov get_option(&str, &(cfg->monarch_timeout)); 219721afaf18SBorislav Petkov } else { 219821afaf18SBorislav Petkov pr_info("mce argument %s ignored. Please use /sys\n", str); 219921afaf18SBorislav Petkov return 0; 220021afaf18SBorislav Petkov } 220121afaf18SBorislav Petkov return 1; 220221afaf18SBorislav Petkov } 220321afaf18SBorislav Petkov __setup("mce", mcheck_enable); 220421afaf18SBorislav Petkov 220521afaf18SBorislav Petkov int __init mcheck_init(void) 220621afaf18SBorislav Petkov { 2207c9c6d216STony Luck mce_register_decode_chain(&early_nb); 22088438b84aSJan H. Schönherr mce_register_decode_chain(&mce_uc_nb); 220921afaf18SBorislav Petkov mce_register_decode_chain(&mce_default_nb); 221021afaf18SBorislav Petkov mcheck_vendor_init_severity(); 221121afaf18SBorislav Petkov 221221afaf18SBorislav Petkov INIT_WORK(&mce_work, mce_gen_pool_process); 221321afaf18SBorislav Petkov init_irq_work(&mce_irq_work, mce_irq_work_cb); 221421afaf18SBorislav Petkov 221521afaf18SBorislav Petkov return 0; 221621afaf18SBorislav Petkov } 221721afaf18SBorislav Petkov 221821afaf18SBorislav Petkov /* 221921afaf18SBorislav Petkov * mce_syscore: PM support 222021afaf18SBorislav Petkov */ 222121afaf18SBorislav Petkov 222221afaf18SBorislav Petkov /* 222321afaf18SBorislav Petkov * Disable machine checks on suspend and shutdown. We can't really handle 222421afaf18SBorislav Petkov * them later. 222521afaf18SBorislav Petkov */ 222621afaf18SBorislav Petkov static void mce_disable_error_reporting(void) 222721afaf18SBorislav Petkov { 2228b4914508SYazen Ghannam struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 222921afaf18SBorislav Petkov int i; 223021afaf18SBorislav Petkov 2231c7d314f3SYazen Ghannam for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 223221afaf18SBorislav Petkov struct mce_bank *b = &mce_banks[i]; 223321afaf18SBorislav Petkov 223421afaf18SBorislav Petkov if (b->init) 223521afaf18SBorislav Petkov wrmsrl(msr_ops.ctl(i), 0); 223621afaf18SBorislav Petkov } 223721afaf18SBorislav Petkov return; 223821afaf18SBorislav Petkov } 223921afaf18SBorislav Petkov 224021afaf18SBorislav Petkov static void vendor_disable_error_reporting(void) 224121afaf18SBorislav Petkov { 224221afaf18SBorislav Petkov /* 22436e898d2bSTony W Wang-oc * Don't clear on Intel or AMD or Hygon or Zhaoxin CPUs. Some of these 22446e898d2bSTony W Wang-oc * MSRs are socket-wide. Disabling them for just a single offlined CPU 22456e898d2bSTony W Wang-oc * is bad, since it will inhibit reporting for all shared resources on 22466e898d2bSTony W Wang-oc * the socket like the last level cache (LLC), the integrated memory 22476e898d2bSTony W Wang-oc * controller (iMC), etc. 224821afaf18SBorislav Petkov */ 224921afaf18SBorislav Petkov if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL || 225021afaf18SBorislav Petkov boot_cpu_data.x86_vendor == X86_VENDOR_HYGON || 22516e898d2bSTony W Wang-oc boot_cpu_data.x86_vendor == X86_VENDOR_AMD || 22526e898d2bSTony W Wang-oc boot_cpu_data.x86_vendor == X86_VENDOR_ZHAOXIN) 225321afaf18SBorislav Petkov return; 225421afaf18SBorislav Petkov 225521afaf18SBorislav Petkov mce_disable_error_reporting(); 225621afaf18SBorislav Petkov } 225721afaf18SBorislav Petkov 225821afaf18SBorislav Petkov static int mce_syscore_suspend(void) 225921afaf18SBorislav Petkov { 226021afaf18SBorislav Petkov vendor_disable_error_reporting(); 226121afaf18SBorislav Petkov return 0; 226221afaf18SBorislav Petkov } 226321afaf18SBorislav Petkov 226421afaf18SBorislav Petkov static void mce_syscore_shutdown(void) 226521afaf18SBorislav Petkov { 226621afaf18SBorislav Petkov vendor_disable_error_reporting(); 226721afaf18SBorislav Petkov } 226821afaf18SBorislav Petkov 226921afaf18SBorislav Petkov /* 227021afaf18SBorislav Petkov * On resume clear all MCE state. Don't want to see leftovers from the BIOS. 227121afaf18SBorislav Petkov * Only one CPU is active at this time, the others get re-added later using 227221afaf18SBorislav Petkov * CPU hotplug: 227321afaf18SBorislav Petkov */ 227421afaf18SBorislav Petkov static void mce_syscore_resume(void) 227521afaf18SBorislav Petkov { 227621afaf18SBorislav Petkov __mcheck_cpu_init_generic(); 227721afaf18SBorislav Petkov __mcheck_cpu_init_vendor(raw_cpu_ptr(&cpu_info)); 227821afaf18SBorislav Petkov __mcheck_cpu_init_clear_banks(); 227921afaf18SBorislav Petkov } 228021afaf18SBorislav Petkov 228121afaf18SBorislav Petkov static struct syscore_ops mce_syscore_ops = { 228221afaf18SBorislav Petkov .suspend = mce_syscore_suspend, 228321afaf18SBorislav Petkov .shutdown = mce_syscore_shutdown, 228421afaf18SBorislav Petkov .resume = mce_syscore_resume, 228521afaf18SBorislav Petkov }; 228621afaf18SBorislav Petkov 228721afaf18SBorislav Petkov /* 228821afaf18SBorislav Petkov * mce_device: Sysfs support 228921afaf18SBorislav Petkov */ 229021afaf18SBorislav Petkov 229121afaf18SBorislav Petkov static void mce_cpu_restart(void *data) 229221afaf18SBorislav Petkov { 229321afaf18SBorislav Petkov if (!mce_available(raw_cpu_ptr(&cpu_info))) 229421afaf18SBorislav Petkov return; 229521afaf18SBorislav Petkov __mcheck_cpu_init_generic(); 229621afaf18SBorislav Petkov __mcheck_cpu_init_clear_banks(); 229721afaf18SBorislav Petkov __mcheck_cpu_init_timer(); 229821afaf18SBorislav Petkov } 229921afaf18SBorislav Petkov 230021afaf18SBorislav Petkov /* Reinit MCEs after user configuration changes */ 230121afaf18SBorislav Petkov static void mce_restart(void) 230221afaf18SBorislav Petkov { 230321afaf18SBorislav Petkov mce_timer_delete_all(); 230421afaf18SBorislav Petkov on_each_cpu(mce_cpu_restart, NULL, 1); 230521afaf18SBorislav Petkov } 230621afaf18SBorislav Petkov 230721afaf18SBorislav Petkov /* Toggle features for corrected errors */ 230821afaf18SBorislav Petkov static void mce_disable_cmci(void *data) 230921afaf18SBorislav Petkov { 231021afaf18SBorislav Petkov if (!mce_available(raw_cpu_ptr(&cpu_info))) 231121afaf18SBorislav Petkov return; 231221afaf18SBorislav Petkov cmci_clear(); 231321afaf18SBorislav Petkov } 231421afaf18SBorislav Petkov 231521afaf18SBorislav Petkov static void mce_enable_ce(void *all) 231621afaf18SBorislav Petkov { 231721afaf18SBorislav Petkov if (!mce_available(raw_cpu_ptr(&cpu_info))) 231821afaf18SBorislav Petkov return; 231921afaf18SBorislav Petkov cmci_reenable(); 232021afaf18SBorislav Petkov cmci_recheck(); 232121afaf18SBorislav Petkov if (all) 232221afaf18SBorislav Petkov __mcheck_cpu_init_timer(); 232321afaf18SBorislav Petkov } 232421afaf18SBorislav Petkov 232521afaf18SBorislav Petkov static struct bus_type mce_subsys = { 232621afaf18SBorislav Petkov .name = "machinecheck", 232721afaf18SBorislav Petkov .dev_name = "machinecheck", 232821afaf18SBorislav Petkov }; 232921afaf18SBorislav Petkov 233021afaf18SBorislav Petkov DEFINE_PER_CPU(struct device *, mce_device); 233121afaf18SBorislav Petkov 2332b4914508SYazen Ghannam static inline struct mce_bank_dev *attr_to_bank(struct device_attribute *attr) 233321afaf18SBorislav Petkov { 2334b4914508SYazen Ghannam return container_of(attr, struct mce_bank_dev, attr); 233521afaf18SBorislav Petkov } 233621afaf18SBorislav Petkov 233721afaf18SBorislav Petkov static ssize_t show_bank(struct device *s, struct device_attribute *attr, 233821afaf18SBorislav Petkov char *buf) 233921afaf18SBorislav Petkov { 2340b4914508SYazen Ghannam u8 bank = attr_to_bank(attr)->bank; 2341b4914508SYazen Ghannam struct mce_bank *b; 2342b4914508SYazen Ghannam 2343c7d314f3SYazen Ghannam if (bank >= per_cpu(mce_num_banks, s->id)) 2344b4914508SYazen Ghannam return -EINVAL; 2345b4914508SYazen Ghannam 2346b4914508SYazen Ghannam b = &per_cpu(mce_banks_array, s->id)[bank]; 2347b4914508SYazen Ghannam 2348068b053dSYazen Ghannam if (!b->init) 2349068b053dSYazen Ghannam return -ENODEV; 2350068b053dSYazen Ghannam 2351b4914508SYazen Ghannam return sprintf(buf, "%llx\n", b->ctl); 235221afaf18SBorislav Petkov } 235321afaf18SBorislav Petkov 235421afaf18SBorislav Petkov static ssize_t set_bank(struct device *s, struct device_attribute *attr, 235521afaf18SBorislav Petkov const char *buf, size_t size) 235621afaf18SBorislav Petkov { 2357b4914508SYazen Ghannam u8 bank = attr_to_bank(attr)->bank; 2358b4914508SYazen Ghannam struct mce_bank *b; 235921afaf18SBorislav Petkov u64 new; 236021afaf18SBorislav Petkov 236121afaf18SBorislav Petkov if (kstrtou64(buf, 0, &new) < 0) 236221afaf18SBorislav Petkov return -EINVAL; 236321afaf18SBorislav Petkov 2364c7d314f3SYazen Ghannam if (bank >= per_cpu(mce_num_banks, s->id)) 2365b4914508SYazen Ghannam return -EINVAL; 2366b4914508SYazen Ghannam 2367b4914508SYazen Ghannam b = &per_cpu(mce_banks_array, s->id)[bank]; 2368b4914508SYazen Ghannam 2369068b053dSYazen Ghannam if (!b->init) 2370068b053dSYazen Ghannam return -ENODEV; 2371068b053dSYazen Ghannam 2372b4914508SYazen Ghannam b->ctl = new; 237321afaf18SBorislav Petkov mce_restart(); 237421afaf18SBorislav Petkov 237521afaf18SBorislav Petkov return size; 237621afaf18SBorislav Petkov } 237721afaf18SBorislav Petkov 237821afaf18SBorislav Petkov static ssize_t set_ignore_ce(struct device *s, 237921afaf18SBorislav Petkov struct device_attribute *attr, 238021afaf18SBorislav Petkov const char *buf, size_t size) 238121afaf18SBorislav Petkov { 238221afaf18SBorislav Petkov u64 new; 238321afaf18SBorislav Petkov 238421afaf18SBorislav Petkov if (kstrtou64(buf, 0, &new) < 0) 238521afaf18SBorislav Petkov return -EINVAL; 238621afaf18SBorislav Petkov 238721afaf18SBorislav Petkov mutex_lock(&mce_sysfs_mutex); 238821afaf18SBorislav Petkov if (mca_cfg.ignore_ce ^ !!new) { 238921afaf18SBorislav Petkov if (new) { 239021afaf18SBorislav Petkov /* disable ce features */ 239121afaf18SBorislav Petkov mce_timer_delete_all(); 239221afaf18SBorislav Petkov on_each_cpu(mce_disable_cmci, NULL, 1); 239321afaf18SBorislav Petkov mca_cfg.ignore_ce = true; 239421afaf18SBorislav Petkov } else { 239521afaf18SBorislav Petkov /* enable ce features */ 239621afaf18SBorislav Petkov mca_cfg.ignore_ce = false; 239721afaf18SBorislav Petkov on_each_cpu(mce_enable_ce, (void *)1, 1); 239821afaf18SBorislav Petkov } 239921afaf18SBorislav Petkov } 240021afaf18SBorislav Petkov mutex_unlock(&mce_sysfs_mutex); 240121afaf18SBorislav Petkov 240221afaf18SBorislav Petkov return size; 240321afaf18SBorislav Petkov } 240421afaf18SBorislav Petkov 240521afaf18SBorislav Petkov static ssize_t set_cmci_disabled(struct device *s, 240621afaf18SBorislav Petkov struct device_attribute *attr, 240721afaf18SBorislav Petkov const char *buf, size_t size) 240821afaf18SBorislav Petkov { 240921afaf18SBorislav Petkov u64 new; 241021afaf18SBorislav Petkov 241121afaf18SBorislav Petkov if (kstrtou64(buf, 0, &new) < 0) 241221afaf18SBorislav Petkov return -EINVAL; 241321afaf18SBorislav Petkov 241421afaf18SBorislav Petkov mutex_lock(&mce_sysfs_mutex); 241521afaf18SBorislav Petkov if (mca_cfg.cmci_disabled ^ !!new) { 241621afaf18SBorislav Petkov if (new) { 241721afaf18SBorislav Petkov /* disable cmci */ 241821afaf18SBorislav Petkov on_each_cpu(mce_disable_cmci, NULL, 1); 241921afaf18SBorislav Petkov mca_cfg.cmci_disabled = true; 242021afaf18SBorislav Petkov } else { 242121afaf18SBorislav Petkov /* enable cmci */ 242221afaf18SBorislav Petkov mca_cfg.cmci_disabled = false; 242321afaf18SBorislav Petkov on_each_cpu(mce_enable_ce, NULL, 1); 242421afaf18SBorislav Petkov } 242521afaf18SBorislav Petkov } 242621afaf18SBorislav Petkov mutex_unlock(&mce_sysfs_mutex); 242721afaf18SBorislav Petkov 242821afaf18SBorislav Petkov return size; 242921afaf18SBorislav Petkov } 243021afaf18SBorislav Petkov 243121afaf18SBorislav Petkov static ssize_t store_int_with_restart(struct device *s, 243221afaf18SBorislav Petkov struct device_attribute *attr, 243321afaf18SBorislav Petkov const char *buf, size_t size) 243421afaf18SBorislav Petkov { 243521afaf18SBorislav Petkov unsigned long old_check_interval = check_interval; 243621afaf18SBorislav Petkov ssize_t ret = device_store_ulong(s, attr, buf, size); 243721afaf18SBorislav Petkov 243821afaf18SBorislav Petkov if (check_interval == old_check_interval) 243921afaf18SBorislav Petkov return ret; 244021afaf18SBorislav Petkov 244121afaf18SBorislav Petkov mutex_lock(&mce_sysfs_mutex); 244221afaf18SBorislav Petkov mce_restart(); 244321afaf18SBorislav Petkov mutex_unlock(&mce_sysfs_mutex); 244421afaf18SBorislav Petkov 244521afaf18SBorislav Petkov return ret; 244621afaf18SBorislav Petkov } 244721afaf18SBorislav Petkov 244821afaf18SBorislav Petkov static DEVICE_INT_ATTR(tolerant, 0644, mca_cfg.tolerant); 244921afaf18SBorislav Petkov static DEVICE_INT_ATTR(monarch_timeout, 0644, mca_cfg.monarch_timeout); 245021afaf18SBorislav Petkov static DEVICE_BOOL_ATTR(dont_log_ce, 0644, mca_cfg.dont_log_ce); 245143505646STony Luck static DEVICE_BOOL_ATTR(print_all, 0644, mca_cfg.print_all); 245221afaf18SBorislav Petkov 245321afaf18SBorislav Petkov static struct dev_ext_attribute dev_attr_check_interval = { 245421afaf18SBorislav Petkov __ATTR(check_interval, 0644, device_show_int, store_int_with_restart), 245521afaf18SBorislav Petkov &check_interval 245621afaf18SBorislav Petkov }; 245721afaf18SBorislav Petkov 245821afaf18SBorislav Petkov static struct dev_ext_attribute dev_attr_ignore_ce = { 245921afaf18SBorislav Petkov __ATTR(ignore_ce, 0644, device_show_bool, set_ignore_ce), 246021afaf18SBorislav Petkov &mca_cfg.ignore_ce 246121afaf18SBorislav Petkov }; 246221afaf18SBorislav Petkov 246321afaf18SBorislav Petkov static struct dev_ext_attribute dev_attr_cmci_disabled = { 246421afaf18SBorislav Petkov __ATTR(cmci_disabled, 0644, device_show_bool, set_cmci_disabled), 246521afaf18SBorislav Petkov &mca_cfg.cmci_disabled 246621afaf18SBorislav Petkov }; 246721afaf18SBorislav Petkov 246821afaf18SBorislav Petkov static struct device_attribute *mce_device_attrs[] = { 246921afaf18SBorislav Petkov &dev_attr_tolerant.attr, 247021afaf18SBorislav Petkov &dev_attr_check_interval.attr, 247121afaf18SBorislav Petkov #ifdef CONFIG_X86_MCELOG_LEGACY 247221afaf18SBorislav Petkov &dev_attr_trigger, 247321afaf18SBorislav Petkov #endif 247421afaf18SBorislav Petkov &dev_attr_monarch_timeout.attr, 247521afaf18SBorislav Petkov &dev_attr_dont_log_ce.attr, 247643505646STony Luck &dev_attr_print_all.attr, 247721afaf18SBorislav Petkov &dev_attr_ignore_ce.attr, 247821afaf18SBorislav Petkov &dev_attr_cmci_disabled.attr, 247921afaf18SBorislav Petkov NULL 248021afaf18SBorislav Petkov }; 248121afaf18SBorislav Petkov 248221afaf18SBorislav Petkov static cpumask_var_t mce_device_initialized; 248321afaf18SBorislav Petkov 248421afaf18SBorislav Petkov static void mce_device_release(struct device *dev) 248521afaf18SBorislav Petkov { 248621afaf18SBorislav Petkov kfree(dev); 248721afaf18SBorislav Petkov } 248821afaf18SBorislav Petkov 2489b4914508SYazen Ghannam /* Per CPU device init. All of the CPUs still share the same bank device: */ 249021afaf18SBorislav Petkov static int mce_device_create(unsigned int cpu) 249121afaf18SBorislav Petkov { 249221afaf18SBorislav Petkov struct device *dev; 249321afaf18SBorislav Petkov int err; 249421afaf18SBorislav Petkov int i, j; 249521afaf18SBorislav Petkov 249621afaf18SBorislav Petkov if (!mce_available(&boot_cpu_data)) 249721afaf18SBorislav Petkov return -EIO; 249821afaf18SBorislav Petkov 249921afaf18SBorislav Petkov dev = per_cpu(mce_device, cpu); 250021afaf18SBorislav Petkov if (dev) 250121afaf18SBorislav Petkov return 0; 250221afaf18SBorislav Petkov 250321afaf18SBorislav Petkov dev = kzalloc(sizeof(*dev), GFP_KERNEL); 250421afaf18SBorislav Petkov if (!dev) 250521afaf18SBorislav Petkov return -ENOMEM; 250621afaf18SBorislav Petkov dev->id = cpu; 250721afaf18SBorislav Petkov dev->bus = &mce_subsys; 250821afaf18SBorislav Petkov dev->release = &mce_device_release; 250921afaf18SBorislav Petkov 251021afaf18SBorislav Petkov err = device_register(dev); 251121afaf18SBorislav Petkov if (err) { 251221afaf18SBorislav Petkov put_device(dev); 251321afaf18SBorislav Petkov return err; 251421afaf18SBorislav Petkov } 251521afaf18SBorislav Petkov 251621afaf18SBorislav Petkov for (i = 0; mce_device_attrs[i]; i++) { 251721afaf18SBorislav Petkov err = device_create_file(dev, mce_device_attrs[i]); 251821afaf18SBorislav Petkov if (err) 251921afaf18SBorislav Petkov goto error; 252021afaf18SBorislav Petkov } 2521c7d314f3SYazen Ghannam for (j = 0; j < per_cpu(mce_num_banks, cpu); j++) { 2522b4914508SYazen Ghannam err = device_create_file(dev, &mce_bank_devs[j].attr); 252321afaf18SBorislav Petkov if (err) 252421afaf18SBorislav Petkov goto error2; 252521afaf18SBorislav Petkov } 252621afaf18SBorislav Petkov cpumask_set_cpu(cpu, mce_device_initialized); 252721afaf18SBorislav Petkov per_cpu(mce_device, cpu) = dev; 252821afaf18SBorislav Petkov 252921afaf18SBorislav Petkov return 0; 253021afaf18SBorislav Petkov error2: 253121afaf18SBorislav Petkov while (--j >= 0) 2532b4914508SYazen Ghannam device_remove_file(dev, &mce_bank_devs[j].attr); 253321afaf18SBorislav Petkov error: 253421afaf18SBorislav Petkov while (--i >= 0) 253521afaf18SBorislav Petkov device_remove_file(dev, mce_device_attrs[i]); 253621afaf18SBorislav Petkov 253721afaf18SBorislav Petkov device_unregister(dev); 253821afaf18SBorislav Petkov 253921afaf18SBorislav Petkov return err; 254021afaf18SBorislav Petkov } 254121afaf18SBorislav Petkov 254221afaf18SBorislav Petkov static void mce_device_remove(unsigned int cpu) 254321afaf18SBorislav Petkov { 254421afaf18SBorislav Petkov struct device *dev = per_cpu(mce_device, cpu); 254521afaf18SBorislav Petkov int i; 254621afaf18SBorislav Petkov 254721afaf18SBorislav Petkov if (!cpumask_test_cpu(cpu, mce_device_initialized)) 254821afaf18SBorislav Petkov return; 254921afaf18SBorislav Petkov 255021afaf18SBorislav Petkov for (i = 0; mce_device_attrs[i]; i++) 255121afaf18SBorislav Petkov device_remove_file(dev, mce_device_attrs[i]); 255221afaf18SBorislav Petkov 2553c7d314f3SYazen Ghannam for (i = 0; i < per_cpu(mce_num_banks, cpu); i++) 2554b4914508SYazen Ghannam device_remove_file(dev, &mce_bank_devs[i].attr); 255521afaf18SBorislav Petkov 255621afaf18SBorislav Petkov device_unregister(dev); 255721afaf18SBorislav Petkov cpumask_clear_cpu(cpu, mce_device_initialized); 255821afaf18SBorislav Petkov per_cpu(mce_device, cpu) = NULL; 255921afaf18SBorislav Petkov } 256021afaf18SBorislav Petkov 256121afaf18SBorislav Petkov /* Make sure there are no machine checks on offlined CPUs. */ 256221afaf18SBorislav Petkov static void mce_disable_cpu(void) 256321afaf18SBorislav Petkov { 256421afaf18SBorislav Petkov if (!mce_available(raw_cpu_ptr(&cpu_info))) 256521afaf18SBorislav Petkov return; 256621afaf18SBorislav Petkov 256721afaf18SBorislav Petkov if (!cpuhp_tasks_frozen) 256821afaf18SBorislav Petkov cmci_clear(); 256921afaf18SBorislav Petkov 257021afaf18SBorislav Petkov vendor_disable_error_reporting(); 257121afaf18SBorislav Petkov } 257221afaf18SBorislav Petkov 257321afaf18SBorislav Petkov static void mce_reenable_cpu(void) 257421afaf18SBorislav Petkov { 2575b4914508SYazen Ghannam struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 257621afaf18SBorislav Petkov int i; 257721afaf18SBorislav Petkov 257821afaf18SBorislav Petkov if (!mce_available(raw_cpu_ptr(&cpu_info))) 257921afaf18SBorislav Petkov return; 258021afaf18SBorislav Petkov 258121afaf18SBorislav Petkov if (!cpuhp_tasks_frozen) 258221afaf18SBorislav Petkov cmci_reenable(); 2583c7d314f3SYazen Ghannam for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 258421afaf18SBorislav Petkov struct mce_bank *b = &mce_banks[i]; 258521afaf18SBorislav Petkov 258621afaf18SBorislav Petkov if (b->init) 258721afaf18SBorislav Petkov wrmsrl(msr_ops.ctl(i), b->ctl); 258821afaf18SBorislav Petkov } 258921afaf18SBorislav Petkov } 259021afaf18SBorislav Petkov 259121afaf18SBorislav Petkov static int mce_cpu_dead(unsigned int cpu) 259221afaf18SBorislav Petkov { 259321afaf18SBorislav Petkov mce_intel_hcpu_update(cpu); 259421afaf18SBorislav Petkov 259521afaf18SBorislav Petkov /* intentionally ignoring frozen here */ 259621afaf18SBorislav Petkov if (!cpuhp_tasks_frozen) 259721afaf18SBorislav Petkov cmci_rediscover(); 259821afaf18SBorislav Petkov return 0; 259921afaf18SBorislav Petkov } 260021afaf18SBorislav Petkov 260121afaf18SBorislav Petkov static int mce_cpu_online(unsigned int cpu) 260221afaf18SBorislav Petkov { 260321afaf18SBorislav Petkov struct timer_list *t = this_cpu_ptr(&mce_timer); 260421afaf18SBorislav Petkov int ret; 260521afaf18SBorislav Petkov 260621afaf18SBorislav Petkov mce_device_create(cpu); 260721afaf18SBorislav Petkov 260821afaf18SBorislav Petkov ret = mce_threshold_create_device(cpu); 260921afaf18SBorislav Petkov if (ret) { 261021afaf18SBorislav Petkov mce_device_remove(cpu); 261121afaf18SBorislav Petkov return ret; 261221afaf18SBorislav Petkov } 261321afaf18SBorislav Petkov mce_reenable_cpu(); 261421afaf18SBorislav Petkov mce_start_timer(t); 261521afaf18SBorislav Petkov return 0; 261621afaf18SBorislav Petkov } 261721afaf18SBorislav Petkov 261821afaf18SBorislav Petkov static int mce_cpu_pre_down(unsigned int cpu) 261921afaf18SBorislav Petkov { 262021afaf18SBorislav Petkov struct timer_list *t = this_cpu_ptr(&mce_timer); 262121afaf18SBorislav Petkov 262221afaf18SBorislav Petkov mce_disable_cpu(); 262321afaf18SBorislav Petkov del_timer_sync(t); 262421afaf18SBorislav Petkov mce_threshold_remove_device(cpu); 262521afaf18SBorislav Petkov mce_device_remove(cpu); 262621afaf18SBorislav Petkov return 0; 262721afaf18SBorislav Petkov } 262821afaf18SBorislav Petkov 262921afaf18SBorislav Petkov static __init void mce_init_banks(void) 263021afaf18SBorislav Petkov { 263121afaf18SBorislav Petkov int i; 263221afaf18SBorislav Petkov 2633b4914508SYazen Ghannam for (i = 0; i < MAX_NR_BANKS; i++) { 2634b4914508SYazen Ghannam struct mce_bank_dev *b = &mce_bank_devs[i]; 263521afaf18SBorislav Petkov struct device_attribute *a = &b->attr; 263621afaf18SBorislav Petkov 2637b4914508SYazen Ghannam b->bank = i; 2638b4914508SYazen Ghannam 263921afaf18SBorislav Petkov sysfs_attr_init(&a->attr); 264021afaf18SBorislav Petkov a->attr.name = b->attrname; 264121afaf18SBorislav Petkov snprintf(b->attrname, ATTR_LEN, "bank%d", i); 264221afaf18SBorislav Petkov 264321afaf18SBorislav Petkov a->attr.mode = 0644; 264421afaf18SBorislav Petkov a->show = show_bank; 264521afaf18SBorislav Petkov a->store = set_bank; 264621afaf18SBorislav Petkov } 264721afaf18SBorislav Petkov } 264821afaf18SBorislav Petkov 26496e7a41c6SThomas Gleixner /* 26506e7a41c6SThomas Gleixner * When running on XEN, this initcall is ordered against the XEN mcelog 26516e7a41c6SThomas Gleixner * initcall: 26526e7a41c6SThomas Gleixner * 26536e7a41c6SThomas Gleixner * device_initcall(xen_late_init_mcelog); 26546e7a41c6SThomas Gleixner * device_initcall_sync(mcheck_init_device); 26556e7a41c6SThomas Gleixner */ 265621afaf18SBorislav Petkov static __init int mcheck_init_device(void) 265721afaf18SBorislav Petkov { 265821afaf18SBorislav Petkov int err; 265921afaf18SBorislav Petkov 266021afaf18SBorislav Petkov /* 266121afaf18SBorislav Petkov * Check if we have a spare virtual bit. This will only become 266221afaf18SBorislav Petkov * a problem if/when we move beyond 5-level page tables. 266321afaf18SBorislav Petkov */ 266421afaf18SBorislav Petkov MAYBE_BUILD_BUG_ON(__VIRTUAL_MASK_SHIFT >= 63); 266521afaf18SBorislav Petkov 266621afaf18SBorislav Petkov if (!mce_available(&boot_cpu_data)) { 266721afaf18SBorislav Petkov err = -EIO; 266821afaf18SBorislav Petkov goto err_out; 266921afaf18SBorislav Petkov } 267021afaf18SBorislav Petkov 267121afaf18SBorislav Petkov if (!zalloc_cpumask_var(&mce_device_initialized, GFP_KERNEL)) { 267221afaf18SBorislav Petkov err = -ENOMEM; 267321afaf18SBorislav Petkov goto err_out; 267421afaf18SBorislav Petkov } 267521afaf18SBorislav Petkov 267621afaf18SBorislav Petkov mce_init_banks(); 267721afaf18SBorislav Petkov 267821afaf18SBorislav Petkov err = subsys_system_register(&mce_subsys, NULL); 267921afaf18SBorislav Petkov if (err) 268021afaf18SBorislav Petkov goto err_out_mem; 268121afaf18SBorislav Petkov 268221afaf18SBorislav Petkov err = cpuhp_setup_state(CPUHP_X86_MCE_DEAD, "x86/mce:dead", NULL, 268321afaf18SBorislav Petkov mce_cpu_dead); 268421afaf18SBorislav Petkov if (err) 268521afaf18SBorislav Petkov goto err_out_mem; 268621afaf18SBorislav Petkov 26876e7a41c6SThomas Gleixner /* 26886e7a41c6SThomas Gleixner * Invokes mce_cpu_online() on all CPUs which are online when 26896e7a41c6SThomas Gleixner * the state is installed. 26906e7a41c6SThomas Gleixner */ 269121afaf18SBorislav Petkov err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/mce:online", 269221afaf18SBorislav Petkov mce_cpu_online, mce_cpu_pre_down); 269321afaf18SBorislav Petkov if (err < 0) 269421afaf18SBorislav Petkov goto err_out_online; 269521afaf18SBorislav Petkov 269621afaf18SBorislav Petkov register_syscore_ops(&mce_syscore_ops); 269721afaf18SBorislav Petkov 269821afaf18SBorislav Petkov return 0; 269921afaf18SBorislav Petkov 270021afaf18SBorislav Petkov err_out_online: 270121afaf18SBorislav Petkov cpuhp_remove_state(CPUHP_X86_MCE_DEAD); 270221afaf18SBorislav Petkov 270321afaf18SBorislav Petkov err_out_mem: 270421afaf18SBorislav Petkov free_cpumask_var(mce_device_initialized); 270521afaf18SBorislav Petkov 270621afaf18SBorislav Petkov err_out: 270721afaf18SBorislav Petkov pr_err("Unable to init MCE device (rc: %d)\n", err); 270821afaf18SBorislav Petkov 270921afaf18SBorislav Petkov return err; 271021afaf18SBorislav Petkov } 271121afaf18SBorislav Petkov device_initcall_sync(mcheck_init_device); 271221afaf18SBorislav Petkov 271321afaf18SBorislav Petkov /* 271421afaf18SBorislav Petkov * Old style boot options parsing. Only for compatibility. 271521afaf18SBorislav Petkov */ 271621afaf18SBorislav Petkov static int __init mcheck_disable(char *str) 271721afaf18SBorislav Petkov { 271821afaf18SBorislav Petkov mca_cfg.disabled = 1; 271921afaf18SBorislav Petkov return 1; 272021afaf18SBorislav Petkov } 272121afaf18SBorislav Petkov __setup("nomce", mcheck_disable); 272221afaf18SBorislav Petkov 272321afaf18SBorislav Petkov #ifdef CONFIG_DEBUG_FS 272421afaf18SBorislav Petkov struct dentry *mce_get_debugfs_dir(void) 272521afaf18SBorislav Petkov { 272621afaf18SBorislav Petkov static struct dentry *dmce; 272721afaf18SBorislav Petkov 272821afaf18SBorislav Petkov if (!dmce) 272921afaf18SBorislav Petkov dmce = debugfs_create_dir("mce", NULL); 273021afaf18SBorislav Petkov 273121afaf18SBorislav Petkov return dmce; 273221afaf18SBorislav Petkov } 273321afaf18SBorislav Petkov 273421afaf18SBorislav Petkov static void mce_reset(void) 273521afaf18SBorislav Petkov { 273621afaf18SBorislav Petkov cpu_missing = 0; 273721afaf18SBorislav Petkov atomic_set(&mce_fake_panicked, 0); 273821afaf18SBorislav Petkov atomic_set(&mce_executing, 0); 273921afaf18SBorislav Petkov atomic_set(&mce_callin, 0); 274021afaf18SBorislav Petkov atomic_set(&global_nwo, 0); 27417bb39313SPaul E. McKenney cpumask_setall(&mce_missing_cpus); 274221afaf18SBorislav Petkov } 274321afaf18SBorislav Petkov 274421afaf18SBorislav Petkov static int fake_panic_get(void *data, u64 *val) 274521afaf18SBorislav Petkov { 274621afaf18SBorislav Petkov *val = fake_panic; 274721afaf18SBorislav Petkov return 0; 274821afaf18SBorislav Petkov } 274921afaf18SBorislav Petkov 275021afaf18SBorislav Petkov static int fake_panic_set(void *data, u64 val) 275121afaf18SBorislav Petkov { 275221afaf18SBorislav Petkov mce_reset(); 275321afaf18SBorislav Petkov fake_panic = val; 275421afaf18SBorislav Petkov return 0; 275521afaf18SBorislav Petkov } 275621afaf18SBorislav Petkov 275728156d76SYueHaibing DEFINE_DEBUGFS_ATTRIBUTE(fake_panic_fops, fake_panic_get, fake_panic_set, 275828156d76SYueHaibing "%llu\n"); 275921afaf18SBorislav Petkov 27606e4f929eSGreg Kroah-Hartman static void __init mcheck_debugfs_init(void) 276121afaf18SBorislav Petkov { 27626e4f929eSGreg Kroah-Hartman struct dentry *dmce; 276321afaf18SBorislav Petkov 276421afaf18SBorislav Petkov dmce = mce_get_debugfs_dir(); 27656e4f929eSGreg Kroah-Hartman debugfs_create_file_unsafe("fake_panic", 0444, dmce, NULL, 27666e4f929eSGreg Kroah-Hartman &fake_panic_fops); 276721afaf18SBorislav Petkov } 276821afaf18SBorislav Petkov #else 27696e4f929eSGreg Kroah-Hartman static void __init mcheck_debugfs_init(void) { } 277021afaf18SBorislav Petkov #endif 277121afaf18SBorislav Petkov 277221afaf18SBorislav Petkov static int __init mcheck_late_init(void) 277321afaf18SBorislav Petkov { 277421afaf18SBorislav Petkov if (mca_cfg.recovery) 2775ec6347bbSDan Williams enable_copy_mc_fragile(); 277621afaf18SBorislav Petkov 277721afaf18SBorislav Petkov mcheck_debugfs_init(); 277821afaf18SBorislav Petkov 277921afaf18SBorislav Petkov /* 278021afaf18SBorislav Petkov * Flush out everything that has been logged during early boot, now that 278121afaf18SBorislav Petkov * everything has been initialized (workqueues, decoders, ...). 278221afaf18SBorislav Petkov */ 278321afaf18SBorislav Petkov mce_schedule_work(); 278421afaf18SBorislav Petkov 278521afaf18SBorislav Petkov return 0; 278621afaf18SBorislav Petkov } 278721afaf18SBorislav Petkov late_initcall(mcheck_late_init); 2788