xref: /openbmc/linux/arch/x86/kernel/cpu/mce/core.c (revision abe9af53)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Machine check handler.
4  *
5  * K8 parts Copyright 2002,2003 Andi Kleen, SuSE Labs.
6  * Rest from unknown author(s).
7  * 2004 Andi Kleen. Rewrote most of it.
8  * Copyright 2008 Intel Corporation
9  * Author: Andi Kleen
10  */
11 
12 #include <linux/thread_info.h>
13 #include <linux/capability.h>
14 #include <linux/miscdevice.h>
15 #include <linux/ratelimit.h>
16 #include <linux/rcupdate.h>
17 #include <linux/kobject.h>
18 #include <linux/uaccess.h>
19 #include <linux/kdebug.h>
20 #include <linux/kernel.h>
21 #include <linux/percpu.h>
22 #include <linux/string.h>
23 #include <linux/device.h>
24 #include <linux/syscore_ops.h>
25 #include <linux/delay.h>
26 #include <linux/ctype.h>
27 #include <linux/sched.h>
28 #include <linux/sysfs.h>
29 #include <linux/types.h>
30 #include <linux/slab.h>
31 #include <linux/init.h>
32 #include <linux/kmod.h>
33 #include <linux/poll.h>
34 #include <linux/nmi.h>
35 #include <linux/cpu.h>
36 #include <linux/ras.h>
37 #include <linux/smp.h>
38 #include <linux/fs.h>
39 #include <linux/mm.h>
40 #include <linux/debugfs.h>
41 #include <linux/irq_work.h>
42 #include <linux/export.h>
43 #include <linux/set_memory.h>
44 #include <linux/sync_core.h>
45 #include <linux/task_work.h>
46 #include <linux/hardirq.h>
47 
48 #include <asm/intel-family.h>
49 #include <asm/processor.h>
50 #include <asm/traps.h>
51 #include <asm/tlbflush.h>
52 #include <asm/mce.h>
53 #include <asm/msr.h>
54 #include <asm/reboot.h>
55 
56 #include "internal.h"
57 
58 /* sysfs synchronization */
59 static DEFINE_MUTEX(mce_sysfs_mutex);
60 
61 #define CREATE_TRACE_POINTS
62 #include <trace/events/mce.h>
63 
64 #define SPINUNIT		100	/* 100ns */
65 
66 DEFINE_PER_CPU(unsigned, mce_exception_count);
67 
68 DEFINE_PER_CPU_READ_MOSTLY(unsigned int, mce_num_banks);
69 
70 struct mce_bank {
71 	u64			ctl;			/* subevents to enable */
72 	bool			init;			/* initialise bank? */
73 };
74 static DEFINE_PER_CPU_READ_MOSTLY(struct mce_bank[MAX_NR_BANKS], mce_banks_array);
75 
76 #define ATTR_LEN               16
77 /* One object for each MCE bank, shared by all CPUs */
78 struct mce_bank_dev {
79 	struct device_attribute	attr;			/* device attribute */
80 	char			attrname[ATTR_LEN];	/* attribute name */
81 	u8			bank;			/* bank number */
82 };
83 static struct mce_bank_dev mce_bank_devs[MAX_NR_BANKS];
84 
85 struct mce_vendor_flags mce_flags __read_mostly;
86 
87 struct mca_config mca_cfg __read_mostly = {
88 	.bootlog  = -1,
89 	/*
90 	 * Tolerant levels:
91 	 * 0: always panic on uncorrected errors, log corrected errors
92 	 * 1: panic or SIGBUS on uncorrected errors, log corrected errors
93 	 * 2: SIGBUS or log uncorrected errors (if possible), log corr. errors
94 	 * 3: never panic or SIGBUS, log all errors (for testing only)
95 	 */
96 	.tolerant = 1,
97 	.monarch_timeout = -1
98 };
99 
100 static DEFINE_PER_CPU(struct mce, mces_seen);
101 static unsigned long mce_need_notify;
102 static int cpu_missing;
103 
104 /*
105  * MCA banks polled by the period polling timer for corrected events.
106  * With Intel CMCI, this only has MCA banks which do not support CMCI (if any).
107  */
108 DEFINE_PER_CPU(mce_banks_t, mce_poll_banks) = {
109 	[0 ... BITS_TO_LONGS(MAX_NR_BANKS)-1] = ~0UL
110 };
111 
112 /*
113  * MCA banks controlled through firmware first for corrected errors.
114  * This is a global list of banks for which we won't enable CMCI and we
115  * won't poll. Firmware controls these banks and is responsible for
116  * reporting corrected errors through GHES. Uncorrected/recoverable
117  * errors are still notified through a machine check.
118  */
119 mce_banks_t mce_banks_ce_disabled;
120 
121 static struct work_struct mce_work;
122 static struct irq_work mce_irq_work;
123 
124 static void (*quirk_no_way_out)(int bank, struct mce *m, struct pt_regs *regs);
125 
126 /*
127  * CPU/chipset specific EDAC code can register a notifier call here to print
128  * MCE errors in a human-readable form.
129  */
130 BLOCKING_NOTIFIER_HEAD(x86_mce_decoder_chain);
131 
132 /* Do initial initialization of a struct mce */
133 noinstr void mce_setup(struct mce *m)
134 {
135 	memset(m, 0, sizeof(struct mce));
136 	m->cpu = m->extcpu = smp_processor_id();
137 	/* need the internal __ version to avoid deadlocks */
138 	m->time = __ktime_get_real_seconds();
139 	m->cpuvendor = boot_cpu_data.x86_vendor;
140 	m->cpuid = cpuid_eax(1);
141 	m->socketid = cpu_data(m->extcpu).phys_proc_id;
142 	m->apicid = cpu_data(m->extcpu).initial_apicid;
143 	m->mcgcap = __rdmsr(MSR_IA32_MCG_CAP);
144 
145 	if (this_cpu_has(X86_FEATURE_INTEL_PPIN))
146 		m->ppin = __rdmsr(MSR_PPIN);
147 	else if (this_cpu_has(X86_FEATURE_AMD_PPIN))
148 		m->ppin = __rdmsr(MSR_AMD_PPIN);
149 
150 	m->microcode = boot_cpu_data.microcode;
151 }
152 
153 DEFINE_PER_CPU(struct mce, injectm);
154 EXPORT_PER_CPU_SYMBOL_GPL(injectm);
155 
156 void mce_log(struct mce *m)
157 {
158 	if (!mce_gen_pool_add(m))
159 		irq_work_queue(&mce_irq_work);
160 }
161 EXPORT_SYMBOL_GPL(mce_log);
162 
163 void mce_register_decode_chain(struct notifier_block *nb)
164 {
165 	if (WARN_ON(nb->priority > MCE_PRIO_MCELOG && nb->priority < MCE_PRIO_EDAC))
166 		return;
167 
168 	blocking_notifier_chain_register(&x86_mce_decoder_chain, nb);
169 }
170 EXPORT_SYMBOL_GPL(mce_register_decode_chain);
171 
172 void mce_unregister_decode_chain(struct notifier_block *nb)
173 {
174 	blocking_notifier_chain_unregister(&x86_mce_decoder_chain, nb);
175 }
176 EXPORT_SYMBOL_GPL(mce_unregister_decode_chain);
177 
178 static inline u32 ctl_reg(int bank)
179 {
180 	return MSR_IA32_MCx_CTL(bank);
181 }
182 
183 static inline u32 status_reg(int bank)
184 {
185 	return MSR_IA32_MCx_STATUS(bank);
186 }
187 
188 static inline u32 addr_reg(int bank)
189 {
190 	return MSR_IA32_MCx_ADDR(bank);
191 }
192 
193 static inline u32 misc_reg(int bank)
194 {
195 	return MSR_IA32_MCx_MISC(bank);
196 }
197 
198 static inline u32 smca_ctl_reg(int bank)
199 {
200 	return MSR_AMD64_SMCA_MCx_CTL(bank);
201 }
202 
203 static inline u32 smca_status_reg(int bank)
204 {
205 	return MSR_AMD64_SMCA_MCx_STATUS(bank);
206 }
207 
208 static inline u32 smca_addr_reg(int bank)
209 {
210 	return MSR_AMD64_SMCA_MCx_ADDR(bank);
211 }
212 
213 static inline u32 smca_misc_reg(int bank)
214 {
215 	return MSR_AMD64_SMCA_MCx_MISC(bank);
216 }
217 
218 struct mca_msr_regs msr_ops = {
219 	.ctl	= ctl_reg,
220 	.status	= status_reg,
221 	.addr	= addr_reg,
222 	.misc	= misc_reg
223 };
224 
225 static void __print_mce(struct mce *m)
226 {
227 	pr_emerg(HW_ERR "CPU %d: Machine Check%s: %Lx Bank %d: %016Lx\n",
228 		 m->extcpu,
229 		 (m->mcgstatus & MCG_STATUS_MCIP ? " Exception" : ""),
230 		 m->mcgstatus, m->bank, m->status);
231 
232 	if (m->ip) {
233 		pr_emerg(HW_ERR "RIP%s %02x:<%016Lx> ",
234 			!(m->mcgstatus & MCG_STATUS_EIPV) ? " !INEXACT!" : "",
235 			m->cs, m->ip);
236 
237 		if (m->cs == __KERNEL_CS)
238 			pr_cont("{%pS}", (void *)(unsigned long)m->ip);
239 		pr_cont("\n");
240 	}
241 
242 	pr_emerg(HW_ERR "TSC %llx ", m->tsc);
243 	if (m->addr)
244 		pr_cont("ADDR %llx ", m->addr);
245 	if (m->misc)
246 		pr_cont("MISC %llx ", m->misc);
247 	if (m->ppin)
248 		pr_cont("PPIN %llx ", m->ppin);
249 
250 	if (mce_flags.smca) {
251 		if (m->synd)
252 			pr_cont("SYND %llx ", m->synd);
253 		if (m->ipid)
254 			pr_cont("IPID %llx ", m->ipid);
255 	}
256 
257 	pr_cont("\n");
258 
259 	/*
260 	 * Note this output is parsed by external tools and old fields
261 	 * should not be changed.
262 	 */
263 	pr_emerg(HW_ERR "PROCESSOR %u:%x TIME %llu SOCKET %u APIC %x microcode %x\n",
264 		m->cpuvendor, m->cpuid, m->time, m->socketid, m->apicid,
265 		m->microcode);
266 }
267 
268 static void print_mce(struct mce *m)
269 {
270 	__print_mce(m);
271 
272 	if (m->cpuvendor != X86_VENDOR_AMD && m->cpuvendor != X86_VENDOR_HYGON)
273 		pr_emerg_ratelimited(HW_ERR "Run the above through 'mcelog --ascii'\n");
274 }
275 
276 #define PANIC_TIMEOUT 5 /* 5 seconds */
277 
278 static atomic_t mce_panicked;
279 
280 static int fake_panic;
281 static atomic_t mce_fake_panicked;
282 
283 /* Panic in progress. Enable interrupts and wait for final IPI */
284 static void wait_for_panic(void)
285 {
286 	long timeout = PANIC_TIMEOUT*USEC_PER_SEC;
287 
288 	preempt_disable();
289 	local_irq_enable();
290 	while (timeout-- > 0)
291 		udelay(1);
292 	if (panic_timeout == 0)
293 		panic_timeout = mca_cfg.panic_timeout;
294 	panic("Panicing machine check CPU died");
295 }
296 
297 static void mce_panic(const char *msg, struct mce *final, char *exp)
298 {
299 	int apei_err = 0;
300 	struct llist_node *pending;
301 	struct mce_evt_llist *l;
302 
303 	if (!fake_panic) {
304 		/*
305 		 * Make sure only one CPU runs in machine check panic
306 		 */
307 		if (atomic_inc_return(&mce_panicked) > 1)
308 			wait_for_panic();
309 		barrier();
310 
311 		bust_spinlocks(1);
312 		console_verbose();
313 	} else {
314 		/* Don't log too much for fake panic */
315 		if (atomic_inc_return(&mce_fake_panicked) > 1)
316 			return;
317 	}
318 	pending = mce_gen_pool_prepare_records();
319 	/* First print corrected ones that are still unlogged */
320 	llist_for_each_entry(l, pending, llnode) {
321 		struct mce *m = &l->mce;
322 		if (!(m->status & MCI_STATUS_UC)) {
323 			print_mce(m);
324 			if (!apei_err)
325 				apei_err = apei_write_mce(m);
326 		}
327 	}
328 	/* Now print uncorrected but with the final one last */
329 	llist_for_each_entry(l, pending, llnode) {
330 		struct mce *m = &l->mce;
331 		if (!(m->status & MCI_STATUS_UC))
332 			continue;
333 		if (!final || mce_cmp(m, final)) {
334 			print_mce(m);
335 			if (!apei_err)
336 				apei_err = apei_write_mce(m);
337 		}
338 	}
339 	if (final) {
340 		print_mce(final);
341 		if (!apei_err)
342 			apei_err = apei_write_mce(final);
343 	}
344 	if (cpu_missing)
345 		pr_emerg(HW_ERR "Some CPUs didn't answer in synchronization\n");
346 	if (exp)
347 		pr_emerg(HW_ERR "Machine check: %s\n", exp);
348 	if (!fake_panic) {
349 		if (panic_timeout == 0)
350 			panic_timeout = mca_cfg.panic_timeout;
351 		panic(msg);
352 	} else
353 		pr_emerg(HW_ERR "Fake kernel panic: %s\n", msg);
354 }
355 
356 /* Support code for software error injection */
357 
358 static int msr_to_offset(u32 msr)
359 {
360 	unsigned bank = __this_cpu_read(injectm.bank);
361 
362 	if (msr == mca_cfg.rip_msr)
363 		return offsetof(struct mce, ip);
364 	if (msr == msr_ops.status(bank))
365 		return offsetof(struct mce, status);
366 	if (msr == msr_ops.addr(bank))
367 		return offsetof(struct mce, addr);
368 	if (msr == msr_ops.misc(bank))
369 		return offsetof(struct mce, misc);
370 	if (msr == MSR_IA32_MCG_STATUS)
371 		return offsetof(struct mce, mcgstatus);
372 	return -1;
373 }
374 
375 __visible bool ex_handler_rdmsr_fault(const struct exception_table_entry *fixup,
376 				      struct pt_regs *regs, int trapnr,
377 				      unsigned long error_code,
378 				      unsigned long fault_addr)
379 {
380 	pr_emerg("MSR access error: RDMSR from 0x%x at rIP: 0x%lx (%pS)\n",
381 		 (unsigned int)regs->cx, regs->ip, (void *)regs->ip);
382 
383 	show_stack_regs(regs);
384 
385 	panic("MCA architectural violation!\n");
386 
387 	while (true)
388 		cpu_relax();
389 
390 	return true;
391 }
392 
393 /* MSR access wrappers used for error injection */
394 static noinstr u64 mce_rdmsrl(u32 msr)
395 {
396 	DECLARE_ARGS(val, low, high);
397 
398 	if (__this_cpu_read(injectm.finished)) {
399 		int offset;
400 		u64 ret;
401 
402 		instrumentation_begin();
403 
404 		offset = msr_to_offset(msr);
405 		if (offset < 0)
406 			ret = 0;
407 		else
408 			ret = *(u64 *)((char *)this_cpu_ptr(&injectm) + offset);
409 
410 		instrumentation_end();
411 
412 		return ret;
413 	}
414 
415 	/*
416 	 * RDMSR on MCA MSRs should not fault. If they do, this is very much an
417 	 * architectural violation and needs to be reported to hw vendor. Panic
418 	 * the box to not allow any further progress.
419 	 */
420 	asm volatile("1: rdmsr\n"
421 		     "2:\n"
422 		     _ASM_EXTABLE_HANDLE(1b, 2b, ex_handler_rdmsr_fault)
423 		     : EAX_EDX_RET(val, low, high) : "c" (msr));
424 
425 
426 	return EAX_EDX_VAL(val, low, high);
427 }
428 
429 __visible bool ex_handler_wrmsr_fault(const struct exception_table_entry *fixup,
430 				      struct pt_regs *regs, int trapnr,
431 				      unsigned long error_code,
432 				      unsigned long fault_addr)
433 {
434 	pr_emerg("MSR access error: WRMSR to 0x%x (tried to write 0x%08x%08x) at rIP: 0x%lx (%pS)\n",
435 		 (unsigned int)regs->cx, (unsigned int)regs->dx, (unsigned int)regs->ax,
436 		  regs->ip, (void *)regs->ip);
437 
438 	show_stack_regs(regs);
439 
440 	panic("MCA architectural violation!\n");
441 
442 	while (true)
443 		cpu_relax();
444 
445 	return true;
446 }
447 
448 static noinstr void mce_wrmsrl(u32 msr, u64 v)
449 {
450 	u32 low, high;
451 
452 	if (__this_cpu_read(injectm.finished)) {
453 		int offset;
454 
455 		instrumentation_begin();
456 
457 		offset = msr_to_offset(msr);
458 		if (offset >= 0)
459 			*(u64 *)((char *)this_cpu_ptr(&injectm) + offset) = v;
460 
461 		instrumentation_end();
462 
463 		return;
464 	}
465 
466 	low  = (u32)v;
467 	high = (u32)(v >> 32);
468 
469 	/* See comment in mce_rdmsrl() */
470 	asm volatile("1: wrmsr\n"
471 		     "2:\n"
472 		     _ASM_EXTABLE_HANDLE(1b, 2b, ex_handler_wrmsr_fault)
473 		     : : "c" (msr), "a"(low), "d" (high) : "memory");
474 }
475 
476 /*
477  * Collect all global (w.r.t. this processor) status about this machine
478  * check into our "mce" struct so that we can use it later to assess
479  * the severity of the problem as we read per-bank specific details.
480  */
481 static inline void mce_gather_info(struct mce *m, struct pt_regs *regs)
482 {
483 	mce_setup(m);
484 
485 	m->mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS);
486 	if (regs) {
487 		/*
488 		 * Get the address of the instruction at the time of
489 		 * the machine check error.
490 		 */
491 		if (m->mcgstatus & (MCG_STATUS_RIPV|MCG_STATUS_EIPV)) {
492 			m->ip = regs->ip;
493 			m->cs = regs->cs;
494 
495 			/*
496 			 * When in VM86 mode make the cs look like ring 3
497 			 * always. This is a lie, but it's better than passing
498 			 * the additional vm86 bit around everywhere.
499 			 */
500 			if (v8086_mode(regs))
501 				m->cs |= 3;
502 		}
503 		/* Use accurate RIP reporting if available. */
504 		if (mca_cfg.rip_msr)
505 			m->ip = mce_rdmsrl(mca_cfg.rip_msr);
506 	}
507 }
508 
509 int mce_available(struct cpuinfo_x86 *c)
510 {
511 	if (mca_cfg.disabled)
512 		return 0;
513 	return cpu_has(c, X86_FEATURE_MCE) && cpu_has(c, X86_FEATURE_MCA);
514 }
515 
516 static void mce_schedule_work(void)
517 {
518 	if (!mce_gen_pool_empty())
519 		schedule_work(&mce_work);
520 }
521 
522 static void mce_irq_work_cb(struct irq_work *entry)
523 {
524 	mce_schedule_work();
525 }
526 
527 /*
528  * Check if the address reported by the CPU is in a format we can parse.
529  * It would be possible to add code for most other cases, but all would
530  * be somewhat complicated (e.g. segment offset would require an instruction
531  * parser). So only support physical addresses up to page granuality for now.
532  */
533 int mce_usable_address(struct mce *m)
534 {
535 	if (!(m->status & MCI_STATUS_ADDRV))
536 		return 0;
537 
538 	/* Checks after this one are Intel/Zhaoxin-specific: */
539 	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL &&
540 	    boot_cpu_data.x86_vendor != X86_VENDOR_ZHAOXIN)
541 		return 1;
542 
543 	if (!(m->status & MCI_STATUS_MISCV))
544 		return 0;
545 
546 	if (MCI_MISC_ADDR_LSB(m->misc) > PAGE_SHIFT)
547 		return 0;
548 
549 	if (MCI_MISC_ADDR_MODE(m->misc) != MCI_MISC_ADDR_PHYS)
550 		return 0;
551 
552 	return 1;
553 }
554 EXPORT_SYMBOL_GPL(mce_usable_address);
555 
556 bool mce_is_memory_error(struct mce *m)
557 {
558 	switch (m->cpuvendor) {
559 	case X86_VENDOR_AMD:
560 	case X86_VENDOR_HYGON:
561 		return amd_mce_is_memory_error(m);
562 
563 	case X86_VENDOR_INTEL:
564 	case X86_VENDOR_ZHAOXIN:
565 		/*
566 		 * Intel SDM Volume 3B - 15.9.2 Compound Error Codes
567 		 *
568 		 * Bit 7 of the MCACOD field of IA32_MCi_STATUS is used for
569 		 * indicating a memory error. Bit 8 is used for indicating a
570 		 * cache hierarchy error. The combination of bit 2 and bit 3
571 		 * is used for indicating a `generic' cache hierarchy error
572 		 * But we can't just blindly check the above bits, because if
573 		 * bit 11 is set, then it is a bus/interconnect error - and
574 		 * either way the above bits just gives more detail on what
575 		 * bus/interconnect error happened. Note that bit 12 can be
576 		 * ignored, as it's the "filter" bit.
577 		 */
578 		return (m->status & 0xef80) == BIT(7) ||
579 		       (m->status & 0xef00) == BIT(8) ||
580 		       (m->status & 0xeffc) == 0xc;
581 
582 	default:
583 		return false;
584 	}
585 }
586 EXPORT_SYMBOL_GPL(mce_is_memory_error);
587 
588 static bool whole_page(struct mce *m)
589 {
590 	if (!mca_cfg.ser || !(m->status & MCI_STATUS_MISCV))
591 		return true;
592 
593 	return MCI_MISC_ADDR_LSB(m->misc) >= PAGE_SHIFT;
594 }
595 
596 bool mce_is_correctable(struct mce *m)
597 {
598 	if (m->cpuvendor == X86_VENDOR_AMD && m->status & MCI_STATUS_DEFERRED)
599 		return false;
600 
601 	if (m->cpuvendor == X86_VENDOR_HYGON && m->status & MCI_STATUS_DEFERRED)
602 		return false;
603 
604 	if (m->status & MCI_STATUS_UC)
605 		return false;
606 
607 	return true;
608 }
609 EXPORT_SYMBOL_GPL(mce_is_correctable);
610 
611 static int mce_early_notifier(struct notifier_block *nb, unsigned long val,
612 			      void *data)
613 {
614 	struct mce *m = (struct mce *)data;
615 
616 	if (!m)
617 		return NOTIFY_DONE;
618 
619 	/* Emit the trace record: */
620 	trace_mce_record(m);
621 
622 	set_bit(0, &mce_need_notify);
623 
624 	mce_notify_irq();
625 
626 	return NOTIFY_DONE;
627 }
628 
629 static struct notifier_block early_nb = {
630 	.notifier_call	= mce_early_notifier,
631 	.priority	= MCE_PRIO_EARLY,
632 };
633 
634 static int uc_decode_notifier(struct notifier_block *nb, unsigned long val,
635 			      void *data)
636 {
637 	struct mce *mce = (struct mce *)data;
638 	unsigned long pfn;
639 
640 	if (!mce || !mce_usable_address(mce))
641 		return NOTIFY_DONE;
642 
643 	if (mce->severity != MCE_AO_SEVERITY &&
644 	    mce->severity != MCE_DEFERRED_SEVERITY)
645 		return NOTIFY_DONE;
646 
647 	pfn = mce->addr >> PAGE_SHIFT;
648 	if (!memory_failure(pfn, 0)) {
649 		set_mce_nospec(pfn, whole_page(mce));
650 		mce->kflags |= MCE_HANDLED_UC;
651 	}
652 
653 	return NOTIFY_OK;
654 }
655 
656 static struct notifier_block mce_uc_nb = {
657 	.notifier_call	= uc_decode_notifier,
658 	.priority	= MCE_PRIO_UC,
659 };
660 
661 static int mce_default_notifier(struct notifier_block *nb, unsigned long val,
662 				void *data)
663 {
664 	struct mce *m = (struct mce *)data;
665 
666 	if (!m)
667 		return NOTIFY_DONE;
668 
669 	if (mca_cfg.print_all || !m->kflags)
670 		__print_mce(m);
671 
672 	return NOTIFY_DONE;
673 }
674 
675 static struct notifier_block mce_default_nb = {
676 	.notifier_call	= mce_default_notifier,
677 	/* lowest prio, we want it to run last. */
678 	.priority	= MCE_PRIO_LOWEST,
679 };
680 
681 /*
682  * Read ADDR and MISC registers.
683  */
684 static void mce_read_aux(struct mce *m, int i)
685 {
686 	if (m->status & MCI_STATUS_MISCV)
687 		m->misc = mce_rdmsrl(msr_ops.misc(i));
688 
689 	if (m->status & MCI_STATUS_ADDRV) {
690 		m->addr = mce_rdmsrl(msr_ops.addr(i));
691 
692 		/*
693 		 * Mask the reported address by the reported granularity.
694 		 */
695 		if (mca_cfg.ser && (m->status & MCI_STATUS_MISCV)) {
696 			u8 shift = MCI_MISC_ADDR_LSB(m->misc);
697 			m->addr >>= shift;
698 			m->addr <<= shift;
699 		}
700 
701 		/*
702 		 * Extract [55:<lsb>] where lsb is the least significant
703 		 * *valid* bit of the address bits.
704 		 */
705 		if (mce_flags.smca) {
706 			u8 lsb = (m->addr >> 56) & 0x3f;
707 
708 			m->addr &= GENMASK_ULL(55, lsb);
709 		}
710 	}
711 
712 	if (mce_flags.smca) {
713 		m->ipid = mce_rdmsrl(MSR_AMD64_SMCA_MCx_IPID(i));
714 
715 		if (m->status & MCI_STATUS_SYNDV)
716 			m->synd = mce_rdmsrl(MSR_AMD64_SMCA_MCx_SYND(i));
717 	}
718 }
719 
720 DEFINE_PER_CPU(unsigned, mce_poll_count);
721 
722 /*
723  * Poll for corrected events or events that happened before reset.
724  * Those are just logged through /dev/mcelog.
725  *
726  * This is executed in standard interrupt context.
727  *
728  * Note: spec recommends to panic for fatal unsignalled
729  * errors here. However this would be quite problematic --
730  * we would need to reimplement the Monarch handling and
731  * it would mess up the exclusion between exception handler
732  * and poll handler -- * so we skip this for now.
733  * These cases should not happen anyways, or only when the CPU
734  * is already totally * confused. In this case it's likely it will
735  * not fully execute the machine check handler either.
736  */
737 bool machine_check_poll(enum mcp_flags flags, mce_banks_t *b)
738 {
739 	struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
740 	bool error_seen = false;
741 	struct mce m;
742 	int i;
743 
744 	this_cpu_inc(mce_poll_count);
745 
746 	mce_gather_info(&m, NULL);
747 
748 	if (flags & MCP_TIMESTAMP)
749 		m.tsc = rdtsc();
750 
751 	for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
752 		if (!mce_banks[i].ctl || !test_bit(i, *b))
753 			continue;
754 
755 		m.misc = 0;
756 		m.addr = 0;
757 		m.bank = i;
758 
759 		barrier();
760 		m.status = mce_rdmsrl(msr_ops.status(i));
761 
762 		/* If this entry is not valid, ignore it */
763 		if (!(m.status & MCI_STATUS_VAL))
764 			continue;
765 
766 		/*
767 		 * If we are logging everything (at CPU online) or this
768 		 * is a corrected error, then we must log it.
769 		 */
770 		if ((flags & MCP_UC) || !(m.status & MCI_STATUS_UC))
771 			goto log_it;
772 
773 		/*
774 		 * Newer Intel systems that support software error
775 		 * recovery need to make additional checks. Other
776 		 * CPUs should skip over uncorrected errors, but log
777 		 * everything else.
778 		 */
779 		if (!mca_cfg.ser) {
780 			if (m.status & MCI_STATUS_UC)
781 				continue;
782 			goto log_it;
783 		}
784 
785 		/* Log "not enabled" (speculative) errors */
786 		if (!(m.status & MCI_STATUS_EN))
787 			goto log_it;
788 
789 		/*
790 		 * Log UCNA (SDM: 15.6.3 "UCR Error Classification")
791 		 * UC == 1 && PCC == 0 && S == 0
792 		 */
793 		if (!(m.status & MCI_STATUS_PCC) && !(m.status & MCI_STATUS_S))
794 			goto log_it;
795 
796 		/*
797 		 * Skip anything else. Presumption is that our read of this
798 		 * bank is racing with a machine check. Leave the log alone
799 		 * for do_machine_check() to deal with it.
800 		 */
801 		continue;
802 
803 log_it:
804 		error_seen = true;
805 
806 		if (flags & MCP_DONTLOG)
807 			goto clear_it;
808 
809 		mce_read_aux(&m, i);
810 		m.severity = mce_severity(&m, NULL, mca_cfg.tolerant, NULL, false);
811 		/*
812 		 * Don't get the IP here because it's unlikely to
813 		 * have anything to do with the actual error location.
814 		 */
815 
816 		if (mca_cfg.dont_log_ce && !mce_usable_address(&m))
817 			goto clear_it;
818 
819 		mce_log(&m);
820 
821 clear_it:
822 		/*
823 		 * Clear state for this bank.
824 		 */
825 		mce_wrmsrl(msr_ops.status(i), 0);
826 	}
827 
828 	/*
829 	 * Don't clear MCG_STATUS here because it's only defined for
830 	 * exceptions.
831 	 */
832 
833 	sync_core();
834 
835 	return error_seen;
836 }
837 EXPORT_SYMBOL_GPL(machine_check_poll);
838 
839 /*
840  * Do a quick check if any of the events requires a panic.
841  * This decides if we keep the events around or clear them.
842  */
843 static int mce_no_way_out(struct mce *m, char **msg, unsigned long *validp,
844 			  struct pt_regs *regs)
845 {
846 	char *tmp = *msg;
847 	int i;
848 
849 	for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
850 		m->status = mce_rdmsrl(msr_ops.status(i));
851 		if (!(m->status & MCI_STATUS_VAL))
852 			continue;
853 
854 		__set_bit(i, validp);
855 		if (quirk_no_way_out)
856 			quirk_no_way_out(i, m, regs);
857 
858 		m->bank = i;
859 		if (mce_severity(m, regs, mca_cfg.tolerant, &tmp, true) >= MCE_PANIC_SEVERITY) {
860 			mce_read_aux(m, i);
861 			*msg = tmp;
862 			return 1;
863 		}
864 	}
865 	return 0;
866 }
867 
868 /*
869  * Variable to establish order between CPUs while scanning.
870  * Each CPU spins initially until executing is equal its number.
871  */
872 static atomic_t mce_executing;
873 
874 /*
875  * Defines order of CPUs on entry. First CPU becomes Monarch.
876  */
877 static atomic_t mce_callin;
878 
879 /*
880  * Check if a timeout waiting for other CPUs happened.
881  */
882 static int mce_timed_out(u64 *t, const char *msg)
883 {
884 	/*
885 	 * The others already did panic for some reason.
886 	 * Bail out like in a timeout.
887 	 * rmb() to tell the compiler that system_state
888 	 * might have been modified by someone else.
889 	 */
890 	rmb();
891 	if (atomic_read(&mce_panicked))
892 		wait_for_panic();
893 	if (!mca_cfg.monarch_timeout)
894 		goto out;
895 	if ((s64)*t < SPINUNIT) {
896 		if (mca_cfg.tolerant <= 1)
897 			mce_panic(msg, NULL, NULL);
898 		cpu_missing = 1;
899 		return 1;
900 	}
901 	*t -= SPINUNIT;
902 out:
903 	touch_nmi_watchdog();
904 	return 0;
905 }
906 
907 /*
908  * The Monarch's reign.  The Monarch is the CPU who entered
909  * the machine check handler first. It waits for the others to
910  * raise the exception too and then grades them. When any
911  * error is fatal panic. Only then let the others continue.
912  *
913  * The other CPUs entering the MCE handler will be controlled by the
914  * Monarch. They are called Subjects.
915  *
916  * This way we prevent any potential data corruption in a unrecoverable case
917  * and also makes sure always all CPU's errors are examined.
918  *
919  * Also this detects the case of a machine check event coming from outer
920  * space (not detected by any CPUs) In this case some external agent wants
921  * us to shut down, so panic too.
922  *
923  * The other CPUs might still decide to panic if the handler happens
924  * in a unrecoverable place, but in this case the system is in a semi-stable
925  * state and won't corrupt anything by itself. It's ok to let the others
926  * continue for a bit first.
927  *
928  * All the spin loops have timeouts; when a timeout happens a CPU
929  * typically elects itself to be Monarch.
930  */
931 static void mce_reign(void)
932 {
933 	int cpu;
934 	struct mce *m = NULL;
935 	int global_worst = 0;
936 	char *msg = NULL;
937 
938 	/*
939 	 * This CPU is the Monarch and the other CPUs have run
940 	 * through their handlers.
941 	 * Grade the severity of the errors of all the CPUs.
942 	 */
943 	for_each_possible_cpu(cpu) {
944 		struct mce *mtmp = &per_cpu(mces_seen, cpu);
945 
946 		if (mtmp->severity > global_worst) {
947 			global_worst = mtmp->severity;
948 			m = &per_cpu(mces_seen, cpu);
949 		}
950 	}
951 
952 	/*
953 	 * Cannot recover? Panic here then.
954 	 * This dumps all the mces in the log buffer and stops the
955 	 * other CPUs.
956 	 */
957 	if (m && global_worst >= MCE_PANIC_SEVERITY && mca_cfg.tolerant < 3) {
958 		/* call mce_severity() to get "msg" for panic */
959 		mce_severity(m, NULL, mca_cfg.tolerant, &msg, true);
960 		mce_panic("Fatal machine check", m, msg);
961 	}
962 
963 	/*
964 	 * For UC somewhere we let the CPU who detects it handle it.
965 	 * Also must let continue the others, otherwise the handling
966 	 * CPU could deadlock on a lock.
967 	 */
968 
969 	/*
970 	 * No machine check event found. Must be some external
971 	 * source or one CPU is hung. Panic.
972 	 */
973 	if (global_worst <= MCE_KEEP_SEVERITY && mca_cfg.tolerant < 3)
974 		mce_panic("Fatal machine check from unknown source", NULL, NULL);
975 
976 	/*
977 	 * Now clear all the mces_seen so that they don't reappear on
978 	 * the next mce.
979 	 */
980 	for_each_possible_cpu(cpu)
981 		memset(&per_cpu(mces_seen, cpu), 0, sizeof(struct mce));
982 }
983 
984 static atomic_t global_nwo;
985 
986 /*
987  * Start of Monarch synchronization. This waits until all CPUs have
988  * entered the exception handler and then determines if any of them
989  * saw a fatal event that requires panic. Then it executes them
990  * in the entry order.
991  * TBD double check parallel CPU hotunplug
992  */
993 static int mce_start(int *no_way_out)
994 {
995 	int order;
996 	int cpus = num_online_cpus();
997 	u64 timeout = (u64)mca_cfg.monarch_timeout * NSEC_PER_USEC;
998 
999 	if (!timeout)
1000 		return -1;
1001 
1002 	atomic_add(*no_way_out, &global_nwo);
1003 	/*
1004 	 * Rely on the implied barrier below, such that global_nwo
1005 	 * is updated before mce_callin.
1006 	 */
1007 	order = atomic_inc_return(&mce_callin);
1008 
1009 	/*
1010 	 * Wait for everyone.
1011 	 */
1012 	while (atomic_read(&mce_callin) != cpus) {
1013 		if (mce_timed_out(&timeout,
1014 				  "Timeout: Not all CPUs entered broadcast exception handler")) {
1015 			atomic_set(&global_nwo, 0);
1016 			return -1;
1017 		}
1018 		ndelay(SPINUNIT);
1019 	}
1020 
1021 	/*
1022 	 * mce_callin should be read before global_nwo
1023 	 */
1024 	smp_rmb();
1025 
1026 	if (order == 1) {
1027 		/*
1028 		 * Monarch: Starts executing now, the others wait.
1029 		 */
1030 		atomic_set(&mce_executing, 1);
1031 	} else {
1032 		/*
1033 		 * Subject: Now start the scanning loop one by one in
1034 		 * the original callin order.
1035 		 * This way when there are any shared banks it will be
1036 		 * only seen by one CPU before cleared, avoiding duplicates.
1037 		 */
1038 		while (atomic_read(&mce_executing) < order) {
1039 			if (mce_timed_out(&timeout,
1040 					  "Timeout: Subject CPUs unable to finish machine check processing")) {
1041 				atomic_set(&global_nwo, 0);
1042 				return -1;
1043 			}
1044 			ndelay(SPINUNIT);
1045 		}
1046 	}
1047 
1048 	/*
1049 	 * Cache the global no_way_out state.
1050 	 */
1051 	*no_way_out = atomic_read(&global_nwo);
1052 
1053 	return order;
1054 }
1055 
1056 /*
1057  * Synchronize between CPUs after main scanning loop.
1058  * This invokes the bulk of the Monarch processing.
1059  */
1060 static int mce_end(int order)
1061 {
1062 	int ret = -1;
1063 	u64 timeout = (u64)mca_cfg.monarch_timeout * NSEC_PER_USEC;
1064 
1065 	if (!timeout)
1066 		goto reset;
1067 	if (order < 0)
1068 		goto reset;
1069 
1070 	/*
1071 	 * Allow others to run.
1072 	 */
1073 	atomic_inc(&mce_executing);
1074 
1075 	if (order == 1) {
1076 		/* CHECKME: Can this race with a parallel hotplug? */
1077 		int cpus = num_online_cpus();
1078 
1079 		/*
1080 		 * Monarch: Wait for everyone to go through their scanning
1081 		 * loops.
1082 		 */
1083 		while (atomic_read(&mce_executing) <= cpus) {
1084 			if (mce_timed_out(&timeout,
1085 					  "Timeout: Monarch CPU unable to finish machine check processing"))
1086 				goto reset;
1087 			ndelay(SPINUNIT);
1088 		}
1089 
1090 		mce_reign();
1091 		barrier();
1092 		ret = 0;
1093 	} else {
1094 		/*
1095 		 * Subject: Wait for Monarch to finish.
1096 		 */
1097 		while (atomic_read(&mce_executing) != 0) {
1098 			if (mce_timed_out(&timeout,
1099 					  "Timeout: Monarch CPU did not finish machine check processing"))
1100 				goto reset;
1101 			ndelay(SPINUNIT);
1102 		}
1103 
1104 		/*
1105 		 * Don't reset anything. That's done by the Monarch.
1106 		 */
1107 		return 0;
1108 	}
1109 
1110 	/*
1111 	 * Reset all global state.
1112 	 */
1113 reset:
1114 	atomic_set(&global_nwo, 0);
1115 	atomic_set(&mce_callin, 0);
1116 	barrier();
1117 
1118 	/*
1119 	 * Let others run again.
1120 	 */
1121 	atomic_set(&mce_executing, 0);
1122 	return ret;
1123 }
1124 
1125 static void mce_clear_state(unsigned long *toclear)
1126 {
1127 	int i;
1128 
1129 	for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
1130 		if (test_bit(i, toclear))
1131 			mce_wrmsrl(msr_ops.status(i), 0);
1132 	}
1133 }
1134 
1135 /*
1136  * Cases where we avoid rendezvous handler timeout:
1137  * 1) If this CPU is offline.
1138  *
1139  * 2) If crashing_cpu was set, e.g. we're entering kdump and we need to
1140  *  skip those CPUs which remain looping in the 1st kernel - see
1141  *  crash_nmi_callback().
1142  *
1143  * Note: there still is a small window between kexec-ing and the new,
1144  * kdump kernel establishing a new #MC handler where a broadcasted MCE
1145  * might not get handled properly.
1146  */
1147 static noinstr bool mce_check_crashing_cpu(void)
1148 {
1149 	unsigned int cpu = smp_processor_id();
1150 
1151 	if (arch_cpu_is_offline(cpu) ||
1152 	    (crashing_cpu != -1 && crashing_cpu != cpu)) {
1153 		u64 mcgstatus;
1154 
1155 		mcgstatus = __rdmsr(MSR_IA32_MCG_STATUS);
1156 
1157 		if (boot_cpu_data.x86_vendor == X86_VENDOR_ZHAOXIN) {
1158 			if (mcgstatus & MCG_STATUS_LMCES)
1159 				return false;
1160 		}
1161 
1162 		if (mcgstatus & MCG_STATUS_RIPV) {
1163 			__wrmsr(MSR_IA32_MCG_STATUS, 0, 0);
1164 			return true;
1165 		}
1166 	}
1167 	return false;
1168 }
1169 
1170 static void __mc_scan_banks(struct mce *m, struct pt_regs *regs, struct mce *final,
1171 			    unsigned long *toclear, unsigned long *valid_banks,
1172 			    int no_way_out, int *worst)
1173 {
1174 	struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
1175 	struct mca_config *cfg = &mca_cfg;
1176 	int severity, i;
1177 
1178 	for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
1179 		__clear_bit(i, toclear);
1180 		if (!test_bit(i, valid_banks))
1181 			continue;
1182 
1183 		if (!mce_banks[i].ctl)
1184 			continue;
1185 
1186 		m->misc = 0;
1187 		m->addr = 0;
1188 		m->bank = i;
1189 
1190 		m->status = mce_rdmsrl(msr_ops.status(i));
1191 		if (!(m->status & MCI_STATUS_VAL))
1192 			continue;
1193 
1194 		/*
1195 		 * Corrected or non-signaled errors are handled by
1196 		 * machine_check_poll(). Leave them alone, unless this panics.
1197 		 */
1198 		if (!(m->status & (cfg->ser ? MCI_STATUS_S : MCI_STATUS_UC)) &&
1199 			!no_way_out)
1200 			continue;
1201 
1202 		/* Set taint even when machine check was not enabled. */
1203 		add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE);
1204 
1205 		severity = mce_severity(m, regs, cfg->tolerant, NULL, true);
1206 
1207 		/*
1208 		 * When machine check was for corrected/deferred handler don't
1209 		 * touch, unless we're panicking.
1210 		 */
1211 		if ((severity == MCE_KEEP_SEVERITY ||
1212 		     severity == MCE_UCNA_SEVERITY) && !no_way_out)
1213 			continue;
1214 
1215 		__set_bit(i, toclear);
1216 
1217 		/* Machine check event was not enabled. Clear, but ignore. */
1218 		if (severity == MCE_NO_SEVERITY)
1219 			continue;
1220 
1221 		mce_read_aux(m, i);
1222 
1223 		/* assuming valid severity level != 0 */
1224 		m->severity = severity;
1225 
1226 		mce_log(m);
1227 
1228 		if (severity > *worst) {
1229 			*final = *m;
1230 			*worst = severity;
1231 		}
1232 	}
1233 
1234 	/* mce_clear_state will clear *final, save locally for use later */
1235 	*m = *final;
1236 }
1237 
1238 static void kill_me_now(struct callback_head *ch)
1239 {
1240 	force_sig(SIGBUS);
1241 }
1242 
1243 static void kill_me_maybe(struct callback_head *cb)
1244 {
1245 	struct task_struct *p = container_of(cb, struct task_struct, mce_kill_me);
1246 	int flags = MF_ACTION_REQUIRED;
1247 
1248 	pr_err("Uncorrected hardware memory error in user-access at %llx", p->mce_addr);
1249 
1250 	if (!p->mce_ripv)
1251 		flags |= MF_MUST_KILL;
1252 
1253 	if (!memory_failure(p->mce_addr >> PAGE_SHIFT, flags) &&
1254 	    !(p->mce_kflags & MCE_IN_KERNEL_COPYIN)) {
1255 		set_mce_nospec(p->mce_addr >> PAGE_SHIFT, p->mce_whole_page);
1256 		sync_core();
1257 		return;
1258 	}
1259 
1260 	if (p->mce_vaddr != (void __user *)-1l) {
1261 		force_sig_mceerr(BUS_MCEERR_AR, p->mce_vaddr, PAGE_SHIFT);
1262 	} else {
1263 		pr_err("Memory error not recovered");
1264 		kill_me_now(cb);
1265 	}
1266 }
1267 
1268 static void queue_task_work(struct mce *m, int kill_it)
1269 {
1270 	current->mce_addr = m->addr;
1271 	current->mce_kflags = m->kflags;
1272 	current->mce_ripv = !!(m->mcgstatus & MCG_STATUS_RIPV);
1273 	current->mce_whole_page = whole_page(m);
1274 
1275 	if (kill_it)
1276 		current->mce_kill_me.func = kill_me_now;
1277 	else
1278 		current->mce_kill_me.func = kill_me_maybe;
1279 
1280 	task_work_add(current, &current->mce_kill_me, TWA_RESUME);
1281 }
1282 
1283 /*
1284  * The actual machine check handler. This only handles real
1285  * exceptions when something got corrupted coming in through int 18.
1286  *
1287  * This is executed in NMI context not subject to normal locking rules. This
1288  * implies that most kernel services cannot be safely used. Don't even
1289  * think about putting a printk in there!
1290  *
1291  * On Intel systems this is entered on all CPUs in parallel through
1292  * MCE broadcast. However some CPUs might be broken beyond repair,
1293  * so be always careful when synchronizing with others.
1294  *
1295  * Tracing and kprobes are disabled: if we interrupted a kernel context
1296  * with IF=1, we need to minimize stack usage.  There are also recursion
1297  * issues: if the machine check was due to a failure of the memory
1298  * backing the user stack, tracing that reads the user stack will cause
1299  * potentially infinite recursion.
1300  */
1301 noinstr void do_machine_check(struct pt_regs *regs)
1302 {
1303 	DECLARE_BITMAP(valid_banks, MAX_NR_BANKS);
1304 	DECLARE_BITMAP(toclear, MAX_NR_BANKS);
1305 	struct mca_config *cfg = &mca_cfg;
1306 	struct mce m, *final;
1307 	char *msg = NULL;
1308 	int worst = 0;
1309 
1310 	/*
1311 	 * Establish sequential order between the CPUs entering the machine
1312 	 * check handler.
1313 	 */
1314 	int order = -1;
1315 
1316 	/*
1317 	 * If no_way_out gets set, there is no safe way to recover from this
1318 	 * MCE.  If mca_cfg.tolerant is cranked up, we'll try anyway.
1319 	 */
1320 	int no_way_out = 0;
1321 
1322 	/*
1323 	 * If kill_it gets set, there might be a way to recover from this
1324 	 * error.
1325 	 */
1326 	int kill_it = 0;
1327 
1328 	/*
1329 	 * MCEs are always local on AMD. Same is determined by MCG_STATUS_LMCES
1330 	 * on Intel.
1331 	 */
1332 	int lmce = 1;
1333 
1334 	this_cpu_inc(mce_exception_count);
1335 
1336 	mce_gather_info(&m, regs);
1337 	m.tsc = rdtsc();
1338 
1339 	final = this_cpu_ptr(&mces_seen);
1340 	*final = m;
1341 
1342 	memset(valid_banks, 0, sizeof(valid_banks));
1343 	no_way_out = mce_no_way_out(&m, &msg, valid_banks, regs);
1344 
1345 	barrier();
1346 
1347 	/*
1348 	 * When no restart IP might need to kill or panic.
1349 	 * Assume the worst for now, but if we find the
1350 	 * severity is MCE_AR_SEVERITY we have other options.
1351 	 */
1352 	if (!(m.mcgstatus & MCG_STATUS_RIPV))
1353 		kill_it = 1;
1354 
1355 	/*
1356 	 * Check if this MCE is signaled to only this logical processor,
1357 	 * on Intel, Zhaoxin only.
1358 	 */
1359 	if (m.cpuvendor == X86_VENDOR_INTEL ||
1360 	    m.cpuvendor == X86_VENDOR_ZHAOXIN)
1361 		lmce = m.mcgstatus & MCG_STATUS_LMCES;
1362 
1363 	/*
1364 	 * Local machine check may already know that we have to panic.
1365 	 * Broadcast machine check begins rendezvous in mce_start()
1366 	 * Go through all banks in exclusion of the other CPUs. This way we
1367 	 * don't report duplicated events on shared banks because the first one
1368 	 * to see it will clear it.
1369 	 */
1370 	if (lmce) {
1371 		if (no_way_out)
1372 			mce_panic("Fatal local machine check", &m, msg);
1373 	} else {
1374 		order = mce_start(&no_way_out);
1375 	}
1376 
1377 	__mc_scan_banks(&m, regs, final, toclear, valid_banks, no_way_out, &worst);
1378 
1379 	if (!no_way_out)
1380 		mce_clear_state(toclear);
1381 
1382 	/*
1383 	 * Do most of the synchronization with other CPUs.
1384 	 * When there's any problem use only local no_way_out state.
1385 	 */
1386 	if (!lmce) {
1387 		if (mce_end(order) < 0)
1388 			no_way_out = worst >= MCE_PANIC_SEVERITY;
1389 	} else {
1390 		/*
1391 		 * If there was a fatal machine check we should have
1392 		 * already called mce_panic earlier in this function.
1393 		 * Since we re-read the banks, we might have found
1394 		 * something new. Check again to see if we found a
1395 		 * fatal error. We call "mce_severity()" again to
1396 		 * make sure we have the right "msg".
1397 		 */
1398 		if (worst >= MCE_PANIC_SEVERITY && mca_cfg.tolerant < 3) {
1399 			mce_severity(&m, regs, cfg->tolerant, &msg, true);
1400 			mce_panic("Local fatal machine check!", &m, msg);
1401 		}
1402 	}
1403 
1404 	/*
1405 	 * If tolerant is at an insane level we drop requests to kill
1406 	 * processes and continue even when there is no way out.
1407 	 */
1408 	if (cfg->tolerant == 3)
1409 		kill_it = 0;
1410 	else if (no_way_out)
1411 		mce_panic("Fatal machine check on current CPU", &m, msg);
1412 
1413 	if (worst > 0)
1414 		irq_work_queue(&mce_irq_work);
1415 
1416 	if (worst != MCE_AR_SEVERITY && !kill_it)
1417 		goto out;
1418 
1419 	/* Fault was in user mode and we need to take some action */
1420 	if ((m.cs & 3) == 3) {
1421 		/* If this triggers there is no way to recover. Die hard. */
1422 		BUG_ON(!on_thread_stack() || !user_mode(regs));
1423 
1424 		queue_task_work(&m, kill_it);
1425 
1426 	} else {
1427 		/*
1428 		 * Handle an MCE which has happened in kernel space but from
1429 		 * which the kernel can recover: ex_has_fault_handler() has
1430 		 * already verified that the rIP at which the error happened is
1431 		 * a rIP from which the kernel can recover (by jumping to
1432 		 * recovery code specified in _ASM_EXTABLE_FAULT()) and the
1433 		 * corresponding exception handler which would do that is the
1434 		 * proper one.
1435 		 */
1436 		if (m.kflags & MCE_IN_KERNEL_RECOV) {
1437 			if (!fixup_exception(regs, X86_TRAP_MC, 0, 0))
1438 				mce_panic("Failed kernel mode recovery", &m, msg);
1439 		}
1440 
1441 		if (m.kflags & MCE_IN_KERNEL_COPYIN)
1442 			queue_task_work(&m, kill_it);
1443 	}
1444 out:
1445 	mce_wrmsrl(MSR_IA32_MCG_STATUS, 0);
1446 }
1447 EXPORT_SYMBOL_GPL(do_machine_check);
1448 
1449 #ifndef CONFIG_MEMORY_FAILURE
1450 int memory_failure(unsigned long pfn, int flags)
1451 {
1452 	/* mce_severity() should not hand us an ACTION_REQUIRED error */
1453 	BUG_ON(flags & MF_ACTION_REQUIRED);
1454 	pr_err("Uncorrected memory error in page 0x%lx ignored\n"
1455 	       "Rebuild kernel with CONFIG_MEMORY_FAILURE=y for smarter handling\n",
1456 	       pfn);
1457 
1458 	return 0;
1459 }
1460 #endif
1461 
1462 /*
1463  * Periodic polling timer for "silent" machine check errors.  If the
1464  * poller finds an MCE, poll 2x faster.  When the poller finds no more
1465  * errors, poll 2x slower (up to check_interval seconds).
1466  */
1467 static unsigned long check_interval = INITIAL_CHECK_INTERVAL;
1468 
1469 static DEFINE_PER_CPU(unsigned long, mce_next_interval); /* in jiffies */
1470 static DEFINE_PER_CPU(struct timer_list, mce_timer);
1471 
1472 static unsigned long mce_adjust_timer_default(unsigned long interval)
1473 {
1474 	return interval;
1475 }
1476 
1477 static unsigned long (*mce_adjust_timer)(unsigned long interval) = mce_adjust_timer_default;
1478 
1479 static void __start_timer(struct timer_list *t, unsigned long interval)
1480 {
1481 	unsigned long when = jiffies + interval;
1482 	unsigned long flags;
1483 
1484 	local_irq_save(flags);
1485 
1486 	if (!timer_pending(t) || time_before(when, t->expires))
1487 		mod_timer(t, round_jiffies(when));
1488 
1489 	local_irq_restore(flags);
1490 }
1491 
1492 static void mce_timer_fn(struct timer_list *t)
1493 {
1494 	struct timer_list *cpu_t = this_cpu_ptr(&mce_timer);
1495 	unsigned long iv;
1496 
1497 	WARN_ON(cpu_t != t);
1498 
1499 	iv = __this_cpu_read(mce_next_interval);
1500 
1501 	if (mce_available(this_cpu_ptr(&cpu_info))) {
1502 		machine_check_poll(0, this_cpu_ptr(&mce_poll_banks));
1503 
1504 		if (mce_intel_cmci_poll()) {
1505 			iv = mce_adjust_timer(iv);
1506 			goto done;
1507 		}
1508 	}
1509 
1510 	/*
1511 	 * Alert userspace if needed. If we logged an MCE, reduce the polling
1512 	 * interval, otherwise increase the polling interval.
1513 	 */
1514 	if (mce_notify_irq())
1515 		iv = max(iv / 2, (unsigned long) HZ/100);
1516 	else
1517 		iv = min(iv * 2, round_jiffies_relative(check_interval * HZ));
1518 
1519 done:
1520 	__this_cpu_write(mce_next_interval, iv);
1521 	__start_timer(t, iv);
1522 }
1523 
1524 /*
1525  * Ensure that the timer is firing in @interval from now.
1526  */
1527 void mce_timer_kick(unsigned long interval)
1528 {
1529 	struct timer_list *t = this_cpu_ptr(&mce_timer);
1530 	unsigned long iv = __this_cpu_read(mce_next_interval);
1531 
1532 	__start_timer(t, interval);
1533 
1534 	if (interval < iv)
1535 		__this_cpu_write(mce_next_interval, interval);
1536 }
1537 
1538 /* Must not be called in IRQ context where del_timer_sync() can deadlock */
1539 static void mce_timer_delete_all(void)
1540 {
1541 	int cpu;
1542 
1543 	for_each_online_cpu(cpu)
1544 		del_timer_sync(&per_cpu(mce_timer, cpu));
1545 }
1546 
1547 /*
1548  * Notify the user(s) about new machine check events.
1549  * Can be called from interrupt context, but not from machine check/NMI
1550  * context.
1551  */
1552 int mce_notify_irq(void)
1553 {
1554 	/* Not more than two messages every minute */
1555 	static DEFINE_RATELIMIT_STATE(ratelimit, 60*HZ, 2);
1556 
1557 	if (test_and_clear_bit(0, &mce_need_notify)) {
1558 		mce_work_trigger();
1559 
1560 		if (__ratelimit(&ratelimit))
1561 			pr_info(HW_ERR "Machine check events logged\n");
1562 
1563 		return 1;
1564 	}
1565 	return 0;
1566 }
1567 EXPORT_SYMBOL_GPL(mce_notify_irq);
1568 
1569 static void __mcheck_cpu_mce_banks_init(void)
1570 {
1571 	struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
1572 	u8 n_banks = this_cpu_read(mce_num_banks);
1573 	int i;
1574 
1575 	for (i = 0; i < n_banks; i++) {
1576 		struct mce_bank *b = &mce_banks[i];
1577 
1578 		/*
1579 		 * Init them all, __mcheck_cpu_apply_quirks() is going to apply
1580 		 * the required vendor quirks before
1581 		 * __mcheck_cpu_init_clear_banks() does the final bank setup.
1582 		 */
1583 		b->ctl = -1ULL;
1584 		b->init = 1;
1585 	}
1586 }
1587 
1588 /*
1589  * Initialize Machine Checks for a CPU.
1590  */
1591 static void __mcheck_cpu_cap_init(void)
1592 {
1593 	u64 cap;
1594 	u8 b;
1595 
1596 	rdmsrl(MSR_IA32_MCG_CAP, cap);
1597 
1598 	b = cap & MCG_BANKCNT_MASK;
1599 
1600 	if (b > MAX_NR_BANKS) {
1601 		pr_warn("CPU%d: Using only %u machine check banks out of %u\n",
1602 			smp_processor_id(), MAX_NR_BANKS, b);
1603 		b = MAX_NR_BANKS;
1604 	}
1605 
1606 	this_cpu_write(mce_num_banks, b);
1607 
1608 	__mcheck_cpu_mce_banks_init();
1609 
1610 	/* Use accurate RIP reporting if available. */
1611 	if ((cap & MCG_EXT_P) && MCG_EXT_CNT(cap) >= 9)
1612 		mca_cfg.rip_msr = MSR_IA32_MCG_EIP;
1613 
1614 	if (cap & MCG_SER_P)
1615 		mca_cfg.ser = 1;
1616 }
1617 
1618 static void __mcheck_cpu_init_generic(void)
1619 {
1620 	enum mcp_flags m_fl = 0;
1621 	mce_banks_t all_banks;
1622 	u64 cap;
1623 
1624 	if (!mca_cfg.bootlog)
1625 		m_fl = MCP_DONTLOG;
1626 
1627 	/*
1628 	 * Log the machine checks left over from the previous reset.
1629 	 */
1630 	bitmap_fill(all_banks, MAX_NR_BANKS);
1631 	machine_check_poll(MCP_UC | m_fl, &all_banks);
1632 
1633 	cr4_set_bits(X86_CR4_MCE);
1634 
1635 	rdmsrl(MSR_IA32_MCG_CAP, cap);
1636 	if (cap & MCG_CTL_P)
1637 		wrmsr(MSR_IA32_MCG_CTL, 0xffffffff, 0xffffffff);
1638 }
1639 
1640 static void __mcheck_cpu_init_clear_banks(void)
1641 {
1642 	struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
1643 	int i;
1644 
1645 	for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
1646 		struct mce_bank *b = &mce_banks[i];
1647 
1648 		if (!b->init)
1649 			continue;
1650 		wrmsrl(msr_ops.ctl(i), b->ctl);
1651 		wrmsrl(msr_ops.status(i), 0);
1652 	}
1653 }
1654 
1655 /*
1656  * Do a final check to see if there are any unused/RAZ banks.
1657  *
1658  * This must be done after the banks have been initialized and any quirks have
1659  * been applied.
1660  *
1661  * Do not call this from any user-initiated flows, e.g. CPU hotplug or sysfs.
1662  * Otherwise, a user who disables a bank will not be able to re-enable it
1663  * without a system reboot.
1664  */
1665 static void __mcheck_cpu_check_banks(void)
1666 {
1667 	struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
1668 	u64 msrval;
1669 	int i;
1670 
1671 	for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
1672 		struct mce_bank *b = &mce_banks[i];
1673 
1674 		if (!b->init)
1675 			continue;
1676 
1677 		rdmsrl(msr_ops.ctl(i), msrval);
1678 		b->init = !!msrval;
1679 	}
1680 }
1681 
1682 /*
1683  * During IFU recovery Sandy Bridge -EP4S processors set the RIPV and
1684  * EIPV bits in MCG_STATUS to zero on the affected logical processor (SDM
1685  * Vol 3B Table 15-20). But this confuses both the code that determines
1686  * whether the machine check occurred in kernel or user mode, and also
1687  * the severity assessment code. Pretend that EIPV was set, and take the
1688  * ip/cs values from the pt_regs that mce_gather_info() ignored earlier.
1689  */
1690 static void quirk_sandybridge_ifu(int bank, struct mce *m, struct pt_regs *regs)
1691 {
1692 	if (bank != 0)
1693 		return;
1694 	if ((m->mcgstatus & (MCG_STATUS_EIPV|MCG_STATUS_RIPV)) != 0)
1695 		return;
1696 	if ((m->status & (MCI_STATUS_OVER|MCI_STATUS_UC|
1697 		          MCI_STATUS_EN|MCI_STATUS_MISCV|MCI_STATUS_ADDRV|
1698 			  MCI_STATUS_PCC|MCI_STATUS_S|MCI_STATUS_AR|
1699 			  MCACOD)) !=
1700 			 (MCI_STATUS_UC|MCI_STATUS_EN|
1701 			  MCI_STATUS_MISCV|MCI_STATUS_ADDRV|MCI_STATUS_S|
1702 			  MCI_STATUS_AR|MCACOD_INSTR))
1703 		return;
1704 
1705 	m->mcgstatus |= MCG_STATUS_EIPV;
1706 	m->ip = regs->ip;
1707 	m->cs = regs->cs;
1708 }
1709 
1710 /* Add per CPU specific workarounds here */
1711 static int __mcheck_cpu_apply_quirks(struct cpuinfo_x86 *c)
1712 {
1713 	struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
1714 	struct mca_config *cfg = &mca_cfg;
1715 
1716 	if (c->x86_vendor == X86_VENDOR_UNKNOWN) {
1717 		pr_info("unknown CPU type - not enabling MCE support\n");
1718 		return -EOPNOTSUPP;
1719 	}
1720 
1721 	/* This should be disabled by the BIOS, but isn't always */
1722 	if (c->x86_vendor == X86_VENDOR_AMD) {
1723 		if (c->x86 == 15 && this_cpu_read(mce_num_banks) > 4) {
1724 			/*
1725 			 * disable GART TBL walk error reporting, which
1726 			 * trips off incorrectly with the IOMMU & 3ware
1727 			 * & Cerberus:
1728 			 */
1729 			clear_bit(10, (unsigned long *)&mce_banks[4].ctl);
1730 		}
1731 		if (c->x86 < 0x11 && cfg->bootlog < 0) {
1732 			/*
1733 			 * Lots of broken BIOS around that don't clear them
1734 			 * by default and leave crap in there. Don't log:
1735 			 */
1736 			cfg->bootlog = 0;
1737 		}
1738 		/*
1739 		 * Various K7s with broken bank 0 around. Always disable
1740 		 * by default.
1741 		 */
1742 		if (c->x86 == 6 && this_cpu_read(mce_num_banks) > 0)
1743 			mce_banks[0].ctl = 0;
1744 
1745 		/*
1746 		 * overflow_recov is supported for F15h Models 00h-0fh
1747 		 * even though we don't have a CPUID bit for it.
1748 		 */
1749 		if (c->x86 == 0x15 && c->x86_model <= 0xf)
1750 			mce_flags.overflow_recov = 1;
1751 
1752 	}
1753 
1754 	if (c->x86_vendor == X86_VENDOR_INTEL) {
1755 		/*
1756 		 * SDM documents that on family 6 bank 0 should not be written
1757 		 * because it aliases to another special BIOS controlled
1758 		 * register.
1759 		 * But it's not aliased anymore on model 0x1a+
1760 		 * Don't ignore bank 0 completely because there could be a
1761 		 * valid event later, merely don't write CTL0.
1762 		 */
1763 
1764 		if (c->x86 == 6 && c->x86_model < 0x1A && this_cpu_read(mce_num_banks) > 0)
1765 			mce_banks[0].init = 0;
1766 
1767 		/*
1768 		 * All newer Intel systems support MCE broadcasting. Enable
1769 		 * synchronization with a one second timeout.
1770 		 */
1771 		if ((c->x86 > 6 || (c->x86 == 6 && c->x86_model >= 0xe)) &&
1772 			cfg->monarch_timeout < 0)
1773 			cfg->monarch_timeout = USEC_PER_SEC;
1774 
1775 		/*
1776 		 * There are also broken BIOSes on some Pentium M and
1777 		 * earlier systems:
1778 		 */
1779 		if (c->x86 == 6 && c->x86_model <= 13 && cfg->bootlog < 0)
1780 			cfg->bootlog = 0;
1781 
1782 		if (c->x86 == 6 && c->x86_model == 45)
1783 			quirk_no_way_out = quirk_sandybridge_ifu;
1784 	}
1785 
1786 	if (c->x86_vendor == X86_VENDOR_ZHAOXIN) {
1787 		/*
1788 		 * All newer Zhaoxin CPUs support MCE broadcasting. Enable
1789 		 * synchronization with a one second timeout.
1790 		 */
1791 		if (c->x86 > 6 || (c->x86_model == 0x19 || c->x86_model == 0x1f)) {
1792 			if (cfg->monarch_timeout < 0)
1793 				cfg->monarch_timeout = USEC_PER_SEC;
1794 		}
1795 	}
1796 
1797 	if (cfg->monarch_timeout < 0)
1798 		cfg->monarch_timeout = 0;
1799 	if (cfg->bootlog != 0)
1800 		cfg->panic_timeout = 30;
1801 
1802 	return 0;
1803 }
1804 
1805 static int __mcheck_cpu_ancient_init(struct cpuinfo_x86 *c)
1806 {
1807 	if (c->x86 != 5)
1808 		return 0;
1809 
1810 	switch (c->x86_vendor) {
1811 	case X86_VENDOR_INTEL:
1812 		intel_p5_mcheck_init(c);
1813 		return 1;
1814 		break;
1815 	case X86_VENDOR_CENTAUR:
1816 		winchip_mcheck_init(c);
1817 		return 1;
1818 		break;
1819 	default:
1820 		return 0;
1821 	}
1822 
1823 	return 0;
1824 }
1825 
1826 /*
1827  * Init basic CPU features needed for early decoding of MCEs.
1828  */
1829 static void __mcheck_cpu_init_early(struct cpuinfo_x86 *c)
1830 {
1831 	if (c->x86_vendor == X86_VENDOR_AMD || c->x86_vendor == X86_VENDOR_HYGON) {
1832 		mce_flags.overflow_recov = !!cpu_has(c, X86_FEATURE_OVERFLOW_RECOV);
1833 		mce_flags.succor	 = !!cpu_has(c, X86_FEATURE_SUCCOR);
1834 		mce_flags.smca		 = !!cpu_has(c, X86_FEATURE_SMCA);
1835 		mce_flags.amd_threshold	 = 1;
1836 
1837 		if (mce_flags.smca) {
1838 			msr_ops.ctl	= smca_ctl_reg;
1839 			msr_ops.status	= smca_status_reg;
1840 			msr_ops.addr	= smca_addr_reg;
1841 			msr_ops.misc	= smca_misc_reg;
1842 		}
1843 	}
1844 }
1845 
1846 static void mce_centaur_feature_init(struct cpuinfo_x86 *c)
1847 {
1848 	struct mca_config *cfg = &mca_cfg;
1849 
1850 	 /*
1851 	  * All newer Centaur CPUs support MCE broadcasting. Enable
1852 	  * synchronization with a one second timeout.
1853 	  */
1854 	if ((c->x86 == 6 && c->x86_model == 0xf && c->x86_stepping >= 0xe) ||
1855 	     c->x86 > 6) {
1856 		if (cfg->monarch_timeout < 0)
1857 			cfg->monarch_timeout = USEC_PER_SEC;
1858 	}
1859 }
1860 
1861 static void mce_zhaoxin_feature_init(struct cpuinfo_x86 *c)
1862 {
1863 	struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
1864 
1865 	/*
1866 	 * These CPUs have MCA bank 8 which reports only one error type called
1867 	 * SVAD (System View Address Decoder). The reporting of that error is
1868 	 * controlled by IA32_MC8.CTL.0.
1869 	 *
1870 	 * If enabled, prefetching on these CPUs will cause SVAD MCE when
1871 	 * virtual machines start and result in a system  panic. Always disable
1872 	 * bank 8 SVAD error by default.
1873 	 */
1874 	if ((c->x86 == 7 && c->x86_model == 0x1b) ||
1875 	    (c->x86_model == 0x19 || c->x86_model == 0x1f)) {
1876 		if (this_cpu_read(mce_num_banks) > 8)
1877 			mce_banks[8].ctl = 0;
1878 	}
1879 
1880 	intel_init_cmci();
1881 	intel_init_lmce();
1882 	mce_adjust_timer = cmci_intel_adjust_timer;
1883 }
1884 
1885 static void mce_zhaoxin_feature_clear(struct cpuinfo_x86 *c)
1886 {
1887 	intel_clear_lmce();
1888 }
1889 
1890 static void __mcheck_cpu_init_vendor(struct cpuinfo_x86 *c)
1891 {
1892 	switch (c->x86_vendor) {
1893 	case X86_VENDOR_INTEL:
1894 		mce_intel_feature_init(c);
1895 		mce_adjust_timer = cmci_intel_adjust_timer;
1896 		break;
1897 
1898 	case X86_VENDOR_AMD: {
1899 		mce_amd_feature_init(c);
1900 		break;
1901 		}
1902 
1903 	case X86_VENDOR_HYGON:
1904 		mce_hygon_feature_init(c);
1905 		break;
1906 
1907 	case X86_VENDOR_CENTAUR:
1908 		mce_centaur_feature_init(c);
1909 		break;
1910 
1911 	case X86_VENDOR_ZHAOXIN:
1912 		mce_zhaoxin_feature_init(c);
1913 		break;
1914 
1915 	default:
1916 		break;
1917 	}
1918 }
1919 
1920 static void __mcheck_cpu_clear_vendor(struct cpuinfo_x86 *c)
1921 {
1922 	switch (c->x86_vendor) {
1923 	case X86_VENDOR_INTEL:
1924 		mce_intel_feature_clear(c);
1925 		break;
1926 
1927 	case X86_VENDOR_ZHAOXIN:
1928 		mce_zhaoxin_feature_clear(c);
1929 		break;
1930 
1931 	default:
1932 		break;
1933 	}
1934 }
1935 
1936 static void mce_start_timer(struct timer_list *t)
1937 {
1938 	unsigned long iv = check_interval * HZ;
1939 
1940 	if (mca_cfg.ignore_ce || !iv)
1941 		return;
1942 
1943 	this_cpu_write(mce_next_interval, iv);
1944 	__start_timer(t, iv);
1945 }
1946 
1947 static void __mcheck_cpu_setup_timer(void)
1948 {
1949 	struct timer_list *t = this_cpu_ptr(&mce_timer);
1950 
1951 	timer_setup(t, mce_timer_fn, TIMER_PINNED);
1952 }
1953 
1954 static void __mcheck_cpu_init_timer(void)
1955 {
1956 	struct timer_list *t = this_cpu_ptr(&mce_timer);
1957 
1958 	timer_setup(t, mce_timer_fn, TIMER_PINNED);
1959 	mce_start_timer(t);
1960 }
1961 
1962 bool filter_mce(struct mce *m)
1963 {
1964 	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
1965 		return amd_filter_mce(m);
1966 	if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
1967 		return intel_filter_mce(m);
1968 
1969 	return false;
1970 }
1971 
1972 /* Handle unconfigured int18 (should never happen) */
1973 static noinstr void unexpected_machine_check(struct pt_regs *regs)
1974 {
1975 	instrumentation_begin();
1976 	pr_err("CPU#%d: Unexpected int18 (Machine Check)\n",
1977 	       smp_processor_id());
1978 	instrumentation_end();
1979 }
1980 
1981 /* Call the installed machine check handler for this CPU setup. */
1982 void (*machine_check_vector)(struct pt_regs *) = unexpected_machine_check;
1983 
1984 static __always_inline void exc_machine_check_kernel(struct pt_regs *regs)
1985 {
1986 	bool irq_state;
1987 
1988 	WARN_ON_ONCE(user_mode(regs));
1989 
1990 	/*
1991 	 * Only required when from kernel mode. See
1992 	 * mce_check_crashing_cpu() for details.
1993 	 */
1994 	if (machine_check_vector == do_machine_check &&
1995 	    mce_check_crashing_cpu())
1996 		return;
1997 
1998 	irq_state = idtentry_enter_nmi(regs);
1999 	/*
2000 	 * The call targets are marked noinstr, but objtool can't figure
2001 	 * that out because it's an indirect call. Annotate it.
2002 	 */
2003 	instrumentation_begin();
2004 	trace_hardirqs_off_finish();
2005 	machine_check_vector(regs);
2006 	if (regs->flags & X86_EFLAGS_IF)
2007 		trace_hardirqs_on_prepare();
2008 	instrumentation_end();
2009 	idtentry_exit_nmi(regs, irq_state);
2010 }
2011 
2012 static __always_inline void exc_machine_check_user(struct pt_regs *regs)
2013 {
2014 	irqentry_enter_from_user_mode(regs);
2015 	instrumentation_begin();
2016 	machine_check_vector(regs);
2017 	instrumentation_end();
2018 	irqentry_exit_to_user_mode(regs);
2019 }
2020 
2021 #ifdef CONFIG_X86_64
2022 /* MCE hit kernel mode */
2023 DEFINE_IDTENTRY_MCE(exc_machine_check)
2024 {
2025 	unsigned long dr7;
2026 
2027 	dr7 = local_db_save();
2028 	exc_machine_check_kernel(regs);
2029 	local_db_restore(dr7);
2030 }
2031 
2032 /* The user mode variant. */
2033 DEFINE_IDTENTRY_MCE_USER(exc_machine_check)
2034 {
2035 	unsigned long dr7;
2036 
2037 	dr7 = local_db_save();
2038 	exc_machine_check_user(regs);
2039 	local_db_restore(dr7);
2040 }
2041 #else
2042 /* 32bit unified entry point */
2043 DEFINE_IDTENTRY_RAW(exc_machine_check)
2044 {
2045 	unsigned long dr7;
2046 
2047 	dr7 = local_db_save();
2048 	if (user_mode(regs))
2049 		exc_machine_check_user(regs);
2050 	else
2051 		exc_machine_check_kernel(regs);
2052 	local_db_restore(dr7);
2053 }
2054 #endif
2055 
2056 /*
2057  * Called for each booted CPU to set up machine checks.
2058  * Must be called with preempt off:
2059  */
2060 void mcheck_cpu_init(struct cpuinfo_x86 *c)
2061 {
2062 	if (mca_cfg.disabled)
2063 		return;
2064 
2065 	if (__mcheck_cpu_ancient_init(c))
2066 		return;
2067 
2068 	if (!mce_available(c))
2069 		return;
2070 
2071 	__mcheck_cpu_cap_init();
2072 
2073 	if (__mcheck_cpu_apply_quirks(c) < 0) {
2074 		mca_cfg.disabled = 1;
2075 		return;
2076 	}
2077 
2078 	if (mce_gen_pool_init()) {
2079 		mca_cfg.disabled = 1;
2080 		pr_emerg("Couldn't allocate MCE records pool!\n");
2081 		return;
2082 	}
2083 
2084 	machine_check_vector = do_machine_check;
2085 
2086 	__mcheck_cpu_init_early(c);
2087 	__mcheck_cpu_init_generic();
2088 	__mcheck_cpu_init_vendor(c);
2089 	__mcheck_cpu_init_clear_banks();
2090 	__mcheck_cpu_check_banks();
2091 	__mcheck_cpu_setup_timer();
2092 }
2093 
2094 /*
2095  * Called for each booted CPU to clear some machine checks opt-ins
2096  */
2097 void mcheck_cpu_clear(struct cpuinfo_x86 *c)
2098 {
2099 	if (mca_cfg.disabled)
2100 		return;
2101 
2102 	if (!mce_available(c))
2103 		return;
2104 
2105 	/*
2106 	 * Possibly to clear general settings generic to x86
2107 	 * __mcheck_cpu_clear_generic(c);
2108 	 */
2109 	__mcheck_cpu_clear_vendor(c);
2110 
2111 }
2112 
2113 static void __mce_disable_bank(void *arg)
2114 {
2115 	int bank = *((int *)arg);
2116 	__clear_bit(bank, this_cpu_ptr(mce_poll_banks));
2117 	cmci_disable_bank(bank);
2118 }
2119 
2120 void mce_disable_bank(int bank)
2121 {
2122 	if (bank >= this_cpu_read(mce_num_banks)) {
2123 		pr_warn(FW_BUG
2124 			"Ignoring request to disable invalid MCA bank %d.\n",
2125 			bank);
2126 		return;
2127 	}
2128 	set_bit(bank, mce_banks_ce_disabled);
2129 	on_each_cpu(__mce_disable_bank, &bank, 1);
2130 }
2131 
2132 /*
2133  * mce=off Disables machine check
2134  * mce=no_cmci Disables CMCI
2135  * mce=no_lmce Disables LMCE
2136  * mce=dont_log_ce Clears corrected events silently, no log created for CEs.
2137  * mce=print_all Print all machine check logs to console
2138  * mce=ignore_ce Disables polling and CMCI, corrected events are not cleared.
2139  * mce=TOLERANCELEVEL[,monarchtimeout] (number, see above)
2140  *	monarchtimeout is how long to wait for other CPUs on machine
2141  *	check, or 0 to not wait
2142  * mce=bootlog Log MCEs from before booting. Disabled by default on AMD Fam10h
2143 	and older.
2144  * mce=nobootlog Don't log MCEs from before booting.
2145  * mce=bios_cmci_threshold Don't program the CMCI threshold
2146  * mce=recovery force enable copy_mc_fragile()
2147  */
2148 static int __init mcheck_enable(char *str)
2149 {
2150 	struct mca_config *cfg = &mca_cfg;
2151 
2152 	if (*str == 0) {
2153 		enable_p5_mce();
2154 		return 1;
2155 	}
2156 	if (*str == '=')
2157 		str++;
2158 	if (!strcmp(str, "off"))
2159 		cfg->disabled = 1;
2160 	else if (!strcmp(str, "no_cmci"))
2161 		cfg->cmci_disabled = true;
2162 	else if (!strcmp(str, "no_lmce"))
2163 		cfg->lmce_disabled = 1;
2164 	else if (!strcmp(str, "dont_log_ce"))
2165 		cfg->dont_log_ce = true;
2166 	else if (!strcmp(str, "print_all"))
2167 		cfg->print_all = true;
2168 	else if (!strcmp(str, "ignore_ce"))
2169 		cfg->ignore_ce = true;
2170 	else if (!strcmp(str, "bootlog") || !strcmp(str, "nobootlog"))
2171 		cfg->bootlog = (str[0] == 'b');
2172 	else if (!strcmp(str, "bios_cmci_threshold"))
2173 		cfg->bios_cmci_threshold = 1;
2174 	else if (!strcmp(str, "recovery"))
2175 		cfg->recovery = 1;
2176 	else if (isdigit(str[0])) {
2177 		if (get_option(&str, &cfg->tolerant) == 2)
2178 			get_option(&str, &(cfg->monarch_timeout));
2179 	} else {
2180 		pr_info("mce argument %s ignored. Please use /sys\n", str);
2181 		return 0;
2182 	}
2183 	return 1;
2184 }
2185 __setup("mce", mcheck_enable);
2186 
2187 int __init mcheck_init(void)
2188 {
2189 	mcheck_intel_therm_init();
2190 	mce_register_decode_chain(&early_nb);
2191 	mce_register_decode_chain(&mce_uc_nb);
2192 	mce_register_decode_chain(&mce_default_nb);
2193 	mcheck_vendor_init_severity();
2194 
2195 	INIT_WORK(&mce_work, mce_gen_pool_process);
2196 	init_irq_work(&mce_irq_work, mce_irq_work_cb);
2197 
2198 	return 0;
2199 }
2200 
2201 /*
2202  * mce_syscore: PM support
2203  */
2204 
2205 /*
2206  * Disable machine checks on suspend and shutdown. We can't really handle
2207  * them later.
2208  */
2209 static void mce_disable_error_reporting(void)
2210 {
2211 	struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
2212 	int i;
2213 
2214 	for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
2215 		struct mce_bank *b = &mce_banks[i];
2216 
2217 		if (b->init)
2218 			wrmsrl(msr_ops.ctl(i), 0);
2219 	}
2220 	return;
2221 }
2222 
2223 static void vendor_disable_error_reporting(void)
2224 {
2225 	/*
2226 	 * Don't clear on Intel or AMD or Hygon or Zhaoxin CPUs. Some of these
2227 	 * MSRs are socket-wide. Disabling them for just a single offlined CPU
2228 	 * is bad, since it will inhibit reporting for all shared resources on
2229 	 * the socket like the last level cache (LLC), the integrated memory
2230 	 * controller (iMC), etc.
2231 	 */
2232 	if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL ||
2233 	    boot_cpu_data.x86_vendor == X86_VENDOR_HYGON ||
2234 	    boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||
2235 	    boot_cpu_data.x86_vendor == X86_VENDOR_ZHAOXIN)
2236 		return;
2237 
2238 	mce_disable_error_reporting();
2239 }
2240 
2241 static int mce_syscore_suspend(void)
2242 {
2243 	vendor_disable_error_reporting();
2244 	return 0;
2245 }
2246 
2247 static void mce_syscore_shutdown(void)
2248 {
2249 	vendor_disable_error_reporting();
2250 }
2251 
2252 /*
2253  * On resume clear all MCE state. Don't want to see leftovers from the BIOS.
2254  * Only one CPU is active at this time, the others get re-added later using
2255  * CPU hotplug:
2256  */
2257 static void mce_syscore_resume(void)
2258 {
2259 	__mcheck_cpu_init_generic();
2260 	__mcheck_cpu_init_vendor(raw_cpu_ptr(&cpu_info));
2261 	__mcheck_cpu_init_clear_banks();
2262 }
2263 
2264 static struct syscore_ops mce_syscore_ops = {
2265 	.suspend	= mce_syscore_suspend,
2266 	.shutdown	= mce_syscore_shutdown,
2267 	.resume		= mce_syscore_resume,
2268 };
2269 
2270 /*
2271  * mce_device: Sysfs support
2272  */
2273 
2274 static void mce_cpu_restart(void *data)
2275 {
2276 	if (!mce_available(raw_cpu_ptr(&cpu_info)))
2277 		return;
2278 	__mcheck_cpu_init_generic();
2279 	__mcheck_cpu_init_clear_banks();
2280 	__mcheck_cpu_init_timer();
2281 }
2282 
2283 /* Reinit MCEs after user configuration changes */
2284 static void mce_restart(void)
2285 {
2286 	mce_timer_delete_all();
2287 	on_each_cpu(mce_cpu_restart, NULL, 1);
2288 }
2289 
2290 /* Toggle features for corrected errors */
2291 static void mce_disable_cmci(void *data)
2292 {
2293 	if (!mce_available(raw_cpu_ptr(&cpu_info)))
2294 		return;
2295 	cmci_clear();
2296 }
2297 
2298 static void mce_enable_ce(void *all)
2299 {
2300 	if (!mce_available(raw_cpu_ptr(&cpu_info)))
2301 		return;
2302 	cmci_reenable();
2303 	cmci_recheck();
2304 	if (all)
2305 		__mcheck_cpu_init_timer();
2306 }
2307 
2308 static struct bus_type mce_subsys = {
2309 	.name		= "machinecheck",
2310 	.dev_name	= "machinecheck",
2311 };
2312 
2313 DEFINE_PER_CPU(struct device *, mce_device);
2314 
2315 static inline struct mce_bank_dev *attr_to_bank(struct device_attribute *attr)
2316 {
2317 	return container_of(attr, struct mce_bank_dev, attr);
2318 }
2319 
2320 static ssize_t show_bank(struct device *s, struct device_attribute *attr,
2321 			 char *buf)
2322 {
2323 	u8 bank = attr_to_bank(attr)->bank;
2324 	struct mce_bank *b;
2325 
2326 	if (bank >= per_cpu(mce_num_banks, s->id))
2327 		return -EINVAL;
2328 
2329 	b = &per_cpu(mce_banks_array, s->id)[bank];
2330 
2331 	if (!b->init)
2332 		return -ENODEV;
2333 
2334 	return sprintf(buf, "%llx\n", b->ctl);
2335 }
2336 
2337 static ssize_t set_bank(struct device *s, struct device_attribute *attr,
2338 			const char *buf, size_t size)
2339 {
2340 	u8 bank = attr_to_bank(attr)->bank;
2341 	struct mce_bank *b;
2342 	u64 new;
2343 
2344 	if (kstrtou64(buf, 0, &new) < 0)
2345 		return -EINVAL;
2346 
2347 	if (bank >= per_cpu(mce_num_banks, s->id))
2348 		return -EINVAL;
2349 
2350 	b = &per_cpu(mce_banks_array, s->id)[bank];
2351 
2352 	if (!b->init)
2353 		return -ENODEV;
2354 
2355 	b->ctl = new;
2356 	mce_restart();
2357 
2358 	return size;
2359 }
2360 
2361 static ssize_t set_ignore_ce(struct device *s,
2362 			     struct device_attribute *attr,
2363 			     const char *buf, size_t size)
2364 {
2365 	u64 new;
2366 
2367 	if (kstrtou64(buf, 0, &new) < 0)
2368 		return -EINVAL;
2369 
2370 	mutex_lock(&mce_sysfs_mutex);
2371 	if (mca_cfg.ignore_ce ^ !!new) {
2372 		if (new) {
2373 			/* disable ce features */
2374 			mce_timer_delete_all();
2375 			on_each_cpu(mce_disable_cmci, NULL, 1);
2376 			mca_cfg.ignore_ce = true;
2377 		} else {
2378 			/* enable ce features */
2379 			mca_cfg.ignore_ce = false;
2380 			on_each_cpu(mce_enable_ce, (void *)1, 1);
2381 		}
2382 	}
2383 	mutex_unlock(&mce_sysfs_mutex);
2384 
2385 	return size;
2386 }
2387 
2388 static ssize_t set_cmci_disabled(struct device *s,
2389 				 struct device_attribute *attr,
2390 				 const char *buf, size_t size)
2391 {
2392 	u64 new;
2393 
2394 	if (kstrtou64(buf, 0, &new) < 0)
2395 		return -EINVAL;
2396 
2397 	mutex_lock(&mce_sysfs_mutex);
2398 	if (mca_cfg.cmci_disabled ^ !!new) {
2399 		if (new) {
2400 			/* disable cmci */
2401 			on_each_cpu(mce_disable_cmci, NULL, 1);
2402 			mca_cfg.cmci_disabled = true;
2403 		} else {
2404 			/* enable cmci */
2405 			mca_cfg.cmci_disabled = false;
2406 			on_each_cpu(mce_enable_ce, NULL, 1);
2407 		}
2408 	}
2409 	mutex_unlock(&mce_sysfs_mutex);
2410 
2411 	return size;
2412 }
2413 
2414 static ssize_t store_int_with_restart(struct device *s,
2415 				      struct device_attribute *attr,
2416 				      const char *buf, size_t size)
2417 {
2418 	unsigned long old_check_interval = check_interval;
2419 	ssize_t ret = device_store_ulong(s, attr, buf, size);
2420 
2421 	if (check_interval == old_check_interval)
2422 		return ret;
2423 
2424 	mutex_lock(&mce_sysfs_mutex);
2425 	mce_restart();
2426 	mutex_unlock(&mce_sysfs_mutex);
2427 
2428 	return ret;
2429 }
2430 
2431 static DEVICE_INT_ATTR(tolerant, 0644, mca_cfg.tolerant);
2432 static DEVICE_INT_ATTR(monarch_timeout, 0644, mca_cfg.monarch_timeout);
2433 static DEVICE_BOOL_ATTR(dont_log_ce, 0644, mca_cfg.dont_log_ce);
2434 static DEVICE_BOOL_ATTR(print_all, 0644, mca_cfg.print_all);
2435 
2436 static struct dev_ext_attribute dev_attr_check_interval = {
2437 	__ATTR(check_interval, 0644, device_show_int, store_int_with_restart),
2438 	&check_interval
2439 };
2440 
2441 static struct dev_ext_attribute dev_attr_ignore_ce = {
2442 	__ATTR(ignore_ce, 0644, device_show_bool, set_ignore_ce),
2443 	&mca_cfg.ignore_ce
2444 };
2445 
2446 static struct dev_ext_attribute dev_attr_cmci_disabled = {
2447 	__ATTR(cmci_disabled, 0644, device_show_bool, set_cmci_disabled),
2448 	&mca_cfg.cmci_disabled
2449 };
2450 
2451 static struct device_attribute *mce_device_attrs[] = {
2452 	&dev_attr_tolerant.attr,
2453 	&dev_attr_check_interval.attr,
2454 #ifdef CONFIG_X86_MCELOG_LEGACY
2455 	&dev_attr_trigger,
2456 #endif
2457 	&dev_attr_monarch_timeout.attr,
2458 	&dev_attr_dont_log_ce.attr,
2459 	&dev_attr_print_all.attr,
2460 	&dev_attr_ignore_ce.attr,
2461 	&dev_attr_cmci_disabled.attr,
2462 	NULL
2463 };
2464 
2465 static cpumask_var_t mce_device_initialized;
2466 
2467 static void mce_device_release(struct device *dev)
2468 {
2469 	kfree(dev);
2470 }
2471 
2472 /* Per CPU device init. All of the CPUs still share the same bank device: */
2473 static int mce_device_create(unsigned int cpu)
2474 {
2475 	struct device *dev;
2476 	int err;
2477 	int i, j;
2478 
2479 	if (!mce_available(&boot_cpu_data))
2480 		return -EIO;
2481 
2482 	dev = per_cpu(mce_device, cpu);
2483 	if (dev)
2484 		return 0;
2485 
2486 	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
2487 	if (!dev)
2488 		return -ENOMEM;
2489 	dev->id  = cpu;
2490 	dev->bus = &mce_subsys;
2491 	dev->release = &mce_device_release;
2492 
2493 	err = device_register(dev);
2494 	if (err) {
2495 		put_device(dev);
2496 		return err;
2497 	}
2498 
2499 	for (i = 0; mce_device_attrs[i]; i++) {
2500 		err = device_create_file(dev, mce_device_attrs[i]);
2501 		if (err)
2502 			goto error;
2503 	}
2504 	for (j = 0; j < per_cpu(mce_num_banks, cpu); j++) {
2505 		err = device_create_file(dev, &mce_bank_devs[j].attr);
2506 		if (err)
2507 			goto error2;
2508 	}
2509 	cpumask_set_cpu(cpu, mce_device_initialized);
2510 	per_cpu(mce_device, cpu) = dev;
2511 
2512 	return 0;
2513 error2:
2514 	while (--j >= 0)
2515 		device_remove_file(dev, &mce_bank_devs[j].attr);
2516 error:
2517 	while (--i >= 0)
2518 		device_remove_file(dev, mce_device_attrs[i]);
2519 
2520 	device_unregister(dev);
2521 
2522 	return err;
2523 }
2524 
2525 static void mce_device_remove(unsigned int cpu)
2526 {
2527 	struct device *dev = per_cpu(mce_device, cpu);
2528 	int i;
2529 
2530 	if (!cpumask_test_cpu(cpu, mce_device_initialized))
2531 		return;
2532 
2533 	for (i = 0; mce_device_attrs[i]; i++)
2534 		device_remove_file(dev, mce_device_attrs[i]);
2535 
2536 	for (i = 0; i < per_cpu(mce_num_banks, cpu); i++)
2537 		device_remove_file(dev, &mce_bank_devs[i].attr);
2538 
2539 	device_unregister(dev);
2540 	cpumask_clear_cpu(cpu, mce_device_initialized);
2541 	per_cpu(mce_device, cpu) = NULL;
2542 }
2543 
2544 /* Make sure there are no machine checks on offlined CPUs. */
2545 static void mce_disable_cpu(void)
2546 {
2547 	if (!mce_available(raw_cpu_ptr(&cpu_info)))
2548 		return;
2549 
2550 	if (!cpuhp_tasks_frozen)
2551 		cmci_clear();
2552 
2553 	vendor_disable_error_reporting();
2554 }
2555 
2556 static void mce_reenable_cpu(void)
2557 {
2558 	struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
2559 	int i;
2560 
2561 	if (!mce_available(raw_cpu_ptr(&cpu_info)))
2562 		return;
2563 
2564 	if (!cpuhp_tasks_frozen)
2565 		cmci_reenable();
2566 	for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
2567 		struct mce_bank *b = &mce_banks[i];
2568 
2569 		if (b->init)
2570 			wrmsrl(msr_ops.ctl(i), b->ctl);
2571 	}
2572 }
2573 
2574 static int mce_cpu_dead(unsigned int cpu)
2575 {
2576 	mce_intel_hcpu_update(cpu);
2577 
2578 	/* intentionally ignoring frozen here */
2579 	if (!cpuhp_tasks_frozen)
2580 		cmci_rediscover();
2581 	return 0;
2582 }
2583 
2584 static int mce_cpu_online(unsigned int cpu)
2585 {
2586 	struct timer_list *t = this_cpu_ptr(&mce_timer);
2587 	int ret;
2588 
2589 	mce_device_create(cpu);
2590 
2591 	ret = mce_threshold_create_device(cpu);
2592 	if (ret) {
2593 		mce_device_remove(cpu);
2594 		return ret;
2595 	}
2596 	mce_reenable_cpu();
2597 	mce_start_timer(t);
2598 	return 0;
2599 }
2600 
2601 static int mce_cpu_pre_down(unsigned int cpu)
2602 {
2603 	struct timer_list *t = this_cpu_ptr(&mce_timer);
2604 
2605 	mce_disable_cpu();
2606 	del_timer_sync(t);
2607 	mce_threshold_remove_device(cpu);
2608 	mce_device_remove(cpu);
2609 	return 0;
2610 }
2611 
2612 static __init void mce_init_banks(void)
2613 {
2614 	int i;
2615 
2616 	for (i = 0; i < MAX_NR_BANKS; i++) {
2617 		struct mce_bank_dev *b = &mce_bank_devs[i];
2618 		struct device_attribute *a = &b->attr;
2619 
2620 		b->bank = i;
2621 
2622 		sysfs_attr_init(&a->attr);
2623 		a->attr.name	= b->attrname;
2624 		snprintf(b->attrname, ATTR_LEN, "bank%d", i);
2625 
2626 		a->attr.mode	= 0644;
2627 		a->show		= show_bank;
2628 		a->store	= set_bank;
2629 	}
2630 }
2631 
2632 /*
2633  * When running on XEN, this initcall is ordered against the XEN mcelog
2634  * initcall:
2635  *
2636  *   device_initcall(xen_late_init_mcelog);
2637  *   device_initcall_sync(mcheck_init_device);
2638  */
2639 static __init int mcheck_init_device(void)
2640 {
2641 	int err;
2642 
2643 	/*
2644 	 * Check if we have a spare virtual bit. This will only become
2645 	 * a problem if/when we move beyond 5-level page tables.
2646 	 */
2647 	MAYBE_BUILD_BUG_ON(__VIRTUAL_MASK_SHIFT >= 63);
2648 
2649 	if (!mce_available(&boot_cpu_data)) {
2650 		err = -EIO;
2651 		goto err_out;
2652 	}
2653 
2654 	if (!zalloc_cpumask_var(&mce_device_initialized, GFP_KERNEL)) {
2655 		err = -ENOMEM;
2656 		goto err_out;
2657 	}
2658 
2659 	mce_init_banks();
2660 
2661 	err = subsys_system_register(&mce_subsys, NULL);
2662 	if (err)
2663 		goto err_out_mem;
2664 
2665 	err = cpuhp_setup_state(CPUHP_X86_MCE_DEAD, "x86/mce:dead", NULL,
2666 				mce_cpu_dead);
2667 	if (err)
2668 		goto err_out_mem;
2669 
2670 	/*
2671 	 * Invokes mce_cpu_online() on all CPUs which are online when
2672 	 * the state is installed.
2673 	 */
2674 	err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/mce:online",
2675 				mce_cpu_online, mce_cpu_pre_down);
2676 	if (err < 0)
2677 		goto err_out_online;
2678 
2679 	register_syscore_ops(&mce_syscore_ops);
2680 
2681 	return 0;
2682 
2683 err_out_online:
2684 	cpuhp_remove_state(CPUHP_X86_MCE_DEAD);
2685 
2686 err_out_mem:
2687 	free_cpumask_var(mce_device_initialized);
2688 
2689 err_out:
2690 	pr_err("Unable to init MCE device (rc: %d)\n", err);
2691 
2692 	return err;
2693 }
2694 device_initcall_sync(mcheck_init_device);
2695 
2696 /*
2697  * Old style boot options parsing. Only for compatibility.
2698  */
2699 static int __init mcheck_disable(char *str)
2700 {
2701 	mca_cfg.disabled = 1;
2702 	return 1;
2703 }
2704 __setup("nomce", mcheck_disable);
2705 
2706 #ifdef CONFIG_DEBUG_FS
2707 struct dentry *mce_get_debugfs_dir(void)
2708 {
2709 	static struct dentry *dmce;
2710 
2711 	if (!dmce)
2712 		dmce = debugfs_create_dir("mce", NULL);
2713 
2714 	return dmce;
2715 }
2716 
2717 static void mce_reset(void)
2718 {
2719 	cpu_missing = 0;
2720 	atomic_set(&mce_fake_panicked, 0);
2721 	atomic_set(&mce_executing, 0);
2722 	atomic_set(&mce_callin, 0);
2723 	atomic_set(&global_nwo, 0);
2724 }
2725 
2726 static int fake_panic_get(void *data, u64 *val)
2727 {
2728 	*val = fake_panic;
2729 	return 0;
2730 }
2731 
2732 static int fake_panic_set(void *data, u64 val)
2733 {
2734 	mce_reset();
2735 	fake_panic = val;
2736 	return 0;
2737 }
2738 
2739 DEFINE_DEBUGFS_ATTRIBUTE(fake_panic_fops, fake_panic_get, fake_panic_set,
2740 			 "%llu\n");
2741 
2742 static void __init mcheck_debugfs_init(void)
2743 {
2744 	struct dentry *dmce;
2745 
2746 	dmce = mce_get_debugfs_dir();
2747 	debugfs_create_file_unsafe("fake_panic", 0444, dmce, NULL,
2748 				   &fake_panic_fops);
2749 }
2750 #else
2751 static void __init mcheck_debugfs_init(void) { }
2752 #endif
2753 
2754 static int __init mcheck_late_init(void)
2755 {
2756 	if (mca_cfg.recovery)
2757 		enable_copy_mc_fragile();
2758 
2759 	mcheck_debugfs_init();
2760 
2761 	/*
2762 	 * Flush out everything that has been logged during early boot, now that
2763 	 * everything has been initialized (workqueues, decoders, ...).
2764 	 */
2765 	mce_schedule_work();
2766 
2767 	return 0;
2768 }
2769 late_initcall(mcheck_late_init);
2770