1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Machine check handler. 4 * 5 * K8 parts Copyright 2002,2003 Andi Kleen, SuSE Labs. 6 * Rest from unknown author(s). 7 * 2004 Andi Kleen. Rewrote most of it. 8 * Copyright 2008 Intel Corporation 9 * Author: Andi Kleen 10 */ 11 12 #include <linux/thread_info.h> 13 #include <linux/capability.h> 14 #include <linux/miscdevice.h> 15 #include <linux/ratelimit.h> 16 #include <linux/rcupdate.h> 17 #include <linux/kobject.h> 18 #include <linux/uaccess.h> 19 #include <linux/kdebug.h> 20 #include <linux/kernel.h> 21 #include <linux/percpu.h> 22 #include <linux/string.h> 23 #include <linux/device.h> 24 #include <linux/syscore_ops.h> 25 #include <linux/delay.h> 26 #include <linux/ctype.h> 27 #include <linux/sched.h> 28 #include <linux/sysfs.h> 29 #include <linux/types.h> 30 #include <linux/slab.h> 31 #include <linux/init.h> 32 #include <linux/kmod.h> 33 #include <linux/poll.h> 34 #include <linux/nmi.h> 35 #include <linux/cpu.h> 36 #include <linux/ras.h> 37 #include <linux/smp.h> 38 #include <linux/fs.h> 39 #include <linux/mm.h> 40 #include <linux/debugfs.h> 41 #include <linux/irq_work.h> 42 #include <linux/export.h> 43 #include <linux/set_memory.h> 44 #include <linux/sync_core.h> 45 #include <linux/task_work.h> 46 #include <linux/hardirq.h> 47 48 #include <asm/intel-family.h> 49 #include <asm/processor.h> 50 #include <asm/traps.h> 51 #include <asm/tlbflush.h> 52 #include <asm/mce.h> 53 #include <asm/msr.h> 54 #include <asm/reboot.h> 55 56 #include "internal.h" 57 58 /* sysfs synchronization */ 59 static DEFINE_MUTEX(mce_sysfs_mutex); 60 61 #define CREATE_TRACE_POINTS 62 #include <trace/events/mce.h> 63 64 #define SPINUNIT 100 /* 100ns */ 65 66 DEFINE_PER_CPU(unsigned, mce_exception_count); 67 68 DEFINE_PER_CPU_READ_MOSTLY(unsigned int, mce_num_banks); 69 70 struct mce_bank { 71 u64 ctl; /* subevents to enable */ 72 bool init; /* initialise bank? */ 73 }; 74 static DEFINE_PER_CPU_READ_MOSTLY(struct mce_bank[MAX_NR_BANKS], mce_banks_array); 75 76 #define ATTR_LEN 16 77 /* One object for each MCE bank, shared by all CPUs */ 78 struct mce_bank_dev { 79 struct device_attribute attr; /* device attribute */ 80 char attrname[ATTR_LEN]; /* attribute name */ 81 u8 bank; /* bank number */ 82 }; 83 static struct mce_bank_dev mce_bank_devs[MAX_NR_BANKS]; 84 85 struct mce_vendor_flags mce_flags __read_mostly; 86 87 struct mca_config mca_cfg __read_mostly = { 88 .bootlog = -1, 89 /* 90 * Tolerant levels: 91 * 0: always panic on uncorrected errors, log corrected errors 92 * 1: panic or SIGBUS on uncorrected errors, log corrected errors 93 * 2: SIGBUS or log uncorrected errors (if possible), log corr. errors 94 * 3: never panic or SIGBUS, log all errors (for testing only) 95 */ 96 .tolerant = 1, 97 .monarch_timeout = -1 98 }; 99 100 static DEFINE_PER_CPU(struct mce, mces_seen); 101 static unsigned long mce_need_notify; 102 103 /* 104 * MCA banks polled by the period polling timer for corrected events. 105 * With Intel CMCI, this only has MCA banks which do not support CMCI (if any). 106 */ 107 DEFINE_PER_CPU(mce_banks_t, mce_poll_banks) = { 108 [0 ... BITS_TO_LONGS(MAX_NR_BANKS)-1] = ~0UL 109 }; 110 111 /* 112 * MCA banks controlled through firmware first for corrected errors. 113 * This is a global list of banks for which we won't enable CMCI and we 114 * won't poll. Firmware controls these banks and is responsible for 115 * reporting corrected errors through GHES. Uncorrected/recoverable 116 * errors are still notified through a machine check. 117 */ 118 mce_banks_t mce_banks_ce_disabled; 119 120 static struct work_struct mce_work; 121 static struct irq_work mce_irq_work; 122 123 /* 124 * CPU/chipset specific EDAC code can register a notifier call here to print 125 * MCE errors in a human-readable form. 126 */ 127 BLOCKING_NOTIFIER_HEAD(x86_mce_decoder_chain); 128 129 /* Do initial initialization of a struct mce */ 130 void mce_setup(struct mce *m) 131 { 132 memset(m, 0, sizeof(struct mce)); 133 m->cpu = m->extcpu = smp_processor_id(); 134 /* need the internal __ version to avoid deadlocks */ 135 m->time = __ktime_get_real_seconds(); 136 m->cpuvendor = boot_cpu_data.x86_vendor; 137 m->cpuid = cpuid_eax(1); 138 m->socketid = cpu_data(m->extcpu).phys_proc_id; 139 m->apicid = cpu_data(m->extcpu).initial_apicid; 140 m->mcgcap = __rdmsr(MSR_IA32_MCG_CAP); 141 m->ppin = cpu_data(m->extcpu).ppin; 142 m->microcode = boot_cpu_data.microcode; 143 } 144 145 DEFINE_PER_CPU(struct mce, injectm); 146 EXPORT_PER_CPU_SYMBOL_GPL(injectm); 147 148 void mce_log(struct mce *m) 149 { 150 if (!mce_gen_pool_add(m)) 151 irq_work_queue(&mce_irq_work); 152 } 153 EXPORT_SYMBOL_GPL(mce_log); 154 155 void mce_register_decode_chain(struct notifier_block *nb) 156 { 157 if (WARN_ON(nb->priority < MCE_PRIO_LOWEST || 158 nb->priority > MCE_PRIO_HIGHEST)) 159 return; 160 161 blocking_notifier_chain_register(&x86_mce_decoder_chain, nb); 162 } 163 EXPORT_SYMBOL_GPL(mce_register_decode_chain); 164 165 void mce_unregister_decode_chain(struct notifier_block *nb) 166 { 167 blocking_notifier_chain_unregister(&x86_mce_decoder_chain, nb); 168 } 169 EXPORT_SYMBOL_GPL(mce_unregister_decode_chain); 170 171 u32 mca_msr_reg(int bank, enum mca_msr reg) 172 { 173 if (mce_flags.smca) { 174 switch (reg) { 175 case MCA_CTL: return MSR_AMD64_SMCA_MCx_CTL(bank); 176 case MCA_ADDR: return MSR_AMD64_SMCA_MCx_ADDR(bank); 177 case MCA_MISC: return MSR_AMD64_SMCA_MCx_MISC(bank); 178 case MCA_STATUS: return MSR_AMD64_SMCA_MCx_STATUS(bank); 179 } 180 } 181 182 switch (reg) { 183 case MCA_CTL: return MSR_IA32_MCx_CTL(bank); 184 case MCA_ADDR: return MSR_IA32_MCx_ADDR(bank); 185 case MCA_MISC: return MSR_IA32_MCx_MISC(bank); 186 case MCA_STATUS: return MSR_IA32_MCx_STATUS(bank); 187 } 188 189 return 0; 190 } 191 192 static void __print_mce(struct mce *m) 193 { 194 pr_emerg(HW_ERR "CPU %d: Machine Check%s: %Lx Bank %d: %016Lx\n", 195 m->extcpu, 196 (m->mcgstatus & MCG_STATUS_MCIP ? " Exception" : ""), 197 m->mcgstatus, m->bank, m->status); 198 199 if (m->ip) { 200 pr_emerg(HW_ERR "RIP%s %02x:<%016Lx> ", 201 !(m->mcgstatus & MCG_STATUS_EIPV) ? " !INEXACT!" : "", 202 m->cs, m->ip); 203 204 if (m->cs == __KERNEL_CS) 205 pr_cont("{%pS}", (void *)(unsigned long)m->ip); 206 pr_cont("\n"); 207 } 208 209 pr_emerg(HW_ERR "TSC %llx ", m->tsc); 210 if (m->addr) 211 pr_cont("ADDR %llx ", m->addr); 212 if (m->misc) 213 pr_cont("MISC %llx ", m->misc); 214 if (m->ppin) 215 pr_cont("PPIN %llx ", m->ppin); 216 217 if (mce_flags.smca) { 218 if (m->synd) 219 pr_cont("SYND %llx ", m->synd); 220 if (m->ipid) 221 pr_cont("IPID %llx ", m->ipid); 222 } 223 224 pr_cont("\n"); 225 226 /* 227 * Note this output is parsed by external tools and old fields 228 * should not be changed. 229 */ 230 pr_emerg(HW_ERR "PROCESSOR %u:%x TIME %llu SOCKET %u APIC %x microcode %x\n", 231 m->cpuvendor, m->cpuid, m->time, m->socketid, m->apicid, 232 m->microcode); 233 } 234 235 static void print_mce(struct mce *m) 236 { 237 __print_mce(m); 238 239 if (m->cpuvendor != X86_VENDOR_AMD && m->cpuvendor != X86_VENDOR_HYGON) 240 pr_emerg_ratelimited(HW_ERR "Run the above through 'mcelog --ascii'\n"); 241 } 242 243 #define PANIC_TIMEOUT 5 /* 5 seconds */ 244 245 static atomic_t mce_panicked; 246 247 static int fake_panic; 248 static atomic_t mce_fake_panicked; 249 250 /* Panic in progress. Enable interrupts and wait for final IPI */ 251 static void wait_for_panic(void) 252 { 253 long timeout = PANIC_TIMEOUT*USEC_PER_SEC; 254 255 preempt_disable(); 256 local_irq_enable(); 257 while (timeout-- > 0) 258 udelay(1); 259 if (panic_timeout == 0) 260 panic_timeout = mca_cfg.panic_timeout; 261 panic("Panicing machine check CPU died"); 262 } 263 264 static noinstr void mce_panic(const char *msg, struct mce *final, char *exp) 265 { 266 struct llist_node *pending; 267 struct mce_evt_llist *l; 268 int apei_err = 0; 269 270 /* 271 * Allow instrumentation around external facilities usage. Not that it 272 * matters a whole lot since the machine is going to panic anyway. 273 */ 274 instrumentation_begin(); 275 276 if (!fake_panic) { 277 /* 278 * Make sure only one CPU runs in machine check panic 279 */ 280 if (atomic_inc_return(&mce_panicked) > 1) 281 wait_for_panic(); 282 barrier(); 283 284 bust_spinlocks(1); 285 console_verbose(); 286 } else { 287 /* Don't log too much for fake panic */ 288 if (atomic_inc_return(&mce_fake_panicked) > 1) 289 goto out; 290 } 291 pending = mce_gen_pool_prepare_records(); 292 /* First print corrected ones that are still unlogged */ 293 llist_for_each_entry(l, pending, llnode) { 294 struct mce *m = &l->mce; 295 if (!(m->status & MCI_STATUS_UC)) { 296 print_mce(m); 297 if (!apei_err) 298 apei_err = apei_write_mce(m); 299 } 300 } 301 /* Now print uncorrected but with the final one last */ 302 llist_for_each_entry(l, pending, llnode) { 303 struct mce *m = &l->mce; 304 if (!(m->status & MCI_STATUS_UC)) 305 continue; 306 if (!final || mce_cmp(m, final)) { 307 print_mce(m); 308 if (!apei_err) 309 apei_err = apei_write_mce(m); 310 } 311 } 312 if (final) { 313 print_mce(final); 314 if (!apei_err) 315 apei_err = apei_write_mce(final); 316 } 317 if (exp) 318 pr_emerg(HW_ERR "Machine check: %s\n", exp); 319 if (!fake_panic) { 320 if (panic_timeout == 0) 321 panic_timeout = mca_cfg.panic_timeout; 322 panic(msg); 323 } else 324 pr_emerg(HW_ERR "Fake kernel panic: %s\n", msg); 325 326 out: 327 instrumentation_end(); 328 } 329 330 /* Support code for software error injection */ 331 332 static int msr_to_offset(u32 msr) 333 { 334 unsigned bank = __this_cpu_read(injectm.bank); 335 336 if (msr == mca_cfg.rip_msr) 337 return offsetof(struct mce, ip); 338 if (msr == mca_msr_reg(bank, MCA_STATUS)) 339 return offsetof(struct mce, status); 340 if (msr == mca_msr_reg(bank, MCA_ADDR)) 341 return offsetof(struct mce, addr); 342 if (msr == mca_msr_reg(bank, MCA_MISC)) 343 return offsetof(struct mce, misc); 344 if (msr == MSR_IA32_MCG_STATUS) 345 return offsetof(struct mce, mcgstatus); 346 return -1; 347 } 348 349 void ex_handler_msr_mce(struct pt_regs *regs, bool wrmsr) 350 { 351 if (wrmsr) { 352 pr_emerg("MSR access error: WRMSR to 0x%x (tried to write 0x%08x%08x) at rIP: 0x%lx (%pS)\n", 353 (unsigned int)regs->cx, (unsigned int)regs->dx, (unsigned int)regs->ax, 354 regs->ip, (void *)regs->ip); 355 } else { 356 pr_emerg("MSR access error: RDMSR from 0x%x at rIP: 0x%lx (%pS)\n", 357 (unsigned int)regs->cx, regs->ip, (void *)regs->ip); 358 } 359 360 show_stack_regs(regs); 361 362 panic("MCA architectural violation!\n"); 363 364 while (true) 365 cpu_relax(); 366 } 367 368 /* MSR access wrappers used for error injection */ 369 noinstr u64 mce_rdmsrl(u32 msr) 370 { 371 DECLARE_ARGS(val, low, high); 372 373 if (__this_cpu_read(injectm.finished)) { 374 int offset; 375 u64 ret; 376 377 instrumentation_begin(); 378 379 offset = msr_to_offset(msr); 380 if (offset < 0) 381 ret = 0; 382 else 383 ret = *(u64 *)((char *)this_cpu_ptr(&injectm) + offset); 384 385 instrumentation_end(); 386 387 return ret; 388 } 389 390 /* 391 * RDMSR on MCA MSRs should not fault. If they do, this is very much an 392 * architectural violation and needs to be reported to hw vendor. Panic 393 * the box to not allow any further progress. 394 */ 395 asm volatile("1: rdmsr\n" 396 "2:\n" 397 _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_RDMSR_IN_MCE) 398 : EAX_EDX_RET(val, low, high) : "c" (msr)); 399 400 401 return EAX_EDX_VAL(val, low, high); 402 } 403 404 static noinstr void mce_wrmsrl(u32 msr, u64 v) 405 { 406 u32 low, high; 407 408 if (__this_cpu_read(injectm.finished)) { 409 int offset; 410 411 instrumentation_begin(); 412 413 offset = msr_to_offset(msr); 414 if (offset >= 0) 415 *(u64 *)((char *)this_cpu_ptr(&injectm) + offset) = v; 416 417 instrumentation_end(); 418 419 return; 420 } 421 422 low = (u32)v; 423 high = (u32)(v >> 32); 424 425 /* See comment in mce_rdmsrl() */ 426 asm volatile("1: wrmsr\n" 427 "2:\n" 428 _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_WRMSR_IN_MCE) 429 : : "c" (msr), "a"(low), "d" (high) : "memory"); 430 } 431 432 /* 433 * Collect all global (w.r.t. this processor) status about this machine 434 * check into our "mce" struct so that we can use it later to assess 435 * the severity of the problem as we read per-bank specific details. 436 */ 437 static noinstr void mce_gather_info(struct mce *m, struct pt_regs *regs) 438 { 439 /* 440 * Enable instrumentation around mce_setup() which calls external 441 * facilities. 442 */ 443 instrumentation_begin(); 444 mce_setup(m); 445 instrumentation_end(); 446 447 m->mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS); 448 if (regs) { 449 /* 450 * Get the address of the instruction at the time of 451 * the machine check error. 452 */ 453 if (m->mcgstatus & (MCG_STATUS_RIPV|MCG_STATUS_EIPV)) { 454 m->ip = regs->ip; 455 m->cs = regs->cs; 456 457 /* 458 * When in VM86 mode make the cs look like ring 3 459 * always. This is a lie, but it's better than passing 460 * the additional vm86 bit around everywhere. 461 */ 462 if (v8086_mode(regs)) 463 m->cs |= 3; 464 } 465 /* Use accurate RIP reporting if available. */ 466 if (mca_cfg.rip_msr) 467 m->ip = mce_rdmsrl(mca_cfg.rip_msr); 468 } 469 } 470 471 int mce_available(struct cpuinfo_x86 *c) 472 { 473 if (mca_cfg.disabled) 474 return 0; 475 return cpu_has(c, X86_FEATURE_MCE) && cpu_has(c, X86_FEATURE_MCA); 476 } 477 478 static void mce_schedule_work(void) 479 { 480 if (!mce_gen_pool_empty()) 481 schedule_work(&mce_work); 482 } 483 484 static void mce_irq_work_cb(struct irq_work *entry) 485 { 486 mce_schedule_work(); 487 } 488 489 /* 490 * Check if the address reported by the CPU is in a format we can parse. 491 * It would be possible to add code for most other cases, but all would 492 * be somewhat complicated (e.g. segment offset would require an instruction 493 * parser). So only support physical addresses up to page granularity for now. 494 */ 495 int mce_usable_address(struct mce *m) 496 { 497 if (!(m->status & MCI_STATUS_ADDRV)) 498 return 0; 499 500 /* Checks after this one are Intel/Zhaoxin-specific: */ 501 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL && 502 boot_cpu_data.x86_vendor != X86_VENDOR_ZHAOXIN) 503 return 1; 504 505 if (!(m->status & MCI_STATUS_MISCV)) 506 return 0; 507 508 if (MCI_MISC_ADDR_LSB(m->misc) > PAGE_SHIFT) 509 return 0; 510 511 if (MCI_MISC_ADDR_MODE(m->misc) != MCI_MISC_ADDR_PHYS) 512 return 0; 513 514 return 1; 515 } 516 EXPORT_SYMBOL_GPL(mce_usable_address); 517 518 bool mce_is_memory_error(struct mce *m) 519 { 520 switch (m->cpuvendor) { 521 case X86_VENDOR_AMD: 522 case X86_VENDOR_HYGON: 523 return amd_mce_is_memory_error(m); 524 525 case X86_VENDOR_INTEL: 526 case X86_VENDOR_ZHAOXIN: 527 /* 528 * Intel SDM Volume 3B - 15.9.2 Compound Error Codes 529 * 530 * Bit 7 of the MCACOD field of IA32_MCi_STATUS is used for 531 * indicating a memory error. Bit 8 is used for indicating a 532 * cache hierarchy error. The combination of bit 2 and bit 3 533 * is used for indicating a `generic' cache hierarchy error 534 * But we can't just blindly check the above bits, because if 535 * bit 11 is set, then it is a bus/interconnect error - and 536 * either way the above bits just gives more detail on what 537 * bus/interconnect error happened. Note that bit 12 can be 538 * ignored, as it's the "filter" bit. 539 */ 540 return (m->status & 0xef80) == BIT(7) || 541 (m->status & 0xef00) == BIT(8) || 542 (m->status & 0xeffc) == 0xc; 543 544 default: 545 return false; 546 } 547 } 548 EXPORT_SYMBOL_GPL(mce_is_memory_error); 549 550 static bool whole_page(struct mce *m) 551 { 552 if (!mca_cfg.ser || !(m->status & MCI_STATUS_MISCV)) 553 return true; 554 555 return MCI_MISC_ADDR_LSB(m->misc) >= PAGE_SHIFT; 556 } 557 558 bool mce_is_correctable(struct mce *m) 559 { 560 if (m->cpuvendor == X86_VENDOR_AMD && m->status & MCI_STATUS_DEFERRED) 561 return false; 562 563 if (m->cpuvendor == X86_VENDOR_HYGON && m->status & MCI_STATUS_DEFERRED) 564 return false; 565 566 if (m->status & MCI_STATUS_UC) 567 return false; 568 569 return true; 570 } 571 EXPORT_SYMBOL_GPL(mce_is_correctable); 572 573 static int mce_early_notifier(struct notifier_block *nb, unsigned long val, 574 void *data) 575 { 576 struct mce *m = (struct mce *)data; 577 578 if (!m) 579 return NOTIFY_DONE; 580 581 /* Emit the trace record: */ 582 trace_mce_record(m); 583 584 set_bit(0, &mce_need_notify); 585 586 mce_notify_irq(); 587 588 return NOTIFY_DONE; 589 } 590 591 static struct notifier_block early_nb = { 592 .notifier_call = mce_early_notifier, 593 .priority = MCE_PRIO_EARLY, 594 }; 595 596 static int uc_decode_notifier(struct notifier_block *nb, unsigned long val, 597 void *data) 598 { 599 struct mce *mce = (struct mce *)data; 600 unsigned long pfn; 601 602 if (!mce || !mce_usable_address(mce)) 603 return NOTIFY_DONE; 604 605 if (mce->severity != MCE_AO_SEVERITY && 606 mce->severity != MCE_DEFERRED_SEVERITY) 607 return NOTIFY_DONE; 608 609 pfn = mce->addr >> PAGE_SHIFT; 610 if (!memory_failure(pfn, 0)) { 611 set_mce_nospec(pfn, whole_page(mce)); 612 mce->kflags |= MCE_HANDLED_UC; 613 } 614 615 return NOTIFY_OK; 616 } 617 618 static struct notifier_block mce_uc_nb = { 619 .notifier_call = uc_decode_notifier, 620 .priority = MCE_PRIO_UC, 621 }; 622 623 static int mce_default_notifier(struct notifier_block *nb, unsigned long val, 624 void *data) 625 { 626 struct mce *m = (struct mce *)data; 627 628 if (!m) 629 return NOTIFY_DONE; 630 631 if (mca_cfg.print_all || !m->kflags) 632 __print_mce(m); 633 634 return NOTIFY_DONE; 635 } 636 637 static struct notifier_block mce_default_nb = { 638 .notifier_call = mce_default_notifier, 639 /* lowest prio, we want it to run last. */ 640 .priority = MCE_PRIO_LOWEST, 641 }; 642 643 /* 644 * Read ADDR and MISC registers. 645 */ 646 static noinstr void mce_read_aux(struct mce *m, int i) 647 { 648 if (m->status & MCI_STATUS_MISCV) 649 m->misc = mce_rdmsrl(mca_msr_reg(i, MCA_MISC)); 650 651 if (m->status & MCI_STATUS_ADDRV) { 652 m->addr = mce_rdmsrl(mca_msr_reg(i, MCA_ADDR)); 653 654 /* 655 * Mask the reported address by the reported granularity. 656 */ 657 if (mca_cfg.ser && (m->status & MCI_STATUS_MISCV)) { 658 u8 shift = MCI_MISC_ADDR_LSB(m->misc); 659 m->addr >>= shift; 660 m->addr <<= shift; 661 } 662 663 /* 664 * Extract [55:<lsb>] where lsb is the least significant 665 * *valid* bit of the address bits. 666 */ 667 if (mce_flags.smca) { 668 u8 lsb = (m->addr >> 56) & 0x3f; 669 670 m->addr &= GENMASK_ULL(55, lsb); 671 } 672 } 673 674 if (mce_flags.smca) { 675 m->ipid = mce_rdmsrl(MSR_AMD64_SMCA_MCx_IPID(i)); 676 677 if (m->status & MCI_STATUS_SYNDV) 678 m->synd = mce_rdmsrl(MSR_AMD64_SMCA_MCx_SYND(i)); 679 } 680 } 681 682 DEFINE_PER_CPU(unsigned, mce_poll_count); 683 684 /* 685 * Poll for corrected events or events that happened before reset. 686 * Those are just logged through /dev/mcelog. 687 * 688 * This is executed in standard interrupt context. 689 * 690 * Note: spec recommends to panic for fatal unsignalled 691 * errors here. However this would be quite problematic -- 692 * we would need to reimplement the Monarch handling and 693 * it would mess up the exclusion between exception handler 694 * and poll handler -- * so we skip this for now. 695 * These cases should not happen anyways, or only when the CPU 696 * is already totally * confused. In this case it's likely it will 697 * not fully execute the machine check handler either. 698 */ 699 bool machine_check_poll(enum mcp_flags flags, mce_banks_t *b) 700 { 701 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 702 bool error_seen = false; 703 struct mce m; 704 int i; 705 706 this_cpu_inc(mce_poll_count); 707 708 mce_gather_info(&m, NULL); 709 710 if (flags & MCP_TIMESTAMP) 711 m.tsc = rdtsc(); 712 713 for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 714 if (!mce_banks[i].ctl || !test_bit(i, *b)) 715 continue; 716 717 m.misc = 0; 718 m.addr = 0; 719 m.bank = i; 720 721 barrier(); 722 m.status = mce_rdmsrl(mca_msr_reg(i, MCA_STATUS)); 723 724 /* If this entry is not valid, ignore it */ 725 if (!(m.status & MCI_STATUS_VAL)) 726 continue; 727 728 /* 729 * If we are logging everything (at CPU online) or this 730 * is a corrected error, then we must log it. 731 */ 732 if ((flags & MCP_UC) || !(m.status & MCI_STATUS_UC)) 733 goto log_it; 734 735 /* 736 * Newer Intel systems that support software error 737 * recovery need to make additional checks. Other 738 * CPUs should skip over uncorrected errors, but log 739 * everything else. 740 */ 741 if (!mca_cfg.ser) { 742 if (m.status & MCI_STATUS_UC) 743 continue; 744 goto log_it; 745 } 746 747 /* Log "not enabled" (speculative) errors */ 748 if (!(m.status & MCI_STATUS_EN)) 749 goto log_it; 750 751 /* 752 * Log UCNA (SDM: 15.6.3 "UCR Error Classification") 753 * UC == 1 && PCC == 0 && S == 0 754 */ 755 if (!(m.status & MCI_STATUS_PCC) && !(m.status & MCI_STATUS_S)) 756 goto log_it; 757 758 /* 759 * Skip anything else. Presumption is that our read of this 760 * bank is racing with a machine check. Leave the log alone 761 * for do_machine_check() to deal with it. 762 */ 763 continue; 764 765 log_it: 766 error_seen = true; 767 768 if (flags & MCP_DONTLOG) 769 goto clear_it; 770 771 mce_read_aux(&m, i); 772 m.severity = mce_severity(&m, NULL, mca_cfg.tolerant, NULL, false); 773 /* 774 * Don't get the IP here because it's unlikely to 775 * have anything to do with the actual error location. 776 */ 777 778 if (mca_cfg.dont_log_ce && !mce_usable_address(&m)) 779 goto clear_it; 780 781 if (flags & MCP_QUEUE_LOG) 782 mce_gen_pool_add(&m); 783 else 784 mce_log(&m); 785 786 clear_it: 787 /* 788 * Clear state for this bank. 789 */ 790 mce_wrmsrl(mca_msr_reg(i, MCA_STATUS), 0); 791 } 792 793 /* 794 * Don't clear MCG_STATUS here because it's only defined for 795 * exceptions. 796 */ 797 798 sync_core(); 799 800 return error_seen; 801 } 802 EXPORT_SYMBOL_GPL(machine_check_poll); 803 804 /* 805 * During IFU recovery Sandy Bridge -EP4S processors set the RIPV and 806 * EIPV bits in MCG_STATUS to zero on the affected logical processor (SDM 807 * Vol 3B Table 15-20). But this confuses both the code that determines 808 * whether the machine check occurred in kernel or user mode, and also 809 * the severity assessment code. Pretend that EIPV was set, and take the 810 * ip/cs values from the pt_regs that mce_gather_info() ignored earlier. 811 */ 812 static void quirk_sandybridge_ifu(int bank, struct mce *m, struct pt_regs *regs) 813 { 814 if (bank != 0) 815 return; 816 if ((m->mcgstatus & (MCG_STATUS_EIPV|MCG_STATUS_RIPV)) != 0) 817 return; 818 if ((m->status & (MCI_STATUS_OVER|MCI_STATUS_UC| 819 MCI_STATUS_EN|MCI_STATUS_MISCV|MCI_STATUS_ADDRV| 820 MCI_STATUS_PCC|MCI_STATUS_S|MCI_STATUS_AR| 821 MCACOD)) != 822 (MCI_STATUS_UC|MCI_STATUS_EN| 823 MCI_STATUS_MISCV|MCI_STATUS_ADDRV|MCI_STATUS_S| 824 MCI_STATUS_AR|MCACOD_INSTR)) 825 return; 826 827 m->mcgstatus |= MCG_STATUS_EIPV; 828 m->ip = regs->ip; 829 m->cs = regs->cs; 830 } 831 832 /* 833 * Do a quick check if any of the events requires a panic. 834 * This decides if we keep the events around or clear them. 835 */ 836 static int mce_no_way_out(struct mce *m, char **msg, unsigned long *validp, 837 struct pt_regs *regs) 838 { 839 char *tmp = *msg; 840 int i; 841 842 for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 843 m->status = mce_rdmsrl(mca_msr_reg(i, MCA_STATUS)); 844 if (!(m->status & MCI_STATUS_VAL)) 845 continue; 846 847 __set_bit(i, validp); 848 if (mce_flags.snb_ifu_quirk) 849 quirk_sandybridge_ifu(i, m, regs); 850 851 m->bank = i; 852 if (mce_severity(m, regs, mca_cfg.tolerant, &tmp, true) >= MCE_PANIC_SEVERITY) { 853 mce_read_aux(m, i); 854 *msg = tmp; 855 return 1; 856 } 857 } 858 return 0; 859 } 860 861 /* 862 * Variable to establish order between CPUs while scanning. 863 * Each CPU spins initially until executing is equal its number. 864 */ 865 static atomic_t mce_executing; 866 867 /* 868 * Defines order of CPUs on entry. First CPU becomes Monarch. 869 */ 870 static atomic_t mce_callin; 871 872 /* 873 * Track which CPUs entered the MCA broadcast synchronization and which not in 874 * order to print holdouts. 875 */ 876 static cpumask_t mce_missing_cpus = CPU_MASK_ALL; 877 878 /* 879 * Check if a timeout waiting for other CPUs happened. 880 */ 881 static noinstr int mce_timed_out(u64 *t, const char *msg) 882 { 883 int ret = 0; 884 885 /* Enable instrumentation around calls to external facilities */ 886 instrumentation_begin(); 887 888 /* 889 * The others already did panic for some reason. 890 * Bail out like in a timeout. 891 * rmb() to tell the compiler that system_state 892 * might have been modified by someone else. 893 */ 894 rmb(); 895 if (atomic_read(&mce_panicked)) 896 wait_for_panic(); 897 if (!mca_cfg.monarch_timeout) 898 goto out; 899 if ((s64)*t < SPINUNIT) { 900 if (mca_cfg.tolerant <= 1) { 901 if (cpumask_and(&mce_missing_cpus, cpu_online_mask, &mce_missing_cpus)) 902 pr_emerg("CPUs not responding to MCE broadcast (may include false positives): %*pbl\n", 903 cpumask_pr_args(&mce_missing_cpus)); 904 mce_panic(msg, NULL, NULL); 905 } 906 ret = 1; 907 goto out; 908 } 909 *t -= SPINUNIT; 910 911 out: 912 touch_nmi_watchdog(); 913 914 instrumentation_end(); 915 916 return ret; 917 } 918 919 /* 920 * The Monarch's reign. The Monarch is the CPU who entered 921 * the machine check handler first. It waits for the others to 922 * raise the exception too and then grades them. When any 923 * error is fatal panic. Only then let the others continue. 924 * 925 * The other CPUs entering the MCE handler will be controlled by the 926 * Monarch. They are called Subjects. 927 * 928 * This way we prevent any potential data corruption in a unrecoverable case 929 * and also makes sure always all CPU's errors are examined. 930 * 931 * Also this detects the case of a machine check event coming from outer 932 * space (not detected by any CPUs) In this case some external agent wants 933 * us to shut down, so panic too. 934 * 935 * The other CPUs might still decide to panic if the handler happens 936 * in a unrecoverable place, but in this case the system is in a semi-stable 937 * state and won't corrupt anything by itself. It's ok to let the others 938 * continue for a bit first. 939 * 940 * All the spin loops have timeouts; when a timeout happens a CPU 941 * typically elects itself to be Monarch. 942 */ 943 static void mce_reign(void) 944 { 945 int cpu; 946 struct mce *m = NULL; 947 int global_worst = 0; 948 char *msg = NULL; 949 950 /* 951 * This CPU is the Monarch and the other CPUs have run 952 * through their handlers. 953 * Grade the severity of the errors of all the CPUs. 954 */ 955 for_each_possible_cpu(cpu) { 956 struct mce *mtmp = &per_cpu(mces_seen, cpu); 957 958 if (mtmp->severity > global_worst) { 959 global_worst = mtmp->severity; 960 m = &per_cpu(mces_seen, cpu); 961 } 962 } 963 964 /* 965 * Cannot recover? Panic here then. 966 * This dumps all the mces in the log buffer and stops the 967 * other CPUs. 968 */ 969 if (m && global_worst >= MCE_PANIC_SEVERITY && mca_cfg.tolerant < 3) { 970 /* call mce_severity() to get "msg" for panic */ 971 mce_severity(m, NULL, mca_cfg.tolerant, &msg, true); 972 mce_panic("Fatal machine check", m, msg); 973 } 974 975 /* 976 * For UC somewhere we let the CPU who detects it handle it. 977 * Also must let continue the others, otherwise the handling 978 * CPU could deadlock on a lock. 979 */ 980 981 /* 982 * No machine check event found. Must be some external 983 * source or one CPU is hung. Panic. 984 */ 985 if (global_worst <= MCE_KEEP_SEVERITY && mca_cfg.tolerant < 3) 986 mce_panic("Fatal machine check from unknown source", NULL, NULL); 987 988 /* 989 * Now clear all the mces_seen so that they don't reappear on 990 * the next mce. 991 */ 992 for_each_possible_cpu(cpu) 993 memset(&per_cpu(mces_seen, cpu), 0, sizeof(struct mce)); 994 } 995 996 static atomic_t global_nwo; 997 998 /* 999 * Start of Monarch synchronization. This waits until all CPUs have 1000 * entered the exception handler and then determines if any of them 1001 * saw a fatal event that requires panic. Then it executes them 1002 * in the entry order. 1003 * TBD double check parallel CPU hotunplug 1004 */ 1005 static noinstr int mce_start(int *no_way_out) 1006 { 1007 u64 timeout = (u64)mca_cfg.monarch_timeout * NSEC_PER_USEC; 1008 int order, ret = -1; 1009 1010 if (!timeout) 1011 return ret; 1012 1013 atomic_add(*no_way_out, &global_nwo); 1014 /* 1015 * Rely on the implied barrier below, such that global_nwo 1016 * is updated before mce_callin. 1017 */ 1018 order = atomic_inc_return(&mce_callin); 1019 cpumask_clear_cpu(smp_processor_id(), &mce_missing_cpus); 1020 1021 /* Enable instrumentation around calls to external facilities */ 1022 instrumentation_begin(); 1023 1024 /* 1025 * Wait for everyone. 1026 */ 1027 while (atomic_read(&mce_callin) != num_online_cpus()) { 1028 if (mce_timed_out(&timeout, 1029 "Timeout: Not all CPUs entered broadcast exception handler")) { 1030 atomic_set(&global_nwo, 0); 1031 goto out; 1032 } 1033 ndelay(SPINUNIT); 1034 } 1035 1036 /* 1037 * mce_callin should be read before global_nwo 1038 */ 1039 smp_rmb(); 1040 1041 if (order == 1) { 1042 /* 1043 * Monarch: Starts executing now, the others wait. 1044 */ 1045 atomic_set(&mce_executing, 1); 1046 } else { 1047 /* 1048 * Subject: Now start the scanning loop one by one in 1049 * the original callin order. 1050 * This way when there are any shared banks it will be 1051 * only seen by one CPU before cleared, avoiding duplicates. 1052 */ 1053 while (atomic_read(&mce_executing) < order) { 1054 if (mce_timed_out(&timeout, 1055 "Timeout: Subject CPUs unable to finish machine check processing")) { 1056 atomic_set(&global_nwo, 0); 1057 goto out; 1058 } 1059 ndelay(SPINUNIT); 1060 } 1061 } 1062 1063 /* 1064 * Cache the global no_way_out state. 1065 */ 1066 *no_way_out = atomic_read(&global_nwo); 1067 1068 ret = order; 1069 1070 out: 1071 instrumentation_end(); 1072 1073 return ret; 1074 } 1075 1076 /* 1077 * Synchronize between CPUs after main scanning loop. 1078 * This invokes the bulk of the Monarch processing. 1079 */ 1080 static noinstr int mce_end(int order) 1081 { 1082 u64 timeout = (u64)mca_cfg.monarch_timeout * NSEC_PER_USEC; 1083 int ret = -1; 1084 1085 /* Allow instrumentation around external facilities. */ 1086 instrumentation_begin(); 1087 1088 if (!timeout) 1089 goto reset; 1090 if (order < 0) 1091 goto reset; 1092 1093 /* 1094 * Allow others to run. 1095 */ 1096 atomic_inc(&mce_executing); 1097 1098 if (order == 1) { 1099 /* 1100 * Monarch: Wait for everyone to go through their scanning 1101 * loops. 1102 */ 1103 while (atomic_read(&mce_executing) <= num_online_cpus()) { 1104 if (mce_timed_out(&timeout, 1105 "Timeout: Monarch CPU unable to finish machine check processing")) 1106 goto reset; 1107 ndelay(SPINUNIT); 1108 } 1109 1110 mce_reign(); 1111 barrier(); 1112 ret = 0; 1113 } else { 1114 /* 1115 * Subject: Wait for Monarch to finish. 1116 */ 1117 while (atomic_read(&mce_executing) != 0) { 1118 if (mce_timed_out(&timeout, 1119 "Timeout: Monarch CPU did not finish machine check processing")) 1120 goto reset; 1121 ndelay(SPINUNIT); 1122 } 1123 1124 /* 1125 * Don't reset anything. That's done by the Monarch. 1126 */ 1127 ret = 0; 1128 goto out; 1129 } 1130 1131 /* 1132 * Reset all global state. 1133 */ 1134 reset: 1135 atomic_set(&global_nwo, 0); 1136 atomic_set(&mce_callin, 0); 1137 cpumask_setall(&mce_missing_cpus); 1138 barrier(); 1139 1140 /* 1141 * Let others run again. 1142 */ 1143 atomic_set(&mce_executing, 0); 1144 1145 out: 1146 instrumentation_end(); 1147 1148 return ret; 1149 } 1150 1151 static void mce_clear_state(unsigned long *toclear) 1152 { 1153 int i; 1154 1155 for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 1156 if (test_bit(i, toclear)) 1157 mce_wrmsrl(mca_msr_reg(i, MCA_STATUS), 0); 1158 } 1159 } 1160 1161 /* 1162 * Cases where we avoid rendezvous handler timeout: 1163 * 1) If this CPU is offline. 1164 * 1165 * 2) If crashing_cpu was set, e.g. we're entering kdump and we need to 1166 * skip those CPUs which remain looping in the 1st kernel - see 1167 * crash_nmi_callback(). 1168 * 1169 * Note: there still is a small window between kexec-ing and the new, 1170 * kdump kernel establishing a new #MC handler where a broadcasted MCE 1171 * might not get handled properly. 1172 */ 1173 static noinstr bool mce_check_crashing_cpu(void) 1174 { 1175 unsigned int cpu = smp_processor_id(); 1176 1177 if (arch_cpu_is_offline(cpu) || 1178 (crashing_cpu != -1 && crashing_cpu != cpu)) { 1179 u64 mcgstatus; 1180 1181 mcgstatus = __rdmsr(MSR_IA32_MCG_STATUS); 1182 1183 if (boot_cpu_data.x86_vendor == X86_VENDOR_ZHAOXIN) { 1184 if (mcgstatus & MCG_STATUS_LMCES) 1185 return false; 1186 } 1187 1188 if (mcgstatus & MCG_STATUS_RIPV) { 1189 __wrmsr(MSR_IA32_MCG_STATUS, 0, 0); 1190 return true; 1191 } 1192 } 1193 return false; 1194 } 1195 1196 static __always_inline int 1197 __mc_scan_banks(struct mce *m, struct pt_regs *regs, struct mce *final, 1198 unsigned long *toclear, unsigned long *valid_banks, int no_way_out, 1199 int *worst) 1200 { 1201 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 1202 struct mca_config *cfg = &mca_cfg; 1203 int severity, i, taint = 0; 1204 1205 for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 1206 __clear_bit(i, toclear); 1207 if (!test_bit(i, valid_banks)) 1208 continue; 1209 1210 if (!mce_banks[i].ctl) 1211 continue; 1212 1213 m->misc = 0; 1214 m->addr = 0; 1215 m->bank = i; 1216 1217 m->status = mce_rdmsrl(mca_msr_reg(i, MCA_STATUS)); 1218 if (!(m->status & MCI_STATUS_VAL)) 1219 continue; 1220 1221 /* 1222 * Corrected or non-signaled errors are handled by 1223 * machine_check_poll(). Leave them alone, unless this panics. 1224 */ 1225 if (!(m->status & (cfg->ser ? MCI_STATUS_S : MCI_STATUS_UC)) && 1226 !no_way_out) 1227 continue; 1228 1229 /* Set taint even when machine check was not enabled. */ 1230 taint++; 1231 1232 severity = mce_severity(m, regs, cfg->tolerant, NULL, true); 1233 1234 /* 1235 * When machine check was for corrected/deferred handler don't 1236 * touch, unless we're panicking. 1237 */ 1238 if ((severity == MCE_KEEP_SEVERITY || 1239 severity == MCE_UCNA_SEVERITY) && !no_way_out) 1240 continue; 1241 1242 __set_bit(i, toclear); 1243 1244 /* Machine check event was not enabled. Clear, but ignore. */ 1245 if (severity == MCE_NO_SEVERITY) 1246 continue; 1247 1248 mce_read_aux(m, i); 1249 1250 /* assuming valid severity level != 0 */ 1251 m->severity = severity; 1252 1253 /* 1254 * Enable instrumentation around the mce_log() call which is 1255 * done in #MC context, where instrumentation is disabled. 1256 */ 1257 instrumentation_begin(); 1258 mce_log(m); 1259 instrumentation_end(); 1260 1261 if (severity > *worst) { 1262 *final = *m; 1263 *worst = severity; 1264 } 1265 } 1266 1267 /* mce_clear_state will clear *final, save locally for use later */ 1268 *m = *final; 1269 1270 return taint; 1271 } 1272 1273 static void kill_me_now(struct callback_head *ch) 1274 { 1275 struct task_struct *p = container_of(ch, struct task_struct, mce_kill_me); 1276 1277 p->mce_count = 0; 1278 force_sig(SIGBUS); 1279 } 1280 1281 static void kill_me_maybe(struct callback_head *cb) 1282 { 1283 struct task_struct *p = container_of(cb, struct task_struct, mce_kill_me); 1284 int flags = MF_ACTION_REQUIRED; 1285 int ret; 1286 1287 p->mce_count = 0; 1288 pr_err("Uncorrected hardware memory error in user-access at %llx", p->mce_addr); 1289 1290 if (!p->mce_ripv) 1291 flags |= MF_MUST_KILL; 1292 1293 ret = memory_failure(p->mce_addr >> PAGE_SHIFT, flags); 1294 if (!ret) { 1295 set_mce_nospec(p->mce_addr >> PAGE_SHIFT, p->mce_whole_page); 1296 sync_core(); 1297 return; 1298 } 1299 1300 /* 1301 * -EHWPOISON from memory_failure() means that it already sent SIGBUS 1302 * to the current process with the proper error info, 1303 * -EOPNOTSUPP means hwpoison_filter() filtered the error event, 1304 * 1305 * In both cases, no further processing is required. 1306 */ 1307 if (ret == -EHWPOISON || ret == -EOPNOTSUPP) 1308 return; 1309 1310 pr_err("Memory error not recovered"); 1311 kill_me_now(cb); 1312 } 1313 1314 static void kill_me_never(struct callback_head *cb) 1315 { 1316 struct task_struct *p = container_of(cb, struct task_struct, mce_kill_me); 1317 1318 p->mce_count = 0; 1319 pr_err("Kernel accessed poison in user space at %llx\n", p->mce_addr); 1320 if (!memory_failure(p->mce_addr >> PAGE_SHIFT, 0)) 1321 set_mce_nospec(p->mce_addr >> PAGE_SHIFT, p->mce_whole_page); 1322 } 1323 1324 static void queue_task_work(struct mce *m, char *msg, void (*func)(struct callback_head *)) 1325 { 1326 int count = ++current->mce_count; 1327 1328 /* First call, save all the details */ 1329 if (count == 1) { 1330 current->mce_addr = m->addr; 1331 current->mce_kflags = m->kflags; 1332 current->mce_ripv = !!(m->mcgstatus & MCG_STATUS_RIPV); 1333 current->mce_whole_page = whole_page(m); 1334 current->mce_kill_me.func = func; 1335 } 1336 1337 /* Ten is likely overkill. Don't expect more than two faults before task_work() */ 1338 if (count > 10) 1339 mce_panic("Too many consecutive machine checks while accessing user data", m, msg); 1340 1341 /* Second or later call, make sure page address matches the one from first call */ 1342 if (count > 1 && (current->mce_addr >> PAGE_SHIFT) != (m->addr >> PAGE_SHIFT)) 1343 mce_panic("Consecutive machine checks to different user pages", m, msg); 1344 1345 /* Do not call task_work_add() more than once */ 1346 if (count > 1) 1347 return; 1348 1349 task_work_add(current, ¤t->mce_kill_me, TWA_RESUME); 1350 } 1351 1352 /* Handle unconfigured int18 (should never happen) */ 1353 static noinstr void unexpected_machine_check(struct pt_regs *regs) 1354 { 1355 instrumentation_begin(); 1356 pr_err("CPU#%d: Unexpected int18 (Machine Check)\n", 1357 smp_processor_id()); 1358 instrumentation_end(); 1359 } 1360 1361 /* 1362 * The actual machine check handler. This only handles real exceptions when 1363 * something got corrupted coming in through int 18. 1364 * 1365 * This is executed in #MC context not subject to normal locking rules. 1366 * This implies that most kernel services cannot be safely used. Don't even 1367 * think about putting a printk in there! 1368 * 1369 * On Intel systems this is entered on all CPUs in parallel through 1370 * MCE broadcast. However some CPUs might be broken beyond repair, 1371 * so be always careful when synchronizing with others. 1372 * 1373 * Tracing and kprobes are disabled: if we interrupted a kernel context 1374 * with IF=1, we need to minimize stack usage. There are also recursion 1375 * issues: if the machine check was due to a failure of the memory 1376 * backing the user stack, tracing that reads the user stack will cause 1377 * potentially infinite recursion. 1378 * 1379 * Currently, the #MC handler calls out to a number of external facilities 1380 * and, therefore, allows instrumentation around them. The optimal thing to 1381 * have would be to do the absolutely minimal work required in #MC context 1382 * and have instrumentation disabled only around that. Further processing can 1383 * then happen in process context where instrumentation is allowed. Achieving 1384 * that requires careful auditing and modifications. Until then, the code 1385 * allows instrumentation temporarily, where required. * 1386 */ 1387 noinstr void do_machine_check(struct pt_regs *regs) 1388 { 1389 int worst = 0, order, no_way_out, kill_current_task, lmce, taint = 0; 1390 DECLARE_BITMAP(valid_banks, MAX_NR_BANKS) = { 0 }; 1391 DECLARE_BITMAP(toclear, MAX_NR_BANKS) = { 0 }; 1392 struct mca_config *cfg = &mca_cfg; 1393 struct mce m, *final; 1394 char *msg = NULL; 1395 1396 if (unlikely(mce_flags.p5)) 1397 return pentium_machine_check(regs); 1398 else if (unlikely(mce_flags.winchip)) 1399 return winchip_machine_check(regs); 1400 else if (unlikely(!mca_cfg.initialized)) 1401 return unexpected_machine_check(regs); 1402 1403 /* 1404 * Establish sequential order between the CPUs entering the machine 1405 * check handler. 1406 */ 1407 order = -1; 1408 1409 /* 1410 * If no_way_out gets set, there is no safe way to recover from this 1411 * MCE. If mca_cfg.tolerant is cranked up, we'll try anyway. 1412 */ 1413 no_way_out = 0; 1414 1415 /* 1416 * If kill_current_task is not set, there might be a way to recover from this 1417 * error. 1418 */ 1419 kill_current_task = 0; 1420 1421 /* 1422 * MCEs are always local on AMD. Same is determined by MCG_STATUS_LMCES 1423 * on Intel. 1424 */ 1425 lmce = 1; 1426 1427 this_cpu_inc(mce_exception_count); 1428 1429 mce_gather_info(&m, regs); 1430 m.tsc = rdtsc(); 1431 1432 final = this_cpu_ptr(&mces_seen); 1433 *final = m; 1434 1435 no_way_out = mce_no_way_out(&m, &msg, valid_banks, regs); 1436 1437 barrier(); 1438 1439 /* 1440 * When no restart IP might need to kill or panic. 1441 * Assume the worst for now, but if we find the 1442 * severity is MCE_AR_SEVERITY we have other options. 1443 */ 1444 if (!(m.mcgstatus & MCG_STATUS_RIPV)) 1445 kill_current_task = (cfg->tolerant == 3) ? 0 : 1; 1446 /* 1447 * Check if this MCE is signaled to only this logical processor, 1448 * on Intel, Zhaoxin only. 1449 */ 1450 if (m.cpuvendor == X86_VENDOR_INTEL || 1451 m.cpuvendor == X86_VENDOR_ZHAOXIN) 1452 lmce = m.mcgstatus & MCG_STATUS_LMCES; 1453 1454 /* 1455 * Local machine check may already know that we have to panic. 1456 * Broadcast machine check begins rendezvous in mce_start() 1457 * Go through all banks in exclusion of the other CPUs. This way we 1458 * don't report duplicated events on shared banks because the first one 1459 * to see it will clear it. 1460 */ 1461 if (lmce) { 1462 if (no_way_out && cfg->tolerant < 3) 1463 mce_panic("Fatal local machine check", &m, msg); 1464 } else { 1465 order = mce_start(&no_way_out); 1466 } 1467 1468 taint = __mc_scan_banks(&m, regs, final, toclear, valid_banks, no_way_out, &worst); 1469 1470 if (!no_way_out) 1471 mce_clear_state(toclear); 1472 1473 /* 1474 * Do most of the synchronization with other CPUs. 1475 * When there's any problem use only local no_way_out state. 1476 */ 1477 if (!lmce) { 1478 if (mce_end(order) < 0) { 1479 if (!no_way_out) 1480 no_way_out = worst >= MCE_PANIC_SEVERITY; 1481 1482 if (no_way_out && cfg->tolerant < 3) 1483 mce_panic("Fatal machine check on current CPU", &m, msg); 1484 } 1485 } else { 1486 /* 1487 * If there was a fatal machine check we should have 1488 * already called mce_panic earlier in this function. 1489 * Since we re-read the banks, we might have found 1490 * something new. Check again to see if we found a 1491 * fatal error. We call "mce_severity()" again to 1492 * make sure we have the right "msg". 1493 */ 1494 if (worst >= MCE_PANIC_SEVERITY && mca_cfg.tolerant < 3) { 1495 mce_severity(&m, regs, cfg->tolerant, &msg, true); 1496 mce_panic("Local fatal machine check!", &m, msg); 1497 } 1498 } 1499 1500 /* 1501 * Enable instrumentation around the external facilities like task_work_add() 1502 * (via queue_task_work()), fixup_exception() etc. For now, that is. Fixing this 1503 * properly would need a lot more involved reorganization. 1504 */ 1505 instrumentation_begin(); 1506 1507 if (taint) 1508 add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE); 1509 1510 if (worst != MCE_AR_SEVERITY && !kill_current_task) 1511 goto out; 1512 1513 /* Fault was in user mode and we need to take some action */ 1514 if ((m.cs & 3) == 3) { 1515 /* If this triggers there is no way to recover. Die hard. */ 1516 BUG_ON(!on_thread_stack() || !user_mode(regs)); 1517 1518 if (kill_current_task) 1519 queue_task_work(&m, msg, kill_me_now); 1520 else 1521 queue_task_work(&m, msg, kill_me_maybe); 1522 1523 } else { 1524 /* 1525 * Handle an MCE which has happened in kernel space but from 1526 * which the kernel can recover: ex_has_fault_handler() has 1527 * already verified that the rIP at which the error happened is 1528 * a rIP from which the kernel can recover (by jumping to 1529 * recovery code specified in _ASM_EXTABLE_FAULT()) and the 1530 * corresponding exception handler which would do that is the 1531 * proper one. 1532 */ 1533 if (m.kflags & MCE_IN_KERNEL_RECOV) { 1534 if (!fixup_exception(regs, X86_TRAP_MC, 0, 0)) 1535 mce_panic("Failed kernel mode recovery", &m, msg); 1536 } 1537 1538 if (m.kflags & MCE_IN_KERNEL_COPYIN) 1539 queue_task_work(&m, msg, kill_me_never); 1540 } 1541 1542 out: 1543 instrumentation_end(); 1544 1545 mce_wrmsrl(MSR_IA32_MCG_STATUS, 0); 1546 } 1547 EXPORT_SYMBOL_GPL(do_machine_check); 1548 1549 #ifndef CONFIG_MEMORY_FAILURE 1550 int memory_failure(unsigned long pfn, int flags) 1551 { 1552 /* mce_severity() should not hand us an ACTION_REQUIRED error */ 1553 BUG_ON(flags & MF_ACTION_REQUIRED); 1554 pr_err("Uncorrected memory error in page 0x%lx ignored\n" 1555 "Rebuild kernel with CONFIG_MEMORY_FAILURE=y for smarter handling\n", 1556 pfn); 1557 1558 return 0; 1559 } 1560 #endif 1561 1562 /* 1563 * Periodic polling timer for "silent" machine check errors. If the 1564 * poller finds an MCE, poll 2x faster. When the poller finds no more 1565 * errors, poll 2x slower (up to check_interval seconds). 1566 */ 1567 static unsigned long check_interval = INITIAL_CHECK_INTERVAL; 1568 1569 static DEFINE_PER_CPU(unsigned long, mce_next_interval); /* in jiffies */ 1570 static DEFINE_PER_CPU(struct timer_list, mce_timer); 1571 1572 static unsigned long mce_adjust_timer_default(unsigned long interval) 1573 { 1574 return interval; 1575 } 1576 1577 static unsigned long (*mce_adjust_timer)(unsigned long interval) = mce_adjust_timer_default; 1578 1579 static void __start_timer(struct timer_list *t, unsigned long interval) 1580 { 1581 unsigned long when = jiffies + interval; 1582 unsigned long flags; 1583 1584 local_irq_save(flags); 1585 1586 if (!timer_pending(t) || time_before(when, t->expires)) 1587 mod_timer(t, round_jiffies(when)); 1588 1589 local_irq_restore(flags); 1590 } 1591 1592 static void mce_timer_fn(struct timer_list *t) 1593 { 1594 struct timer_list *cpu_t = this_cpu_ptr(&mce_timer); 1595 unsigned long iv; 1596 1597 WARN_ON(cpu_t != t); 1598 1599 iv = __this_cpu_read(mce_next_interval); 1600 1601 if (mce_available(this_cpu_ptr(&cpu_info))) { 1602 machine_check_poll(0, this_cpu_ptr(&mce_poll_banks)); 1603 1604 if (mce_intel_cmci_poll()) { 1605 iv = mce_adjust_timer(iv); 1606 goto done; 1607 } 1608 } 1609 1610 /* 1611 * Alert userspace if needed. If we logged an MCE, reduce the polling 1612 * interval, otherwise increase the polling interval. 1613 */ 1614 if (mce_notify_irq()) 1615 iv = max(iv / 2, (unsigned long) HZ/100); 1616 else 1617 iv = min(iv * 2, round_jiffies_relative(check_interval * HZ)); 1618 1619 done: 1620 __this_cpu_write(mce_next_interval, iv); 1621 __start_timer(t, iv); 1622 } 1623 1624 /* 1625 * Ensure that the timer is firing in @interval from now. 1626 */ 1627 void mce_timer_kick(unsigned long interval) 1628 { 1629 struct timer_list *t = this_cpu_ptr(&mce_timer); 1630 unsigned long iv = __this_cpu_read(mce_next_interval); 1631 1632 __start_timer(t, interval); 1633 1634 if (interval < iv) 1635 __this_cpu_write(mce_next_interval, interval); 1636 } 1637 1638 /* Must not be called in IRQ context where del_timer_sync() can deadlock */ 1639 static void mce_timer_delete_all(void) 1640 { 1641 int cpu; 1642 1643 for_each_online_cpu(cpu) 1644 del_timer_sync(&per_cpu(mce_timer, cpu)); 1645 } 1646 1647 /* 1648 * Notify the user(s) about new machine check events. 1649 * Can be called from interrupt context, but not from machine check/NMI 1650 * context. 1651 */ 1652 int mce_notify_irq(void) 1653 { 1654 /* Not more than two messages every minute */ 1655 static DEFINE_RATELIMIT_STATE(ratelimit, 60*HZ, 2); 1656 1657 if (test_and_clear_bit(0, &mce_need_notify)) { 1658 mce_work_trigger(); 1659 1660 if (__ratelimit(&ratelimit)) 1661 pr_info(HW_ERR "Machine check events logged\n"); 1662 1663 return 1; 1664 } 1665 return 0; 1666 } 1667 EXPORT_SYMBOL_GPL(mce_notify_irq); 1668 1669 static void __mcheck_cpu_mce_banks_init(void) 1670 { 1671 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 1672 u8 n_banks = this_cpu_read(mce_num_banks); 1673 int i; 1674 1675 for (i = 0; i < n_banks; i++) { 1676 struct mce_bank *b = &mce_banks[i]; 1677 1678 /* 1679 * Init them all, __mcheck_cpu_apply_quirks() is going to apply 1680 * the required vendor quirks before 1681 * __mcheck_cpu_init_clear_banks() does the final bank setup. 1682 */ 1683 b->ctl = -1ULL; 1684 b->init = true; 1685 } 1686 } 1687 1688 /* 1689 * Initialize Machine Checks for a CPU. 1690 */ 1691 static void __mcheck_cpu_cap_init(void) 1692 { 1693 u64 cap; 1694 u8 b; 1695 1696 rdmsrl(MSR_IA32_MCG_CAP, cap); 1697 1698 b = cap & MCG_BANKCNT_MASK; 1699 1700 if (b > MAX_NR_BANKS) { 1701 pr_warn("CPU%d: Using only %u machine check banks out of %u\n", 1702 smp_processor_id(), MAX_NR_BANKS, b); 1703 b = MAX_NR_BANKS; 1704 } 1705 1706 this_cpu_write(mce_num_banks, b); 1707 1708 __mcheck_cpu_mce_banks_init(); 1709 1710 /* Use accurate RIP reporting if available. */ 1711 if ((cap & MCG_EXT_P) && MCG_EXT_CNT(cap) >= 9) 1712 mca_cfg.rip_msr = MSR_IA32_MCG_EIP; 1713 1714 if (cap & MCG_SER_P) 1715 mca_cfg.ser = 1; 1716 } 1717 1718 static void __mcheck_cpu_init_generic(void) 1719 { 1720 enum mcp_flags m_fl = 0; 1721 mce_banks_t all_banks; 1722 u64 cap; 1723 1724 if (!mca_cfg.bootlog) 1725 m_fl = MCP_DONTLOG; 1726 1727 /* 1728 * Log the machine checks left over from the previous reset. Log them 1729 * only, do not start processing them. That will happen in mcheck_late_init() 1730 * when all consumers have been registered on the notifier chain. 1731 */ 1732 bitmap_fill(all_banks, MAX_NR_BANKS); 1733 machine_check_poll(MCP_UC | MCP_QUEUE_LOG | m_fl, &all_banks); 1734 1735 cr4_set_bits(X86_CR4_MCE); 1736 1737 rdmsrl(MSR_IA32_MCG_CAP, cap); 1738 if (cap & MCG_CTL_P) 1739 wrmsr(MSR_IA32_MCG_CTL, 0xffffffff, 0xffffffff); 1740 } 1741 1742 static void __mcheck_cpu_init_clear_banks(void) 1743 { 1744 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 1745 int i; 1746 1747 for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 1748 struct mce_bank *b = &mce_banks[i]; 1749 1750 if (!b->init) 1751 continue; 1752 wrmsrl(mca_msr_reg(i, MCA_CTL), b->ctl); 1753 wrmsrl(mca_msr_reg(i, MCA_STATUS), 0); 1754 } 1755 } 1756 1757 /* 1758 * Do a final check to see if there are any unused/RAZ banks. 1759 * 1760 * This must be done after the banks have been initialized and any quirks have 1761 * been applied. 1762 * 1763 * Do not call this from any user-initiated flows, e.g. CPU hotplug or sysfs. 1764 * Otherwise, a user who disables a bank will not be able to re-enable it 1765 * without a system reboot. 1766 */ 1767 static void __mcheck_cpu_check_banks(void) 1768 { 1769 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 1770 u64 msrval; 1771 int i; 1772 1773 for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 1774 struct mce_bank *b = &mce_banks[i]; 1775 1776 if (!b->init) 1777 continue; 1778 1779 rdmsrl(mca_msr_reg(i, MCA_CTL), msrval); 1780 b->init = !!msrval; 1781 } 1782 } 1783 1784 /* Add per CPU specific workarounds here */ 1785 static int __mcheck_cpu_apply_quirks(struct cpuinfo_x86 *c) 1786 { 1787 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 1788 struct mca_config *cfg = &mca_cfg; 1789 1790 if (c->x86_vendor == X86_VENDOR_UNKNOWN) { 1791 pr_info("unknown CPU type - not enabling MCE support\n"); 1792 return -EOPNOTSUPP; 1793 } 1794 1795 /* This should be disabled by the BIOS, but isn't always */ 1796 if (c->x86_vendor == X86_VENDOR_AMD) { 1797 if (c->x86 == 15 && this_cpu_read(mce_num_banks) > 4) { 1798 /* 1799 * disable GART TBL walk error reporting, which 1800 * trips off incorrectly with the IOMMU & 3ware 1801 * & Cerberus: 1802 */ 1803 clear_bit(10, (unsigned long *)&mce_banks[4].ctl); 1804 } 1805 if (c->x86 < 0x11 && cfg->bootlog < 0) { 1806 /* 1807 * Lots of broken BIOS around that don't clear them 1808 * by default and leave crap in there. Don't log: 1809 */ 1810 cfg->bootlog = 0; 1811 } 1812 /* 1813 * Various K7s with broken bank 0 around. Always disable 1814 * by default. 1815 */ 1816 if (c->x86 == 6 && this_cpu_read(mce_num_banks) > 0) 1817 mce_banks[0].ctl = 0; 1818 1819 /* 1820 * overflow_recov is supported for F15h Models 00h-0fh 1821 * even though we don't have a CPUID bit for it. 1822 */ 1823 if (c->x86 == 0x15 && c->x86_model <= 0xf) 1824 mce_flags.overflow_recov = 1; 1825 1826 } 1827 1828 if (c->x86_vendor == X86_VENDOR_INTEL) { 1829 /* 1830 * SDM documents that on family 6 bank 0 should not be written 1831 * because it aliases to another special BIOS controlled 1832 * register. 1833 * But it's not aliased anymore on model 0x1a+ 1834 * Don't ignore bank 0 completely because there could be a 1835 * valid event later, merely don't write CTL0. 1836 */ 1837 1838 if (c->x86 == 6 && c->x86_model < 0x1A && this_cpu_read(mce_num_banks) > 0) 1839 mce_banks[0].init = false; 1840 1841 /* 1842 * All newer Intel systems support MCE broadcasting. Enable 1843 * synchronization with a one second timeout. 1844 */ 1845 if ((c->x86 > 6 || (c->x86 == 6 && c->x86_model >= 0xe)) && 1846 cfg->monarch_timeout < 0) 1847 cfg->monarch_timeout = USEC_PER_SEC; 1848 1849 /* 1850 * There are also broken BIOSes on some Pentium M and 1851 * earlier systems: 1852 */ 1853 if (c->x86 == 6 && c->x86_model <= 13 && cfg->bootlog < 0) 1854 cfg->bootlog = 0; 1855 1856 if (c->x86 == 6 && c->x86_model == 45) 1857 mce_flags.snb_ifu_quirk = 1; 1858 } 1859 1860 if (c->x86_vendor == X86_VENDOR_ZHAOXIN) { 1861 /* 1862 * All newer Zhaoxin CPUs support MCE broadcasting. Enable 1863 * synchronization with a one second timeout. 1864 */ 1865 if (c->x86 > 6 || (c->x86_model == 0x19 || c->x86_model == 0x1f)) { 1866 if (cfg->monarch_timeout < 0) 1867 cfg->monarch_timeout = USEC_PER_SEC; 1868 } 1869 } 1870 1871 if (cfg->monarch_timeout < 0) 1872 cfg->monarch_timeout = 0; 1873 if (cfg->bootlog != 0) 1874 cfg->panic_timeout = 30; 1875 1876 return 0; 1877 } 1878 1879 static int __mcheck_cpu_ancient_init(struct cpuinfo_x86 *c) 1880 { 1881 if (c->x86 != 5) 1882 return 0; 1883 1884 switch (c->x86_vendor) { 1885 case X86_VENDOR_INTEL: 1886 intel_p5_mcheck_init(c); 1887 mce_flags.p5 = 1; 1888 return 1; 1889 case X86_VENDOR_CENTAUR: 1890 winchip_mcheck_init(c); 1891 mce_flags.winchip = 1; 1892 return 1; 1893 default: 1894 return 0; 1895 } 1896 1897 return 0; 1898 } 1899 1900 /* 1901 * Init basic CPU features needed for early decoding of MCEs. 1902 */ 1903 static void __mcheck_cpu_init_early(struct cpuinfo_x86 *c) 1904 { 1905 if (c->x86_vendor == X86_VENDOR_AMD || c->x86_vendor == X86_VENDOR_HYGON) { 1906 mce_flags.overflow_recov = !!cpu_has(c, X86_FEATURE_OVERFLOW_RECOV); 1907 mce_flags.succor = !!cpu_has(c, X86_FEATURE_SUCCOR); 1908 mce_flags.smca = !!cpu_has(c, X86_FEATURE_SMCA); 1909 mce_flags.amd_threshold = 1; 1910 } 1911 } 1912 1913 static void mce_centaur_feature_init(struct cpuinfo_x86 *c) 1914 { 1915 struct mca_config *cfg = &mca_cfg; 1916 1917 /* 1918 * All newer Centaur CPUs support MCE broadcasting. Enable 1919 * synchronization with a one second timeout. 1920 */ 1921 if ((c->x86 == 6 && c->x86_model == 0xf && c->x86_stepping >= 0xe) || 1922 c->x86 > 6) { 1923 if (cfg->monarch_timeout < 0) 1924 cfg->monarch_timeout = USEC_PER_SEC; 1925 } 1926 } 1927 1928 static void mce_zhaoxin_feature_init(struct cpuinfo_x86 *c) 1929 { 1930 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 1931 1932 /* 1933 * These CPUs have MCA bank 8 which reports only one error type called 1934 * SVAD (System View Address Decoder). The reporting of that error is 1935 * controlled by IA32_MC8.CTL.0. 1936 * 1937 * If enabled, prefetching on these CPUs will cause SVAD MCE when 1938 * virtual machines start and result in a system panic. Always disable 1939 * bank 8 SVAD error by default. 1940 */ 1941 if ((c->x86 == 7 && c->x86_model == 0x1b) || 1942 (c->x86_model == 0x19 || c->x86_model == 0x1f)) { 1943 if (this_cpu_read(mce_num_banks) > 8) 1944 mce_banks[8].ctl = 0; 1945 } 1946 1947 intel_init_cmci(); 1948 intel_init_lmce(); 1949 mce_adjust_timer = cmci_intel_adjust_timer; 1950 } 1951 1952 static void mce_zhaoxin_feature_clear(struct cpuinfo_x86 *c) 1953 { 1954 intel_clear_lmce(); 1955 } 1956 1957 static void __mcheck_cpu_init_vendor(struct cpuinfo_x86 *c) 1958 { 1959 switch (c->x86_vendor) { 1960 case X86_VENDOR_INTEL: 1961 mce_intel_feature_init(c); 1962 mce_adjust_timer = cmci_intel_adjust_timer; 1963 break; 1964 1965 case X86_VENDOR_AMD: { 1966 mce_amd_feature_init(c); 1967 break; 1968 } 1969 1970 case X86_VENDOR_HYGON: 1971 mce_hygon_feature_init(c); 1972 break; 1973 1974 case X86_VENDOR_CENTAUR: 1975 mce_centaur_feature_init(c); 1976 break; 1977 1978 case X86_VENDOR_ZHAOXIN: 1979 mce_zhaoxin_feature_init(c); 1980 break; 1981 1982 default: 1983 break; 1984 } 1985 } 1986 1987 static void __mcheck_cpu_clear_vendor(struct cpuinfo_x86 *c) 1988 { 1989 switch (c->x86_vendor) { 1990 case X86_VENDOR_INTEL: 1991 mce_intel_feature_clear(c); 1992 break; 1993 1994 case X86_VENDOR_ZHAOXIN: 1995 mce_zhaoxin_feature_clear(c); 1996 break; 1997 1998 default: 1999 break; 2000 } 2001 } 2002 2003 static void mce_start_timer(struct timer_list *t) 2004 { 2005 unsigned long iv = check_interval * HZ; 2006 2007 if (mca_cfg.ignore_ce || !iv) 2008 return; 2009 2010 this_cpu_write(mce_next_interval, iv); 2011 __start_timer(t, iv); 2012 } 2013 2014 static void __mcheck_cpu_setup_timer(void) 2015 { 2016 struct timer_list *t = this_cpu_ptr(&mce_timer); 2017 2018 timer_setup(t, mce_timer_fn, TIMER_PINNED); 2019 } 2020 2021 static void __mcheck_cpu_init_timer(void) 2022 { 2023 struct timer_list *t = this_cpu_ptr(&mce_timer); 2024 2025 timer_setup(t, mce_timer_fn, TIMER_PINNED); 2026 mce_start_timer(t); 2027 } 2028 2029 bool filter_mce(struct mce *m) 2030 { 2031 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) 2032 return amd_filter_mce(m); 2033 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) 2034 return intel_filter_mce(m); 2035 2036 return false; 2037 } 2038 2039 static __always_inline void exc_machine_check_kernel(struct pt_regs *regs) 2040 { 2041 irqentry_state_t irq_state; 2042 2043 WARN_ON_ONCE(user_mode(regs)); 2044 2045 /* 2046 * Only required when from kernel mode. See 2047 * mce_check_crashing_cpu() for details. 2048 */ 2049 if (mca_cfg.initialized && mce_check_crashing_cpu()) 2050 return; 2051 2052 irq_state = irqentry_nmi_enter(regs); 2053 2054 do_machine_check(regs); 2055 2056 irqentry_nmi_exit(regs, irq_state); 2057 } 2058 2059 static __always_inline void exc_machine_check_user(struct pt_regs *regs) 2060 { 2061 irqentry_enter_from_user_mode(regs); 2062 2063 do_machine_check(regs); 2064 2065 irqentry_exit_to_user_mode(regs); 2066 } 2067 2068 #ifdef CONFIG_X86_64 2069 /* MCE hit kernel mode */ 2070 DEFINE_IDTENTRY_MCE(exc_machine_check) 2071 { 2072 unsigned long dr7; 2073 2074 dr7 = local_db_save(); 2075 exc_machine_check_kernel(regs); 2076 local_db_restore(dr7); 2077 } 2078 2079 /* The user mode variant. */ 2080 DEFINE_IDTENTRY_MCE_USER(exc_machine_check) 2081 { 2082 unsigned long dr7; 2083 2084 dr7 = local_db_save(); 2085 exc_machine_check_user(regs); 2086 local_db_restore(dr7); 2087 } 2088 #else 2089 /* 32bit unified entry point */ 2090 DEFINE_IDTENTRY_RAW(exc_machine_check) 2091 { 2092 unsigned long dr7; 2093 2094 dr7 = local_db_save(); 2095 if (user_mode(regs)) 2096 exc_machine_check_user(regs); 2097 else 2098 exc_machine_check_kernel(regs); 2099 local_db_restore(dr7); 2100 } 2101 #endif 2102 2103 /* 2104 * Called for each booted CPU to set up machine checks. 2105 * Must be called with preempt off: 2106 */ 2107 void mcheck_cpu_init(struct cpuinfo_x86 *c) 2108 { 2109 if (mca_cfg.disabled) 2110 return; 2111 2112 if (__mcheck_cpu_ancient_init(c)) 2113 return; 2114 2115 if (!mce_available(c)) 2116 return; 2117 2118 __mcheck_cpu_cap_init(); 2119 2120 if (__mcheck_cpu_apply_quirks(c) < 0) { 2121 mca_cfg.disabled = 1; 2122 return; 2123 } 2124 2125 if (mce_gen_pool_init()) { 2126 mca_cfg.disabled = 1; 2127 pr_emerg("Couldn't allocate MCE records pool!\n"); 2128 return; 2129 } 2130 2131 mca_cfg.initialized = 1; 2132 2133 __mcheck_cpu_init_early(c); 2134 __mcheck_cpu_init_generic(); 2135 __mcheck_cpu_init_vendor(c); 2136 __mcheck_cpu_init_clear_banks(); 2137 __mcheck_cpu_check_banks(); 2138 __mcheck_cpu_setup_timer(); 2139 } 2140 2141 /* 2142 * Called for each booted CPU to clear some machine checks opt-ins 2143 */ 2144 void mcheck_cpu_clear(struct cpuinfo_x86 *c) 2145 { 2146 if (mca_cfg.disabled) 2147 return; 2148 2149 if (!mce_available(c)) 2150 return; 2151 2152 /* 2153 * Possibly to clear general settings generic to x86 2154 * __mcheck_cpu_clear_generic(c); 2155 */ 2156 __mcheck_cpu_clear_vendor(c); 2157 2158 } 2159 2160 static void __mce_disable_bank(void *arg) 2161 { 2162 int bank = *((int *)arg); 2163 __clear_bit(bank, this_cpu_ptr(mce_poll_banks)); 2164 cmci_disable_bank(bank); 2165 } 2166 2167 void mce_disable_bank(int bank) 2168 { 2169 if (bank >= this_cpu_read(mce_num_banks)) { 2170 pr_warn(FW_BUG 2171 "Ignoring request to disable invalid MCA bank %d.\n", 2172 bank); 2173 return; 2174 } 2175 set_bit(bank, mce_banks_ce_disabled); 2176 on_each_cpu(__mce_disable_bank, &bank, 1); 2177 } 2178 2179 /* 2180 * mce=off Disables machine check 2181 * mce=no_cmci Disables CMCI 2182 * mce=no_lmce Disables LMCE 2183 * mce=dont_log_ce Clears corrected events silently, no log created for CEs. 2184 * mce=print_all Print all machine check logs to console 2185 * mce=ignore_ce Disables polling and CMCI, corrected events are not cleared. 2186 * mce=TOLERANCELEVEL[,monarchtimeout] (number, see above) 2187 * monarchtimeout is how long to wait for other CPUs on machine 2188 * check, or 0 to not wait 2189 * mce=bootlog Log MCEs from before booting. Disabled by default on AMD Fam10h 2190 and older. 2191 * mce=nobootlog Don't log MCEs from before booting. 2192 * mce=bios_cmci_threshold Don't program the CMCI threshold 2193 * mce=recovery force enable copy_mc_fragile() 2194 */ 2195 static int __init mcheck_enable(char *str) 2196 { 2197 struct mca_config *cfg = &mca_cfg; 2198 2199 if (*str == 0) { 2200 enable_p5_mce(); 2201 return 1; 2202 } 2203 if (*str == '=') 2204 str++; 2205 if (!strcmp(str, "off")) 2206 cfg->disabled = 1; 2207 else if (!strcmp(str, "no_cmci")) 2208 cfg->cmci_disabled = true; 2209 else if (!strcmp(str, "no_lmce")) 2210 cfg->lmce_disabled = 1; 2211 else if (!strcmp(str, "dont_log_ce")) 2212 cfg->dont_log_ce = true; 2213 else if (!strcmp(str, "print_all")) 2214 cfg->print_all = true; 2215 else if (!strcmp(str, "ignore_ce")) 2216 cfg->ignore_ce = true; 2217 else if (!strcmp(str, "bootlog") || !strcmp(str, "nobootlog")) 2218 cfg->bootlog = (str[0] == 'b'); 2219 else if (!strcmp(str, "bios_cmci_threshold")) 2220 cfg->bios_cmci_threshold = 1; 2221 else if (!strcmp(str, "recovery")) 2222 cfg->recovery = 1; 2223 else if (isdigit(str[0])) { 2224 if (get_option(&str, &cfg->tolerant) == 2) 2225 get_option(&str, &(cfg->monarch_timeout)); 2226 } else { 2227 pr_info("mce argument %s ignored. Please use /sys\n", str); 2228 return 0; 2229 } 2230 return 1; 2231 } 2232 __setup("mce", mcheck_enable); 2233 2234 int __init mcheck_init(void) 2235 { 2236 mce_register_decode_chain(&early_nb); 2237 mce_register_decode_chain(&mce_uc_nb); 2238 mce_register_decode_chain(&mce_default_nb); 2239 2240 INIT_WORK(&mce_work, mce_gen_pool_process); 2241 init_irq_work(&mce_irq_work, mce_irq_work_cb); 2242 2243 return 0; 2244 } 2245 2246 /* 2247 * mce_syscore: PM support 2248 */ 2249 2250 /* 2251 * Disable machine checks on suspend and shutdown. We can't really handle 2252 * them later. 2253 */ 2254 static void mce_disable_error_reporting(void) 2255 { 2256 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 2257 int i; 2258 2259 for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 2260 struct mce_bank *b = &mce_banks[i]; 2261 2262 if (b->init) 2263 wrmsrl(mca_msr_reg(i, MCA_CTL), 0); 2264 } 2265 return; 2266 } 2267 2268 static void vendor_disable_error_reporting(void) 2269 { 2270 /* 2271 * Don't clear on Intel or AMD or Hygon or Zhaoxin CPUs. Some of these 2272 * MSRs are socket-wide. Disabling them for just a single offlined CPU 2273 * is bad, since it will inhibit reporting for all shared resources on 2274 * the socket like the last level cache (LLC), the integrated memory 2275 * controller (iMC), etc. 2276 */ 2277 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL || 2278 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON || 2279 boot_cpu_data.x86_vendor == X86_VENDOR_AMD || 2280 boot_cpu_data.x86_vendor == X86_VENDOR_ZHAOXIN) 2281 return; 2282 2283 mce_disable_error_reporting(); 2284 } 2285 2286 static int mce_syscore_suspend(void) 2287 { 2288 vendor_disable_error_reporting(); 2289 return 0; 2290 } 2291 2292 static void mce_syscore_shutdown(void) 2293 { 2294 vendor_disable_error_reporting(); 2295 } 2296 2297 /* 2298 * On resume clear all MCE state. Don't want to see leftovers from the BIOS. 2299 * Only one CPU is active at this time, the others get re-added later using 2300 * CPU hotplug: 2301 */ 2302 static void mce_syscore_resume(void) 2303 { 2304 __mcheck_cpu_init_generic(); 2305 __mcheck_cpu_init_vendor(raw_cpu_ptr(&cpu_info)); 2306 __mcheck_cpu_init_clear_banks(); 2307 } 2308 2309 static struct syscore_ops mce_syscore_ops = { 2310 .suspend = mce_syscore_suspend, 2311 .shutdown = mce_syscore_shutdown, 2312 .resume = mce_syscore_resume, 2313 }; 2314 2315 /* 2316 * mce_device: Sysfs support 2317 */ 2318 2319 static void mce_cpu_restart(void *data) 2320 { 2321 if (!mce_available(raw_cpu_ptr(&cpu_info))) 2322 return; 2323 __mcheck_cpu_init_generic(); 2324 __mcheck_cpu_init_clear_banks(); 2325 __mcheck_cpu_init_timer(); 2326 } 2327 2328 /* Reinit MCEs after user configuration changes */ 2329 static void mce_restart(void) 2330 { 2331 mce_timer_delete_all(); 2332 on_each_cpu(mce_cpu_restart, NULL, 1); 2333 } 2334 2335 /* Toggle features for corrected errors */ 2336 static void mce_disable_cmci(void *data) 2337 { 2338 if (!mce_available(raw_cpu_ptr(&cpu_info))) 2339 return; 2340 cmci_clear(); 2341 } 2342 2343 static void mce_enable_ce(void *all) 2344 { 2345 if (!mce_available(raw_cpu_ptr(&cpu_info))) 2346 return; 2347 cmci_reenable(); 2348 cmci_recheck(); 2349 if (all) 2350 __mcheck_cpu_init_timer(); 2351 } 2352 2353 static struct bus_type mce_subsys = { 2354 .name = "machinecheck", 2355 .dev_name = "machinecheck", 2356 }; 2357 2358 DEFINE_PER_CPU(struct device *, mce_device); 2359 2360 static inline struct mce_bank_dev *attr_to_bank(struct device_attribute *attr) 2361 { 2362 return container_of(attr, struct mce_bank_dev, attr); 2363 } 2364 2365 static ssize_t show_bank(struct device *s, struct device_attribute *attr, 2366 char *buf) 2367 { 2368 u8 bank = attr_to_bank(attr)->bank; 2369 struct mce_bank *b; 2370 2371 if (bank >= per_cpu(mce_num_banks, s->id)) 2372 return -EINVAL; 2373 2374 b = &per_cpu(mce_banks_array, s->id)[bank]; 2375 2376 if (!b->init) 2377 return -ENODEV; 2378 2379 return sprintf(buf, "%llx\n", b->ctl); 2380 } 2381 2382 static ssize_t set_bank(struct device *s, struct device_attribute *attr, 2383 const char *buf, size_t size) 2384 { 2385 u8 bank = attr_to_bank(attr)->bank; 2386 struct mce_bank *b; 2387 u64 new; 2388 2389 if (kstrtou64(buf, 0, &new) < 0) 2390 return -EINVAL; 2391 2392 if (bank >= per_cpu(mce_num_banks, s->id)) 2393 return -EINVAL; 2394 2395 b = &per_cpu(mce_banks_array, s->id)[bank]; 2396 2397 if (!b->init) 2398 return -ENODEV; 2399 2400 b->ctl = new; 2401 mce_restart(); 2402 2403 return size; 2404 } 2405 2406 static ssize_t set_ignore_ce(struct device *s, 2407 struct device_attribute *attr, 2408 const char *buf, size_t size) 2409 { 2410 u64 new; 2411 2412 if (kstrtou64(buf, 0, &new) < 0) 2413 return -EINVAL; 2414 2415 mutex_lock(&mce_sysfs_mutex); 2416 if (mca_cfg.ignore_ce ^ !!new) { 2417 if (new) { 2418 /* disable ce features */ 2419 mce_timer_delete_all(); 2420 on_each_cpu(mce_disable_cmci, NULL, 1); 2421 mca_cfg.ignore_ce = true; 2422 } else { 2423 /* enable ce features */ 2424 mca_cfg.ignore_ce = false; 2425 on_each_cpu(mce_enable_ce, (void *)1, 1); 2426 } 2427 } 2428 mutex_unlock(&mce_sysfs_mutex); 2429 2430 return size; 2431 } 2432 2433 static ssize_t set_cmci_disabled(struct device *s, 2434 struct device_attribute *attr, 2435 const char *buf, size_t size) 2436 { 2437 u64 new; 2438 2439 if (kstrtou64(buf, 0, &new) < 0) 2440 return -EINVAL; 2441 2442 mutex_lock(&mce_sysfs_mutex); 2443 if (mca_cfg.cmci_disabled ^ !!new) { 2444 if (new) { 2445 /* disable cmci */ 2446 on_each_cpu(mce_disable_cmci, NULL, 1); 2447 mca_cfg.cmci_disabled = true; 2448 } else { 2449 /* enable cmci */ 2450 mca_cfg.cmci_disabled = false; 2451 on_each_cpu(mce_enable_ce, NULL, 1); 2452 } 2453 } 2454 mutex_unlock(&mce_sysfs_mutex); 2455 2456 return size; 2457 } 2458 2459 static ssize_t store_int_with_restart(struct device *s, 2460 struct device_attribute *attr, 2461 const char *buf, size_t size) 2462 { 2463 unsigned long old_check_interval = check_interval; 2464 ssize_t ret = device_store_ulong(s, attr, buf, size); 2465 2466 if (check_interval == old_check_interval) 2467 return ret; 2468 2469 mutex_lock(&mce_sysfs_mutex); 2470 mce_restart(); 2471 mutex_unlock(&mce_sysfs_mutex); 2472 2473 return ret; 2474 } 2475 2476 static DEVICE_INT_ATTR(tolerant, 0644, mca_cfg.tolerant); 2477 static DEVICE_INT_ATTR(monarch_timeout, 0644, mca_cfg.monarch_timeout); 2478 static DEVICE_BOOL_ATTR(dont_log_ce, 0644, mca_cfg.dont_log_ce); 2479 static DEVICE_BOOL_ATTR(print_all, 0644, mca_cfg.print_all); 2480 2481 static struct dev_ext_attribute dev_attr_check_interval = { 2482 __ATTR(check_interval, 0644, device_show_int, store_int_with_restart), 2483 &check_interval 2484 }; 2485 2486 static struct dev_ext_attribute dev_attr_ignore_ce = { 2487 __ATTR(ignore_ce, 0644, device_show_bool, set_ignore_ce), 2488 &mca_cfg.ignore_ce 2489 }; 2490 2491 static struct dev_ext_attribute dev_attr_cmci_disabled = { 2492 __ATTR(cmci_disabled, 0644, device_show_bool, set_cmci_disabled), 2493 &mca_cfg.cmci_disabled 2494 }; 2495 2496 static struct device_attribute *mce_device_attrs[] = { 2497 &dev_attr_tolerant.attr, 2498 &dev_attr_check_interval.attr, 2499 #ifdef CONFIG_X86_MCELOG_LEGACY 2500 &dev_attr_trigger, 2501 #endif 2502 &dev_attr_monarch_timeout.attr, 2503 &dev_attr_dont_log_ce.attr, 2504 &dev_attr_print_all.attr, 2505 &dev_attr_ignore_ce.attr, 2506 &dev_attr_cmci_disabled.attr, 2507 NULL 2508 }; 2509 2510 static cpumask_var_t mce_device_initialized; 2511 2512 static void mce_device_release(struct device *dev) 2513 { 2514 kfree(dev); 2515 } 2516 2517 /* Per CPU device init. All of the CPUs still share the same bank device: */ 2518 static int mce_device_create(unsigned int cpu) 2519 { 2520 struct device *dev; 2521 int err; 2522 int i, j; 2523 2524 if (!mce_available(&boot_cpu_data)) 2525 return -EIO; 2526 2527 dev = per_cpu(mce_device, cpu); 2528 if (dev) 2529 return 0; 2530 2531 dev = kzalloc(sizeof(*dev), GFP_KERNEL); 2532 if (!dev) 2533 return -ENOMEM; 2534 dev->id = cpu; 2535 dev->bus = &mce_subsys; 2536 dev->release = &mce_device_release; 2537 2538 err = device_register(dev); 2539 if (err) { 2540 put_device(dev); 2541 return err; 2542 } 2543 2544 for (i = 0; mce_device_attrs[i]; i++) { 2545 err = device_create_file(dev, mce_device_attrs[i]); 2546 if (err) 2547 goto error; 2548 } 2549 for (j = 0; j < per_cpu(mce_num_banks, cpu); j++) { 2550 err = device_create_file(dev, &mce_bank_devs[j].attr); 2551 if (err) 2552 goto error2; 2553 } 2554 cpumask_set_cpu(cpu, mce_device_initialized); 2555 per_cpu(mce_device, cpu) = dev; 2556 2557 return 0; 2558 error2: 2559 while (--j >= 0) 2560 device_remove_file(dev, &mce_bank_devs[j].attr); 2561 error: 2562 while (--i >= 0) 2563 device_remove_file(dev, mce_device_attrs[i]); 2564 2565 device_unregister(dev); 2566 2567 return err; 2568 } 2569 2570 static void mce_device_remove(unsigned int cpu) 2571 { 2572 struct device *dev = per_cpu(mce_device, cpu); 2573 int i; 2574 2575 if (!cpumask_test_cpu(cpu, mce_device_initialized)) 2576 return; 2577 2578 for (i = 0; mce_device_attrs[i]; i++) 2579 device_remove_file(dev, mce_device_attrs[i]); 2580 2581 for (i = 0; i < per_cpu(mce_num_banks, cpu); i++) 2582 device_remove_file(dev, &mce_bank_devs[i].attr); 2583 2584 device_unregister(dev); 2585 cpumask_clear_cpu(cpu, mce_device_initialized); 2586 per_cpu(mce_device, cpu) = NULL; 2587 } 2588 2589 /* Make sure there are no machine checks on offlined CPUs. */ 2590 static void mce_disable_cpu(void) 2591 { 2592 if (!mce_available(raw_cpu_ptr(&cpu_info))) 2593 return; 2594 2595 if (!cpuhp_tasks_frozen) 2596 cmci_clear(); 2597 2598 vendor_disable_error_reporting(); 2599 } 2600 2601 static void mce_reenable_cpu(void) 2602 { 2603 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 2604 int i; 2605 2606 if (!mce_available(raw_cpu_ptr(&cpu_info))) 2607 return; 2608 2609 if (!cpuhp_tasks_frozen) 2610 cmci_reenable(); 2611 for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 2612 struct mce_bank *b = &mce_banks[i]; 2613 2614 if (b->init) 2615 wrmsrl(mca_msr_reg(i, MCA_CTL), b->ctl); 2616 } 2617 } 2618 2619 static int mce_cpu_dead(unsigned int cpu) 2620 { 2621 mce_intel_hcpu_update(cpu); 2622 2623 /* intentionally ignoring frozen here */ 2624 if (!cpuhp_tasks_frozen) 2625 cmci_rediscover(); 2626 return 0; 2627 } 2628 2629 static int mce_cpu_online(unsigned int cpu) 2630 { 2631 struct timer_list *t = this_cpu_ptr(&mce_timer); 2632 int ret; 2633 2634 mce_device_create(cpu); 2635 2636 ret = mce_threshold_create_device(cpu); 2637 if (ret) { 2638 mce_device_remove(cpu); 2639 return ret; 2640 } 2641 mce_reenable_cpu(); 2642 mce_start_timer(t); 2643 return 0; 2644 } 2645 2646 static int mce_cpu_pre_down(unsigned int cpu) 2647 { 2648 struct timer_list *t = this_cpu_ptr(&mce_timer); 2649 2650 mce_disable_cpu(); 2651 del_timer_sync(t); 2652 mce_threshold_remove_device(cpu); 2653 mce_device_remove(cpu); 2654 return 0; 2655 } 2656 2657 static __init void mce_init_banks(void) 2658 { 2659 int i; 2660 2661 for (i = 0; i < MAX_NR_BANKS; i++) { 2662 struct mce_bank_dev *b = &mce_bank_devs[i]; 2663 struct device_attribute *a = &b->attr; 2664 2665 b->bank = i; 2666 2667 sysfs_attr_init(&a->attr); 2668 a->attr.name = b->attrname; 2669 snprintf(b->attrname, ATTR_LEN, "bank%d", i); 2670 2671 a->attr.mode = 0644; 2672 a->show = show_bank; 2673 a->store = set_bank; 2674 } 2675 } 2676 2677 /* 2678 * When running on XEN, this initcall is ordered against the XEN mcelog 2679 * initcall: 2680 * 2681 * device_initcall(xen_late_init_mcelog); 2682 * device_initcall_sync(mcheck_init_device); 2683 */ 2684 static __init int mcheck_init_device(void) 2685 { 2686 int err; 2687 2688 /* 2689 * Check if we have a spare virtual bit. This will only become 2690 * a problem if/when we move beyond 5-level page tables. 2691 */ 2692 MAYBE_BUILD_BUG_ON(__VIRTUAL_MASK_SHIFT >= 63); 2693 2694 if (!mce_available(&boot_cpu_data)) { 2695 err = -EIO; 2696 goto err_out; 2697 } 2698 2699 if (!zalloc_cpumask_var(&mce_device_initialized, GFP_KERNEL)) { 2700 err = -ENOMEM; 2701 goto err_out; 2702 } 2703 2704 mce_init_banks(); 2705 2706 err = subsys_system_register(&mce_subsys, NULL); 2707 if (err) 2708 goto err_out_mem; 2709 2710 err = cpuhp_setup_state(CPUHP_X86_MCE_DEAD, "x86/mce:dead", NULL, 2711 mce_cpu_dead); 2712 if (err) 2713 goto err_out_mem; 2714 2715 /* 2716 * Invokes mce_cpu_online() on all CPUs which are online when 2717 * the state is installed. 2718 */ 2719 err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/mce:online", 2720 mce_cpu_online, mce_cpu_pre_down); 2721 if (err < 0) 2722 goto err_out_online; 2723 2724 register_syscore_ops(&mce_syscore_ops); 2725 2726 return 0; 2727 2728 err_out_online: 2729 cpuhp_remove_state(CPUHP_X86_MCE_DEAD); 2730 2731 err_out_mem: 2732 free_cpumask_var(mce_device_initialized); 2733 2734 err_out: 2735 pr_err("Unable to init MCE device (rc: %d)\n", err); 2736 2737 return err; 2738 } 2739 device_initcall_sync(mcheck_init_device); 2740 2741 /* 2742 * Old style boot options parsing. Only for compatibility. 2743 */ 2744 static int __init mcheck_disable(char *str) 2745 { 2746 mca_cfg.disabled = 1; 2747 return 1; 2748 } 2749 __setup("nomce", mcheck_disable); 2750 2751 #ifdef CONFIG_DEBUG_FS 2752 struct dentry *mce_get_debugfs_dir(void) 2753 { 2754 static struct dentry *dmce; 2755 2756 if (!dmce) 2757 dmce = debugfs_create_dir("mce", NULL); 2758 2759 return dmce; 2760 } 2761 2762 static void mce_reset(void) 2763 { 2764 atomic_set(&mce_fake_panicked, 0); 2765 atomic_set(&mce_executing, 0); 2766 atomic_set(&mce_callin, 0); 2767 atomic_set(&global_nwo, 0); 2768 cpumask_setall(&mce_missing_cpus); 2769 } 2770 2771 static int fake_panic_get(void *data, u64 *val) 2772 { 2773 *val = fake_panic; 2774 return 0; 2775 } 2776 2777 static int fake_panic_set(void *data, u64 val) 2778 { 2779 mce_reset(); 2780 fake_panic = val; 2781 return 0; 2782 } 2783 2784 DEFINE_DEBUGFS_ATTRIBUTE(fake_panic_fops, fake_panic_get, fake_panic_set, 2785 "%llu\n"); 2786 2787 static void __init mcheck_debugfs_init(void) 2788 { 2789 struct dentry *dmce; 2790 2791 dmce = mce_get_debugfs_dir(); 2792 debugfs_create_file_unsafe("fake_panic", 0444, dmce, NULL, 2793 &fake_panic_fops); 2794 } 2795 #else 2796 static void __init mcheck_debugfs_init(void) { } 2797 #endif 2798 2799 static int __init mcheck_late_init(void) 2800 { 2801 if (mca_cfg.recovery) 2802 enable_copy_mc_fragile(); 2803 2804 mcheck_debugfs_init(); 2805 2806 /* 2807 * Flush out everything that has been logged during early boot, now that 2808 * everything has been initialized (workqueues, decoders, ...). 2809 */ 2810 mce_schedule_work(); 2811 2812 return 0; 2813 } 2814 late_initcall(mcheck_late_init); 2815