1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Machine check handler. 4 * 5 * K8 parts Copyright 2002,2003 Andi Kleen, SuSE Labs. 6 * Rest from unknown author(s). 7 * 2004 Andi Kleen. Rewrote most of it. 8 * Copyright 2008 Intel Corporation 9 * Author: Andi Kleen 10 */ 11 12 #include <linux/thread_info.h> 13 #include <linux/capability.h> 14 #include <linux/miscdevice.h> 15 #include <linux/ratelimit.h> 16 #include <linux/rcupdate.h> 17 #include <linux/kobject.h> 18 #include <linux/uaccess.h> 19 #include <linux/kdebug.h> 20 #include <linux/kernel.h> 21 #include <linux/percpu.h> 22 #include <linux/string.h> 23 #include <linux/device.h> 24 #include <linux/syscore_ops.h> 25 #include <linux/delay.h> 26 #include <linux/ctype.h> 27 #include <linux/sched.h> 28 #include <linux/sysfs.h> 29 #include <linux/types.h> 30 #include <linux/slab.h> 31 #include <linux/init.h> 32 #include <linux/kmod.h> 33 #include <linux/poll.h> 34 #include <linux/nmi.h> 35 #include <linux/cpu.h> 36 #include <linux/ras.h> 37 #include <linux/smp.h> 38 #include <linux/fs.h> 39 #include <linux/mm.h> 40 #include <linux/debugfs.h> 41 #include <linux/irq_work.h> 42 #include <linux/export.h> 43 #include <linux/jump_label.h> 44 #include <linux/set_memory.h> 45 #include <linux/sync_core.h> 46 #include <linux/task_work.h> 47 #include <linux/hardirq.h> 48 49 #include <asm/intel-family.h> 50 #include <asm/processor.h> 51 #include <asm/traps.h> 52 #include <asm/tlbflush.h> 53 #include <asm/mce.h> 54 #include <asm/msr.h> 55 #include <asm/reboot.h> 56 57 #include "internal.h" 58 59 /* sysfs synchronization */ 60 static DEFINE_MUTEX(mce_sysfs_mutex); 61 62 #define CREATE_TRACE_POINTS 63 #include <trace/events/mce.h> 64 65 #define SPINUNIT 100 /* 100ns */ 66 67 DEFINE_PER_CPU(unsigned, mce_exception_count); 68 69 DEFINE_PER_CPU_READ_MOSTLY(unsigned int, mce_num_banks); 70 71 struct mce_bank { 72 u64 ctl; /* subevents to enable */ 73 bool init; /* initialise bank? */ 74 }; 75 static DEFINE_PER_CPU_READ_MOSTLY(struct mce_bank[MAX_NR_BANKS], mce_banks_array); 76 77 #define ATTR_LEN 16 78 /* One object for each MCE bank, shared by all CPUs */ 79 struct mce_bank_dev { 80 struct device_attribute attr; /* device attribute */ 81 char attrname[ATTR_LEN]; /* attribute name */ 82 u8 bank; /* bank number */ 83 }; 84 static struct mce_bank_dev mce_bank_devs[MAX_NR_BANKS]; 85 86 struct mce_vendor_flags mce_flags __read_mostly; 87 88 struct mca_config mca_cfg __read_mostly = { 89 .bootlog = -1, 90 /* 91 * Tolerant levels: 92 * 0: always panic on uncorrected errors, log corrected errors 93 * 1: panic or SIGBUS on uncorrected errors, log corrected errors 94 * 2: SIGBUS or log uncorrected errors (if possible), log corr. errors 95 * 3: never panic or SIGBUS, log all errors (for testing only) 96 */ 97 .tolerant = 1, 98 .monarch_timeout = -1 99 }; 100 101 static DEFINE_PER_CPU(struct mce, mces_seen); 102 static unsigned long mce_need_notify; 103 static int cpu_missing; 104 105 /* 106 * MCA banks polled by the period polling timer for corrected events. 107 * With Intel CMCI, this only has MCA banks which do not support CMCI (if any). 108 */ 109 DEFINE_PER_CPU(mce_banks_t, mce_poll_banks) = { 110 [0 ... BITS_TO_LONGS(MAX_NR_BANKS)-1] = ~0UL 111 }; 112 113 /* 114 * MCA banks controlled through firmware first for corrected errors. 115 * This is a global list of banks for which we won't enable CMCI and we 116 * won't poll. Firmware controls these banks and is responsible for 117 * reporting corrected errors through GHES. Uncorrected/recoverable 118 * errors are still notified through a machine check. 119 */ 120 mce_banks_t mce_banks_ce_disabled; 121 122 static struct work_struct mce_work; 123 static struct irq_work mce_irq_work; 124 125 static void (*quirk_no_way_out)(int bank, struct mce *m, struct pt_regs *regs); 126 127 /* 128 * CPU/chipset specific EDAC code can register a notifier call here to print 129 * MCE errors in a human-readable form. 130 */ 131 BLOCKING_NOTIFIER_HEAD(x86_mce_decoder_chain); 132 133 /* Do initial initialization of a struct mce */ 134 noinstr void mce_setup(struct mce *m) 135 { 136 memset(m, 0, sizeof(struct mce)); 137 m->cpu = m->extcpu = smp_processor_id(); 138 /* need the internal __ version to avoid deadlocks */ 139 m->time = __ktime_get_real_seconds(); 140 m->cpuvendor = boot_cpu_data.x86_vendor; 141 m->cpuid = cpuid_eax(1); 142 m->socketid = cpu_data(m->extcpu).phys_proc_id; 143 m->apicid = cpu_data(m->extcpu).initial_apicid; 144 m->mcgcap = __rdmsr(MSR_IA32_MCG_CAP); 145 146 if (this_cpu_has(X86_FEATURE_INTEL_PPIN)) 147 m->ppin = __rdmsr(MSR_PPIN); 148 else if (this_cpu_has(X86_FEATURE_AMD_PPIN)) 149 m->ppin = __rdmsr(MSR_AMD_PPIN); 150 151 m->microcode = boot_cpu_data.microcode; 152 } 153 154 DEFINE_PER_CPU(struct mce, injectm); 155 EXPORT_PER_CPU_SYMBOL_GPL(injectm); 156 157 void mce_log(struct mce *m) 158 { 159 if (!mce_gen_pool_add(m)) 160 irq_work_queue(&mce_irq_work); 161 } 162 EXPORT_SYMBOL_GPL(mce_log); 163 164 void mce_register_decode_chain(struct notifier_block *nb) 165 { 166 if (WARN_ON(nb->priority > MCE_PRIO_MCELOG && nb->priority < MCE_PRIO_EDAC)) 167 return; 168 169 blocking_notifier_chain_register(&x86_mce_decoder_chain, nb); 170 } 171 EXPORT_SYMBOL_GPL(mce_register_decode_chain); 172 173 void mce_unregister_decode_chain(struct notifier_block *nb) 174 { 175 blocking_notifier_chain_unregister(&x86_mce_decoder_chain, nb); 176 } 177 EXPORT_SYMBOL_GPL(mce_unregister_decode_chain); 178 179 static inline u32 ctl_reg(int bank) 180 { 181 return MSR_IA32_MCx_CTL(bank); 182 } 183 184 static inline u32 status_reg(int bank) 185 { 186 return MSR_IA32_MCx_STATUS(bank); 187 } 188 189 static inline u32 addr_reg(int bank) 190 { 191 return MSR_IA32_MCx_ADDR(bank); 192 } 193 194 static inline u32 misc_reg(int bank) 195 { 196 return MSR_IA32_MCx_MISC(bank); 197 } 198 199 static inline u32 smca_ctl_reg(int bank) 200 { 201 return MSR_AMD64_SMCA_MCx_CTL(bank); 202 } 203 204 static inline u32 smca_status_reg(int bank) 205 { 206 return MSR_AMD64_SMCA_MCx_STATUS(bank); 207 } 208 209 static inline u32 smca_addr_reg(int bank) 210 { 211 return MSR_AMD64_SMCA_MCx_ADDR(bank); 212 } 213 214 static inline u32 smca_misc_reg(int bank) 215 { 216 return MSR_AMD64_SMCA_MCx_MISC(bank); 217 } 218 219 struct mca_msr_regs msr_ops = { 220 .ctl = ctl_reg, 221 .status = status_reg, 222 .addr = addr_reg, 223 .misc = misc_reg 224 }; 225 226 static void __print_mce(struct mce *m) 227 { 228 pr_emerg(HW_ERR "CPU %d: Machine Check%s: %Lx Bank %d: %016Lx\n", 229 m->extcpu, 230 (m->mcgstatus & MCG_STATUS_MCIP ? " Exception" : ""), 231 m->mcgstatus, m->bank, m->status); 232 233 if (m->ip) { 234 pr_emerg(HW_ERR "RIP%s %02x:<%016Lx> ", 235 !(m->mcgstatus & MCG_STATUS_EIPV) ? " !INEXACT!" : "", 236 m->cs, m->ip); 237 238 if (m->cs == __KERNEL_CS) 239 pr_cont("{%pS}", (void *)(unsigned long)m->ip); 240 pr_cont("\n"); 241 } 242 243 pr_emerg(HW_ERR "TSC %llx ", m->tsc); 244 if (m->addr) 245 pr_cont("ADDR %llx ", m->addr); 246 if (m->misc) 247 pr_cont("MISC %llx ", m->misc); 248 if (m->ppin) 249 pr_cont("PPIN %llx ", m->ppin); 250 251 if (mce_flags.smca) { 252 if (m->synd) 253 pr_cont("SYND %llx ", m->synd); 254 if (m->ipid) 255 pr_cont("IPID %llx ", m->ipid); 256 } 257 258 pr_cont("\n"); 259 260 /* 261 * Note this output is parsed by external tools and old fields 262 * should not be changed. 263 */ 264 pr_emerg(HW_ERR "PROCESSOR %u:%x TIME %llu SOCKET %u APIC %x microcode %x\n", 265 m->cpuvendor, m->cpuid, m->time, m->socketid, m->apicid, 266 m->microcode); 267 } 268 269 static void print_mce(struct mce *m) 270 { 271 __print_mce(m); 272 273 if (m->cpuvendor != X86_VENDOR_AMD && m->cpuvendor != X86_VENDOR_HYGON) 274 pr_emerg_ratelimited(HW_ERR "Run the above through 'mcelog --ascii'\n"); 275 } 276 277 #define PANIC_TIMEOUT 5 /* 5 seconds */ 278 279 static atomic_t mce_panicked; 280 281 static int fake_panic; 282 static atomic_t mce_fake_panicked; 283 284 /* Panic in progress. Enable interrupts and wait for final IPI */ 285 static void wait_for_panic(void) 286 { 287 long timeout = PANIC_TIMEOUT*USEC_PER_SEC; 288 289 preempt_disable(); 290 local_irq_enable(); 291 while (timeout-- > 0) 292 udelay(1); 293 if (panic_timeout == 0) 294 panic_timeout = mca_cfg.panic_timeout; 295 panic("Panicing machine check CPU died"); 296 } 297 298 static void mce_panic(const char *msg, struct mce *final, char *exp) 299 { 300 int apei_err = 0; 301 struct llist_node *pending; 302 struct mce_evt_llist *l; 303 304 if (!fake_panic) { 305 /* 306 * Make sure only one CPU runs in machine check panic 307 */ 308 if (atomic_inc_return(&mce_panicked) > 1) 309 wait_for_panic(); 310 barrier(); 311 312 bust_spinlocks(1); 313 console_verbose(); 314 } else { 315 /* Don't log too much for fake panic */ 316 if (atomic_inc_return(&mce_fake_panicked) > 1) 317 return; 318 } 319 pending = mce_gen_pool_prepare_records(); 320 /* First print corrected ones that are still unlogged */ 321 llist_for_each_entry(l, pending, llnode) { 322 struct mce *m = &l->mce; 323 if (!(m->status & MCI_STATUS_UC)) { 324 print_mce(m); 325 if (!apei_err) 326 apei_err = apei_write_mce(m); 327 } 328 } 329 /* Now print uncorrected but with the final one last */ 330 llist_for_each_entry(l, pending, llnode) { 331 struct mce *m = &l->mce; 332 if (!(m->status & MCI_STATUS_UC)) 333 continue; 334 if (!final || mce_cmp(m, final)) { 335 print_mce(m); 336 if (!apei_err) 337 apei_err = apei_write_mce(m); 338 } 339 } 340 if (final) { 341 print_mce(final); 342 if (!apei_err) 343 apei_err = apei_write_mce(final); 344 } 345 if (cpu_missing) 346 pr_emerg(HW_ERR "Some CPUs didn't answer in synchronization\n"); 347 if (exp) 348 pr_emerg(HW_ERR "Machine check: %s\n", exp); 349 if (!fake_panic) { 350 if (panic_timeout == 0) 351 panic_timeout = mca_cfg.panic_timeout; 352 panic(msg); 353 } else 354 pr_emerg(HW_ERR "Fake kernel panic: %s\n", msg); 355 } 356 357 /* Support code for software error injection */ 358 359 static int msr_to_offset(u32 msr) 360 { 361 unsigned bank = __this_cpu_read(injectm.bank); 362 363 if (msr == mca_cfg.rip_msr) 364 return offsetof(struct mce, ip); 365 if (msr == msr_ops.status(bank)) 366 return offsetof(struct mce, status); 367 if (msr == msr_ops.addr(bank)) 368 return offsetof(struct mce, addr); 369 if (msr == msr_ops.misc(bank)) 370 return offsetof(struct mce, misc); 371 if (msr == MSR_IA32_MCG_STATUS) 372 return offsetof(struct mce, mcgstatus); 373 return -1; 374 } 375 376 /* MSR access wrappers used for error injection */ 377 static u64 mce_rdmsrl(u32 msr) 378 { 379 u64 v; 380 381 if (__this_cpu_read(injectm.finished)) { 382 int offset = msr_to_offset(msr); 383 384 if (offset < 0) 385 return 0; 386 return *(u64 *)((char *)this_cpu_ptr(&injectm) + offset); 387 } 388 389 if (rdmsrl_safe(msr, &v)) { 390 WARN_ONCE(1, "mce: Unable to read MSR 0x%x!\n", msr); 391 /* 392 * Return zero in case the access faulted. This should 393 * not happen normally but can happen if the CPU does 394 * something weird, or if the code is buggy. 395 */ 396 v = 0; 397 } 398 399 return v; 400 } 401 402 static void mce_wrmsrl(u32 msr, u64 v) 403 { 404 if (__this_cpu_read(injectm.finished)) { 405 int offset = msr_to_offset(msr); 406 407 if (offset >= 0) 408 *(u64 *)((char *)this_cpu_ptr(&injectm) + offset) = v; 409 return; 410 } 411 wrmsrl(msr, v); 412 } 413 414 /* 415 * Collect all global (w.r.t. this processor) status about this machine 416 * check into our "mce" struct so that we can use it later to assess 417 * the severity of the problem as we read per-bank specific details. 418 */ 419 static inline void mce_gather_info(struct mce *m, struct pt_regs *regs) 420 { 421 mce_setup(m); 422 423 m->mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS); 424 if (regs) { 425 /* 426 * Get the address of the instruction at the time of 427 * the machine check error. 428 */ 429 if (m->mcgstatus & (MCG_STATUS_RIPV|MCG_STATUS_EIPV)) { 430 m->ip = regs->ip; 431 m->cs = regs->cs; 432 433 /* 434 * When in VM86 mode make the cs look like ring 3 435 * always. This is a lie, but it's better than passing 436 * the additional vm86 bit around everywhere. 437 */ 438 if (v8086_mode(regs)) 439 m->cs |= 3; 440 } 441 /* Use accurate RIP reporting if available. */ 442 if (mca_cfg.rip_msr) 443 m->ip = mce_rdmsrl(mca_cfg.rip_msr); 444 } 445 } 446 447 int mce_available(struct cpuinfo_x86 *c) 448 { 449 if (mca_cfg.disabled) 450 return 0; 451 return cpu_has(c, X86_FEATURE_MCE) && cpu_has(c, X86_FEATURE_MCA); 452 } 453 454 static void mce_schedule_work(void) 455 { 456 if (!mce_gen_pool_empty()) 457 schedule_work(&mce_work); 458 } 459 460 static void mce_irq_work_cb(struct irq_work *entry) 461 { 462 mce_schedule_work(); 463 } 464 465 /* 466 * Check if the address reported by the CPU is in a format we can parse. 467 * It would be possible to add code for most other cases, but all would 468 * be somewhat complicated (e.g. segment offset would require an instruction 469 * parser). So only support physical addresses up to page granuality for now. 470 */ 471 int mce_usable_address(struct mce *m) 472 { 473 if (!(m->status & MCI_STATUS_ADDRV)) 474 return 0; 475 476 /* Checks after this one are Intel/Zhaoxin-specific: */ 477 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL && 478 boot_cpu_data.x86_vendor != X86_VENDOR_ZHAOXIN) 479 return 1; 480 481 if (!(m->status & MCI_STATUS_MISCV)) 482 return 0; 483 484 if (MCI_MISC_ADDR_LSB(m->misc) > PAGE_SHIFT) 485 return 0; 486 487 if (MCI_MISC_ADDR_MODE(m->misc) != MCI_MISC_ADDR_PHYS) 488 return 0; 489 490 return 1; 491 } 492 EXPORT_SYMBOL_GPL(mce_usable_address); 493 494 bool mce_is_memory_error(struct mce *m) 495 { 496 switch (m->cpuvendor) { 497 case X86_VENDOR_AMD: 498 case X86_VENDOR_HYGON: 499 return amd_mce_is_memory_error(m); 500 501 case X86_VENDOR_INTEL: 502 case X86_VENDOR_ZHAOXIN: 503 /* 504 * Intel SDM Volume 3B - 15.9.2 Compound Error Codes 505 * 506 * Bit 7 of the MCACOD field of IA32_MCi_STATUS is used for 507 * indicating a memory error. Bit 8 is used for indicating a 508 * cache hierarchy error. The combination of bit 2 and bit 3 509 * is used for indicating a `generic' cache hierarchy error 510 * But we can't just blindly check the above bits, because if 511 * bit 11 is set, then it is a bus/interconnect error - and 512 * either way the above bits just gives more detail on what 513 * bus/interconnect error happened. Note that bit 12 can be 514 * ignored, as it's the "filter" bit. 515 */ 516 return (m->status & 0xef80) == BIT(7) || 517 (m->status & 0xef00) == BIT(8) || 518 (m->status & 0xeffc) == 0xc; 519 520 default: 521 return false; 522 } 523 } 524 EXPORT_SYMBOL_GPL(mce_is_memory_error); 525 526 static bool whole_page(struct mce *m) 527 { 528 if (!mca_cfg.ser || !(m->status & MCI_STATUS_MISCV)) 529 return true; 530 531 return MCI_MISC_ADDR_LSB(m->misc) >= PAGE_SHIFT; 532 } 533 534 bool mce_is_correctable(struct mce *m) 535 { 536 if (m->cpuvendor == X86_VENDOR_AMD && m->status & MCI_STATUS_DEFERRED) 537 return false; 538 539 if (m->cpuvendor == X86_VENDOR_HYGON && m->status & MCI_STATUS_DEFERRED) 540 return false; 541 542 if (m->status & MCI_STATUS_UC) 543 return false; 544 545 return true; 546 } 547 EXPORT_SYMBOL_GPL(mce_is_correctable); 548 549 static int mce_early_notifier(struct notifier_block *nb, unsigned long val, 550 void *data) 551 { 552 struct mce *m = (struct mce *)data; 553 554 if (!m) 555 return NOTIFY_DONE; 556 557 /* Emit the trace record: */ 558 trace_mce_record(m); 559 560 set_bit(0, &mce_need_notify); 561 562 mce_notify_irq(); 563 564 return NOTIFY_DONE; 565 } 566 567 static struct notifier_block early_nb = { 568 .notifier_call = mce_early_notifier, 569 .priority = MCE_PRIO_EARLY, 570 }; 571 572 static int uc_decode_notifier(struct notifier_block *nb, unsigned long val, 573 void *data) 574 { 575 struct mce *mce = (struct mce *)data; 576 unsigned long pfn; 577 578 if (!mce || !mce_usable_address(mce)) 579 return NOTIFY_DONE; 580 581 if (mce->severity != MCE_AO_SEVERITY && 582 mce->severity != MCE_DEFERRED_SEVERITY) 583 return NOTIFY_DONE; 584 585 pfn = mce->addr >> PAGE_SHIFT; 586 if (!memory_failure(pfn, 0)) { 587 set_mce_nospec(pfn, whole_page(mce)); 588 mce->kflags |= MCE_HANDLED_UC; 589 } 590 591 return NOTIFY_OK; 592 } 593 594 static struct notifier_block mce_uc_nb = { 595 .notifier_call = uc_decode_notifier, 596 .priority = MCE_PRIO_UC, 597 }; 598 599 static int mce_default_notifier(struct notifier_block *nb, unsigned long val, 600 void *data) 601 { 602 struct mce *m = (struct mce *)data; 603 604 if (!m) 605 return NOTIFY_DONE; 606 607 if (mca_cfg.print_all || !m->kflags) 608 __print_mce(m); 609 610 return NOTIFY_DONE; 611 } 612 613 static struct notifier_block mce_default_nb = { 614 .notifier_call = mce_default_notifier, 615 /* lowest prio, we want it to run last. */ 616 .priority = MCE_PRIO_LOWEST, 617 }; 618 619 /* 620 * Read ADDR and MISC registers. 621 */ 622 static void mce_read_aux(struct mce *m, int i) 623 { 624 if (m->status & MCI_STATUS_MISCV) 625 m->misc = mce_rdmsrl(msr_ops.misc(i)); 626 627 if (m->status & MCI_STATUS_ADDRV) { 628 m->addr = mce_rdmsrl(msr_ops.addr(i)); 629 630 /* 631 * Mask the reported address by the reported granularity. 632 */ 633 if (mca_cfg.ser && (m->status & MCI_STATUS_MISCV)) { 634 u8 shift = MCI_MISC_ADDR_LSB(m->misc); 635 m->addr >>= shift; 636 m->addr <<= shift; 637 } 638 639 /* 640 * Extract [55:<lsb>] where lsb is the least significant 641 * *valid* bit of the address bits. 642 */ 643 if (mce_flags.smca) { 644 u8 lsb = (m->addr >> 56) & 0x3f; 645 646 m->addr &= GENMASK_ULL(55, lsb); 647 } 648 } 649 650 if (mce_flags.smca) { 651 m->ipid = mce_rdmsrl(MSR_AMD64_SMCA_MCx_IPID(i)); 652 653 if (m->status & MCI_STATUS_SYNDV) 654 m->synd = mce_rdmsrl(MSR_AMD64_SMCA_MCx_SYND(i)); 655 } 656 } 657 658 DEFINE_PER_CPU(unsigned, mce_poll_count); 659 660 /* 661 * Poll for corrected events or events that happened before reset. 662 * Those are just logged through /dev/mcelog. 663 * 664 * This is executed in standard interrupt context. 665 * 666 * Note: spec recommends to panic for fatal unsignalled 667 * errors here. However this would be quite problematic -- 668 * we would need to reimplement the Monarch handling and 669 * it would mess up the exclusion between exception handler 670 * and poll handler -- * so we skip this for now. 671 * These cases should not happen anyways, or only when the CPU 672 * is already totally * confused. In this case it's likely it will 673 * not fully execute the machine check handler either. 674 */ 675 bool machine_check_poll(enum mcp_flags flags, mce_banks_t *b) 676 { 677 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 678 bool error_seen = false; 679 struct mce m; 680 int i; 681 682 this_cpu_inc(mce_poll_count); 683 684 mce_gather_info(&m, NULL); 685 686 if (flags & MCP_TIMESTAMP) 687 m.tsc = rdtsc(); 688 689 for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 690 if (!mce_banks[i].ctl || !test_bit(i, *b)) 691 continue; 692 693 m.misc = 0; 694 m.addr = 0; 695 m.bank = i; 696 697 barrier(); 698 m.status = mce_rdmsrl(msr_ops.status(i)); 699 700 /* If this entry is not valid, ignore it */ 701 if (!(m.status & MCI_STATUS_VAL)) 702 continue; 703 704 /* 705 * If we are logging everything (at CPU online) or this 706 * is a corrected error, then we must log it. 707 */ 708 if ((flags & MCP_UC) || !(m.status & MCI_STATUS_UC)) 709 goto log_it; 710 711 /* 712 * Newer Intel systems that support software error 713 * recovery need to make additional checks. Other 714 * CPUs should skip over uncorrected errors, but log 715 * everything else. 716 */ 717 if (!mca_cfg.ser) { 718 if (m.status & MCI_STATUS_UC) 719 continue; 720 goto log_it; 721 } 722 723 /* Log "not enabled" (speculative) errors */ 724 if (!(m.status & MCI_STATUS_EN)) 725 goto log_it; 726 727 /* 728 * Log UCNA (SDM: 15.6.3 "UCR Error Classification") 729 * UC == 1 && PCC == 0 && S == 0 730 */ 731 if (!(m.status & MCI_STATUS_PCC) && !(m.status & MCI_STATUS_S)) 732 goto log_it; 733 734 /* 735 * Skip anything else. Presumption is that our read of this 736 * bank is racing with a machine check. Leave the log alone 737 * for do_machine_check() to deal with it. 738 */ 739 continue; 740 741 log_it: 742 error_seen = true; 743 744 if (flags & MCP_DONTLOG) 745 goto clear_it; 746 747 mce_read_aux(&m, i); 748 m.severity = mce_severity(&m, mca_cfg.tolerant, NULL, false); 749 /* 750 * Don't get the IP here because it's unlikely to 751 * have anything to do with the actual error location. 752 */ 753 754 if (mca_cfg.dont_log_ce && !mce_usable_address(&m)) 755 goto clear_it; 756 757 mce_log(&m); 758 759 clear_it: 760 /* 761 * Clear state for this bank. 762 */ 763 mce_wrmsrl(msr_ops.status(i), 0); 764 } 765 766 /* 767 * Don't clear MCG_STATUS here because it's only defined for 768 * exceptions. 769 */ 770 771 sync_core(); 772 773 return error_seen; 774 } 775 EXPORT_SYMBOL_GPL(machine_check_poll); 776 777 /* 778 * Do a quick check if any of the events requires a panic. 779 * This decides if we keep the events around or clear them. 780 */ 781 static int mce_no_way_out(struct mce *m, char **msg, unsigned long *validp, 782 struct pt_regs *regs) 783 { 784 char *tmp = *msg; 785 int i; 786 787 for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 788 m->status = mce_rdmsrl(msr_ops.status(i)); 789 if (!(m->status & MCI_STATUS_VAL)) 790 continue; 791 792 __set_bit(i, validp); 793 if (quirk_no_way_out) 794 quirk_no_way_out(i, m, regs); 795 796 m->bank = i; 797 if (mce_severity(m, mca_cfg.tolerant, &tmp, true) >= MCE_PANIC_SEVERITY) { 798 mce_read_aux(m, i); 799 *msg = tmp; 800 return 1; 801 } 802 } 803 return 0; 804 } 805 806 /* 807 * Variable to establish order between CPUs while scanning. 808 * Each CPU spins initially until executing is equal its number. 809 */ 810 static atomic_t mce_executing; 811 812 /* 813 * Defines order of CPUs on entry. First CPU becomes Monarch. 814 */ 815 static atomic_t mce_callin; 816 817 /* 818 * Check if a timeout waiting for other CPUs happened. 819 */ 820 static int mce_timed_out(u64 *t, const char *msg) 821 { 822 /* 823 * The others already did panic for some reason. 824 * Bail out like in a timeout. 825 * rmb() to tell the compiler that system_state 826 * might have been modified by someone else. 827 */ 828 rmb(); 829 if (atomic_read(&mce_panicked)) 830 wait_for_panic(); 831 if (!mca_cfg.monarch_timeout) 832 goto out; 833 if ((s64)*t < SPINUNIT) { 834 if (mca_cfg.tolerant <= 1) 835 mce_panic(msg, NULL, NULL); 836 cpu_missing = 1; 837 return 1; 838 } 839 *t -= SPINUNIT; 840 out: 841 touch_nmi_watchdog(); 842 return 0; 843 } 844 845 /* 846 * The Monarch's reign. The Monarch is the CPU who entered 847 * the machine check handler first. It waits for the others to 848 * raise the exception too and then grades them. When any 849 * error is fatal panic. Only then let the others continue. 850 * 851 * The other CPUs entering the MCE handler will be controlled by the 852 * Monarch. They are called Subjects. 853 * 854 * This way we prevent any potential data corruption in a unrecoverable case 855 * and also makes sure always all CPU's errors are examined. 856 * 857 * Also this detects the case of a machine check event coming from outer 858 * space (not detected by any CPUs) In this case some external agent wants 859 * us to shut down, so panic too. 860 * 861 * The other CPUs might still decide to panic if the handler happens 862 * in a unrecoverable place, but in this case the system is in a semi-stable 863 * state and won't corrupt anything by itself. It's ok to let the others 864 * continue for a bit first. 865 * 866 * All the spin loops have timeouts; when a timeout happens a CPU 867 * typically elects itself to be Monarch. 868 */ 869 static void mce_reign(void) 870 { 871 int cpu; 872 struct mce *m = NULL; 873 int global_worst = 0; 874 char *msg = NULL; 875 char *nmsg = NULL; 876 877 /* 878 * This CPU is the Monarch and the other CPUs have run 879 * through their handlers. 880 * Grade the severity of the errors of all the CPUs. 881 */ 882 for_each_possible_cpu(cpu) { 883 int severity = mce_severity(&per_cpu(mces_seen, cpu), 884 mca_cfg.tolerant, 885 &nmsg, true); 886 if (severity > global_worst) { 887 msg = nmsg; 888 global_worst = severity; 889 m = &per_cpu(mces_seen, cpu); 890 } 891 } 892 893 /* 894 * Cannot recover? Panic here then. 895 * This dumps all the mces in the log buffer and stops the 896 * other CPUs. 897 */ 898 if (m && global_worst >= MCE_PANIC_SEVERITY && mca_cfg.tolerant < 3) 899 mce_panic("Fatal machine check", m, msg); 900 901 /* 902 * For UC somewhere we let the CPU who detects it handle it. 903 * Also must let continue the others, otherwise the handling 904 * CPU could deadlock on a lock. 905 */ 906 907 /* 908 * No machine check event found. Must be some external 909 * source or one CPU is hung. Panic. 910 */ 911 if (global_worst <= MCE_KEEP_SEVERITY && mca_cfg.tolerant < 3) 912 mce_panic("Fatal machine check from unknown source", NULL, NULL); 913 914 /* 915 * Now clear all the mces_seen so that they don't reappear on 916 * the next mce. 917 */ 918 for_each_possible_cpu(cpu) 919 memset(&per_cpu(mces_seen, cpu), 0, sizeof(struct mce)); 920 } 921 922 static atomic_t global_nwo; 923 924 /* 925 * Start of Monarch synchronization. This waits until all CPUs have 926 * entered the exception handler and then determines if any of them 927 * saw a fatal event that requires panic. Then it executes them 928 * in the entry order. 929 * TBD double check parallel CPU hotunplug 930 */ 931 static int mce_start(int *no_way_out) 932 { 933 int order; 934 int cpus = num_online_cpus(); 935 u64 timeout = (u64)mca_cfg.monarch_timeout * NSEC_PER_USEC; 936 937 if (!timeout) 938 return -1; 939 940 atomic_add(*no_way_out, &global_nwo); 941 /* 942 * Rely on the implied barrier below, such that global_nwo 943 * is updated before mce_callin. 944 */ 945 order = atomic_inc_return(&mce_callin); 946 947 /* 948 * Wait for everyone. 949 */ 950 while (atomic_read(&mce_callin) != cpus) { 951 if (mce_timed_out(&timeout, 952 "Timeout: Not all CPUs entered broadcast exception handler")) { 953 atomic_set(&global_nwo, 0); 954 return -1; 955 } 956 ndelay(SPINUNIT); 957 } 958 959 /* 960 * mce_callin should be read before global_nwo 961 */ 962 smp_rmb(); 963 964 if (order == 1) { 965 /* 966 * Monarch: Starts executing now, the others wait. 967 */ 968 atomic_set(&mce_executing, 1); 969 } else { 970 /* 971 * Subject: Now start the scanning loop one by one in 972 * the original callin order. 973 * This way when there are any shared banks it will be 974 * only seen by one CPU before cleared, avoiding duplicates. 975 */ 976 while (atomic_read(&mce_executing) < order) { 977 if (mce_timed_out(&timeout, 978 "Timeout: Subject CPUs unable to finish machine check processing")) { 979 atomic_set(&global_nwo, 0); 980 return -1; 981 } 982 ndelay(SPINUNIT); 983 } 984 } 985 986 /* 987 * Cache the global no_way_out state. 988 */ 989 *no_way_out = atomic_read(&global_nwo); 990 991 return order; 992 } 993 994 /* 995 * Synchronize between CPUs after main scanning loop. 996 * This invokes the bulk of the Monarch processing. 997 */ 998 static int mce_end(int order) 999 { 1000 int ret = -1; 1001 u64 timeout = (u64)mca_cfg.monarch_timeout * NSEC_PER_USEC; 1002 1003 if (!timeout) 1004 goto reset; 1005 if (order < 0) 1006 goto reset; 1007 1008 /* 1009 * Allow others to run. 1010 */ 1011 atomic_inc(&mce_executing); 1012 1013 if (order == 1) { 1014 /* CHECKME: Can this race with a parallel hotplug? */ 1015 int cpus = num_online_cpus(); 1016 1017 /* 1018 * Monarch: Wait for everyone to go through their scanning 1019 * loops. 1020 */ 1021 while (atomic_read(&mce_executing) <= cpus) { 1022 if (mce_timed_out(&timeout, 1023 "Timeout: Monarch CPU unable to finish machine check processing")) 1024 goto reset; 1025 ndelay(SPINUNIT); 1026 } 1027 1028 mce_reign(); 1029 barrier(); 1030 ret = 0; 1031 } else { 1032 /* 1033 * Subject: Wait for Monarch to finish. 1034 */ 1035 while (atomic_read(&mce_executing) != 0) { 1036 if (mce_timed_out(&timeout, 1037 "Timeout: Monarch CPU did not finish machine check processing")) 1038 goto reset; 1039 ndelay(SPINUNIT); 1040 } 1041 1042 /* 1043 * Don't reset anything. That's done by the Monarch. 1044 */ 1045 return 0; 1046 } 1047 1048 /* 1049 * Reset all global state. 1050 */ 1051 reset: 1052 atomic_set(&global_nwo, 0); 1053 atomic_set(&mce_callin, 0); 1054 barrier(); 1055 1056 /* 1057 * Let others run again. 1058 */ 1059 atomic_set(&mce_executing, 0); 1060 return ret; 1061 } 1062 1063 static void mce_clear_state(unsigned long *toclear) 1064 { 1065 int i; 1066 1067 for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 1068 if (test_bit(i, toclear)) 1069 mce_wrmsrl(msr_ops.status(i), 0); 1070 } 1071 } 1072 1073 /* 1074 * Cases where we avoid rendezvous handler timeout: 1075 * 1) If this CPU is offline. 1076 * 1077 * 2) If crashing_cpu was set, e.g. we're entering kdump and we need to 1078 * skip those CPUs which remain looping in the 1st kernel - see 1079 * crash_nmi_callback(). 1080 * 1081 * Note: there still is a small window between kexec-ing and the new, 1082 * kdump kernel establishing a new #MC handler where a broadcasted MCE 1083 * might not get handled properly. 1084 */ 1085 static noinstr bool mce_check_crashing_cpu(void) 1086 { 1087 unsigned int cpu = smp_processor_id(); 1088 1089 if (arch_cpu_is_offline(cpu) || 1090 (crashing_cpu != -1 && crashing_cpu != cpu)) { 1091 u64 mcgstatus; 1092 1093 mcgstatus = __rdmsr(MSR_IA32_MCG_STATUS); 1094 1095 if (boot_cpu_data.x86_vendor == X86_VENDOR_ZHAOXIN) { 1096 if (mcgstatus & MCG_STATUS_LMCES) 1097 return false; 1098 } 1099 1100 if (mcgstatus & MCG_STATUS_RIPV) { 1101 __wrmsr(MSR_IA32_MCG_STATUS, 0, 0); 1102 return true; 1103 } 1104 } 1105 return false; 1106 } 1107 1108 static void __mc_scan_banks(struct mce *m, struct mce *final, 1109 unsigned long *toclear, unsigned long *valid_banks, 1110 int no_way_out, int *worst) 1111 { 1112 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 1113 struct mca_config *cfg = &mca_cfg; 1114 int severity, i; 1115 1116 for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 1117 __clear_bit(i, toclear); 1118 if (!test_bit(i, valid_banks)) 1119 continue; 1120 1121 if (!mce_banks[i].ctl) 1122 continue; 1123 1124 m->misc = 0; 1125 m->addr = 0; 1126 m->bank = i; 1127 1128 m->status = mce_rdmsrl(msr_ops.status(i)); 1129 if (!(m->status & MCI_STATUS_VAL)) 1130 continue; 1131 1132 /* 1133 * Corrected or non-signaled errors are handled by 1134 * machine_check_poll(). Leave them alone, unless this panics. 1135 */ 1136 if (!(m->status & (cfg->ser ? MCI_STATUS_S : MCI_STATUS_UC)) && 1137 !no_way_out) 1138 continue; 1139 1140 /* Set taint even when machine check was not enabled. */ 1141 add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE); 1142 1143 severity = mce_severity(m, cfg->tolerant, NULL, true); 1144 1145 /* 1146 * When machine check was for corrected/deferred handler don't 1147 * touch, unless we're panicking. 1148 */ 1149 if ((severity == MCE_KEEP_SEVERITY || 1150 severity == MCE_UCNA_SEVERITY) && !no_way_out) 1151 continue; 1152 1153 __set_bit(i, toclear); 1154 1155 /* Machine check event was not enabled. Clear, but ignore. */ 1156 if (severity == MCE_NO_SEVERITY) 1157 continue; 1158 1159 mce_read_aux(m, i); 1160 1161 /* assuming valid severity level != 0 */ 1162 m->severity = severity; 1163 1164 mce_log(m); 1165 1166 if (severity > *worst) { 1167 *final = *m; 1168 *worst = severity; 1169 } 1170 } 1171 1172 /* mce_clear_state will clear *final, save locally for use later */ 1173 *m = *final; 1174 } 1175 1176 static void kill_me_now(struct callback_head *ch) 1177 { 1178 force_sig(SIGBUS); 1179 } 1180 1181 static void kill_me_maybe(struct callback_head *cb) 1182 { 1183 struct task_struct *p = container_of(cb, struct task_struct, mce_kill_me); 1184 int flags = MF_ACTION_REQUIRED; 1185 1186 pr_err("Uncorrected hardware memory error in user-access at %llx", p->mce_addr); 1187 1188 if (!p->mce_ripv) 1189 flags |= MF_MUST_KILL; 1190 1191 if (!memory_failure(p->mce_addr >> PAGE_SHIFT, flags)) { 1192 set_mce_nospec(p->mce_addr >> PAGE_SHIFT, p->mce_whole_page); 1193 return; 1194 } 1195 1196 pr_err("Memory error not recovered"); 1197 kill_me_now(cb); 1198 } 1199 1200 /* 1201 * The actual machine check handler. This only handles real 1202 * exceptions when something got corrupted coming in through int 18. 1203 * 1204 * This is executed in NMI context not subject to normal locking rules. This 1205 * implies that most kernel services cannot be safely used. Don't even 1206 * think about putting a printk in there! 1207 * 1208 * On Intel systems this is entered on all CPUs in parallel through 1209 * MCE broadcast. However some CPUs might be broken beyond repair, 1210 * so be always careful when synchronizing with others. 1211 * 1212 * Tracing and kprobes are disabled: if we interrupted a kernel context 1213 * with IF=1, we need to minimize stack usage. There are also recursion 1214 * issues: if the machine check was due to a failure of the memory 1215 * backing the user stack, tracing that reads the user stack will cause 1216 * potentially infinite recursion. 1217 */ 1218 noinstr void do_machine_check(struct pt_regs *regs) 1219 { 1220 DECLARE_BITMAP(valid_banks, MAX_NR_BANKS); 1221 DECLARE_BITMAP(toclear, MAX_NR_BANKS); 1222 struct mca_config *cfg = &mca_cfg; 1223 struct mce m, *final; 1224 char *msg = NULL; 1225 int worst = 0; 1226 1227 /* 1228 * Establish sequential order between the CPUs entering the machine 1229 * check handler. 1230 */ 1231 int order = -1; 1232 1233 /* 1234 * If no_way_out gets set, there is no safe way to recover from this 1235 * MCE. If mca_cfg.tolerant is cranked up, we'll try anyway. 1236 */ 1237 int no_way_out = 0; 1238 1239 /* 1240 * If kill_it gets set, there might be a way to recover from this 1241 * error. 1242 */ 1243 int kill_it = 0; 1244 1245 /* 1246 * MCEs are always local on AMD. Same is determined by MCG_STATUS_LMCES 1247 * on Intel. 1248 */ 1249 int lmce = 1; 1250 1251 this_cpu_inc(mce_exception_count); 1252 1253 mce_gather_info(&m, regs); 1254 m.tsc = rdtsc(); 1255 1256 final = this_cpu_ptr(&mces_seen); 1257 *final = m; 1258 1259 memset(valid_banks, 0, sizeof(valid_banks)); 1260 no_way_out = mce_no_way_out(&m, &msg, valid_banks, regs); 1261 1262 barrier(); 1263 1264 /* 1265 * When no restart IP might need to kill or panic. 1266 * Assume the worst for now, but if we find the 1267 * severity is MCE_AR_SEVERITY we have other options. 1268 */ 1269 if (!(m.mcgstatus & MCG_STATUS_RIPV)) 1270 kill_it = 1; 1271 1272 /* 1273 * Check if this MCE is signaled to only this logical processor, 1274 * on Intel, Zhaoxin only. 1275 */ 1276 if (m.cpuvendor == X86_VENDOR_INTEL || 1277 m.cpuvendor == X86_VENDOR_ZHAOXIN) 1278 lmce = m.mcgstatus & MCG_STATUS_LMCES; 1279 1280 /* 1281 * Local machine check may already know that we have to panic. 1282 * Broadcast machine check begins rendezvous in mce_start() 1283 * Go through all banks in exclusion of the other CPUs. This way we 1284 * don't report duplicated events on shared banks because the first one 1285 * to see it will clear it. 1286 */ 1287 if (lmce) { 1288 if (no_way_out) 1289 mce_panic("Fatal local machine check", &m, msg); 1290 } else { 1291 order = mce_start(&no_way_out); 1292 } 1293 1294 __mc_scan_banks(&m, final, toclear, valid_banks, no_way_out, &worst); 1295 1296 if (!no_way_out) 1297 mce_clear_state(toclear); 1298 1299 /* 1300 * Do most of the synchronization with other CPUs. 1301 * When there's any problem use only local no_way_out state. 1302 */ 1303 if (!lmce) { 1304 if (mce_end(order) < 0) 1305 no_way_out = worst >= MCE_PANIC_SEVERITY; 1306 } else { 1307 /* 1308 * If there was a fatal machine check we should have 1309 * already called mce_panic earlier in this function. 1310 * Since we re-read the banks, we might have found 1311 * something new. Check again to see if we found a 1312 * fatal error. We call "mce_severity()" again to 1313 * make sure we have the right "msg". 1314 */ 1315 if (worst >= MCE_PANIC_SEVERITY && mca_cfg.tolerant < 3) { 1316 mce_severity(&m, cfg->tolerant, &msg, true); 1317 mce_panic("Local fatal machine check!", &m, msg); 1318 } 1319 } 1320 1321 /* 1322 * If tolerant is at an insane level we drop requests to kill 1323 * processes and continue even when there is no way out. 1324 */ 1325 if (cfg->tolerant == 3) 1326 kill_it = 0; 1327 else if (no_way_out) 1328 mce_panic("Fatal machine check on current CPU", &m, msg); 1329 1330 if (worst > 0) 1331 irq_work_queue(&mce_irq_work); 1332 1333 mce_wrmsrl(MSR_IA32_MCG_STATUS, 0); 1334 1335 sync_core(); 1336 1337 if (worst != MCE_AR_SEVERITY && !kill_it) 1338 return; 1339 1340 /* Fault was in user mode and we need to take some action */ 1341 if ((m.cs & 3) == 3) { 1342 /* If this triggers there is no way to recover. Die hard. */ 1343 BUG_ON(!on_thread_stack() || !user_mode(regs)); 1344 1345 current->mce_addr = m.addr; 1346 current->mce_ripv = !!(m.mcgstatus & MCG_STATUS_RIPV); 1347 current->mce_whole_page = whole_page(&m); 1348 current->mce_kill_me.func = kill_me_maybe; 1349 if (kill_it) 1350 current->mce_kill_me.func = kill_me_now; 1351 task_work_add(current, ¤t->mce_kill_me, true); 1352 } else { 1353 /* 1354 * Handle an MCE which has happened in kernel space but from 1355 * which the kernel can recover: ex_has_fault_handler() has 1356 * already verified that the rIP at which the error happened is 1357 * a rIP from which the kernel can recover (by jumping to 1358 * recovery code specified in _ASM_EXTABLE_FAULT()) and the 1359 * corresponding exception handler which would do that is the 1360 * proper one. 1361 */ 1362 if (m.kflags & MCE_IN_KERNEL_RECOV) { 1363 if (!fixup_exception(regs, X86_TRAP_MC, 0, 0)) 1364 mce_panic("Failed kernel mode recovery", &m, msg); 1365 } 1366 } 1367 } 1368 EXPORT_SYMBOL_GPL(do_machine_check); 1369 1370 #ifndef CONFIG_MEMORY_FAILURE 1371 int memory_failure(unsigned long pfn, int flags) 1372 { 1373 /* mce_severity() should not hand us an ACTION_REQUIRED error */ 1374 BUG_ON(flags & MF_ACTION_REQUIRED); 1375 pr_err("Uncorrected memory error in page 0x%lx ignored\n" 1376 "Rebuild kernel with CONFIG_MEMORY_FAILURE=y for smarter handling\n", 1377 pfn); 1378 1379 return 0; 1380 } 1381 #endif 1382 1383 /* 1384 * Periodic polling timer for "silent" machine check errors. If the 1385 * poller finds an MCE, poll 2x faster. When the poller finds no more 1386 * errors, poll 2x slower (up to check_interval seconds). 1387 */ 1388 static unsigned long check_interval = INITIAL_CHECK_INTERVAL; 1389 1390 static DEFINE_PER_CPU(unsigned long, mce_next_interval); /* in jiffies */ 1391 static DEFINE_PER_CPU(struct timer_list, mce_timer); 1392 1393 static unsigned long mce_adjust_timer_default(unsigned long interval) 1394 { 1395 return interval; 1396 } 1397 1398 static unsigned long (*mce_adjust_timer)(unsigned long interval) = mce_adjust_timer_default; 1399 1400 static void __start_timer(struct timer_list *t, unsigned long interval) 1401 { 1402 unsigned long when = jiffies + interval; 1403 unsigned long flags; 1404 1405 local_irq_save(flags); 1406 1407 if (!timer_pending(t) || time_before(when, t->expires)) 1408 mod_timer(t, round_jiffies(when)); 1409 1410 local_irq_restore(flags); 1411 } 1412 1413 static void mce_timer_fn(struct timer_list *t) 1414 { 1415 struct timer_list *cpu_t = this_cpu_ptr(&mce_timer); 1416 unsigned long iv; 1417 1418 WARN_ON(cpu_t != t); 1419 1420 iv = __this_cpu_read(mce_next_interval); 1421 1422 if (mce_available(this_cpu_ptr(&cpu_info))) { 1423 machine_check_poll(0, this_cpu_ptr(&mce_poll_banks)); 1424 1425 if (mce_intel_cmci_poll()) { 1426 iv = mce_adjust_timer(iv); 1427 goto done; 1428 } 1429 } 1430 1431 /* 1432 * Alert userspace if needed. If we logged an MCE, reduce the polling 1433 * interval, otherwise increase the polling interval. 1434 */ 1435 if (mce_notify_irq()) 1436 iv = max(iv / 2, (unsigned long) HZ/100); 1437 else 1438 iv = min(iv * 2, round_jiffies_relative(check_interval * HZ)); 1439 1440 done: 1441 __this_cpu_write(mce_next_interval, iv); 1442 __start_timer(t, iv); 1443 } 1444 1445 /* 1446 * Ensure that the timer is firing in @interval from now. 1447 */ 1448 void mce_timer_kick(unsigned long interval) 1449 { 1450 struct timer_list *t = this_cpu_ptr(&mce_timer); 1451 unsigned long iv = __this_cpu_read(mce_next_interval); 1452 1453 __start_timer(t, interval); 1454 1455 if (interval < iv) 1456 __this_cpu_write(mce_next_interval, interval); 1457 } 1458 1459 /* Must not be called in IRQ context where del_timer_sync() can deadlock */ 1460 static void mce_timer_delete_all(void) 1461 { 1462 int cpu; 1463 1464 for_each_online_cpu(cpu) 1465 del_timer_sync(&per_cpu(mce_timer, cpu)); 1466 } 1467 1468 /* 1469 * Notify the user(s) about new machine check events. 1470 * Can be called from interrupt context, but not from machine check/NMI 1471 * context. 1472 */ 1473 int mce_notify_irq(void) 1474 { 1475 /* Not more than two messages every minute */ 1476 static DEFINE_RATELIMIT_STATE(ratelimit, 60*HZ, 2); 1477 1478 if (test_and_clear_bit(0, &mce_need_notify)) { 1479 mce_work_trigger(); 1480 1481 if (__ratelimit(&ratelimit)) 1482 pr_info(HW_ERR "Machine check events logged\n"); 1483 1484 return 1; 1485 } 1486 return 0; 1487 } 1488 EXPORT_SYMBOL_GPL(mce_notify_irq); 1489 1490 static void __mcheck_cpu_mce_banks_init(void) 1491 { 1492 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 1493 u8 n_banks = this_cpu_read(mce_num_banks); 1494 int i; 1495 1496 for (i = 0; i < n_banks; i++) { 1497 struct mce_bank *b = &mce_banks[i]; 1498 1499 /* 1500 * Init them all, __mcheck_cpu_apply_quirks() is going to apply 1501 * the required vendor quirks before 1502 * __mcheck_cpu_init_clear_banks() does the final bank setup. 1503 */ 1504 b->ctl = -1ULL; 1505 b->init = 1; 1506 } 1507 } 1508 1509 /* 1510 * Initialize Machine Checks for a CPU. 1511 */ 1512 static void __mcheck_cpu_cap_init(void) 1513 { 1514 u64 cap; 1515 u8 b; 1516 1517 rdmsrl(MSR_IA32_MCG_CAP, cap); 1518 1519 b = cap & MCG_BANKCNT_MASK; 1520 1521 if (b > MAX_NR_BANKS) { 1522 pr_warn("CPU%d: Using only %u machine check banks out of %u\n", 1523 smp_processor_id(), MAX_NR_BANKS, b); 1524 b = MAX_NR_BANKS; 1525 } 1526 1527 this_cpu_write(mce_num_banks, b); 1528 1529 __mcheck_cpu_mce_banks_init(); 1530 1531 /* Use accurate RIP reporting if available. */ 1532 if ((cap & MCG_EXT_P) && MCG_EXT_CNT(cap) >= 9) 1533 mca_cfg.rip_msr = MSR_IA32_MCG_EIP; 1534 1535 if (cap & MCG_SER_P) 1536 mca_cfg.ser = 1; 1537 } 1538 1539 static void __mcheck_cpu_init_generic(void) 1540 { 1541 enum mcp_flags m_fl = 0; 1542 mce_banks_t all_banks; 1543 u64 cap; 1544 1545 if (!mca_cfg.bootlog) 1546 m_fl = MCP_DONTLOG; 1547 1548 /* 1549 * Log the machine checks left over from the previous reset. 1550 */ 1551 bitmap_fill(all_banks, MAX_NR_BANKS); 1552 machine_check_poll(MCP_UC | m_fl, &all_banks); 1553 1554 cr4_set_bits(X86_CR4_MCE); 1555 1556 rdmsrl(MSR_IA32_MCG_CAP, cap); 1557 if (cap & MCG_CTL_P) 1558 wrmsr(MSR_IA32_MCG_CTL, 0xffffffff, 0xffffffff); 1559 } 1560 1561 static void __mcheck_cpu_init_clear_banks(void) 1562 { 1563 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 1564 int i; 1565 1566 for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 1567 struct mce_bank *b = &mce_banks[i]; 1568 1569 if (!b->init) 1570 continue; 1571 wrmsrl(msr_ops.ctl(i), b->ctl); 1572 wrmsrl(msr_ops.status(i), 0); 1573 } 1574 } 1575 1576 /* 1577 * Do a final check to see if there are any unused/RAZ banks. 1578 * 1579 * This must be done after the banks have been initialized and any quirks have 1580 * been applied. 1581 * 1582 * Do not call this from any user-initiated flows, e.g. CPU hotplug or sysfs. 1583 * Otherwise, a user who disables a bank will not be able to re-enable it 1584 * without a system reboot. 1585 */ 1586 static void __mcheck_cpu_check_banks(void) 1587 { 1588 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 1589 u64 msrval; 1590 int i; 1591 1592 for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 1593 struct mce_bank *b = &mce_banks[i]; 1594 1595 if (!b->init) 1596 continue; 1597 1598 rdmsrl(msr_ops.ctl(i), msrval); 1599 b->init = !!msrval; 1600 } 1601 } 1602 1603 /* 1604 * During IFU recovery Sandy Bridge -EP4S processors set the RIPV and 1605 * EIPV bits in MCG_STATUS to zero on the affected logical processor (SDM 1606 * Vol 3B Table 15-20). But this confuses both the code that determines 1607 * whether the machine check occurred in kernel or user mode, and also 1608 * the severity assessment code. Pretend that EIPV was set, and take the 1609 * ip/cs values from the pt_regs that mce_gather_info() ignored earlier. 1610 */ 1611 static void quirk_sandybridge_ifu(int bank, struct mce *m, struct pt_regs *regs) 1612 { 1613 if (bank != 0) 1614 return; 1615 if ((m->mcgstatus & (MCG_STATUS_EIPV|MCG_STATUS_RIPV)) != 0) 1616 return; 1617 if ((m->status & (MCI_STATUS_OVER|MCI_STATUS_UC| 1618 MCI_STATUS_EN|MCI_STATUS_MISCV|MCI_STATUS_ADDRV| 1619 MCI_STATUS_PCC|MCI_STATUS_S|MCI_STATUS_AR| 1620 MCACOD)) != 1621 (MCI_STATUS_UC|MCI_STATUS_EN| 1622 MCI_STATUS_MISCV|MCI_STATUS_ADDRV|MCI_STATUS_S| 1623 MCI_STATUS_AR|MCACOD_INSTR)) 1624 return; 1625 1626 m->mcgstatus |= MCG_STATUS_EIPV; 1627 m->ip = regs->ip; 1628 m->cs = regs->cs; 1629 } 1630 1631 /* Add per CPU specific workarounds here */ 1632 static int __mcheck_cpu_apply_quirks(struct cpuinfo_x86 *c) 1633 { 1634 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 1635 struct mca_config *cfg = &mca_cfg; 1636 1637 if (c->x86_vendor == X86_VENDOR_UNKNOWN) { 1638 pr_info("unknown CPU type - not enabling MCE support\n"); 1639 return -EOPNOTSUPP; 1640 } 1641 1642 /* This should be disabled by the BIOS, but isn't always */ 1643 if (c->x86_vendor == X86_VENDOR_AMD) { 1644 if (c->x86 == 15 && this_cpu_read(mce_num_banks) > 4) { 1645 /* 1646 * disable GART TBL walk error reporting, which 1647 * trips off incorrectly with the IOMMU & 3ware 1648 * & Cerberus: 1649 */ 1650 clear_bit(10, (unsigned long *)&mce_banks[4].ctl); 1651 } 1652 if (c->x86 < 0x11 && cfg->bootlog < 0) { 1653 /* 1654 * Lots of broken BIOS around that don't clear them 1655 * by default and leave crap in there. Don't log: 1656 */ 1657 cfg->bootlog = 0; 1658 } 1659 /* 1660 * Various K7s with broken bank 0 around. Always disable 1661 * by default. 1662 */ 1663 if (c->x86 == 6 && this_cpu_read(mce_num_banks) > 0) 1664 mce_banks[0].ctl = 0; 1665 1666 /* 1667 * overflow_recov is supported for F15h Models 00h-0fh 1668 * even though we don't have a CPUID bit for it. 1669 */ 1670 if (c->x86 == 0x15 && c->x86_model <= 0xf) 1671 mce_flags.overflow_recov = 1; 1672 1673 } 1674 1675 if (c->x86_vendor == X86_VENDOR_INTEL) { 1676 /* 1677 * SDM documents that on family 6 bank 0 should not be written 1678 * because it aliases to another special BIOS controlled 1679 * register. 1680 * But it's not aliased anymore on model 0x1a+ 1681 * Don't ignore bank 0 completely because there could be a 1682 * valid event later, merely don't write CTL0. 1683 */ 1684 1685 if (c->x86 == 6 && c->x86_model < 0x1A && this_cpu_read(mce_num_banks) > 0) 1686 mce_banks[0].init = 0; 1687 1688 /* 1689 * All newer Intel systems support MCE broadcasting. Enable 1690 * synchronization with a one second timeout. 1691 */ 1692 if ((c->x86 > 6 || (c->x86 == 6 && c->x86_model >= 0xe)) && 1693 cfg->monarch_timeout < 0) 1694 cfg->monarch_timeout = USEC_PER_SEC; 1695 1696 /* 1697 * There are also broken BIOSes on some Pentium M and 1698 * earlier systems: 1699 */ 1700 if (c->x86 == 6 && c->x86_model <= 13 && cfg->bootlog < 0) 1701 cfg->bootlog = 0; 1702 1703 if (c->x86 == 6 && c->x86_model == 45) 1704 quirk_no_way_out = quirk_sandybridge_ifu; 1705 } 1706 1707 if (c->x86_vendor == X86_VENDOR_ZHAOXIN) { 1708 /* 1709 * All newer Zhaoxin CPUs support MCE broadcasting. Enable 1710 * synchronization with a one second timeout. 1711 */ 1712 if (c->x86 > 6 || (c->x86_model == 0x19 || c->x86_model == 0x1f)) { 1713 if (cfg->monarch_timeout < 0) 1714 cfg->monarch_timeout = USEC_PER_SEC; 1715 } 1716 } 1717 1718 if (cfg->monarch_timeout < 0) 1719 cfg->monarch_timeout = 0; 1720 if (cfg->bootlog != 0) 1721 cfg->panic_timeout = 30; 1722 1723 return 0; 1724 } 1725 1726 static int __mcheck_cpu_ancient_init(struct cpuinfo_x86 *c) 1727 { 1728 if (c->x86 != 5) 1729 return 0; 1730 1731 switch (c->x86_vendor) { 1732 case X86_VENDOR_INTEL: 1733 intel_p5_mcheck_init(c); 1734 return 1; 1735 break; 1736 case X86_VENDOR_CENTAUR: 1737 winchip_mcheck_init(c); 1738 return 1; 1739 break; 1740 default: 1741 return 0; 1742 } 1743 1744 return 0; 1745 } 1746 1747 /* 1748 * Init basic CPU features needed for early decoding of MCEs. 1749 */ 1750 static void __mcheck_cpu_init_early(struct cpuinfo_x86 *c) 1751 { 1752 if (c->x86_vendor == X86_VENDOR_AMD || c->x86_vendor == X86_VENDOR_HYGON) { 1753 mce_flags.overflow_recov = !!cpu_has(c, X86_FEATURE_OVERFLOW_RECOV); 1754 mce_flags.succor = !!cpu_has(c, X86_FEATURE_SUCCOR); 1755 mce_flags.smca = !!cpu_has(c, X86_FEATURE_SMCA); 1756 mce_flags.amd_threshold = 1; 1757 1758 if (mce_flags.smca) { 1759 msr_ops.ctl = smca_ctl_reg; 1760 msr_ops.status = smca_status_reg; 1761 msr_ops.addr = smca_addr_reg; 1762 msr_ops.misc = smca_misc_reg; 1763 } 1764 } 1765 } 1766 1767 static void mce_centaur_feature_init(struct cpuinfo_x86 *c) 1768 { 1769 struct mca_config *cfg = &mca_cfg; 1770 1771 /* 1772 * All newer Centaur CPUs support MCE broadcasting. Enable 1773 * synchronization with a one second timeout. 1774 */ 1775 if ((c->x86 == 6 && c->x86_model == 0xf && c->x86_stepping >= 0xe) || 1776 c->x86 > 6) { 1777 if (cfg->monarch_timeout < 0) 1778 cfg->monarch_timeout = USEC_PER_SEC; 1779 } 1780 } 1781 1782 static void mce_zhaoxin_feature_init(struct cpuinfo_x86 *c) 1783 { 1784 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 1785 1786 /* 1787 * These CPUs have MCA bank 8 which reports only one error type called 1788 * SVAD (System View Address Decoder). The reporting of that error is 1789 * controlled by IA32_MC8.CTL.0. 1790 * 1791 * If enabled, prefetching on these CPUs will cause SVAD MCE when 1792 * virtual machines start and result in a system panic. Always disable 1793 * bank 8 SVAD error by default. 1794 */ 1795 if ((c->x86 == 7 && c->x86_model == 0x1b) || 1796 (c->x86_model == 0x19 || c->x86_model == 0x1f)) { 1797 if (this_cpu_read(mce_num_banks) > 8) 1798 mce_banks[8].ctl = 0; 1799 } 1800 1801 intel_init_cmci(); 1802 intel_init_lmce(); 1803 mce_adjust_timer = cmci_intel_adjust_timer; 1804 } 1805 1806 static void mce_zhaoxin_feature_clear(struct cpuinfo_x86 *c) 1807 { 1808 intel_clear_lmce(); 1809 } 1810 1811 static void __mcheck_cpu_init_vendor(struct cpuinfo_x86 *c) 1812 { 1813 switch (c->x86_vendor) { 1814 case X86_VENDOR_INTEL: 1815 mce_intel_feature_init(c); 1816 mce_adjust_timer = cmci_intel_adjust_timer; 1817 break; 1818 1819 case X86_VENDOR_AMD: { 1820 mce_amd_feature_init(c); 1821 break; 1822 } 1823 1824 case X86_VENDOR_HYGON: 1825 mce_hygon_feature_init(c); 1826 break; 1827 1828 case X86_VENDOR_CENTAUR: 1829 mce_centaur_feature_init(c); 1830 break; 1831 1832 case X86_VENDOR_ZHAOXIN: 1833 mce_zhaoxin_feature_init(c); 1834 break; 1835 1836 default: 1837 break; 1838 } 1839 } 1840 1841 static void __mcheck_cpu_clear_vendor(struct cpuinfo_x86 *c) 1842 { 1843 switch (c->x86_vendor) { 1844 case X86_VENDOR_INTEL: 1845 mce_intel_feature_clear(c); 1846 break; 1847 1848 case X86_VENDOR_ZHAOXIN: 1849 mce_zhaoxin_feature_clear(c); 1850 break; 1851 1852 default: 1853 break; 1854 } 1855 } 1856 1857 static void mce_start_timer(struct timer_list *t) 1858 { 1859 unsigned long iv = check_interval * HZ; 1860 1861 if (mca_cfg.ignore_ce || !iv) 1862 return; 1863 1864 this_cpu_write(mce_next_interval, iv); 1865 __start_timer(t, iv); 1866 } 1867 1868 static void __mcheck_cpu_setup_timer(void) 1869 { 1870 struct timer_list *t = this_cpu_ptr(&mce_timer); 1871 1872 timer_setup(t, mce_timer_fn, TIMER_PINNED); 1873 } 1874 1875 static void __mcheck_cpu_init_timer(void) 1876 { 1877 struct timer_list *t = this_cpu_ptr(&mce_timer); 1878 1879 timer_setup(t, mce_timer_fn, TIMER_PINNED); 1880 mce_start_timer(t); 1881 } 1882 1883 bool filter_mce(struct mce *m) 1884 { 1885 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) 1886 return amd_filter_mce(m); 1887 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) 1888 return intel_filter_mce(m); 1889 1890 return false; 1891 } 1892 1893 /* Handle unconfigured int18 (should never happen) */ 1894 static noinstr void unexpected_machine_check(struct pt_regs *regs) 1895 { 1896 instrumentation_begin(); 1897 pr_err("CPU#%d: Unexpected int18 (Machine Check)\n", 1898 smp_processor_id()); 1899 instrumentation_end(); 1900 } 1901 1902 /* Call the installed machine check handler for this CPU setup. */ 1903 void (*machine_check_vector)(struct pt_regs *) = unexpected_machine_check; 1904 1905 static __always_inline void exc_machine_check_kernel(struct pt_regs *regs) 1906 { 1907 WARN_ON_ONCE(user_mode(regs)); 1908 1909 /* 1910 * Only required when from kernel mode. See 1911 * mce_check_crashing_cpu() for details. 1912 */ 1913 if (machine_check_vector == do_machine_check && 1914 mce_check_crashing_cpu()) 1915 return; 1916 1917 nmi_enter(); 1918 /* 1919 * The call targets are marked noinstr, but objtool can't figure 1920 * that out because it's an indirect call. Annotate it. 1921 */ 1922 instrumentation_begin(); 1923 trace_hardirqs_off_finish(); 1924 machine_check_vector(regs); 1925 if (regs->flags & X86_EFLAGS_IF) 1926 trace_hardirqs_on_prepare(); 1927 instrumentation_end(); 1928 nmi_exit(); 1929 } 1930 1931 static __always_inline void exc_machine_check_user(struct pt_regs *regs) 1932 { 1933 irqentry_enter_from_user_mode(regs); 1934 instrumentation_begin(); 1935 machine_check_vector(regs); 1936 instrumentation_end(); 1937 irqentry_exit_to_user_mode(regs); 1938 } 1939 1940 #ifdef CONFIG_X86_64 1941 /* MCE hit kernel mode */ 1942 DEFINE_IDTENTRY_MCE(exc_machine_check) 1943 { 1944 unsigned long dr7; 1945 1946 dr7 = local_db_save(); 1947 exc_machine_check_kernel(regs); 1948 local_db_restore(dr7); 1949 } 1950 1951 /* The user mode variant. */ 1952 DEFINE_IDTENTRY_MCE_USER(exc_machine_check) 1953 { 1954 unsigned long dr7; 1955 1956 dr7 = local_db_save(); 1957 exc_machine_check_user(regs); 1958 local_db_restore(dr7); 1959 } 1960 #else 1961 /* 32bit unified entry point */ 1962 DEFINE_IDTENTRY_RAW(exc_machine_check) 1963 { 1964 unsigned long dr7; 1965 1966 dr7 = local_db_save(); 1967 if (user_mode(regs)) 1968 exc_machine_check_user(regs); 1969 else 1970 exc_machine_check_kernel(regs); 1971 local_db_restore(dr7); 1972 } 1973 #endif 1974 1975 /* 1976 * Called for each booted CPU to set up machine checks. 1977 * Must be called with preempt off: 1978 */ 1979 void mcheck_cpu_init(struct cpuinfo_x86 *c) 1980 { 1981 if (mca_cfg.disabled) 1982 return; 1983 1984 if (__mcheck_cpu_ancient_init(c)) 1985 return; 1986 1987 if (!mce_available(c)) 1988 return; 1989 1990 __mcheck_cpu_cap_init(); 1991 1992 if (__mcheck_cpu_apply_quirks(c) < 0) { 1993 mca_cfg.disabled = 1; 1994 return; 1995 } 1996 1997 if (mce_gen_pool_init()) { 1998 mca_cfg.disabled = 1; 1999 pr_emerg("Couldn't allocate MCE records pool!\n"); 2000 return; 2001 } 2002 2003 machine_check_vector = do_machine_check; 2004 2005 __mcheck_cpu_init_early(c); 2006 __mcheck_cpu_init_generic(); 2007 __mcheck_cpu_init_vendor(c); 2008 __mcheck_cpu_init_clear_banks(); 2009 __mcheck_cpu_check_banks(); 2010 __mcheck_cpu_setup_timer(); 2011 } 2012 2013 /* 2014 * Called for each booted CPU to clear some machine checks opt-ins 2015 */ 2016 void mcheck_cpu_clear(struct cpuinfo_x86 *c) 2017 { 2018 if (mca_cfg.disabled) 2019 return; 2020 2021 if (!mce_available(c)) 2022 return; 2023 2024 /* 2025 * Possibly to clear general settings generic to x86 2026 * __mcheck_cpu_clear_generic(c); 2027 */ 2028 __mcheck_cpu_clear_vendor(c); 2029 2030 } 2031 2032 static void __mce_disable_bank(void *arg) 2033 { 2034 int bank = *((int *)arg); 2035 __clear_bit(bank, this_cpu_ptr(mce_poll_banks)); 2036 cmci_disable_bank(bank); 2037 } 2038 2039 void mce_disable_bank(int bank) 2040 { 2041 if (bank >= this_cpu_read(mce_num_banks)) { 2042 pr_warn(FW_BUG 2043 "Ignoring request to disable invalid MCA bank %d.\n", 2044 bank); 2045 return; 2046 } 2047 set_bit(bank, mce_banks_ce_disabled); 2048 on_each_cpu(__mce_disable_bank, &bank, 1); 2049 } 2050 2051 /* 2052 * mce=off Disables machine check 2053 * mce=no_cmci Disables CMCI 2054 * mce=no_lmce Disables LMCE 2055 * mce=dont_log_ce Clears corrected events silently, no log created for CEs. 2056 * mce=print_all Print all machine check logs to console 2057 * mce=ignore_ce Disables polling and CMCI, corrected events are not cleared. 2058 * mce=TOLERANCELEVEL[,monarchtimeout] (number, see above) 2059 * monarchtimeout is how long to wait for other CPUs on machine 2060 * check, or 0 to not wait 2061 * mce=bootlog Log MCEs from before booting. Disabled by default on AMD Fam10h 2062 and older. 2063 * mce=nobootlog Don't log MCEs from before booting. 2064 * mce=bios_cmci_threshold Don't program the CMCI threshold 2065 * mce=recovery force enable memcpy_mcsafe() 2066 */ 2067 static int __init mcheck_enable(char *str) 2068 { 2069 struct mca_config *cfg = &mca_cfg; 2070 2071 if (*str == 0) { 2072 enable_p5_mce(); 2073 return 1; 2074 } 2075 if (*str == '=') 2076 str++; 2077 if (!strcmp(str, "off")) 2078 cfg->disabled = 1; 2079 else if (!strcmp(str, "no_cmci")) 2080 cfg->cmci_disabled = true; 2081 else if (!strcmp(str, "no_lmce")) 2082 cfg->lmce_disabled = 1; 2083 else if (!strcmp(str, "dont_log_ce")) 2084 cfg->dont_log_ce = true; 2085 else if (!strcmp(str, "print_all")) 2086 cfg->print_all = true; 2087 else if (!strcmp(str, "ignore_ce")) 2088 cfg->ignore_ce = true; 2089 else if (!strcmp(str, "bootlog") || !strcmp(str, "nobootlog")) 2090 cfg->bootlog = (str[0] == 'b'); 2091 else if (!strcmp(str, "bios_cmci_threshold")) 2092 cfg->bios_cmci_threshold = 1; 2093 else if (!strcmp(str, "recovery")) 2094 cfg->recovery = 1; 2095 else if (isdigit(str[0])) { 2096 if (get_option(&str, &cfg->tolerant) == 2) 2097 get_option(&str, &(cfg->monarch_timeout)); 2098 } else { 2099 pr_info("mce argument %s ignored. Please use /sys\n", str); 2100 return 0; 2101 } 2102 return 1; 2103 } 2104 __setup("mce", mcheck_enable); 2105 2106 int __init mcheck_init(void) 2107 { 2108 mcheck_intel_therm_init(); 2109 mce_register_decode_chain(&early_nb); 2110 mce_register_decode_chain(&mce_uc_nb); 2111 mce_register_decode_chain(&mce_default_nb); 2112 mcheck_vendor_init_severity(); 2113 2114 INIT_WORK(&mce_work, mce_gen_pool_process); 2115 init_irq_work(&mce_irq_work, mce_irq_work_cb); 2116 2117 return 0; 2118 } 2119 2120 /* 2121 * mce_syscore: PM support 2122 */ 2123 2124 /* 2125 * Disable machine checks on suspend and shutdown. We can't really handle 2126 * them later. 2127 */ 2128 static void mce_disable_error_reporting(void) 2129 { 2130 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 2131 int i; 2132 2133 for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 2134 struct mce_bank *b = &mce_banks[i]; 2135 2136 if (b->init) 2137 wrmsrl(msr_ops.ctl(i), 0); 2138 } 2139 return; 2140 } 2141 2142 static void vendor_disable_error_reporting(void) 2143 { 2144 /* 2145 * Don't clear on Intel or AMD or Hygon or Zhaoxin CPUs. Some of these 2146 * MSRs are socket-wide. Disabling them for just a single offlined CPU 2147 * is bad, since it will inhibit reporting for all shared resources on 2148 * the socket like the last level cache (LLC), the integrated memory 2149 * controller (iMC), etc. 2150 */ 2151 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL || 2152 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON || 2153 boot_cpu_data.x86_vendor == X86_VENDOR_AMD || 2154 boot_cpu_data.x86_vendor == X86_VENDOR_ZHAOXIN) 2155 return; 2156 2157 mce_disable_error_reporting(); 2158 } 2159 2160 static int mce_syscore_suspend(void) 2161 { 2162 vendor_disable_error_reporting(); 2163 return 0; 2164 } 2165 2166 static void mce_syscore_shutdown(void) 2167 { 2168 vendor_disable_error_reporting(); 2169 } 2170 2171 /* 2172 * On resume clear all MCE state. Don't want to see leftovers from the BIOS. 2173 * Only one CPU is active at this time, the others get re-added later using 2174 * CPU hotplug: 2175 */ 2176 static void mce_syscore_resume(void) 2177 { 2178 __mcheck_cpu_init_generic(); 2179 __mcheck_cpu_init_vendor(raw_cpu_ptr(&cpu_info)); 2180 __mcheck_cpu_init_clear_banks(); 2181 } 2182 2183 static struct syscore_ops mce_syscore_ops = { 2184 .suspend = mce_syscore_suspend, 2185 .shutdown = mce_syscore_shutdown, 2186 .resume = mce_syscore_resume, 2187 }; 2188 2189 /* 2190 * mce_device: Sysfs support 2191 */ 2192 2193 static void mce_cpu_restart(void *data) 2194 { 2195 if (!mce_available(raw_cpu_ptr(&cpu_info))) 2196 return; 2197 __mcheck_cpu_init_generic(); 2198 __mcheck_cpu_init_clear_banks(); 2199 __mcheck_cpu_init_timer(); 2200 } 2201 2202 /* Reinit MCEs after user configuration changes */ 2203 static void mce_restart(void) 2204 { 2205 mce_timer_delete_all(); 2206 on_each_cpu(mce_cpu_restart, NULL, 1); 2207 } 2208 2209 /* Toggle features for corrected errors */ 2210 static void mce_disable_cmci(void *data) 2211 { 2212 if (!mce_available(raw_cpu_ptr(&cpu_info))) 2213 return; 2214 cmci_clear(); 2215 } 2216 2217 static void mce_enable_ce(void *all) 2218 { 2219 if (!mce_available(raw_cpu_ptr(&cpu_info))) 2220 return; 2221 cmci_reenable(); 2222 cmci_recheck(); 2223 if (all) 2224 __mcheck_cpu_init_timer(); 2225 } 2226 2227 static struct bus_type mce_subsys = { 2228 .name = "machinecheck", 2229 .dev_name = "machinecheck", 2230 }; 2231 2232 DEFINE_PER_CPU(struct device *, mce_device); 2233 2234 static inline struct mce_bank_dev *attr_to_bank(struct device_attribute *attr) 2235 { 2236 return container_of(attr, struct mce_bank_dev, attr); 2237 } 2238 2239 static ssize_t show_bank(struct device *s, struct device_attribute *attr, 2240 char *buf) 2241 { 2242 u8 bank = attr_to_bank(attr)->bank; 2243 struct mce_bank *b; 2244 2245 if (bank >= per_cpu(mce_num_banks, s->id)) 2246 return -EINVAL; 2247 2248 b = &per_cpu(mce_banks_array, s->id)[bank]; 2249 2250 if (!b->init) 2251 return -ENODEV; 2252 2253 return sprintf(buf, "%llx\n", b->ctl); 2254 } 2255 2256 static ssize_t set_bank(struct device *s, struct device_attribute *attr, 2257 const char *buf, size_t size) 2258 { 2259 u8 bank = attr_to_bank(attr)->bank; 2260 struct mce_bank *b; 2261 u64 new; 2262 2263 if (kstrtou64(buf, 0, &new) < 0) 2264 return -EINVAL; 2265 2266 if (bank >= per_cpu(mce_num_banks, s->id)) 2267 return -EINVAL; 2268 2269 b = &per_cpu(mce_banks_array, s->id)[bank]; 2270 2271 if (!b->init) 2272 return -ENODEV; 2273 2274 b->ctl = new; 2275 mce_restart(); 2276 2277 return size; 2278 } 2279 2280 static ssize_t set_ignore_ce(struct device *s, 2281 struct device_attribute *attr, 2282 const char *buf, size_t size) 2283 { 2284 u64 new; 2285 2286 if (kstrtou64(buf, 0, &new) < 0) 2287 return -EINVAL; 2288 2289 mutex_lock(&mce_sysfs_mutex); 2290 if (mca_cfg.ignore_ce ^ !!new) { 2291 if (new) { 2292 /* disable ce features */ 2293 mce_timer_delete_all(); 2294 on_each_cpu(mce_disable_cmci, NULL, 1); 2295 mca_cfg.ignore_ce = true; 2296 } else { 2297 /* enable ce features */ 2298 mca_cfg.ignore_ce = false; 2299 on_each_cpu(mce_enable_ce, (void *)1, 1); 2300 } 2301 } 2302 mutex_unlock(&mce_sysfs_mutex); 2303 2304 return size; 2305 } 2306 2307 static ssize_t set_cmci_disabled(struct device *s, 2308 struct device_attribute *attr, 2309 const char *buf, size_t size) 2310 { 2311 u64 new; 2312 2313 if (kstrtou64(buf, 0, &new) < 0) 2314 return -EINVAL; 2315 2316 mutex_lock(&mce_sysfs_mutex); 2317 if (mca_cfg.cmci_disabled ^ !!new) { 2318 if (new) { 2319 /* disable cmci */ 2320 on_each_cpu(mce_disable_cmci, NULL, 1); 2321 mca_cfg.cmci_disabled = true; 2322 } else { 2323 /* enable cmci */ 2324 mca_cfg.cmci_disabled = false; 2325 on_each_cpu(mce_enable_ce, NULL, 1); 2326 } 2327 } 2328 mutex_unlock(&mce_sysfs_mutex); 2329 2330 return size; 2331 } 2332 2333 static ssize_t store_int_with_restart(struct device *s, 2334 struct device_attribute *attr, 2335 const char *buf, size_t size) 2336 { 2337 unsigned long old_check_interval = check_interval; 2338 ssize_t ret = device_store_ulong(s, attr, buf, size); 2339 2340 if (check_interval == old_check_interval) 2341 return ret; 2342 2343 mutex_lock(&mce_sysfs_mutex); 2344 mce_restart(); 2345 mutex_unlock(&mce_sysfs_mutex); 2346 2347 return ret; 2348 } 2349 2350 static DEVICE_INT_ATTR(tolerant, 0644, mca_cfg.tolerant); 2351 static DEVICE_INT_ATTR(monarch_timeout, 0644, mca_cfg.monarch_timeout); 2352 static DEVICE_BOOL_ATTR(dont_log_ce, 0644, mca_cfg.dont_log_ce); 2353 static DEVICE_BOOL_ATTR(print_all, 0644, mca_cfg.print_all); 2354 2355 static struct dev_ext_attribute dev_attr_check_interval = { 2356 __ATTR(check_interval, 0644, device_show_int, store_int_with_restart), 2357 &check_interval 2358 }; 2359 2360 static struct dev_ext_attribute dev_attr_ignore_ce = { 2361 __ATTR(ignore_ce, 0644, device_show_bool, set_ignore_ce), 2362 &mca_cfg.ignore_ce 2363 }; 2364 2365 static struct dev_ext_attribute dev_attr_cmci_disabled = { 2366 __ATTR(cmci_disabled, 0644, device_show_bool, set_cmci_disabled), 2367 &mca_cfg.cmci_disabled 2368 }; 2369 2370 static struct device_attribute *mce_device_attrs[] = { 2371 &dev_attr_tolerant.attr, 2372 &dev_attr_check_interval.attr, 2373 #ifdef CONFIG_X86_MCELOG_LEGACY 2374 &dev_attr_trigger, 2375 #endif 2376 &dev_attr_monarch_timeout.attr, 2377 &dev_attr_dont_log_ce.attr, 2378 &dev_attr_print_all.attr, 2379 &dev_attr_ignore_ce.attr, 2380 &dev_attr_cmci_disabled.attr, 2381 NULL 2382 }; 2383 2384 static cpumask_var_t mce_device_initialized; 2385 2386 static void mce_device_release(struct device *dev) 2387 { 2388 kfree(dev); 2389 } 2390 2391 /* Per CPU device init. All of the CPUs still share the same bank device: */ 2392 static int mce_device_create(unsigned int cpu) 2393 { 2394 struct device *dev; 2395 int err; 2396 int i, j; 2397 2398 if (!mce_available(&boot_cpu_data)) 2399 return -EIO; 2400 2401 dev = per_cpu(mce_device, cpu); 2402 if (dev) 2403 return 0; 2404 2405 dev = kzalloc(sizeof(*dev), GFP_KERNEL); 2406 if (!dev) 2407 return -ENOMEM; 2408 dev->id = cpu; 2409 dev->bus = &mce_subsys; 2410 dev->release = &mce_device_release; 2411 2412 err = device_register(dev); 2413 if (err) { 2414 put_device(dev); 2415 return err; 2416 } 2417 2418 for (i = 0; mce_device_attrs[i]; i++) { 2419 err = device_create_file(dev, mce_device_attrs[i]); 2420 if (err) 2421 goto error; 2422 } 2423 for (j = 0; j < per_cpu(mce_num_banks, cpu); j++) { 2424 err = device_create_file(dev, &mce_bank_devs[j].attr); 2425 if (err) 2426 goto error2; 2427 } 2428 cpumask_set_cpu(cpu, mce_device_initialized); 2429 per_cpu(mce_device, cpu) = dev; 2430 2431 return 0; 2432 error2: 2433 while (--j >= 0) 2434 device_remove_file(dev, &mce_bank_devs[j].attr); 2435 error: 2436 while (--i >= 0) 2437 device_remove_file(dev, mce_device_attrs[i]); 2438 2439 device_unregister(dev); 2440 2441 return err; 2442 } 2443 2444 static void mce_device_remove(unsigned int cpu) 2445 { 2446 struct device *dev = per_cpu(mce_device, cpu); 2447 int i; 2448 2449 if (!cpumask_test_cpu(cpu, mce_device_initialized)) 2450 return; 2451 2452 for (i = 0; mce_device_attrs[i]; i++) 2453 device_remove_file(dev, mce_device_attrs[i]); 2454 2455 for (i = 0; i < per_cpu(mce_num_banks, cpu); i++) 2456 device_remove_file(dev, &mce_bank_devs[i].attr); 2457 2458 device_unregister(dev); 2459 cpumask_clear_cpu(cpu, mce_device_initialized); 2460 per_cpu(mce_device, cpu) = NULL; 2461 } 2462 2463 /* Make sure there are no machine checks on offlined CPUs. */ 2464 static void mce_disable_cpu(void) 2465 { 2466 if (!mce_available(raw_cpu_ptr(&cpu_info))) 2467 return; 2468 2469 if (!cpuhp_tasks_frozen) 2470 cmci_clear(); 2471 2472 vendor_disable_error_reporting(); 2473 } 2474 2475 static void mce_reenable_cpu(void) 2476 { 2477 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array); 2478 int i; 2479 2480 if (!mce_available(raw_cpu_ptr(&cpu_info))) 2481 return; 2482 2483 if (!cpuhp_tasks_frozen) 2484 cmci_reenable(); 2485 for (i = 0; i < this_cpu_read(mce_num_banks); i++) { 2486 struct mce_bank *b = &mce_banks[i]; 2487 2488 if (b->init) 2489 wrmsrl(msr_ops.ctl(i), b->ctl); 2490 } 2491 } 2492 2493 static int mce_cpu_dead(unsigned int cpu) 2494 { 2495 mce_intel_hcpu_update(cpu); 2496 2497 /* intentionally ignoring frozen here */ 2498 if (!cpuhp_tasks_frozen) 2499 cmci_rediscover(); 2500 return 0; 2501 } 2502 2503 static int mce_cpu_online(unsigned int cpu) 2504 { 2505 struct timer_list *t = this_cpu_ptr(&mce_timer); 2506 int ret; 2507 2508 mce_device_create(cpu); 2509 2510 ret = mce_threshold_create_device(cpu); 2511 if (ret) { 2512 mce_device_remove(cpu); 2513 return ret; 2514 } 2515 mce_reenable_cpu(); 2516 mce_start_timer(t); 2517 return 0; 2518 } 2519 2520 static int mce_cpu_pre_down(unsigned int cpu) 2521 { 2522 struct timer_list *t = this_cpu_ptr(&mce_timer); 2523 2524 mce_disable_cpu(); 2525 del_timer_sync(t); 2526 mce_threshold_remove_device(cpu); 2527 mce_device_remove(cpu); 2528 return 0; 2529 } 2530 2531 static __init void mce_init_banks(void) 2532 { 2533 int i; 2534 2535 for (i = 0; i < MAX_NR_BANKS; i++) { 2536 struct mce_bank_dev *b = &mce_bank_devs[i]; 2537 struct device_attribute *a = &b->attr; 2538 2539 b->bank = i; 2540 2541 sysfs_attr_init(&a->attr); 2542 a->attr.name = b->attrname; 2543 snprintf(b->attrname, ATTR_LEN, "bank%d", i); 2544 2545 a->attr.mode = 0644; 2546 a->show = show_bank; 2547 a->store = set_bank; 2548 } 2549 } 2550 2551 /* 2552 * When running on XEN, this initcall is ordered against the XEN mcelog 2553 * initcall: 2554 * 2555 * device_initcall(xen_late_init_mcelog); 2556 * device_initcall_sync(mcheck_init_device); 2557 */ 2558 static __init int mcheck_init_device(void) 2559 { 2560 int err; 2561 2562 /* 2563 * Check if we have a spare virtual bit. This will only become 2564 * a problem if/when we move beyond 5-level page tables. 2565 */ 2566 MAYBE_BUILD_BUG_ON(__VIRTUAL_MASK_SHIFT >= 63); 2567 2568 if (!mce_available(&boot_cpu_data)) { 2569 err = -EIO; 2570 goto err_out; 2571 } 2572 2573 if (!zalloc_cpumask_var(&mce_device_initialized, GFP_KERNEL)) { 2574 err = -ENOMEM; 2575 goto err_out; 2576 } 2577 2578 mce_init_banks(); 2579 2580 err = subsys_system_register(&mce_subsys, NULL); 2581 if (err) 2582 goto err_out_mem; 2583 2584 err = cpuhp_setup_state(CPUHP_X86_MCE_DEAD, "x86/mce:dead", NULL, 2585 mce_cpu_dead); 2586 if (err) 2587 goto err_out_mem; 2588 2589 /* 2590 * Invokes mce_cpu_online() on all CPUs which are online when 2591 * the state is installed. 2592 */ 2593 err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/mce:online", 2594 mce_cpu_online, mce_cpu_pre_down); 2595 if (err < 0) 2596 goto err_out_online; 2597 2598 register_syscore_ops(&mce_syscore_ops); 2599 2600 return 0; 2601 2602 err_out_online: 2603 cpuhp_remove_state(CPUHP_X86_MCE_DEAD); 2604 2605 err_out_mem: 2606 free_cpumask_var(mce_device_initialized); 2607 2608 err_out: 2609 pr_err("Unable to init MCE device (rc: %d)\n", err); 2610 2611 return err; 2612 } 2613 device_initcall_sync(mcheck_init_device); 2614 2615 /* 2616 * Old style boot options parsing. Only for compatibility. 2617 */ 2618 static int __init mcheck_disable(char *str) 2619 { 2620 mca_cfg.disabled = 1; 2621 return 1; 2622 } 2623 __setup("nomce", mcheck_disable); 2624 2625 #ifdef CONFIG_DEBUG_FS 2626 struct dentry *mce_get_debugfs_dir(void) 2627 { 2628 static struct dentry *dmce; 2629 2630 if (!dmce) 2631 dmce = debugfs_create_dir("mce", NULL); 2632 2633 return dmce; 2634 } 2635 2636 static void mce_reset(void) 2637 { 2638 cpu_missing = 0; 2639 atomic_set(&mce_fake_panicked, 0); 2640 atomic_set(&mce_executing, 0); 2641 atomic_set(&mce_callin, 0); 2642 atomic_set(&global_nwo, 0); 2643 } 2644 2645 static int fake_panic_get(void *data, u64 *val) 2646 { 2647 *val = fake_panic; 2648 return 0; 2649 } 2650 2651 static int fake_panic_set(void *data, u64 val) 2652 { 2653 mce_reset(); 2654 fake_panic = val; 2655 return 0; 2656 } 2657 2658 DEFINE_DEBUGFS_ATTRIBUTE(fake_panic_fops, fake_panic_get, fake_panic_set, 2659 "%llu\n"); 2660 2661 static void __init mcheck_debugfs_init(void) 2662 { 2663 struct dentry *dmce; 2664 2665 dmce = mce_get_debugfs_dir(); 2666 debugfs_create_file_unsafe("fake_panic", 0444, dmce, NULL, 2667 &fake_panic_fops); 2668 } 2669 #else 2670 static void __init mcheck_debugfs_init(void) { } 2671 #endif 2672 2673 DEFINE_STATIC_KEY_FALSE(mcsafe_key); 2674 EXPORT_SYMBOL_GPL(mcsafe_key); 2675 2676 static int __init mcheck_late_init(void) 2677 { 2678 if (mca_cfg.recovery) 2679 static_branch_inc(&mcsafe_key); 2680 2681 mcheck_debugfs_init(); 2682 2683 /* 2684 * Flush out everything that has been logged during early boot, now that 2685 * everything has been initialized (workqueues, decoders, ...). 2686 */ 2687 mce_schedule_work(); 2688 2689 return 0; 2690 } 2691 late_initcall(mcheck_late_init); 2692