1 /* 2 * APEI Generic Hardware Error Source support 3 * 4 * Generic Hardware Error Source provides a way to report platform 5 * hardware errors (such as that from chipset). It works in so called 6 * "Firmware First" mode, that is, hardware errors are reported to 7 * firmware firstly, then reported to Linux by firmware. This way, 8 * some non-standard hardware error registers or non-standard hardware 9 * link can be checked by firmware to produce more hardware error 10 * information for Linux. 11 * 12 * For more information about Generic Hardware Error Source, please 13 * refer to ACPI Specification version 4.0, section 17.3.2.6 14 * 15 * Copyright 2010,2011 Intel Corp. 16 * Author: Huang Ying <ying.huang@intel.com> 17 * 18 * This program is free software; you can redistribute it and/or 19 * modify it under the terms of the GNU General Public License version 20 * 2 as published by the Free Software Foundation; 21 * 22 * This program is distributed in the hope that it will be useful, 23 * but WITHOUT ANY WARRANTY; without even the implied warranty of 24 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 25 * GNU General Public License for more details. 26 */ 27 28 #include <linux/kernel.h> 29 #include <linux/moduleparam.h> 30 #include <linux/init.h> 31 #include <linux/acpi.h> 32 #include <linux/io.h> 33 #include <linux/interrupt.h> 34 #include <linux/timer.h> 35 #include <linux/cper.h> 36 #include <linux/kdebug.h> 37 #include <linux/platform_device.h> 38 #include <linux/mutex.h> 39 #include <linux/ratelimit.h> 40 #include <linux/vmalloc.h> 41 #include <linux/irq_work.h> 42 #include <linux/llist.h> 43 #include <linux/genalloc.h> 44 #include <linux/pci.h> 45 #include <linux/aer.h> 46 #include <linux/nmi.h> 47 #include <linux/sched/clock.h> 48 #include <linux/uuid.h> 49 #include <linux/ras.h> 50 51 #include <acpi/actbl1.h> 52 #include <acpi/ghes.h> 53 #include <acpi/apei.h> 54 #include <asm/tlbflush.h> 55 #include <ras/ras_event.h> 56 57 #include "apei-internal.h" 58 59 #define GHES_PFX "GHES: " 60 61 #define GHES_ESTATUS_MAX_SIZE 65536 62 #define GHES_ESOURCE_PREALLOC_MAX_SIZE 65536 63 64 #define GHES_ESTATUS_POOL_MIN_ALLOC_ORDER 3 65 66 /* This is just an estimation for memory pool allocation */ 67 #define GHES_ESTATUS_CACHE_AVG_SIZE 512 68 69 #define GHES_ESTATUS_CACHES_SIZE 4 70 71 #define GHES_ESTATUS_IN_CACHE_MAX_NSEC 10000000000ULL 72 /* Prevent too many caches are allocated because of RCU */ 73 #define GHES_ESTATUS_CACHE_ALLOCED_MAX (GHES_ESTATUS_CACHES_SIZE * 3 / 2) 74 75 #define GHES_ESTATUS_CACHE_LEN(estatus_len) \ 76 (sizeof(struct ghes_estatus_cache) + (estatus_len)) 77 #define GHES_ESTATUS_FROM_CACHE(estatus_cache) \ 78 ((struct acpi_hest_generic_status *) \ 79 ((struct ghes_estatus_cache *)(estatus_cache) + 1)) 80 81 #define GHES_ESTATUS_NODE_LEN(estatus_len) \ 82 (sizeof(struct ghes_estatus_node) + (estatus_len)) 83 #define GHES_ESTATUS_FROM_NODE(estatus_node) \ 84 ((struct acpi_hest_generic_status *) \ 85 ((struct ghes_estatus_node *)(estatus_node) + 1)) 86 87 static inline bool is_hest_type_generic_v2(struct ghes *ghes) 88 { 89 return ghes->generic->header.type == ACPI_HEST_TYPE_GENERIC_ERROR_V2; 90 } 91 92 /* 93 * This driver isn't really modular, however for the time being, 94 * continuing to use module_param is the easiest way to remain 95 * compatible with existing boot arg use cases. 96 */ 97 bool ghes_disable; 98 module_param_named(disable, ghes_disable, bool, 0); 99 100 /* 101 * All error sources notified with HED (Hardware Error Device) share a 102 * single notifier callback, so they need to be linked and checked one 103 * by one. This holds true for NMI too. 104 * 105 * RCU is used for these lists, so ghes_list_mutex is only used for 106 * list changing, not for traversing. 107 */ 108 static LIST_HEAD(ghes_hed); 109 static DEFINE_MUTEX(ghes_list_mutex); 110 111 /* 112 * Because the memory area used to transfer hardware error information 113 * from BIOS to Linux can be determined only in NMI, IRQ or timer 114 * handler, but general ioremap can not be used in atomic context, so 115 * a special version of atomic ioremap is implemented for that. 116 */ 117 118 /* 119 * Two virtual pages are used, one for IRQ/PROCESS context, the other for 120 * NMI context (optionally). 121 */ 122 #define GHES_IOREMAP_PAGES 2 123 #define GHES_IOREMAP_IRQ_PAGE(base) (base) 124 #define GHES_IOREMAP_NMI_PAGE(base) ((base) + PAGE_SIZE) 125 126 /* virtual memory area for atomic ioremap */ 127 static struct vm_struct *ghes_ioremap_area; 128 /* 129 * These 2 spinlock is used to prevent atomic ioremap virtual memory 130 * area from being mapped simultaneously. 131 */ 132 static DEFINE_RAW_SPINLOCK(ghes_ioremap_lock_nmi); 133 static DEFINE_SPINLOCK(ghes_ioremap_lock_irq); 134 135 static struct gen_pool *ghes_estatus_pool; 136 static unsigned long ghes_estatus_pool_size_request; 137 138 static struct ghes_estatus_cache *ghes_estatus_caches[GHES_ESTATUS_CACHES_SIZE]; 139 static atomic_t ghes_estatus_cache_alloced; 140 141 static int ghes_panic_timeout __read_mostly = 30; 142 143 static int ghes_ioremap_init(void) 144 { 145 ghes_ioremap_area = __get_vm_area(PAGE_SIZE * GHES_IOREMAP_PAGES, 146 VM_IOREMAP, VMALLOC_START, VMALLOC_END); 147 if (!ghes_ioremap_area) { 148 pr_err(GHES_PFX "Failed to allocate virtual memory area for atomic ioremap.\n"); 149 return -ENOMEM; 150 } 151 152 return 0; 153 } 154 155 static void ghes_ioremap_exit(void) 156 { 157 free_vm_area(ghes_ioremap_area); 158 } 159 160 static void __iomem *ghes_ioremap_pfn_nmi(u64 pfn) 161 { 162 unsigned long vaddr; 163 phys_addr_t paddr; 164 pgprot_t prot; 165 166 vaddr = (unsigned long)GHES_IOREMAP_NMI_PAGE(ghes_ioremap_area->addr); 167 168 paddr = pfn << PAGE_SHIFT; 169 prot = arch_apei_get_mem_attribute(paddr); 170 ioremap_page_range(vaddr, vaddr + PAGE_SIZE, paddr, prot); 171 172 return (void __iomem *)vaddr; 173 } 174 175 static void __iomem *ghes_ioremap_pfn_irq(u64 pfn) 176 { 177 unsigned long vaddr, paddr; 178 pgprot_t prot; 179 180 vaddr = (unsigned long)GHES_IOREMAP_IRQ_PAGE(ghes_ioremap_area->addr); 181 182 paddr = pfn << PAGE_SHIFT; 183 prot = arch_apei_get_mem_attribute(paddr); 184 185 ioremap_page_range(vaddr, vaddr + PAGE_SIZE, paddr, prot); 186 187 return (void __iomem *)vaddr; 188 } 189 190 static void ghes_iounmap_nmi(void __iomem *vaddr_ptr) 191 { 192 unsigned long vaddr = (unsigned long __force)vaddr_ptr; 193 void *base = ghes_ioremap_area->addr; 194 195 BUG_ON(vaddr != (unsigned long)GHES_IOREMAP_NMI_PAGE(base)); 196 unmap_kernel_range_noflush(vaddr, PAGE_SIZE); 197 arch_apei_flush_tlb_one(vaddr); 198 } 199 200 static void ghes_iounmap_irq(void __iomem *vaddr_ptr) 201 { 202 unsigned long vaddr = (unsigned long __force)vaddr_ptr; 203 void *base = ghes_ioremap_area->addr; 204 205 BUG_ON(vaddr != (unsigned long)GHES_IOREMAP_IRQ_PAGE(base)); 206 unmap_kernel_range_noflush(vaddr, PAGE_SIZE); 207 arch_apei_flush_tlb_one(vaddr); 208 } 209 210 static int ghes_estatus_pool_init(void) 211 { 212 ghes_estatus_pool = gen_pool_create(GHES_ESTATUS_POOL_MIN_ALLOC_ORDER, -1); 213 if (!ghes_estatus_pool) 214 return -ENOMEM; 215 return 0; 216 } 217 218 static void ghes_estatus_pool_free_chunk_page(struct gen_pool *pool, 219 struct gen_pool_chunk *chunk, 220 void *data) 221 { 222 free_page(chunk->start_addr); 223 } 224 225 static void ghes_estatus_pool_exit(void) 226 { 227 gen_pool_for_each_chunk(ghes_estatus_pool, 228 ghes_estatus_pool_free_chunk_page, NULL); 229 gen_pool_destroy(ghes_estatus_pool); 230 } 231 232 static int ghes_estatus_pool_expand(unsigned long len) 233 { 234 unsigned long i, pages, size, addr; 235 int ret; 236 237 ghes_estatus_pool_size_request += PAGE_ALIGN(len); 238 size = gen_pool_size(ghes_estatus_pool); 239 if (size >= ghes_estatus_pool_size_request) 240 return 0; 241 pages = (ghes_estatus_pool_size_request - size) / PAGE_SIZE; 242 for (i = 0; i < pages; i++) { 243 addr = __get_free_page(GFP_KERNEL); 244 if (!addr) 245 return -ENOMEM; 246 ret = gen_pool_add(ghes_estatus_pool, addr, PAGE_SIZE, -1); 247 if (ret) 248 return ret; 249 } 250 251 return 0; 252 } 253 254 static int map_gen_v2(struct ghes *ghes) 255 { 256 return apei_map_generic_address(&ghes->generic_v2->read_ack_register); 257 } 258 259 static void unmap_gen_v2(struct ghes *ghes) 260 { 261 apei_unmap_generic_address(&ghes->generic_v2->read_ack_register); 262 } 263 264 static struct ghes *ghes_new(struct acpi_hest_generic *generic) 265 { 266 struct ghes *ghes; 267 unsigned int error_block_length; 268 int rc; 269 270 ghes = kzalloc(sizeof(*ghes), GFP_KERNEL); 271 if (!ghes) 272 return ERR_PTR(-ENOMEM); 273 274 ghes->generic = generic; 275 if (is_hest_type_generic_v2(ghes)) { 276 rc = map_gen_v2(ghes); 277 if (rc) 278 goto err_free; 279 } 280 281 rc = apei_map_generic_address(&generic->error_status_address); 282 if (rc) 283 goto err_unmap_read_ack_addr; 284 error_block_length = generic->error_block_length; 285 if (error_block_length > GHES_ESTATUS_MAX_SIZE) { 286 pr_warning(FW_WARN GHES_PFX 287 "Error status block length is too long: %u for " 288 "generic hardware error source: %d.\n", 289 error_block_length, generic->header.source_id); 290 error_block_length = GHES_ESTATUS_MAX_SIZE; 291 } 292 ghes->estatus = kmalloc(error_block_length, GFP_KERNEL); 293 if (!ghes->estatus) { 294 rc = -ENOMEM; 295 goto err_unmap_status_addr; 296 } 297 298 return ghes; 299 300 err_unmap_status_addr: 301 apei_unmap_generic_address(&generic->error_status_address); 302 err_unmap_read_ack_addr: 303 if (is_hest_type_generic_v2(ghes)) 304 unmap_gen_v2(ghes); 305 err_free: 306 kfree(ghes); 307 return ERR_PTR(rc); 308 } 309 310 static void ghes_fini(struct ghes *ghes) 311 { 312 kfree(ghes->estatus); 313 apei_unmap_generic_address(&ghes->generic->error_status_address); 314 if (is_hest_type_generic_v2(ghes)) 315 unmap_gen_v2(ghes); 316 } 317 318 static inline int ghes_severity(int severity) 319 { 320 switch (severity) { 321 case CPER_SEV_INFORMATIONAL: 322 return GHES_SEV_NO; 323 case CPER_SEV_CORRECTED: 324 return GHES_SEV_CORRECTED; 325 case CPER_SEV_RECOVERABLE: 326 return GHES_SEV_RECOVERABLE; 327 case CPER_SEV_FATAL: 328 return GHES_SEV_PANIC; 329 default: 330 /* Unknown, go panic */ 331 return GHES_SEV_PANIC; 332 } 333 } 334 335 static void ghes_copy_tofrom_phys(void *buffer, u64 paddr, u32 len, 336 int from_phys) 337 { 338 void __iomem *vaddr; 339 unsigned long flags = 0; 340 int in_nmi = in_nmi(); 341 u64 offset; 342 u32 trunk; 343 344 while (len > 0) { 345 offset = paddr - (paddr & PAGE_MASK); 346 if (in_nmi) { 347 raw_spin_lock(&ghes_ioremap_lock_nmi); 348 vaddr = ghes_ioremap_pfn_nmi(paddr >> PAGE_SHIFT); 349 } else { 350 spin_lock_irqsave(&ghes_ioremap_lock_irq, flags); 351 vaddr = ghes_ioremap_pfn_irq(paddr >> PAGE_SHIFT); 352 } 353 trunk = PAGE_SIZE - offset; 354 trunk = min(trunk, len); 355 if (from_phys) 356 memcpy_fromio(buffer, vaddr + offset, trunk); 357 else 358 memcpy_toio(vaddr + offset, buffer, trunk); 359 len -= trunk; 360 paddr += trunk; 361 buffer += trunk; 362 if (in_nmi) { 363 ghes_iounmap_nmi(vaddr); 364 raw_spin_unlock(&ghes_ioremap_lock_nmi); 365 } else { 366 ghes_iounmap_irq(vaddr); 367 spin_unlock_irqrestore(&ghes_ioremap_lock_irq, flags); 368 } 369 } 370 } 371 372 static int ghes_read_estatus(struct ghes *ghes, int silent) 373 { 374 struct acpi_hest_generic *g = ghes->generic; 375 u64 buf_paddr; 376 u32 len; 377 int rc; 378 379 rc = apei_read(&buf_paddr, &g->error_status_address); 380 if (rc) { 381 if (!silent && printk_ratelimit()) 382 pr_warning(FW_WARN GHES_PFX 383 "Failed to read error status block address for hardware error source: %d.\n", 384 g->header.source_id); 385 return -EIO; 386 } 387 if (!buf_paddr) 388 return -ENOENT; 389 390 ghes_copy_tofrom_phys(ghes->estatus, buf_paddr, 391 sizeof(*ghes->estatus), 1); 392 if (!ghes->estatus->block_status) 393 return -ENOENT; 394 395 ghes->buffer_paddr = buf_paddr; 396 ghes->flags |= GHES_TO_CLEAR; 397 398 rc = -EIO; 399 len = cper_estatus_len(ghes->estatus); 400 if (len < sizeof(*ghes->estatus)) 401 goto err_read_block; 402 if (len > ghes->generic->error_block_length) 403 goto err_read_block; 404 if (cper_estatus_check_header(ghes->estatus)) 405 goto err_read_block; 406 ghes_copy_tofrom_phys(ghes->estatus + 1, 407 buf_paddr + sizeof(*ghes->estatus), 408 len - sizeof(*ghes->estatus), 1); 409 if (cper_estatus_check(ghes->estatus)) 410 goto err_read_block; 411 rc = 0; 412 413 err_read_block: 414 if (rc && !silent && printk_ratelimit()) 415 pr_warning(FW_WARN GHES_PFX 416 "Failed to read error status block!\n"); 417 return rc; 418 } 419 420 static void ghes_clear_estatus(struct ghes *ghes) 421 { 422 ghes->estatus->block_status = 0; 423 if (!(ghes->flags & GHES_TO_CLEAR)) 424 return; 425 ghes_copy_tofrom_phys(ghes->estatus, ghes->buffer_paddr, 426 sizeof(ghes->estatus->block_status), 0); 427 ghes->flags &= ~GHES_TO_CLEAR; 428 } 429 430 static void ghes_handle_memory_failure(struct acpi_hest_generic_data *gdata, int sev) 431 { 432 #ifdef CONFIG_ACPI_APEI_MEMORY_FAILURE 433 unsigned long pfn; 434 int flags = -1; 435 int sec_sev = ghes_severity(gdata->error_severity); 436 struct cper_sec_mem_err *mem_err = acpi_hest_get_payload(gdata); 437 438 if (!(mem_err->validation_bits & CPER_MEM_VALID_PA)) 439 return; 440 441 pfn = mem_err->physical_addr >> PAGE_SHIFT; 442 if (!pfn_valid(pfn)) { 443 pr_warn_ratelimited(FW_WARN GHES_PFX 444 "Invalid address in generic error data: %#llx\n", 445 mem_err->physical_addr); 446 return; 447 } 448 449 /* iff following two events can be handled properly by now */ 450 if (sec_sev == GHES_SEV_CORRECTED && 451 (gdata->flags & CPER_SEC_ERROR_THRESHOLD_EXCEEDED)) 452 flags = MF_SOFT_OFFLINE; 453 if (sev == GHES_SEV_RECOVERABLE && sec_sev == GHES_SEV_RECOVERABLE) 454 flags = 0; 455 456 if (flags != -1) 457 memory_failure_queue(pfn, 0, flags); 458 #endif 459 } 460 461 static void ghes_do_proc(struct ghes *ghes, 462 const struct acpi_hest_generic_status *estatus) 463 { 464 int sev, sec_sev; 465 struct acpi_hest_generic_data *gdata; 466 guid_t *sec_type; 467 guid_t *fru_id = &NULL_UUID_LE; 468 char *fru_text = ""; 469 470 sev = ghes_severity(estatus->error_severity); 471 apei_estatus_for_each_section(estatus, gdata) { 472 sec_type = (guid_t *)gdata->section_type; 473 sec_sev = ghes_severity(gdata->error_severity); 474 if (gdata->validation_bits & CPER_SEC_VALID_FRU_ID) 475 fru_id = (guid_t *)gdata->fru_id; 476 477 if (gdata->validation_bits & CPER_SEC_VALID_FRU_TEXT) 478 fru_text = gdata->fru_text; 479 480 if (guid_equal(sec_type, &CPER_SEC_PLATFORM_MEM)) { 481 struct cper_sec_mem_err *mem_err = acpi_hest_get_payload(gdata); 482 483 ghes_edac_report_mem_error(ghes, sev, mem_err); 484 485 arch_apei_report_mem_error(sev, mem_err); 486 ghes_handle_memory_failure(gdata, sev); 487 } 488 #ifdef CONFIG_ACPI_APEI_PCIEAER 489 else if (guid_equal(sec_type, &CPER_SEC_PCIE)) { 490 struct cper_sec_pcie *pcie_err = acpi_hest_get_payload(gdata); 491 492 if (sev == GHES_SEV_RECOVERABLE && 493 sec_sev == GHES_SEV_RECOVERABLE && 494 pcie_err->validation_bits & CPER_PCIE_VALID_DEVICE_ID && 495 pcie_err->validation_bits & CPER_PCIE_VALID_AER_INFO) { 496 unsigned int devfn; 497 int aer_severity; 498 499 devfn = PCI_DEVFN(pcie_err->device_id.device, 500 pcie_err->device_id.function); 501 aer_severity = cper_severity_to_aer(gdata->error_severity); 502 503 /* 504 * If firmware reset the component to contain 505 * the error, we must reinitialize it before 506 * use, so treat it as a fatal AER error. 507 */ 508 if (gdata->flags & CPER_SEC_RESET) 509 aer_severity = AER_FATAL; 510 511 aer_recover_queue(pcie_err->device_id.segment, 512 pcie_err->device_id.bus, 513 devfn, aer_severity, 514 (struct aer_capability_regs *) 515 pcie_err->aer_info); 516 } 517 518 } 519 #endif 520 else if (guid_equal(sec_type, &CPER_SEC_PROC_ARM)) { 521 struct cper_sec_proc_arm *err = acpi_hest_get_payload(gdata); 522 523 log_arm_hw_error(err); 524 } else { 525 void *err = acpi_hest_get_payload(gdata); 526 527 log_non_standard_event(sec_type, fru_id, fru_text, 528 sec_sev, err, 529 gdata->error_data_length); 530 } 531 } 532 } 533 534 static void __ghes_print_estatus(const char *pfx, 535 const struct acpi_hest_generic *generic, 536 const struct acpi_hest_generic_status *estatus) 537 { 538 static atomic_t seqno; 539 unsigned int curr_seqno; 540 char pfx_seq[64]; 541 542 if (pfx == NULL) { 543 if (ghes_severity(estatus->error_severity) <= 544 GHES_SEV_CORRECTED) 545 pfx = KERN_WARNING; 546 else 547 pfx = KERN_ERR; 548 } 549 curr_seqno = atomic_inc_return(&seqno); 550 snprintf(pfx_seq, sizeof(pfx_seq), "%s{%u}" HW_ERR, pfx, curr_seqno); 551 printk("%s""Hardware error from APEI Generic Hardware Error Source: %d\n", 552 pfx_seq, generic->header.source_id); 553 cper_estatus_print(pfx_seq, estatus); 554 } 555 556 static int ghes_print_estatus(const char *pfx, 557 const struct acpi_hest_generic *generic, 558 const struct acpi_hest_generic_status *estatus) 559 { 560 /* Not more than 2 messages every 5 seconds */ 561 static DEFINE_RATELIMIT_STATE(ratelimit_corrected, 5*HZ, 2); 562 static DEFINE_RATELIMIT_STATE(ratelimit_uncorrected, 5*HZ, 2); 563 struct ratelimit_state *ratelimit; 564 565 if (ghes_severity(estatus->error_severity) <= GHES_SEV_CORRECTED) 566 ratelimit = &ratelimit_corrected; 567 else 568 ratelimit = &ratelimit_uncorrected; 569 if (__ratelimit(ratelimit)) { 570 __ghes_print_estatus(pfx, generic, estatus); 571 return 1; 572 } 573 return 0; 574 } 575 576 /* 577 * GHES error status reporting throttle, to report more kinds of 578 * errors, instead of just most frequently occurred errors. 579 */ 580 static int ghes_estatus_cached(struct acpi_hest_generic_status *estatus) 581 { 582 u32 len; 583 int i, cached = 0; 584 unsigned long long now; 585 struct ghes_estatus_cache *cache; 586 struct acpi_hest_generic_status *cache_estatus; 587 588 len = cper_estatus_len(estatus); 589 rcu_read_lock(); 590 for (i = 0; i < GHES_ESTATUS_CACHES_SIZE; i++) { 591 cache = rcu_dereference(ghes_estatus_caches[i]); 592 if (cache == NULL) 593 continue; 594 if (len != cache->estatus_len) 595 continue; 596 cache_estatus = GHES_ESTATUS_FROM_CACHE(cache); 597 if (memcmp(estatus, cache_estatus, len)) 598 continue; 599 atomic_inc(&cache->count); 600 now = sched_clock(); 601 if (now - cache->time_in < GHES_ESTATUS_IN_CACHE_MAX_NSEC) 602 cached = 1; 603 break; 604 } 605 rcu_read_unlock(); 606 return cached; 607 } 608 609 static struct ghes_estatus_cache *ghes_estatus_cache_alloc( 610 struct acpi_hest_generic *generic, 611 struct acpi_hest_generic_status *estatus) 612 { 613 int alloced; 614 u32 len, cache_len; 615 struct ghes_estatus_cache *cache; 616 struct acpi_hest_generic_status *cache_estatus; 617 618 alloced = atomic_add_return(1, &ghes_estatus_cache_alloced); 619 if (alloced > GHES_ESTATUS_CACHE_ALLOCED_MAX) { 620 atomic_dec(&ghes_estatus_cache_alloced); 621 return NULL; 622 } 623 len = cper_estatus_len(estatus); 624 cache_len = GHES_ESTATUS_CACHE_LEN(len); 625 cache = (void *)gen_pool_alloc(ghes_estatus_pool, cache_len); 626 if (!cache) { 627 atomic_dec(&ghes_estatus_cache_alloced); 628 return NULL; 629 } 630 cache_estatus = GHES_ESTATUS_FROM_CACHE(cache); 631 memcpy(cache_estatus, estatus, len); 632 cache->estatus_len = len; 633 atomic_set(&cache->count, 0); 634 cache->generic = generic; 635 cache->time_in = sched_clock(); 636 return cache; 637 } 638 639 static void ghes_estatus_cache_free(struct ghes_estatus_cache *cache) 640 { 641 u32 len; 642 643 len = cper_estatus_len(GHES_ESTATUS_FROM_CACHE(cache)); 644 len = GHES_ESTATUS_CACHE_LEN(len); 645 gen_pool_free(ghes_estatus_pool, (unsigned long)cache, len); 646 atomic_dec(&ghes_estatus_cache_alloced); 647 } 648 649 static void ghes_estatus_cache_rcu_free(struct rcu_head *head) 650 { 651 struct ghes_estatus_cache *cache; 652 653 cache = container_of(head, struct ghes_estatus_cache, rcu); 654 ghes_estatus_cache_free(cache); 655 } 656 657 static void ghes_estatus_cache_add( 658 struct acpi_hest_generic *generic, 659 struct acpi_hest_generic_status *estatus) 660 { 661 int i, slot = -1, count; 662 unsigned long long now, duration, period, max_period = 0; 663 struct ghes_estatus_cache *cache, *slot_cache = NULL, *new_cache; 664 665 new_cache = ghes_estatus_cache_alloc(generic, estatus); 666 if (new_cache == NULL) 667 return; 668 rcu_read_lock(); 669 now = sched_clock(); 670 for (i = 0; i < GHES_ESTATUS_CACHES_SIZE; i++) { 671 cache = rcu_dereference(ghes_estatus_caches[i]); 672 if (cache == NULL) { 673 slot = i; 674 slot_cache = NULL; 675 break; 676 } 677 duration = now - cache->time_in; 678 if (duration >= GHES_ESTATUS_IN_CACHE_MAX_NSEC) { 679 slot = i; 680 slot_cache = cache; 681 break; 682 } 683 count = atomic_read(&cache->count); 684 period = duration; 685 do_div(period, (count + 1)); 686 if (period > max_period) { 687 max_period = period; 688 slot = i; 689 slot_cache = cache; 690 } 691 } 692 /* new_cache must be put into array after its contents are written */ 693 smp_wmb(); 694 if (slot != -1 && cmpxchg(ghes_estatus_caches + slot, 695 slot_cache, new_cache) == slot_cache) { 696 if (slot_cache) 697 call_rcu(&slot_cache->rcu, ghes_estatus_cache_rcu_free); 698 } else 699 ghes_estatus_cache_free(new_cache); 700 rcu_read_unlock(); 701 } 702 703 static int ghes_ack_error(struct acpi_hest_generic_v2 *gv2) 704 { 705 int rc; 706 u64 val = 0; 707 708 rc = apei_read(&val, &gv2->read_ack_register); 709 if (rc) 710 return rc; 711 712 val &= gv2->read_ack_preserve << gv2->read_ack_register.bit_offset; 713 val |= gv2->read_ack_write << gv2->read_ack_register.bit_offset; 714 715 return apei_write(val, &gv2->read_ack_register); 716 } 717 718 static void __ghes_panic(struct ghes *ghes) 719 { 720 __ghes_print_estatus(KERN_EMERG, ghes->generic, ghes->estatus); 721 722 /* reboot to log the error! */ 723 if (!panic_timeout) 724 panic_timeout = ghes_panic_timeout; 725 panic("Fatal hardware error!"); 726 } 727 728 static int ghes_proc(struct ghes *ghes) 729 { 730 int rc; 731 732 rc = ghes_read_estatus(ghes, 0); 733 if (rc) 734 goto out; 735 736 if (ghes_severity(ghes->estatus->error_severity) >= GHES_SEV_PANIC) { 737 __ghes_panic(ghes); 738 } 739 740 if (!ghes_estatus_cached(ghes->estatus)) { 741 if (ghes_print_estatus(NULL, ghes->generic, ghes->estatus)) 742 ghes_estatus_cache_add(ghes->generic, ghes->estatus); 743 } 744 ghes_do_proc(ghes, ghes->estatus); 745 746 /* 747 * GHESv2 type HEST entries introduce support for error acknowledgment, 748 * so only acknowledge the error if this support is present. 749 */ 750 if (is_hest_type_generic_v2(ghes)) { 751 rc = ghes_ack_error(ghes->generic_v2); 752 if (rc) 753 return rc; 754 } 755 out: 756 ghes_clear_estatus(ghes); 757 return rc; 758 } 759 760 static void ghes_add_timer(struct ghes *ghes) 761 { 762 struct acpi_hest_generic *g = ghes->generic; 763 unsigned long expire; 764 765 if (!g->notify.poll_interval) { 766 pr_warning(FW_WARN GHES_PFX "Poll interval is 0 for generic hardware error source: %d, disabled.\n", 767 g->header.source_id); 768 return; 769 } 770 expire = jiffies + msecs_to_jiffies(g->notify.poll_interval); 771 ghes->timer.expires = round_jiffies_relative(expire); 772 add_timer(&ghes->timer); 773 } 774 775 static void ghes_poll_func(unsigned long data) 776 { 777 struct ghes *ghes = (void *)data; 778 779 ghes_proc(ghes); 780 if (!(ghes->flags & GHES_EXITING)) 781 ghes_add_timer(ghes); 782 } 783 784 static irqreturn_t ghes_irq_func(int irq, void *data) 785 { 786 struct ghes *ghes = data; 787 int rc; 788 789 rc = ghes_proc(ghes); 790 if (rc) 791 return IRQ_NONE; 792 793 return IRQ_HANDLED; 794 } 795 796 static int ghes_notify_hed(struct notifier_block *this, unsigned long event, 797 void *data) 798 { 799 struct ghes *ghes; 800 int ret = NOTIFY_DONE; 801 802 rcu_read_lock(); 803 list_for_each_entry_rcu(ghes, &ghes_hed, list) { 804 if (!ghes_proc(ghes)) 805 ret = NOTIFY_OK; 806 } 807 rcu_read_unlock(); 808 809 return ret; 810 } 811 812 static struct notifier_block ghes_notifier_hed = { 813 .notifier_call = ghes_notify_hed, 814 }; 815 816 #ifdef CONFIG_ACPI_APEI_SEA 817 static LIST_HEAD(ghes_sea); 818 819 /* 820 * Return 0 only if one of the SEA error sources successfully reported an error 821 * record sent from the firmware. 822 */ 823 int ghes_notify_sea(void) 824 { 825 struct ghes *ghes; 826 int ret = -ENOENT; 827 828 rcu_read_lock(); 829 list_for_each_entry_rcu(ghes, &ghes_sea, list) { 830 if (!ghes_proc(ghes)) 831 ret = 0; 832 } 833 rcu_read_unlock(); 834 return ret; 835 } 836 837 static void ghes_sea_add(struct ghes *ghes) 838 { 839 mutex_lock(&ghes_list_mutex); 840 list_add_rcu(&ghes->list, &ghes_sea); 841 mutex_unlock(&ghes_list_mutex); 842 } 843 844 static void ghes_sea_remove(struct ghes *ghes) 845 { 846 mutex_lock(&ghes_list_mutex); 847 list_del_rcu(&ghes->list); 848 mutex_unlock(&ghes_list_mutex); 849 synchronize_rcu(); 850 } 851 #else /* CONFIG_ACPI_APEI_SEA */ 852 static inline void ghes_sea_add(struct ghes *ghes) 853 { 854 pr_err(GHES_PFX "ID: %d, trying to add SEA notification which is not supported\n", 855 ghes->generic->header.source_id); 856 } 857 858 static inline void ghes_sea_remove(struct ghes *ghes) 859 { 860 pr_err(GHES_PFX "ID: %d, trying to remove SEA notification which is not supported\n", 861 ghes->generic->header.source_id); 862 } 863 #endif /* CONFIG_ACPI_APEI_SEA */ 864 865 #ifdef CONFIG_HAVE_ACPI_APEI_NMI 866 /* 867 * printk is not safe in NMI context. So in NMI handler, we allocate 868 * required memory from lock-less memory allocator 869 * (ghes_estatus_pool), save estatus into it, put them into lock-less 870 * list (ghes_estatus_llist), then delay printk into IRQ context via 871 * irq_work (ghes_proc_irq_work). ghes_estatus_size_request record 872 * required pool size by all NMI error source. 873 */ 874 static struct llist_head ghes_estatus_llist; 875 static struct irq_work ghes_proc_irq_work; 876 877 /* 878 * NMI may be triggered on any CPU, so ghes_in_nmi is used for 879 * having only one concurrent reader. 880 */ 881 static atomic_t ghes_in_nmi = ATOMIC_INIT(0); 882 883 static LIST_HEAD(ghes_nmi); 884 885 static void ghes_proc_in_irq(struct irq_work *irq_work) 886 { 887 struct llist_node *llnode, *next; 888 struct ghes_estatus_node *estatus_node; 889 struct acpi_hest_generic *generic; 890 struct acpi_hest_generic_status *estatus; 891 u32 len, node_len; 892 893 llnode = llist_del_all(&ghes_estatus_llist); 894 /* 895 * Because the time order of estatus in list is reversed, 896 * revert it back to proper order. 897 */ 898 llnode = llist_reverse_order(llnode); 899 while (llnode) { 900 next = llnode->next; 901 estatus_node = llist_entry(llnode, struct ghes_estatus_node, 902 llnode); 903 estatus = GHES_ESTATUS_FROM_NODE(estatus_node); 904 len = cper_estatus_len(estatus); 905 node_len = GHES_ESTATUS_NODE_LEN(len); 906 ghes_do_proc(estatus_node->ghes, estatus); 907 if (!ghes_estatus_cached(estatus)) { 908 generic = estatus_node->generic; 909 if (ghes_print_estatus(NULL, generic, estatus)) 910 ghes_estatus_cache_add(generic, estatus); 911 } 912 gen_pool_free(ghes_estatus_pool, (unsigned long)estatus_node, 913 node_len); 914 llnode = next; 915 } 916 } 917 918 static void ghes_print_queued_estatus(void) 919 { 920 struct llist_node *llnode; 921 struct ghes_estatus_node *estatus_node; 922 struct acpi_hest_generic *generic; 923 struct acpi_hest_generic_status *estatus; 924 u32 len, node_len; 925 926 llnode = llist_del_all(&ghes_estatus_llist); 927 /* 928 * Because the time order of estatus in list is reversed, 929 * revert it back to proper order. 930 */ 931 llnode = llist_reverse_order(llnode); 932 while (llnode) { 933 estatus_node = llist_entry(llnode, struct ghes_estatus_node, 934 llnode); 935 estatus = GHES_ESTATUS_FROM_NODE(estatus_node); 936 len = cper_estatus_len(estatus); 937 node_len = GHES_ESTATUS_NODE_LEN(len); 938 generic = estatus_node->generic; 939 ghes_print_estatus(NULL, generic, estatus); 940 llnode = llnode->next; 941 } 942 } 943 944 /* Save estatus for further processing in IRQ context */ 945 static void __process_error(struct ghes *ghes) 946 { 947 #ifdef CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG 948 u32 len, node_len; 949 struct ghes_estatus_node *estatus_node; 950 struct acpi_hest_generic_status *estatus; 951 952 if (ghes_estatus_cached(ghes->estatus)) 953 return; 954 955 len = cper_estatus_len(ghes->estatus); 956 node_len = GHES_ESTATUS_NODE_LEN(len); 957 958 estatus_node = (void *)gen_pool_alloc(ghes_estatus_pool, node_len); 959 if (!estatus_node) 960 return; 961 962 estatus_node->ghes = ghes; 963 estatus_node->generic = ghes->generic; 964 estatus = GHES_ESTATUS_FROM_NODE(estatus_node); 965 memcpy(estatus, ghes->estatus, len); 966 llist_add(&estatus_node->llnode, &ghes_estatus_llist); 967 #endif 968 } 969 970 static int ghes_notify_nmi(unsigned int cmd, struct pt_regs *regs) 971 { 972 struct ghes *ghes; 973 int sev, ret = NMI_DONE; 974 975 if (!atomic_add_unless(&ghes_in_nmi, 1, 1)) 976 return ret; 977 978 list_for_each_entry_rcu(ghes, &ghes_nmi, list) { 979 if (ghes_read_estatus(ghes, 1)) { 980 ghes_clear_estatus(ghes); 981 continue; 982 } else { 983 ret = NMI_HANDLED; 984 } 985 986 sev = ghes_severity(ghes->estatus->error_severity); 987 if (sev >= GHES_SEV_PANIC) { 988 oops_begin(); 989 ghes_print_queued_estatus(); 990 __ghes_panic(ghes); 991 } 992 993 if (!(ghes->flags & GHES_TO_CLEAR)) 994 continue; 995 996 __process_error(ghes); 997 ghes_clear_estatus(ghes); 998 } 999 1000 #ifdef CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG 1001 if (ret == NMI_HANDLED) 1002 irq_work_queue(&ghes_proc_irq_work); 1003 #endif 1004 atomic_dec(&ghes_in_nmi); 1005 return ret; 1006 } 1007 1008 static unsigned long ghes_esource_prealloc_size( 1009 const struct acpi_hest_generic *generic) 1010 { 1011 unsigned long block_length, prealloc_records, prealloc_size; 1012 1013 block_length = min_t(unsigned long, generic->error_block_length, 1014 GHES_ESTATUS_MAX_SIZE); 1015 prealloc_records = max_t(unsigned long, 1016 generic->records_to_preallocate, 1); 1017 prealloc_size = min_t(unsigned long, block_length * prealloc_records, 1018 GHES_ESOURCE_PREALLOC_MAX_SIZE); 1019 1020 return prealloc_size; 1021 } 1022 1023 static void ghes_estatus_pool_shrink(unsigned long len) 1024 { 1025 ghes_estatus_pool_size_request -= PAGE_ALIGN(len); 1026 } 1027 1028 static void ghes_nmi_add(struct ghes *ghes) 1029 { 1030 unsigned long len; 1031 1032 len = ghes_esource_prealloc_size(ghes->generic); 1033 ghes_estatus_pool_expand(len); 1034 mutex_lock(&ghes_list_mutex); 1035 if (list_empty(&ghes_nmi)) 1036 register_nmi_handler(NMI_LOCAL, ghes_notify_nmi, 0, "ghes"); 1037 list_add_rcu(&ghes->list, &ghes_nmi); 1038 mutex_unlock(&ghes_list_mutex); 1039 } 1040 1041 static void ghes_nmi_remove(struct ghes *ghes) 1042 { 1043 unsigned long len; 1044 1045 mutex_lock(&ghes_list_mutex); 1046 list_del_rcu(&ghes->list); 1047 if (list_empty(&ghes_nmi)) 1048 unregister_nmi_handler(NMI_LOCAL, "ghes"); 1049 mutex_unlock(&ghes_list_mutex); 1050 /* 1051 * To synchronize with NMI handler, ghes can only be 1052 * freed after NMI handler finishes. 1053 */ 1054 synchronize_rcu(); 1055 len = ghes_esource_prealloc_size(ghes->generic); 1056 ghes_estatus_pool_shrink(len); 1057 } 1058 1059 static void ghes_nmi_init_cxt(void) 1060 { 1061 init_irq_work(&ghes_proc_irq_work, ghes_proc_in_irq); 1062 } 1063 #else /* CONFIG_HAVE_ACPI_APEI_NMI */ 1064 static inline void ghes_nmi_add(struct ghes *ghes) 1065 { 1066 pr_err(GHES_PFX "ID: %d, trying to add NMI notification which is not supported!\n", 1067 ghes->generic->header.source_id); 1068 BUG(); 1069 } 1070 1071 static inline void ghes_nmi_remove(struct ghes *ghes) 1072 { 1073 pr_err(GHES_PFX "ID: %d, trying to remove NMI notification which is not supported!\n", 1074 ghes->generic->header.source_id); 1075 BUG(); 1076 } 1077 1078 static inline void ghes_nmi_init_cxt(void) 1079 { 1080 } 1081 #endif /* CONFIG_HAVE_ACPI_APEI_NMI */ 1082 1083 static int ghes_probe(struct platform_device *ghes_dev) 1084 { 1085 struct acpi_hest_generic *generic; 1086 struct ghes *ghes = NULL; 1087 1088 int rc = -EINVAL; 1089 1090 generic = *(struct acpi_hest_generic **)ghes_dev->dev.platform_data; 1091 if (!generic->enabled) 1092 return -ENODEV; 1093 1094 switch (generic->notify.type) { 1095 case ACPI_HEST_NOTIFY_POLLED: 1096 case ACPI_HEST_NOTIFY_EXTERNAL: 1097 case ACPI_HEST_NOTIFY_SCI: 1098 case ACPI_HEST_NOTIFY_GSIV: 1099 case ACPI_HEST_NOTIFY_GPIO: 1100 break; 1101 1102 case ACPI_HEST_NOTIFY_SEA: 1103 if (!IS_ENABLED(CONFIG_ACPI_APEI_SEA)) { 1104 pr_warn(GHES_PFX "Generic hardware error source: %d notified via SEA is not supported\n", 1105 generic->header.source_id); 1106 rc = -ENOTSUPP; 1107 goto err; 1108 } 1109 break; 1110 case ACPI_HEST_NOTIFY_NMI: 1111 if (!IS_ENABLED(CONFIG_HAVE_ACPI_APEI_NMI)) { 1112 pr_warn(GHES_PFX "Generic hardware error source: %d notified via NMI interrupt is not supported!\n", 1113 generic->header.source_id); 1114 goto err; 1115 } 1116 break; 1117 case ACPI_HEST_NOTIFY_LOCAL: 1118 pr_warning(GHES_PFX "Generic hardware error source: %d notified via local interrupt is not supported!\n", 1119 generic->header.source_id); 1120 goto err; 1121 default: 1122 pr_warning(FW_WARN GHES_PFX "Unknown notification type: %u for generic hardware error source: %d\n", 1123 generic->notify.type, generic->header.source_id); 1124 goto err; 1125 } 1126 1127 rc = -EIO; 1128 if (generic->error_block_length < 1129 sizeof(struct acpi_hest_generic_status)) { 1130 pr_warning(FW_BUG GHES_PFX "Invalid error block length: %u for generic hardware error source: %d\n", 1131 generic->error_block_length, 1132 generic->header.source_id); 1133 goto err; 1134 } 1135 ghes = ghes_new(generic); 1136 if (IS_ERR(ghes)) { 1137 rc = PTR_ERR(ghes); 1138 ghes = NULL; 1139 goto err; 1140 } 1141 1142 rc = ghes_edac_register(ghes, &ghes_dev->dev); 1143 if (rc < 0) 1144 goto err; 1145 1146 switch (generic->notify.type) { 1147 case ACPI_HEST_NOTIFY_POLLED: 1148 setup_deferrable_timer(&ghes->timer, ghes_poll_func, 1149 (unsigned long)ghes); 1150 ghes_add_timer(ghes); 1151 break; 1152 case ACPI_HEST_NOTIFY_EXTERNAL: 1153 /* External interrupt vector is GSI */ 1154 rc = acpi_gsi_to_irq(generic->notify.vector, &ghes->irq); 1155 if (rc) { 1156 pr_err(GHES_PFX "Failed to map GSI to IRQ for generic hardware error source: %d\n", 1157 generic->header.source_id); 1158 goto err_edac_unreg; 1159 } 1160 rc = request_irq(ghes->irq, ghes_irq_func, 0, "GHES IRQ", ghes); 1161 if (rc) { 1162 pr_err(GHES_PFX "Failed to register IRQ for generic hardware error source: %d\n", 1163 generic->header.source_id); 1164 goto err_edac_unreg; 1165 } 1166 break; 1167 1168 case ACPI_HEST_NOTIFY_SCI: 1169 case ACPI_HEST_NOTIFY_GSIV: 1170 case ACPI_HEST_NOTIFY_GPIO: 1171 mutex_lock(&ghes_list_mutex); 1172 if (list_empty(&ghes_hed)) 1173 register_acpi_hed_notifier(&ghes_notifier_hed); 1174 list_add_rcu(&ghes->list, &ghes_hed); 1175 mutex_unlock(&ghes_list_mutex); 1176 break; 1177 1178 case ACPI_HEST_NOTIFY_SEA: 1179 ghes_sea_add(ghes); 1180 break; 1181 case ACPI_HEST_NOTIFY_NMI: 1182 ghes_nmi_add(ghes); 1183 break; 1184 default: 1185 BUG(); 1186 } 1187 platform_set_drvdata(ghes_dev, ghes); 1188 1189 /* Handle any pending errors right away */ 1190 ghes_proc(ghes); 1191 1192 return 0; 1193 err_edac_unreg: 1194 ghes_edac_unregister(ghes); 1195 err: 1196 if (ghes) { 1197 ghes_fini(ghes); 1198 kfree(ghes); 1199 } 1200 return rc; 1201 } 1202 1203 static int ghes_remove(struct platform_device *ghes_dev) 1204 { 1205 struct ghes *ghes; 1206 struct acpi_hest_generic *generic; 1207 1208 ghes = platform_get_drvdata(ghes_dev); 1209 generic = ghes->generic; 1210 1211 ghes->flags |= GHES_EXITING; 1212 switch (generic->notify.type) { 1213 case ACPI_HEST_NOTIFY_POLLED: 1214 del_timer_sync(&ghes->timer); 1215 break; 1216 case ACPI_HEST_NOTIFY_EXTERNAL: 1217 free_irq(ghes->irq, ghes); 1218 break; 1219 1220 case ACPI_HEST_NOTIFY_SCI: 1221 case ACPI_HEST_NOTIFY_GSIV: 1222 case ACPI_HEST_NOTIFY_GPIO: 1223 mutex_lock(&ghes_list_mutex); 1224 list_del_rcu(&ghes->list); 1225 if (list_empty(&ghes_hed)) 1226 unregister_acpi_hed_notifier(&ghes_notifier_hed); 1227 mutex_unlock(&ghes_list_mutex); 1228 synchronize_rcu(); 1229 break; 1230 1231 case ACPI_HEST_NOTIFY_SEA: 1232 ghes_sea_remove(ghes); 1233 break; 1234 case ACPI_HEST_NOTIFY_NMI: 1235 ghes_nmi_remove(ghes); 1236 break; 1237 default: 1238 BUG(); 1239 break; 1240 } 1241 1242 ghes_fini(ghes); 1243 1244 ghes_edac_unregister(ghes); 1245 1246 kfree(ghes); 1247 1248 platform_set_drvdata(ghes_dev, NULL); 1249 1250 return 0; 1251 } 1252 1253 static struct platform_driver ghes_platform_driver = { 1254 .driver = { 1255 .name = "GHES", 1256 }, 1257 .probe = ghes_probe, 1258 .remove = ghes_remove, 1259 }; 1260 1261 static int __init ghes_init(void) 1262 { 1263 int rc; 1264 1265 if (acpi_disabled) 1266 return -ENODEV; 1267 1268 if (hest_disable) { 1269 pr_info(GHES_PFX "HEST is not enabled!\n"); 1270 return -EINVAL; 1271 } 1272 1273 if (ghes_disable) { 1274 pr_info(GHES_PFX "GHES is not enabled!\n"); 1275 return -EINVAL; 1276 } 1277 1278 ghes_nmi_init_cxt(); 1279 1280 rc = ghes_ioremap_init(); 1281 if (rc) 1282 goto err; 1283 1284 rc = ghes_estatus_pool_init(); 1285 if (rc) 1286 goto err_ioremap_exit; 1287 1288 rc = ghes_estatus_pool_expand(GHES_ESTATUS_CACHE_AVG_SIZE * 1289 GHES_ESTATUS_CACHE_ALLOCED_MAX); 1290 if (rc) 1291 goto err_pool_exit; 1292 1293 rc = platform_driver_register(&ghes_platform_driver); 1294 if (rc) 1295 goto err_pool_exit; 1296 1297 rc = apei_osc_setup(); 1298 if (rc == 0 && osc_sb_apei_support_acked) 1299 pr_info(GHES_PFX "APEI firmware first mode is enabled by APEI bit and WHEA _OSC.\n"); 1300 else if (rc == 0 && !osc_sb_apei_support_acked) 1301 pr_info(GHES_PFX "APEI firmware first mode is enabled by WHEA _OSC.\n"); 1302 else if (rc && osc_sb_apei_support_acked) 1303 pr_info(GHES_PFX "APEI firmware first mode is enabled by APEI bit.\n"); 1304 else 1305 pr_info(GHES_PFX "Failed to enable APEI firmware first mode.\n"); 1306 1307 return 0; 1308 err_pool_exit: 1309 ghes_estatus_pool_exit(); 1310 err_ioremap_exit: 1311 ghes_ioremap_exit(); 1312 err: 1313 return rc; 1314 } 1315 device_initcall(ghes_init); 1316