1 /* 2 * APEI Generic Hardware Error Source support 3 * 4 * Generic Hardware Error Source provides a way to report platform 5 * hardware errors (such as that from chipset). It works in so called 6 * "Firmware First" mode, that is, hardware errors are reported to 7 * firmware firstly, then reported to Linux by firmware. This way, 8 * some non-standard hardware error registers or non-standard hardware 9 * link can be checked by firmware to produce more hardware error 10 * information for Linux. 11 * 12 * For more information about Generic Hardware Error Source, please 13 * refer to ACPI Specification version 4.0, section 17.3.2.6 14 * 15 * Copyright 2010,2011 Intel Corp. 16 * Author: Huang Ying <ying.huang@intel.com> 17 * 18 * This program is free software; you can redistribute it and/or 19 * modify it under the terms of the GNU General Public License version 20 * 2 as published by the Free Software Foundation; 21 * 22 * This program is distributed in the hope that it will be useful, 23 * but WITHOUT ANY WARRANTY; without even the implied warranty of 24 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 25 * GNU General Public License for more details. 26 */ 27 28 #include <linux/kernel.h> 29 #include <linux/moduleparam.h> 30 #include <linux/init.h> 31 #include <linux/acpi.h> 32 #include <linux/io.h> 33 #include <linux/interrupt.h> 34 #include <linux/timer.h> 35 #include <linux/cper.h> 36 #include <linux/kdebug.h> 37 #include <linux/platform_device.h> 38 #include <linux/mutex.h> 39 #include <linux/ratelimit.h> 40 #include <linux/vmalloc.h> 41 #include <linux/irq_work.h> 42 #include <linux/llist.h> 43 #include <linux/genalloc.h> 44 #include <linux/pci.h> 45 #include <linux/aer.h> 46 #include <linux/nmi.h> 47 #include <linux/sched/clock.h> 48 #include <linux/uuid.h> 49 #include <linux/ras.h> 50 51 #include <acpi/actbl1.h> 52 #include <acpi/ghes.h> 53 #include <acpi/apei.h> 54 #include <asm/tlbflush.h> 55 #include <ras/ras_event.h> 56 57 #include "apei-internal.h" 58 59 #define GHES_PFX "GHES: " 60 61 #define GHES_ESTATUS_MAX_SIZE 65536 62 #define GHES_ESOURCE_PREALLOC_MAX_SIZE 65536 63 64 #define GHES_ESTATUS_POOL_MIN_ALLOC_ORDER 3 65 66 /* This is just an estimation for memory pool allocation */ 67 #define GHES_ESTATUS_CACHE_AVG_SIZE 512 68 69 #define GHES_ESTATUS_CACHES_SIZE 4 70 71 #define GHES_ESTATUS_IN_CACHE_MAX_NSEC 10000000000ULL 72 /* Prevent too many caches are allocated because of RCU */ 73 #define GHES_ESTATUS_CACHE_ALLOCED_MAX (GHES_ESTATUS_CACHES_SIZE * 3 / 2) 74 75 #define GHES_ESTATUS_CACHE_LEN(estatus_len) \ 76 (sizeof(struct ghes_estatus_cache) + (estatus_len)) 77 #define GHES_ESTATUS_FROM_CACHE(estatus_cache) \ 78 ((struct acpi_hest_generic_status *) \ 79 ((struct ghes_estatus_cache *)(estatus_cache) + 1)) 80 81 #define GHES_ESTATUS_NODE_LEN(estatus_len) \ 82 (sizeof(struct ghes_estatus_node) + (estatus_len)) 83 #define GHES_ESTATUS_FROM_NODE(estatus_node) \ 84 ((struct acpi_hest_generic_status *) \ 85 ((struct ghes_estatus_node *)(estatus_node) + 1)) 86 87 static inline bool is_hest_type_generic_v2(struct ghes *ghes) 88 { 89 return ghes->generic->header.type == ACPI_HEST_TYPE_GENERIC_ERROR_V2; 90 } 91 92 /* 93 * This driver isn't really modular, however for the time being, 94 * continuing to use module_param is the easiest way to remain 95 * compatible with existing boot arg use cases. 96 */ 97 bool ghes_disable; 98 module_param_named(disable, ghes_disable, bool, 0); 99 100 /* 101 * All error sources notified with HED (Hardware Error Device) share a 102 * single notifier callback, so they need to be linked and checked one 103 * by one. This holds true for NMI too. 104 * 105 * RCU is used for these lists, so ghes_list_mutex is only used for 106 * list changing, not for traversing. 107 */ 108 static LIST_HEAD(ghes_hed); 109 static DEFINE_MUTEX(ghes_list_mutex); 110 111 /* 112 * Because the memory area used to transfer hardware error information 113 * from BIOS to Linux can be determined only in NMI, IRQ or timer 114 * handler, but general ioremap can not be used in atomic context, so 115 * a special version of atomic ioremap is implemented for that. 116 */ 117 118 /* 119 * Two virtual pages are used, one for IRQ/PROCESS context, the other for 120 * NMI context (optionally). 121 */ 122 #define GHES_IOREMAP_PAGES 2 123 #define GHES_IOREMAP_IRQ_PAGE(base) (base) 124 #define GHES_IOREMAP_NMI_PAGE(base) ((base) + PAGE_SIZE) 125 126 /* virtual memory area for atomic ioremap */ 127 static struct vm_struct *ghes_ioremap_area; 128 /* 129 * These 2 spinlock is used to prevent atomic ioremap virtual memory 130 * area from being mapped simultaneously. 131 */ 132 static DEFINE_RAW_SPINLOCK(ghes_ioremap_lock_nmi); 133 static DEFINE_SPINLOCK(ghes_ioremap_lock_irq); 134 135 static struct gen_pool *ghes_estatus_pool; 136 static unsigned long ghes_estatus_pool_size_request; 137 138 static struct ghes_estatus_cache *ghes_estatus_caches[GHES_ESTATUS_CACHES_SIZE]; 139 static atomic_t ghes_estatus_cache_alloced; 140 141 static int ghes_panic_timeout __read_mostly = 30; 142 143 static int ghes_ioremap_init(void) 144 { 145 ghes_ioremap_area = __get_vm_area(PAGE_SIZE * GHES_IOREMAP_PAGES, 146 VM_IOREMAP, VMALLOC_START, VMALLOC_END); 147 if (!ghes_ioremap_area) { 148 pr_err(GHES_PFX "Failed to allocate virtual memory area for atomic ioremap.\n"); 149 return -ENOMEM; 150 } 151 152 return 0; 153 } 154 155 static void ghes_ioremap_exit(void) 156 { 157 free_vm_area(ghes_ioremap_area); 158 } 159 160 static void __iomem *ghes_ioremap_pfn_nmi(u64 pfn) 161 { 162 unsigned long vaddr; 163 phys_addr_t paddr; 164 pgprot_t prot; 165 166 vaddr = (unsigned long)GHES_IOREMAP_NMI_PAGE(ghes_ioremap_area->addr); 167 168 paddr = pfn << PAGE_SHIFT; 169 prot = arch_apei_get_mem_attribute(paddr); 170 ioremap_page_range(vaddr, vaddr + PAGE_SIZE, paddr, prot); 171 172 return (void __iomem *)vaddr; 173 } 174 175 static void __iomem *ghes_ioremap_pfn_irq(u64 pfn) 176 { 177 unsigned long vaddr, paddr; 178 pgprot_t prot; 179 180 vaddr = (unsigned long)GHES_IOREMAP_IRQ_PAGE(ghes_ioremap_area->addr); 181 182 paddr = pfn << PAGE_SHIFT; 183 prot = arch_apei_get_mem_attribute(paddr); 184 185 ioremap_page_range(vaddr, vaddr + PAGE_SIZE, paddr, prot); 186 187 return (void __iomem *)vaddr; 188 } 189 190 static void ghes_iounmap_nmi(void __iomem *vaddr_ptr) 191 { 192 unsigned long vaddr = (unsigned long __force)vaddr_ptr; 193 void *base = ghes_ioremap_area->addr; 194 195 BUG_ON(vaddr != (unsigned long)GHES_IOREMAP_NMI_PAGE(base)); 196 unmap_kernel_range_noflush(vaddr, PAGE_SIZE); 197 arch_apei_flush_tlb_one(vaddr); 198 } 199 200 static void ghes_iounmap_irq(void __iomem *vaddr_ptr) 201 { 202 unsigned long vaddr = (unsigned long __force)vaddr_ptr; 203 void *base = ghes_ioremap_area->addr; 204 205 BUG_ON(vaddr != (unsigned long)GHES_IOREMAP_IRQ_PAGE(base)); 206 unmap_kernel_range_noflush(vaddr, PAGE_SIZE); 207 arch_apei_flush_tlb_one(vaddr); 208 } 209 210 static int ghes_estatus_pool_init(void) 211 { 212 ghes_estatus_pool = gen_pool_create(GHES_ESTATUS_POOL_MIN_ALLOC_ORDER, -1); 213 if (!ghes_estatus_pool) 214 return -ENOMEM; 215 return 0; 216 } 217 218 static void ghes_estatus_pool_free_chunk_page(struct gen_pool *pool, 219 struct gen_pool_chunk *chunk, 220 void *data) 221 { 222 free_page(chunk->start_addr); 223 } 224 225 static void ghes_estatus_pool_exit(void) 226 { 227 gen_pool_for_each_chunk(ghes_estatus_pool, 228 ghes_estatus_pool_free_chunk_page, NULL); 229 gen_pool_destroy(ghes_estatus_pool); 230 } 231 232 static int ghes_estatus_pool_expand(unsigned long len) 233 { 234 unsigned long i, pages, size, addr; 235 int ret; 236 237 ghes_estatus_pool_size_request += PAGE_ALIGN(len); 238 size = gen_pool_size(ghes_estatus_pool); 239 if (size >= ghes_estatus_pool_size_request) 240 return 0; 241 pages = (ghes_estatus_pool_size_request - size) / PAGE_SIZE; 242 for (i = 0; i < pages; i++) { 243 addr = __get_free_page(GFP_KERNEL); 244 if (!addr) 245 return -ENOMEM; 246 ret = gen_pool_add(ghes_estatus_pool, addr, PAGE_SIZE, -1); 247 if (ret) 248 return ret; 249 } 250 251 return 0; 252 } 253 254 static int map_gen_v2(struct ghes *ghes) 255 { 256 return apei_map_generic_address(&ghes->generic_v2->read_ack_register); 257 } 258 259 static void unmap_gen_v2(struct ghes *ghes) 260 { 261 apei_unmap_generic_address(&ghes->generic_v2->read_ack_register); 262 } 263 264 static struct ghes *ghes_new(struct acpi_hest_generic *generic) 265 { 266 struct ghes *ghes; 267 unsigned int error_block_length; 268 int rc; 269 270 ghes = kzalloc(sizeof(*ghes), GFP_KERNEL); 271 if (!ghes) 272 return ERR_PTR(-ENOMEM); 273 274 ghes->generic = generic; 275 if (is_hest_type_generic_v2(ghes)) { 276 rc = map_gen_v2(ghes); 277 if (rc) 278 goto err_free; 279 } 280 281 rc = apei_map_generic_address(&generic->error_status_address); 282 if (rc) 283 goto err_unmap_read_ack_addr; 284 error_block_length = generic->error_block_length; 285 if (error_block_length > GHES_ESTATUS_MAX_SIZE) { 286 pr_warning(FW_WARN GHES_PFX 287 "Error status block length is too long: %u for " 288 "generic hardware error source: %d.\n", 289 error_block_length, generic->header.source_id); 290 error_block_length = GHES_ESTATUS_MAX_SIZE; 291 } 292 ghes->estatus = kmalloc(error_block_length, GFP_KERNEL); 293 if (!ghes->estatus) { 294 rc = -ENOMEM; 295 goto err_unmap_status_addr; 296 } 297 298 return ghes; 299 300 err_unmap_status_addr: 301 apei_unmap_generic_address(&generic->error_status_address); 302 err_unmap_read_ack_addr: 303 if (is_hest_type_generic_v2(ghes)) 304 unmap_gen_v2(ghes); 305 err_free: 306 kfree(ghes); 307 return ERR_PTR(rc); 308 } 309 310 static void ghes_fini(struct ghes *ghes) 311 { 312 kfree(ghes->estatus); 313 apei_unmap_generic_address(&ghes->generic->error_status_address); 314 if (is_hest_type_generic_v2(ghes)) 315 unmap_gen_v2(ghes); 316 } 317 318 static inline int ghes_severity(int severity) 319 { 320 switch (severity) { 321 case CPER_SEV_INFORMATIONAL: 322 return GHES_SEV_NO; 323 case CPER_SEV_CORRECTED: 324 return GHES_SEV_CORRECTED; 325 case CPER_SEV_RECOVERABLE: 326 return GHES_SEV_RECOVERABLE; 327 case CPER_SEV_FATAL: 328 return GHES_SEV_PANIC; 329 default: 330 /* Unknown, go panic */ 331 return GHES_SEV_PANIC; 332 } 333 } 334 335 static void ghes_copy_tofrom_phys(void *buffer, u64 paddr, u32 len, 336 int from_phys) 337 { 338 void __iomem *vaddr; 339 unsigned long flags = 0; 340 int in_nmi = in_nmi(); 341 u64 offset; 342 u32 trunk; 343 344 while (len > 0) { 345 offset = paddr - (paddr & PAGE_MASK); 346 if (in_nmi) { 347 raw_spin_lock(&ghes_ioremap_lock_nmi); 348 vaddr = ghes_ioremap_pfn_nmi(paddr >> PAGE_SHIFT); 349 } else { 350 spin_lock_irqsave(&ghes_ioremap_lock_irq, flags); 351 vaddr = ghes_ioremap_pfn_irq(paddr >> PAGE_SHIFT); 352 } 353 trunk = PAGE_SIZE - offset; 354 trunk = min(trunk, len); 355 if (from_phys) 356 memcpy_fromio(buffer, vaddr + offset, trunk); 357 else 358 memcpy_toio(vaddr + offset, buffer, trunk); 359 len -= trunk; 360 paddr += trunk; 361 buffer += trunk; 362 if (in_nmi) { 363 ghes_iounmap_nmi(vaddr); 364 raw_spin_unlock(&ghes_ioremap_lock_nmi); 365 } else { 366 ghes_iounmap_irq(vaddr); 367 spin_unlock_irqrestore(&ghes_ioremap_lock_irq, flags); 368 } 369 } 370 } 371 372 static int ghes_read_estatus(struct ghes *ghes, int silent) 373 { 374 struct acpi_hest_generic *g = ghes->generic; 375 u64 buf_paddr; 376 u32 len; 377 int rc; 378 379 rc = apei_read(&buf_paddr, &g->error_status_address); 380 if (rc) { 381 if (!silent && printk_ratelimit()) 382 pr_warning(FW_WARN GHES_PFX 383 "Failed to read error status block address for hardware error source: %d.\n", 384 g->header.source_id); 385 return -EIO; 386 } 387 if (!buf_paddr) 388 return -ENOENT; 389 390 ghes_copy_tofrom_phys(ghes->estatus, buf_paddr, 391 sizeof(*ghes->estatus), 1); 392 if (!ghes->estatus->block_status) 393 return -ENOENT; 394 395 ghes->buffer_paddr = buf_paddr; 396 ghes->flags |= GHES_TO_CLEAR; 397 398 rc = -EIO; 399 len = cper_estatus_len(ghes->estatus); 400 if (len < sizeof(*ghes->estatus)) 401 goto err_read_block; 402 if (len > ghes->generic->error_block_length) 403 goto err_read_block; 404 if (cper_estatus_check_header(ghes->estatus)) 405 goto err_read_block; 406 ghes_copy_tofrom_phys(ghes->estatus + 1, 407 buf_paddr + sizeof(*ghes->estatus), 408 len - sizeof(*ghes->estatus), 1); 409 if (cper_estatus_check(ghes->estatus)) 410 goto err_read_block; 411 rc = 0; 412 413 err_read_block: 414 if (rc && !silent && printk_ratelimit()) 415 pr_warning(FW_WARN GHES_PFX 416 "Failed to read error status block!\n"); 417 return rc; 418 } 419 420 static void ghes_clear_estatus(struct ghes *ghes) 421 { 422 ghes->estatus->block_status = 0; 423 if (!(ghes->flags & GHES_TO_CLEAR)) 424 return; 425 ghes_copy_tofrom_phys(ghes->estatus, ghes->buffer_paddr, 426 sizeof(ghes->estatus->block_status), 0); 427 ghes->flags &= ~GHES_TO_CLEAR; 428 } 429 430 static void ghes_handle_memory_failure(struct acpi_hest_generic_data *gdata, int sev) 431 { 432 #ifdef CONFIG_ACPI_APEI_MEMORY_FAILURE 433 unsigned long pfn; 434 int flags = -1; 435 int sec_sev = ghes_severity(gdata->error_severity); 436 struct cper_sec_mem_err *mem_err = acpi_hest_get_payload(gdata); 437 438 if (!(mem_err->validation_bits & CPER_MEM_VALID_PA)) 439 return; 440 441 pfn = mem_err->physical_addr >> PAGE_SHIFT; 442 if (!pfn_valid(pfn)) { 443 pr_warn_ratelimited(FW_WARN GHES_PFX 444 "Invalid address in generic error data: %#llx\n", 445 mem_err->physical_addr); 446 return; 447 } 448 449 /* iff following two events can be handled properly by now */ 450 if (sec_sev == GHES_SEV_CORRECTED && 451 (gdata->flags & CPER_SEC_ERROR_THRESHOLD_EXCEEDED)) 452 flags = MF_SOFT_OFFLINE; 453 if (sev == GHES_SEV_RECOVERABLE && sec_sev == GHES_SEV_RECOVERABLE) 454 flags = 0; 455 456 if (flags != -1) 457 memory_failure_queue(pfn, 0, flags); 458 #endif 459 } 460 461 static void ghes_do_proc(struct ghes *ghes, 462 const struct acpi_hest_generic_status *estatus) 463 { 464 int sev, sec_sev; 465 struct acpi_hest_generic_data *gdata; 466 guid_t *sec_type; 467 guid_t *fru_id = &NULL_UUID_LE; 468 char *fru_text = ""; 469 470 sev = ghes_severity(estatus->error_severity); 471 apei_estatus_for_each_section(estatus, gdata) { 472 sec_type = (guid_t *)gdata->section_type; 473 sec_sev = ghes_severity(gdata->error_severity); 474 if (gdata->validation_bits & CPER_SEC_VALID_FRU_ID) 475 fru_id = (guid_t *)gdata->fru_id; 476 477 if (gdata->validation_bits & CPER_SEC_VALID_FRU_TEXT) 478 fru_text = gdata->fru_text; 479 480 if (guid_equal(sec_type, &CPER_SEC_PLATFORM_MEM)) { 481 struct cper_sec_mem_err *mem_err = acpi_hest_get_payload(gdata); 482 483 ghes_edac_report_mem_error(ghes, sev, mem_err); 484 485 arch_apei_report_mem_error(sev, mem_err); 486 ghes_handle_memory_failure(gdata, sev); 487 } 488 #ifdef CONFIG_ACPI_APEI_PCIEAER 489 else if (guid_equal(sec_type, &CPER_SEC_PCIE)) { 490 struct cper_sec_pcie *pcie_err = acpi_hest_get_payload(gdata); 491 492 if (sev == GHES_SEV_RECOVERABLE && 493 sec_sev == GHES_SEV_RECOVERABLE && 494 pcie_err->validation_bits & CPER_PCIE_VALID_DEVICE_ID && 495 pcie_err->validation_bits & CPER_PCIE_VALID_AER_INFO) { 496 unsigned int devfn; 497 int aer_severity; 498 499 devfn = PCI_DEVFN(pcie_err->device_id.device, 500 pcie_err->device_id.function); 501 aer_severity = cper_severity_to_aer(gdata->error_severity); 502 503 /* 504 * If firmware reset the component to contain 505 * the error, we must reinitialize it before 506 * use, so treat it as a fatal AER error. 507 */ 508 if (gdata->flags & CPER_SEC_RESET) 509 aer_severity = AER_FATAL; 510 511 aer_recover_queue(pcie_err->device_id.segment, 512 pcie_err->device_id.bus, 513 devfn, aer_severity, 514 (struct aer_capability_regs *) 515 pcie_err->aer_info); 516 } 517 518 } 519 #endif 520 else if (guid_equal(sec_type, &CPER_SEC_PROC_ARM)) { 521 struct cper_sec_proc_arm *err = acpi_hest_get_payload(gdata); 522 523 log_arm_hw_error(err); 524 } else { 525 void *err = acpi_hest_get_payload(gdata); 526 527 log_non_standard_event(sec_type, fru_id, fru_text, 528 sec_sev, err, 529 gdata->error_data_length); 530 } 531 } 532 } 533 534 static void __ghes_print_estatus(const char *pfx, 535 const struct acpi_hest_generic *generic, 536 const struct acpi_hest_generic_status *estatus) 537 { 538 static atomic_t seqno; 539 unsigned int curr_seqno; 540 char pfx_seq[64]; 541 542 if (pfx == NULL) { 543 if (ghes_severity(estatus->error_severity) <= 544 GHES_SEV_CORRECTED) 545 pfx = KERN_WARNING; 546 else 547 pfx = KERN_ERR; 548 } 549 curr_seqno = atomic_inc_return(&seqno); 550 snprintf(pfx_seq, sizeof(pfx_seq), "%s{%u}" HW_ERR, pfx, curr_seqno); 551 printk("%s""Hardware error from APEI Generic Hardware Error Source: %d\n", 552 pfx_seq, generic->header.source_id); 553 cper_estatus_print(pfx_seq, estatus); 554 } 555 556 static int ghes_print_estatus(const char *pfx, 557 const struct acpi_hest_generic *generic, 558 const struct acpi_hest_generic_status *estatus) 559 { 560 /* Not more than 2 messages every 5 seconds */ 561 static DEFINE_RATELIMIT_STATE(ratelimit_corrected, 5*HZ, 2); 562 static DEFINE_RATELIMIT_STATE(ratelimit_uncorrected, 5*HZ, 2); 563 struct ratelimit_state *ratelimit; 564 565 if (ghes_severity(estatus->error_severity) <= GHES_SEV_CORRECTED) 566 ratelimit = &ratelimit_corrected; 567 else 568 ratelimit = &ratelimit_uncorrected; 569 if (__ratelimit(ratelimit)) { 570 __ghes_print_estatus(pfx, generic, estatus); 571 return 1; 572 } 573 return 0; 574 } 575 576 /* 577 * GHES error status reporting throttle, to report more kinds of 578 * errors, instead of just most frequently occurred errors. 579 */ 580 static int ghes_estatus_cached(struct acpi_hest_generic_status *estatus) 581 { 582 u32 len; 583 int i, cached = 0; 584 unsigned long long now; 585 struct ghes_estatus_cache *cache; 586 struct acpi_hest_generic_status *cache_estatus; 587 588 len = cper_estatus_len(estatus); 589 rcu_read_lock(); 590 for (i = 0; i < GHES_ESTATUS_CACHES_SIZE; i++) { 591 cache = rcu_dereference(ghes_estatus_caches[i]); 592 if (cache == NULL) 593 continue; 594 if (len != cache->estatus_len) 595 continue; 596 cache_estatus = GHES_ESTATUS_FROM_CACHE(cache); 597 if (memcmp(estatus, cache_estatus, len)) 598 continue; 599 atomic_inc(&cache->count); 600 now = sched_clock(); 601 if (now - cache->time_in < GHES_ESTATUS_IN_CACHE_MAX_NSEC) 602 cached = 1; 603 break; 604 } 605 rcu_read_unlock(); 606 return cached; 607 } 608 609 static struct ghes_estatus_cache *ghes_estatus_cache_alloc( 610 struct acpi_hest_generic *generic, 611 struct acpi_hest_generic_status *estatus) 612 { 613 int alloced; 614 u32 len, cache_len; 615 struct ghes_estatus_cache *cache; 616 struct acpi_hest_generic_status *cache_estatus; 617 618 alloced = atomic_add_return(1, &ghes_estatus_cache_alloced); 619 if (alloced > GHES_ESTATUS_CACHE_ALLOCED_MAX) { 620 atomic_dec(&ghes_estatus_cache_alloced); 621 return NULL; 622 } 623 len = cper_estatus_len(estatus); 624 cache_len = GHES_ESTATUS_CACHE_LEN(len); 625 cache = (void *)gen_pool_alloc(ghes_estatus_pool, cache_len); 626 if (!cache) { 627 atomic_dec(&ghes_estatus_cache_alloced); 628 return NULL; 629 } 630 cache_estatus = GHES_ESTATUS_FROM_CACHE(cache); 631 memcpy(cache_estatus, estatus, len); 632 cache->estatus_len = len; 633 atomic_set(&cache->count, 0); 634 cache->generic = generic; 635 cache->time_in = sched_clock(); 636 return cache; 637 } 638 639 static void ghes_estatus_cache_free(struct ghes_estatus_cache *cache) 640 { 641 u32 len; 642 643 len = cper_estatus_len(GHES_ESTATUS_FROM_CACHE(cache)); 644 len = GHES_ESTATUS_CACHE_LEN(len); 645 gen_pool_free(ghes_estatus_pool, (unsigned long)cache, len); 646 atomic_dec(&ghes_estatus_cache_alloced); 647 } 648 649 static void ghes_estatus_cache_rcu_free(struct rcu_head *head) 650 { 651 struct ghes_estatus_cache *cache; 652 653 cache = container_of(head, struct ghes_estatus_cache, rcu); 654 ghes_estatus_cache_free(cache); 655 } 656 657 static void ghes_estatus_cache_add( 658 struct acpi_hest_generic *generic, 659 struct acpi_hest_generic_status *estatus) 660 { 661 int i, slot = -1, count; 662 unsigned long long now, duration, period, max_period = 0; 663 struct ghes_estatus_cache *cache, *slot_cache = NULL, *new_cache; 664 665 new_cache = ghes_estatus_cache_alloc(generic, estatus); 666 if (new_cache == NULL) 667 return; 668 rcu_read_lock(); 669 now = sched_clock(); 670 for (i = 0; i < GHES_ESTATUS_CACHES_SIZE; i++) { 671 cache = rcu_dereference(ghes_estatus_caches[i]); 672 if (cache == NULL) { 673 slot = i; 674 slot_cache = NULL; 675 break; 676 } 677 duration = now - cache->time_in; 678 if (duration >= GHES_ESTATUS_IN_CACHE_MAX_NSEC) { 679 slot = i; 680 slot_cache = cache; 681 break; 682 } 683 count = atomic_read(&cache->count); 684 period = duration; 685 do_div(period, (count + 1)); 686 if (period > max_period) { 687 max_period = period; 688 slot = i; 689 slot_cache = cache; 690 } 691 } 692 /* new_cache must be put into array after its contents are written */ 693 smp_wmb(); 694 if (slot != -1 && cmpxchg(ghes_estatus_caches + slot, 695 slot_cache, new_cache) == slot_cache) { 696 if (slot_cache) 697 call_rcu(&slot_cache->rcu, ghes_estatus_cache_rcu_free); 698 } else 699 ghes_estatus_cache_free(new_cache); 700 rcu_read_unlock(); 701 } 702 703 static int ghes_ack_error(struct acpi_hest_generic_v2 *gv2) 704 { 705 int rc; 706 u64 val = 0; 707 708 rc = apei_read(&val, &gv2->read_ack_register); 709 if (rc) 710 return rc; 711 712 val &= gv2->read_ack_preserve << gv2->read_ack_register.bit_offset; 713 val |= gv2->read_ack_write << gv2->read_ack_register.bit_offset; 714 715 return apei_write(val, &gv2->read_ack_register); 716 } 717 718 static void __ghes_panic(struct ghes *ghes) 719 { 720 __ghes_print_estatus(KERN_EMERG, ghes->generic, ghes->estatus); 721 722 /* reboot to log the error! */ 723 if (!panic_timeout) 724 panic_timeout = ghes_panic_timeout; 725 panic("Fatal hardware error!"); 726 } 727 728 static int ghes_proc(struct ghes *ghes) 729 { 730 int rc; 731 732 rc = ghes_read_estatus(ghes, 0); 733 if (rc) 734 goto out; 735 736 if (ghes_severity(ghes->estatus->error_severity) >= GHES_SEV_PANIC) { 737 __ghes_panic(ghes); 738 } 739 740 if (!ghes_estatus_cached(ghes->estatus)) { 741 if (ghes_print_estatus(NULL, ghes->generic, ghes->estatus)) 742 ghes_estatus_cache_add(ghes->generic, ghes->estatus); 743 } 744 ghes_do_proc(ghes, ghes->estatus); 745 746 out: 747 ghes_clear_estatus(ghes); 748 749 if (rc == -ENOENT) 750 return rc; 751 752 /* 753 * GHESv2 type HEST entries introduce support for error acknowledgment, 754 * so only acknowledge the error if this support is present. 755 */ 756 if (is_hest_type_generic_v2(ghes)) 757 return ghes_ack_error(ghes->generic_v2); 758 759 return rc; 760 } 761 762 static void ghes_add_timer(struct ghes *ghes) 763 { 764 struct acpi_hest_generic *g = ghes->generic; 765 unsigned long expire; 766 767 if (!g->notify.poll_interval) { 768 pr_warning(FW_WARN GHES_PFX "Poll interval is 0 for generic hardware error source: %d, disabled.\n", 769 g->header.source_id); 770 return; 771 } 772 expire = jiffies + msecs_to_jiffies(g->notify.poll_interval); 773 ghes->timer.expires = round_jiffies_relative(expire); 774 add_timer(&ghes->timer); 775 } 776 777 static void ghes_poll_func(unsigned long data) 778 { 779 struct ghes *ghes = (void *)data; 780 781 ghes_proc(ghes); 782 if (!(ghes->flags & GHES_EXITING)) 783 ghes_add_timer(ghes); 784 } 785 786 static irqreturn_t ghes_irq_func(int irq, void *data) 787 { 788 struct ghes *ghes = data; 789 int rc; 790 791 rc = ghes_proc(ghes); 792 if (rc) 793 return IRQ_NONE; 794 795 return IRQ_HANDLED; 796 } 797 798 static int ghes_notify_hed(struct notifier_block *this, unsigned long event, 799 void *data) 800 { 801 struct ghes *ghes; 802 int ret = NOTIFY_DONE; 803 804 rcu_read_lock(); 805 list_for_each_entry_rcu(ghes, &ghes_hed, list) { 806 if (!ghes_proc(ghes)) 807 ret = NOTIFY_OK; 808 } 809 rcu_read_unlock(); 810 811 return ret; 812 } 813 814 static struct notifier_block ghes_notifier_hed = { 815 .notifier_call = ghes_notify_hed, 816 }; 817 818 #ifdef CONFIG_ACPI_APEI_SEA 819 static LIST_HEAD(ghes_sea); 820 821 /* 822 * Return 0 only if one of the SEA error sources successfully reported an error 823 * record sent from the firmware. 824 */ 825 int ghes_notify_sea(void) 826 { 827 struct ghes *ghes; 828 int ret = -ENOENT; 829 830 rcu_read_lock(); 831 list_for_each_entry_rcu(ghes, &ghes_sea, list) { 832 if (!ghes_proc(ghes)) 833 ret = 0; 834 } 835 rcu_read_unlock(); 836 return ret; 837 } 838 839 static void ghes_sea_add(struct ghes *ghes) 840 { 841 mutex_lock(&ghes_list_mutex); 842 list_add_rcu(&ghes->list, &ghes_sea); 843 mutex_unlock(&ghes_list_mutex); 844 } 845 846 static void ghes_sea_remove(struct ghes *ghes) 847 { 848 mutex_lock(&ghes_list_mutex); 849 list_del_rcu(&ghes->list); 850 mutex_unlock(&ghes_list_mutex); 851 synchronize_rcu(); 852 } 853 #else /* CONFIG_ACPI_APEI_SEA */ 854 static inline void ghes_sea_add(struct ghes *ghes) 855 { 856 pr_err(GHES_PFX "ID: %d, trying to add SEA notification which is not supported\n", 857 ghes->generic->header.source_id); 858 } 859 860 static inline void ghes_sea_remove(struct ghes *ghes) 861 { 862 pr_err(GHES_PFX "ID: %d, trying to remove SEA notification which is not supported\n", 863 ghes->generic->header.source_id); 864 } 865 #endif /* CONFIG_ACPI_APEI_SEA */ 866 867 #ifdef CONFIG_HAVE_ACPI_APEI_NMI 868 /* 869 * printk is not safe in NMI context. So in NMI handler, we allocate 870 * required memory from lock-less memory allocator 871 * (ghes_estatus_pool), save estatus into it, put them into lock-less 872 * list (ghes_estatus_llist), then delay printk into IRQ context via 873 * irq_work (ghes_proc_irq_work). ghes_estatus_size_request record 874 * required pool size by all NMI error source. 875 */ 876 static struct llist_head ghes_estatus_llist; 877 static struct irq_work ghes_proc_irq_work; 878 879 /* 880 * NMI may be triggered on any CPU, so ghes_in_nmi is used for 881 * having only one concurrent reader. 882 */ 883 static atomic_t ghes_in_nmi = ATOMIC_INIT(0); 884 885 static LIST_HEAD(ghes_nmi); 886 887 static void ghes_proc_in_irq(struct irq_work *irq_work) 888 { 889 struct llist_node *llnode, *next; 890 struct ghes_estatus_node *estatus_node; 891 struct acpi_hest_generic *generic; 892 struct acpi_hest_generic_status *estatus; 893 u32 len, node_len; 894 895 llnode = llist_del_all(&ghes_estatus_llist); 896 /* 897 * Because the time order of estatus in list is reversed, 898 * revert it back to proper order. 899 */ 900 llnode = llist_reverse_order(llnode); 901 while (llnode) { 902 next = llnode->next; 903 estatus_node = llist_entry(llnode, struct ghes_estatus_node, 904 llnode); 905 estatus = GHES_ESTATUS_FROM_NODE(estatus_node); 906 len = cper_estatus_len(estatus); 907 node_len = GHES_ESTATUS_NODE_LEN(len); 908 ghes_do_proc(estatus_node->ghes, estatus); 909 if (!ghes_estatus_cached(estatus)) { 910 generic = estatus_node->generic; 911 if (ghes_print_estatus(NULL, generic, estatus)) 912 ghes_estatus_cache_add(generic, estatus); 913 } 914 gen_pool_free(ghes_estatus_pool, (unsigned long)estatus_node, 915 node_len); 916 llnode = next; 917 } 918 } 919 920 static void ghes_print_queued_estatus(void) 921 { 922 struct llist_node *llnode; 923 struct ghes_estatus_node *estatus_node; 924 struct acpi_hest_generic *generic; 925 struct acpi_hest_generic_status *estatus; 926 u32 len, node_len; 927 928 llnode = llist_del_all(&ghes_estatus_llist); 929 /* 930 * Because the time order of estatus in list is reversed, 931 * revert it back to proper order. 932 */ 933 llnode = llist_reverse_order(llnode); 934 while (llnode) { 935 estatus_node = llist_entry(llnode, struct ghes_estatus_node, 936 llnode); 937 estatus = GHES_ESTATUS_FROM_NODE(estatus_node); 938 len = cper_estatus_len(estatus); 939 node_len = GHES_ESTATUS_NODE_LEN(len); 940 generic = estatus_node->generic; 941 ghes_print_estatus(NULL, generic, estatus); 942 llnode = llnode->next; 943 } 944 } 945 946 /* Save estatus for further processing in IRQ context */ 947 static void __process_error(struct ghes *ghes) 948 { 949 #ifdef CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG 950 u32 len, node_len; 951 struct ghes_estatus_node *estatus_node; 952 struct acpi_hest_generic_status *estatus; 953 954 if (ghes_estatus_cached(ghes->estatus)) 955 return; 956 957 len = cper_estatus_len(ghes->estatus); 958 node_len = GHES_ESTATUS_NODE_LEN(len); 959 960 estatus_node = (void *)gen_pool_alloc(ghes_estatus_pool, node_len); 961 if (!estatus_node) 962 return; 963 964 estatus_node->ghes = ghes; 965 estatus_node->generic = ghes->generic; 966 estatus = GHES_ESTATUS_FROM_NODE(estatus_node); 967 memcpy(estatus, ghes->estatus, len); 968 llist_add(&estatus_node->llnode, &ghes_estatus_llist); 969 #endif 970 } 971 972 static int ghes_notify_nmi(unsigned int cmd, struct pt_regs *regs) 973 { 974 struct ghes *ghes; 975 int sev, ret = NMI_DONE; 976 977 if (!atomic_add_unless(&ghes_in_nmi, 1, 1)) 978 return ret; 979 980 list_for_each_entry_rcu(ghes, &ghes_nmi, list) { 981 if (ghes_read_estatus(ghes, 1)) { 982 ghes_clear_estatus(ghes); 983 continue; 984 } else { 985 ret = NMI_HANDLED; 986 } 987 988 sev = ghes_severity(ghes->estatus->error_severity); 989 if (sev >= GHES_SEV_PANIC) { 990 oops_begin(); 991 ghes_print_queued_estatus(); 992 __ghes_panic(ghes); 993 } 994 995 if (!(ghes->flags & GHES_TO_CLEAR)) 996 continue; 997 998 __process_error(ghes); 999 ghes_clear_estatus(ghes); 1000 } 1001 1002 #ifdef CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG 1003 if (ret == NMI_HANDLED) 1004 irq_work_queue(&ghes_proc_irq_work); 1005 #endif 1006 atomic_dec(&ghes_in_nmi); 1007 return ret; 1008 } 1009 1010 static unsigned long ghes_esource_prealloc_size( 1011 const struct acpi_hest_generic *generic) 1012 { 1013 unsigned long block_length, prealloc_records, prealloc_size; 1014 1015 block_length = min_t(unsigned long, generic->error_block_length, 1016 GHES_ESTATUS_MAX_SIZE); 1017 prealloc_records = max_t(unsigned long, 1018 generic->records_to_preallocate, 1); 1019 prealloc_size = min_t(unsigned long, block_length * prealloc_records, 1020 GHES_ESOURCE_PREALLOC_MAX_SIZE); 1021 1022 return prealloc_size; 1023 } 1024 1025 static void ghes_estatus_pool_shrink(unsigned long len) 1026 { 1027 ghes_estatus_pool_size_request -= PAGE_ALIGN(len); 1028 } 1029 1030 static void ghes_nmi_add(struct ghes *ghes) 1031 { 1032 unsigned long len; 1033 1034 len = ghes_esource_prealloc_size(ghes->generic); 1035 ghes_estatus_pool_expand(len); 1036 mutex_lock(&ghes_list_mutex); 1037 if (list_empty(&ghes_nmi)) 1038 register_nmi_handler(NMI_LOCAL, ghes_notify_nmi, 0, "ghes"); 1039 list_add_rcu(&ghes->list, &ghes_nmi); 1040 mutex_unlock(&ghes_list_mutex); 1041 } 1042 1043 static void ghes_nmi_remove(struct ghes *ghes) 1044 { 1045 unsigned long len; 1046 1047 mutex_lock(&ghes_list_mutex); 1048 list_del_rcu(&ghes->list); 1049 if (list_empty(&ghes_nmi)) 1050 unregister_nmi_handler(NMI_LOCAL, "ghes"); 1051 mutex_unlock(&ghes_list_mutex); 1052 /* 1053 * To synchronize with NMI handler, ghes can only be 1054 * freed after NMI handler finishes. 1055 */ 1056 synchronize_rcu(); 1057 len = ghes_esource_prealloc_size(ghes->generic); 1058 ghes_estatus_pool_shrink(len); 1059 } 1060 1061 static void ghes_nmi_init_cxt(void) 1062 { 1063 init_irq_work(&ghes_proc_irq_work, ghes_proc_in_irq); 1064 } 1065 #else /* CONFIG_HAVE_ACPI_APEI_NMI */ 1066 static inline void ghes_nmi_add(struct ghes *ghes) 1067 { 1068 pr_err(GHES_PFX "ID: %d, trying to add NMI notification which is not supported!\n", 1069 ghes->generic->header.source_id); 1070 BUG(); 1071 } 1072 1073 static inline void ghes_nmi_remove(struct ghes *ghes) 1074 { 1075 pr_err(GHES_PFX "ID: %d, trying to remove NMI notification which is not supported!\n", 1076 ghes->generic->header.source_id); 1077 BUG(); 1078 } 1079 1080 static inline void ghes_nmi_init_cxt(void) 1081 { 1082 } 1083 #endif /* CONFIG_HAVE_ACPI_APEI_NMI */ 1084 1085 static int ghes_probe(struct platform_device *ghes_dev) 1086 { 1087 struct acpi_hest_generic *generic; 1088 struct ghes *ghes = NULL; 1089 1090 int rc = -EINVAL; 1091 1092 generic = *(struct acpi_hest_generic **)ghes_dev->dev.platform_data; 1093 if (!generic->enabled) 1094 return -ENODEV; 1095 1096 switch (generic->notify.type) { 1097 case ACPI_HEST_NOTIFY_POLLED: 1098 case ACPI_HEST_NOTIFY_EXTERNAL: 1099 case ACPI_HEST_NOTIFY_SCI: 1100 case ACPI_HEST_NOTIFY_GSIV: 1101 case ACPI_HEST_NOTIFY_GPIO: 1102 break; 1103 1104 case ACPI_HEST_NOTIFY_SEA: 1105 if (!IS_ENABLED(CONFIG_ACPI_APEI_SEA)) { 1106 pr_warn(GHES_PFX "Generic hardware error source: %d notified via SEA is not supported\n", 1107 generic->header.source_id); 1108 rc = -ENOTSUPP; 1109 goto err; 1110 } 1111 break; 1112 case ACPI_HEST_NOTIFY_NMI: 1113 if (!IS_ENABLED(CONFIG_HAVE_ACPI_APEI_NMI)) { 1114 pr_warn(GHES_PFX "Generic hardware error source: %d notified via NMI interrupt is not supported!\n", 1115 generic->header.source_id); 1116 goto err; 1117 } 1118 break; 1119 case ACPI_HEST_NOTIFY_LOCAL: 1120 pr_warning(GHES_PFX "Generic hardware error source: %d notified via local interrupt is not supported!\n", 1121 generic->header.source_id); 1122 goto err; 1123 default: 1124 pr_warning(FW_WARN GHES_PFX "Unknown notification type: %u for generic hardware error source: %d\n", 1125 generic->notify.type, generic->header.source_id); 1126 goto err; 1127 } 1128 1129 rc = -EIO; 1130 if (generic->error_block_length < 1131 sizeof(struct acpi_hest_generic_status)) { 1132 pr_warning(FW_BUG GHES_PFX "Invalid error block length: %u for generic hardware error source: %d\n", 1133 generic->error_block_length, 1134 generic->header.source_id); 1135 goto err; 1136 } 1137 ghes = ghes_new(generic); 1138 if (IS_ERR(ghes)) { 1139 rc = PTR_ERR(ghes); 1140 ghes = NULL; 1141 goto err; 1142 } 1143 1144 rc = ghes_edac_register(ghes, &ghes_dev->dev); 1145 if (rc < 0) 1146 goto err; 1147 1148 switch (generic->notify.type) { 1149 case ACPI_HEST_NOTIFY_POLLED: 1150 setup_deferrable_timer(&ghes->timer, ghes_poll_func, 1151 (unsigned long)ghes); 1152 ghes_add_timer(ghes); 1153 break; 1154 case ACPI_HEST_NOTIFY_EXTERNAL: 1155 /* External interrupt vector is GSI */ 1156 rc = acpi_gsi_to_irq(generic->notify.vector, &ghes->irq); 1157 if (rc) { 1158 pr_err(GHES_PFX "Failed to map GSI to IRQ for generic hardware error source: %d\n", 1159 generic->header.source_id); 1160 goto err_edac_unreg; 1161 } 1162 rc = request_irq(ghes->irq, ghes_irq_func, IRQF_SHARED, 1163 "GHES IRQ", ghes); 1164 if (rc) { 1165 pr_err(GHES_PFX "Failed to register IRQ for generic hardware error source: %d\n", 1166 generic->header.source_id); 1167 goto err_edac_unreg; 1168 } 1169 break; 1170 1171 case ACPI_HEST_NOTIFY_SCI: 1172 case ACPI_HEST_NOTIFY_GSIV: 1173 case ACPI_HEST_NOTIFY_GPIO: 1174 mutex_lock(&ghes_list_mutex); 1175 if (list_empty(&ghes_hed)) 1176 register_acpi_hed_notifier(&ghes_notifier_hed); 1177 list_add_rcu(&ghes->list, &ghes_hed); 1178 mutex_unlock(&ghes_list_mutex); 1179 break; 1180 1181 case ACPI_HEST_NOTIFY_SEA: 1182 ghes_sea_add(ghes); 1183 break; 1184 case ACPI_HEST_NOTIFY_NMI: 1185 ghes_nmi_add(ghes); 1186 break; 1187 default: 1188 BUG(); 1189 } 1190 platform_set_drvdata(ghes_dev, ghes); 1191 1192 /* Handle any pending errors right away */ 1193 ghes_proc(ghes); 1194 1195 return 0; 1196 err_edac_unreg: 1197 ghes_edac_unregister(ghes); 1198 err: 1199 if (ghes) { 1200 ghes_fini(ghes); 1201 kfree(ghes); 1202 } 1203 return rc; 1204 } 1205 1206 static int ghes_remove(struct platform_device *ghes_dev) 1207 { 1208 struct ghes *ghes; 1209 struct acpi_hest_generic *generic; 1210 1211 ghes = platform_get_drvdata(ghes_dev); 1212 generic = ghes->generic; 1213 1214 ghes->flags |= GHES_EXITING; 1215 switch (generic->notify.type) { 1216 case ACPI_HEST_NOTIFY_POLLED: 1217 del_timer_sync(&ghes->timer); 1218 break; 1219 case ACPI_HEST_NOTIFY_EXTERNAL: 1220 free_irq(ghes->irq, ghes); 1221 break; 1222 1223 case ACPI_HEST_NOTIFY_SCI: 1224 case ACPI_HEST_NOTIFY_GSIV: 1225 case ACPI_HEST_NOTIFY_GPIO: 1226 mutex_lock(&ghes_list_mutex); 1227 list_del_rcu(&ghes->list); 1228 if (list_empty(&ghes_hed)) 1229 unregister_acpi_hed_notifier(&ghes_notifier_hed); 1230 mutex_unlock(&ghes_list_mutex); 1231 synchronize_rcu(); 1232 break; 1233 1234 case ACPI_HEST_NOTIFY_SEA: 1235 ghes_sea_remove(ghes); 1236 break; 1237 case ACPI_HEST_NOTIFY_NMI: 1238 ghes_nmi_remove(ghes); 1239 break; 1240 default: 1241 BUG(); 1242 break; 1243 } 1244 1245 ghes_fini(ghes); 1246 1247 ghes_edac_unregister(ghes); 1248 1249 kfree(ghes); 1250 1251 platform_set_drvdata(ghes_dev, NULL); 1252 1253 return 0; 1254 } 1255 1256 static struct platform_driver ghes_platform_driver = { 1257 .driver = { 1258 .name = "GHES", 1259 }, 1260 .probe = ghes_probe, 1261 .remove = ghes_remove, 1262 }; 1263 1264 static int __init ghes_init(void) 1265 { 1266 int rc; 1267 1268 if (acpi_disabled) 1269 return -ENODEV; 1270 1271 switch (hest_disable) { 1272 case HEST_NOT_FOUND: 1273 return -ENODEV; 1274 case HEST_DISABLED: 1275 pr_info(GHES_PFX "HEST is not enabled!\n"); 1276 return -EINVAL; 1277 default: 1278 break; 1279 } 1280 1281 if (ghes_disable) { 1282 pr_info(GHES_PFX "GHES is not enabled!\n"); 1283 return -EINVAL; 1284 } 1285 1286 ghes_nmi_init_cxt(); 1287 1288 rc = ghes_ioremap_init(); 1289 if (rc) 1290 goto err; 1291 1292 rc = ghes_estatus_pool_init(); 1293 if (rc) 1294 goto err_ioremap_exit; 1295 1296 rc = ghes_estatus_pool_expand(GHES_ESTATUS_CACHE_AVG_SIZE * 1297 GHES_ESTATUS_CACHE_ALLOCED_MAX); 1298 if (rc) 1299 goto err_pool_exit; 1300 1301 rc = platform_driver_register(&ghes_platform_driver); 1302 if (rc) 1303 goto err_pool_exit; 1304 1305 rc = apei_osc_setup(); 1306 if (rc == 0 && osc_sb_apei_support_acked) 1307 pr_info(GHES_PFX "APEI firmware first mode is enabled by APEI bit and WHEA _OSC.\n"); 1308 else if (rc == 0 && !osc_sb_apei_support_acked) 1309 pr_info(GHES_PFX "APEI firmware first mode is enabled by WHEA _OSC.\n"); 1310 else if (rc && osc_sb_apei_support_acked) 1311 pr_info(GHES_PFX "APEI firmware first mode is enabled by APEI bit.\n"); 1312 else 1313 pr_info(GHES_PFX "Failed to enable APEI firmware first mode.\n"); 1314 1315 return 0; 1316 err_pool_exit: 1317 ghes_estatus_pool_exit(); 1318 err_ioremap_exit: 1319 ghes_ioremap_exit(); 1320 err: 1321 return rc; 1322 } 1323 device_initcall(ghes_init); 1324