1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Driver for Intel client SoC with integrated memory controller using IBECC 4 * 5 * Copyright (C) 2020 Intel Corporation 6 * 7 * The In-Band ECC (IBECC) IP provides ECC protection to all or specific 8 * regions of the physical memory space. It's used for memory controllers 9 * that don't support the out-of-band ECC which often needs an additional 10 * storage device to each channel for storing ECC data. 11 */ 12 13 #include <linux/module.h> 14 #include <linux/init.h> 15 #include <linux/pci.h> 16 #include <linux/slab.h> 17 #include <linux/irq_work.h> 18 #include <linux/llist.h> 19 #include <linux/genalloc.h> 20 #include <linux/edac.h> 21 #include <linux/bits.h> 22 #include <linux/io.h> 23 #include <asm/mach_traps.h> 24 #include <asm/nmi.h> 25 #include <asm/mce.h> 26 27 #include "edac_mc.h" 28 #include "edac_module.h" 29 30 #define IGEN6_REVISION "v2.5" 31 32 #define EDAC_MOD_STR "igen6_edac" 33 #define IGEN6_NMI_NAME "igen6_ibecc" 34 35 /* Debug macros */ 36 #define igen6_printk(level, fmt, arg...) \ 37 edac_printk(level, "igen6", fmt, ##arg) 38 39 #define igen6_mc_printk(mci, level, fmt, arg...) \ 40 edac_mc_chipset_printk(mci, level, "igen6", fmt, ##arg) 41 42 #define GET_BITFIELD(v, lo, hi) (((v) & GENMASK_ULL(hi, lo)) >> (lo)) 43 44 #define NUM_IMC 2 /* Max memory controllers */ 45 #define NUM_CHANNELS 2 /* Max channels */ 46 #define NUM_DIMMS 2 /* Max DIMMs per channel */ 47 48 #define _4GB BIT_ULL(32) 49 50 /* Size of physical memory */ 51 #define TOM_OFFSET 0xa0 52 /* Top of low usable DRAM */ 53 #define TOLUD_OFFSET 0xbc 54 /* Capability register C */ 55 #define CAPID_C_OFFSET 0xec 56 #define CAPID_C_IBECC BIT(15) 57 58 /* Capability register E */ 59 #define CAPID_E_OFFSET 0xf0 60 #define CAPID_E_IBECC BIT(12) 61 62 /* Error Status */ 63 #define ERRSTS_OFFSET 0xc8 64 #define ERRSTS_CE BIT_ULL(6) 65 #define ERRSTS_UE BIT_ULL(7) 66 67 /* Error Command */ 68 #define ERRCMD_OFFSET 0xca 69 #define ERRCMD_CE BIT_ULL(6) 70 #define ERRCMD_UE BIT_ULL(7) 71 72 /* IBECC MMIO base address */ 73 #define IBECC_BASE (res_cfg->ibecc_base) 74 #define IBECC_ACTIVATE_OFFSET IBECC_BASE 75 #define IBECC_ACTIVATE_EN BIT(0) 76 77 /* IBECC error log */ 78 #define ECC_ERROR_LOG_OFFSET (IBECC_BASE + res_cfg->ibecc_error_log_offset) 79 #define ECC_ERROR_LOG_CE BIT_ULL(62) 80 #define ECC_ERROR_LOG_UE BIT_ULL(63) 81 #define ECC_ERROR_LOG_ADDR_SHIFT 5 82 #define ECC_ERROR_LOG_ADDR(v) GET_BITFIELD(v, 5, 38) 83 #define ECC_ERROR_LOG_SYND(v) GET_BITFIELD(v, 46, 61) 84 85 /* Host MMIO base address */ 86 #define MCHBAR_OFFSET 0x48 87 #define MCHBAR_EN BIT_ULL(0) 88 #define MCHBAR_BASE(v) (GET_BITFIELD(v, 16, 38) << 16) 89 #define MCHBAR_SIZE 0x10000 90 91 /* Parameters for the channel decode stage */ 92 #define IMC_BASE (res_cfg->imc_base) 93 #define MAD_INTER_CHANNEL_OFFSET IMC_BASE 94 #define MAD_INTER_CHANNEL_DDR_TYPE(v) GET_BITFIELD(v, 0, 2) 95 #define MAD_INTER_CHANNEL_ECHM(v) GET_BITFIELD(v, 3, 3) 96 #define MAD_INTER_CHANNEL_CH_L_MAP(v) GET_BITFIELD(v, 4, 4) 97 #define MAD_INTER_CHANNEL_CH_S_SIZE(v) ((u64)GET_BITFIELD(v, 12, 19) << 29) 98 99 /* Parameters for DRAM decode stage */ 100 #define MAD_INTRA_CH0_OFFSET (IMC_BASE + 4) 101 #define MAD_INTRA_CH_DIMM_L_MAP(v) GET_BITFIELD(v, 0, 0) 102 103 /* DIMM characteristics */ 104 #define MAD_DIMM_CH0_OFFSET (IMC_BASE + 0xc) 105 #define MAD_DIMM_CH_DIMM_L_SIZE(v) ((u64)GET_BITFIELD(v, 0, 6) << 29) 106 #define MAD_DIMM_CH_DLW(v) GET_BITFIELD(v, 7, 8) 107 #define MAD_DIMM_CH_DIMM_S_SIZE(v) ((u64)GET_BITFIELD(v, 16, 22) << 29) 108 #define MAD_DIMM_CH_DSW(v) GET_BITFIELD(v, 24, 25) 109 110 /* Hash for memory controller selection */ 111 #define MAD_MC_HASH_OFFSET (IMC_BASE + 0x1b8) 112 #define MAC_MC_HASH_LSB(v) GET_BITFIELD(v, 1, 3) 113 114 /* Hash for channel selection */ 115 #define CHANNEL_HASH_OFFSET (IMC_BASE + 0x24) 116 /* Hash for enhanced channel selection */ 117 #define CHANNEL_EHASH_OFFSET (IMC_BASE + 0x28) 118 #define CHANNEL_HASH_MASK(v) (GET_BITFIELD(v, 6, 19) << 6) 119 #define CHANNEL_HASH_LSB_MASK_BIT(v) GET_BITFIELD(v, 24, 26) 120 #define CHANNEL_HASH_MODE(v) GET_BITFIELD(v, 28, 28) 121 122 /* Parameters for memory slice decode stage */ 123 #define MEM_SLICE_HASH_MASK(v) (GET_BITFIELD(v, 6, 19) << 6) 124 #define MEM_SLICE_HASH_LSB_MASK_BIT(v) GET_BITFIELD(v, 24, 26) 125 126 static struct res_config { 127 bool machine_check; 128 int num_imc; 129 u32 imc_base; 130 u32 cmf_base; 131 u32 cmf_size; 132 u32 ms_hash_offset; 133 u32 ibecc_base; 134 u32 ibecc_error_log_offset; 135 bool (*ibecc_available)(struct pci_dev *pdev); 136 /* Convert error address logged in IBECC to system physical address */ 137 u64 (*err_addr_to_sys_addr)(u64 eaddr, int mc); 138 /* Convert error address logged in IBECC to integrated memory controller address */ 139 u64 (*err_addr_to_imc_addr)(u64 eaddr, int mc); 140 } *res_cfg; 141 142 struct igen6_imc { 143 int mc; 144 struct mem_ctl_info *mci; 145 struct pci_dev *pdev; 146 struct device dev; 147 void __iomem *window; 148 u64 size; 149 u64 ch_s_size; 150 int ch_l_map; 151 u64 dimm_s_size[NUM_CHANNELS]; 152 u64 dimm_l_size[NUM_CHANNELS]; 153 int dimm_l_map[NUM_CHANNELS]; 154 }; 155 156 static struct igen6_pvt { 157 struct igen6_imc imc[NUM_IMC]; 158 u64 ms_hash; 159 u64 ms_s_size; 160 int ms_l_map; 161 } *igen6_pvt; 162 163 /* The top of low usable DRAM */ 164 static u32 igen6_tolud; 165 /* The size of physical memory */ 166 static u64 igen6_tom; 167 168 struct decoded_addr { 169 int mc; 170 u64 imc_addr; 171 u64 sys_addr; 172 int channel_idx; 173 u64 channel_addr; 174 int sub_channel_idx; 175 u64 sub_channel_addr; 176 }; 177 178 struct ecclog_node { 179 struct llist_node llnode; 180 int mc; 181 u64 ecclog; 182 }; 183 184 /* 185 * In the NMI handler, the driver uses the lock-less memory allocator 186 * to allocate memory to store the IBECC error logs and links the logs 187 * to the lock-less list. Delay printk() and the work of error reporting 188 * to EDAC core in a worker. 189 */ 190 #define ECCLOG_POOL_SIZE PAGE_SIZE 191 static LLIST_HEAD(ecclog_llist); 192 static struct gen_pool *ecclog_pool; 193 static char ecclog_buf[ECCLOG_POOL_SIZE]; 194 static struct irq_work ecclog_irq_work; 195 static struct work_struct ecclog_work; 196 197 /* Compute die IDs for Elkhart Lake with IBECC */ 198 #define DID_EHL_SKU5 0x4514 199 #define DID_EHL_SKU6 0x4528 200 #define DID_EHL_SKU7 0x452a 201 #define DID_EHL_SKU8 0x4516 202 #define DID_EHL_SKU9 0x452c 203 #define DID_EHL_SKU10 0x452e 204 #define DID_EHL_SKU11 0x4532 205 #define DID_EHL_SKU12 0x4518 206 #define DID_EHL_SKU13 0x451a 207 #define DID_EHL_SKU14 0x4534 208 #define DID_EHL_SKU15 0x4536 209 210 /* Compute die IDs for ICL-NNPI with IBECC */ 211 #define DID_ICL_SKU8 0x4581 212 #define DID_ICL_SKU10 0x4585 213 #define DID_ICL_SKU11 0x4589 214 #define DID_ICL_SKU12 0x458d 215 216 /* Compute die IDs for Tiger Lake with IBECC */ 217 #define DID_TGL_SKU 0x9a14 218 219 /* Compute die IDs for Alder Lake with IBECC */ 220 #define DID_ADL_SKU1 0x4601 221 #define DID_ADL_SKU2 0x4602 222 #define DID_ADL_SKU3 0x4621 223 #define DID_ADL_SKU4 0x4641 224 225 static bool ehl_ibecc_available(struct pci_dev *pdev) 226 { 227 u32 v; 228 229 if (pci_read_config_dword(pdev, CAPID_C_OFFSET, &v)) 230 return false; 231 232 return !!(CAPID_C_IBECC & v); 233 } 234 235 static u64 ehl_err_addr_to_sys_addr(u64 eaddr, int mc) 236 { 237 return eaddr; 238 } 239 240 static u64 ehl_err_addr_to_imc_addr(u64 eaddr, int mc) 241 { 242 if (eaddr < igen6_tolud) 243 return eaddr; 244 245 if (igen6_tom <= _4GB) 246 return eaddr + igen6_tolud - _4GB; 247 248 if (eaddr < _4GB) 249 return eaddr + igen6_tolud - igen6_tom; 250 251 return eaddr; 252 } 253 254 static bool icl_ibecc_available(struct pci_dev *pdev) 255 { 256 u32 v; 257 258 if (pci_read_config_dword(pdev, CAPID_C_OFFSET, &v)) 259 return false; 260 261 return !(CAPID_C_IBECC & v) && 262 (boot_cpu_data.x86_stepping >= 1); 263 } 264 265 static bool tgl_ibecc_available(struct pci_dev *pdev) 266 { 267 u32 v; 268 269 if (pci_read_config_dword(pdev, CAPID_E_OFFSET, &v)) 270 return false; 271 272 return !(CAPID_E_IBECC & v); 273 } 274 275 static u64 mem_addr_to_sys_addr(u64 maddr) 276 { 277 if (maddr < igen6_tolud) 278 return maddr; 279 280 if (igen6_tom <= _4GB) 281 return maddr - igen6_tolud + _4GB; 282 283 if (maddr < _4GB) 284 return maddr - igen6_tolud + igen6_tom; 285 286 return maddr; 287 } 288 289 static u64 mem_slice_hash(u64 addr, u64 mask, u64 hash_init, int intlv_bit) 290 { 291 u64 hash_addr = addr & mask, hash = hash_init; 292 u64 intlv = (addr >> intlv_bit) & 1; 293 int i; 294 295 for (i = 6; i < 20; i++) 296 hash ^= (hash_addr >> i) & 1; 297 298 return hash ^ intlv; 299 } 300 301 static u64 tgl_err_addr_to_mem_addr(u64 eaddr, int mc) 302 { 303 u64 maddr, hash, mask, ms_s_size; 304 int intlv_bit; 305 u32 ms_hash; 306 307 ms_s_size = igen6_pvt->ms_s_size; 308 if (eaddr >= ms_s_size) 309 return eaddr + ms_s_size; 310 311 ms_hash = igen6_pvt->ms_hash; 312 313 mask = MEM_SLICE_HASH_MASK(ms_hash); 314 intlv_bit = MEM_SLICE_HASH_LSB_MASK_BIT(ms_hash) + 6; 315 316 maddr = GET_BITFIELD(eaddr, intlv_bit, 63) << (intlv_bit + 1) | 317 GET_BITFIELD(eaddr, 0, intlv_bit - 1); 318 319 hash = mem_slice_hash(maddr, mask, mc, intlv_bit); 320 321 return maddr | (hash << intlv_bit); 322 } 323 324 static u64 tgl_err_addr_to_sys_addr(u64 eaddr, int mc) 325 { 326 u64 maddr = tgl_err_addr_to_mem_addr(eaddr, mc); 327 328 return mem_addr_to_sys_addr(maddr); 329 } 330 331 static u64 tgl_err_addr_to_imc_addr(u64 eaddr, int mc) 332 { 333 return eaddr; 334 } 335 336 static u64 adl_err_addr_to_sys_addr(u64 eaddr, int mc) 337 { 338 return mem_addr_to_sys_addr(eaddr); 339 } 340 341 static u64 adl_err_addr_to_imc_addr(u64 eaddr, int mc) 342 { 343 u64 imc_addr, ms_s_size = igen6_pvt->ms_s_size; 344 struct igen6_imc *imc = &igen6_pvt->imc[mc]; 345 int intlv_bit; 346 u32 mc_hash; 347 348 if (eaddr >= 2 * ms_s_size) 349 return eaddr - ms_s_size; 350 351 mc_hash = readl(imc->window + MAD_MC_HASH_OFFSET); 352 353 intlv_bit = MAC_MC_HASH_LSB(mc_hash) + 6; 354 355 imc_addr = GET_BITFIELD(eaddr, intlv_bit + 1, 63) << intlv_bit | 356 GET_BITFIELD(eaddr, 0, intlv_bit - 1); 357 358 return imc_addr; 359 } 360 361 static struct res_config ehl_cfg = { 362 .num_imc = 1, 363 .imc_base = 0x5000, 364 .ibecc_base = 0xdc00, 365 .ibecc_available = ehl_ibecc_available, 366 .ibecc_error_log_offset = 0x170, 367 .err_addr_to_sys_addr = ehl_err_addr_to_sys_addr, 368 .err_addr_to_imc_addr = ehl_err_addr_to_imc_addr, 369 }; 370 371 static struct res_config icl_cfg = { 372 .num_imc = 1, 373 .imc_base = 0x5000, 374 .ibecc_base = 0xd800, 375 .ibecc_error_log_offset = 0x170, 376 .ibecc_available = icl_ibecc_available, 377 .err_addr_to_sys_addr = ehl_err_addr_to_sys_addr, 378 .err_addr_to_imc_addr = ehl_err_addr_to_imc_addr, 379 }; 380 381 static struct res_config tgl_cfg = { 382 .machine_check = true, 383 .num_imc = 2, 384 .imc_base = 0x5000, 385 .cmf_base = 0x11000, 386 .cmf_size = 0x800, 387 .ms_hash_offset = 0xac, 388 .ibecc_base = 0xd400, 389 .ibecc_error_log_offset = 0x170, 390 .ibecc_available = tgl_ibecc_available, 391 .err_addr_to_sys_addr = tgl_err_addr_to_sys_addr, 392 .err_addr_to_imc_addr = tgl_err_addr_to_imc_addr, 393 }; 394 395 static struct res_config adl_cfg = { 396 .machine_check = true, 397 .num_imc = 2, 398 .imc_base = 0xd800, 399 .ibecc_base = 0xd400, 400 .ibecc_error_log_offset = 0x68, 401 .ibecc_available = tgl_ibecc_available, 402 .err_addr_to_sys_addr = adl_err_addr_to_sys_addr, 403 .err_addr_to_imc_addr = adl_err_addr_to_imc_addr, 404 }; 405 406 static const struct pci_device_id igen6_pci_tbl[] = { 407 { PCI_VDEVICE(INTEL, DID_EHL_SKU5), (kernel_ulong_t)&ehl_cfg }, 408 { PCI_VDEVICE(INTEL, DID_EHL_SKU6), (kernel_ulong_t)&ehl_cfg }, 409 { PCI_VDEVICE(INTEL, DID_EHL_SKU7), (kernel_ulong_t)&ehl_cfg }, 410 { PCI_VDEVICE(INTEL, DID_EHL_SKU8), (kernel_ulong_t)&ehl_cfg }, 411 { PCI_VDEVICE(INTEL, DID_EHL_SKU9), (kernel_ulong_t)&ehl_cfg }, 412 { PCI_VDEVICE(INTEL, DID_EHL_SKU10), (kernel_ulong_t)&ehl_cfg }, 413 { PCI_VDEVICE(INTEL, DID_EHL_SKU11), (kernel_ulong_t)&ehl_cfg }, 414 { PCI_VDEVICE(INTEL, DID_EHL_SKU12), (kernel_ulong_t)&ehl_cfg }, 415 { PCI_VDEVICE(INTEL, DID_EHL_SKU13), (kernel_ulong_t)&ehl_cfg }, 416 { PCI_VDEVICE(INTEL, DID_EHL_SKU14), (kernel_ulong_t)&ehl_cfg }, 417 { PCI_VDEVICE(INTEL, DID_EHL_SKU15), (kernel_ulong_t)&ehl_cfg }, 418 { PCI_VDEVICE(INTEL, DID_ICL_SKU8), (kernel_ulong_t)&icl_cfg }, 419 { PCI_VDEVICE(INTEL, DID_ICL_SKU10), (kernel_ulong_t)&icl_cfg }, 420 { PCI_VDEVICE(INTEL, DID_ICL_SKU11), (kernel_ulong_t)&icl_cfg }, 421 { PCI_VDEVICE(INTEL, DID_ICL_SKU12), (kernel_ulong_t)&icl_cfg }, 422 { PCI_VDEVICE(INTEL, DID_TGL_SKU), (kernel_ulong_t)&tgl_cfg }, 423 { PCI_VDEVICE(INTEL, DID_ADL_SKU1), (kernel_ulong_t)&adl_cfg }, 424 { PCI_VDEVICE(INTEL, DID_ADL_SKU2), (kernel_ulong_t)&adl_cfg }, 425 { PCI_VDEVICE(INTEL, DID_ADL_SKU3), (kernel_ulong_t)&adl_cfg }, 426 { PCI_VDEVICE(INTEL, DID_ADL_SKU4), (kernel_ulong_t)&adl_cfg }, 427 { }, 428 }; 429 MODULE_DEVICE_TABLE(pci, igen6_pci_tbl); 430 431 static enum dev_type get_width(int dimm_l, u32 mad_dimm) 432 { 433 u32 w = dimm_l ? MAD_DIMM_CH_DLW(mad_dimm) : 434 MAD_DIMM_CH_DSW(mad_dimm); 435 436 switch (w) { 437 case 0: 438 return DEV_X8; 439 case 1: 440 return DEV_X16; 441 case 2: 442 return DEV_X32; 443 default: 444 return DEV_UNKNOWN; 445 } 446 } 447 448 static enum mem_type get_memory_type(u32 mad_inter) 449 { 450 u32 t = MAD_INTER_CHANNEL_DDR_TYPE(mad_inter); 451 452 switch (t) { 453 case 0: 454 return MEM_DDR4; 455 case 1: 456 return MEM_DDR3; 457 case 2: 458 return MEM_LPDDR3; 459 case 3: 460 return MEM_LPDDR4; 461 case 4: 462 return MEM_WIO2; 463 default: 464 return MEM_UNKNOWN; 465 } 466 } 467 468 static int decode_chan_idx(u64 addr, u64 mask, int intlv_bit) 469 { 470 u64 hash_addr = addr & mask, hash = 0; 471 u64 intlv = (addr >> intlv_bit) & 1; 472 int i; 473 474 for (i = 6; i < 20; i++) 475 hash ^= (hash_addr >> i) & 1; 476 477 return (int)hash ^ intlv; 478 } 479 480 static u64 decode_channel_addr(u64 addr, int intlv_bit) 481 { 482 u64 channel_addr; 483 484 /* Remove the interleave bit and shift upper part down to fill gap */ 485 channel_addr = GET_BITFIELD(addr, intlv_bit + 1, 63) << intlv_bit; 486 channel_addr |= GET_BITFIELD(addr, 0, intlv_bit - 1); 487 488 return channel_addr; 489 } 490 491 static void decode_addr(u64 addr, u32 hash, u64 s_size, int l_map, 492 int *idx, u64 *sub_addr) 493 { 494 int intlv_bit = CHANNEL_HASH_LSB_MASK_BIT(hash) + 6; 495 496 if (addr > 2 * s_size) { 497 *sub_addr = addr - s_size; 498 *idx = l_map; 499 return; 500 } 501 502 if (CHANNEL_HASH_MODE(hash)) { 503 *sub_addr = decode_channel_addr(addr, intlv_bit); 504 *idx = decode_chan_idx(addr, CHANNEL_HASH_MASK(hash), intlv_bit); 505 } else { 506 *sub_addr = decode_channel_addr(addr, 6); 507 *idx = GET_BITFIELD(addr, 6, 6); 508 } 509 } 510 511 static int igen6_decode(struct decoded_addr *res) 512 { 513 struct igen6_imc *imc = &igen6_pvt->imc[res->mc]; 514 u64 addr = res->imc_addr, sub_addr, s_size; 515 int idx, l_map; 516 u32 hash; 517 518 if (addr >= igen6_tom) { 519 edac_dbg(0, "Address 0x%llx out of range\n", addr); 520 return -EINVAL; 521 } 522 523 /* Decode channel */ 524 hash = readl(imc->window + CHANNEL_HASH_OFFSET); 525 s_size = imc->ch_s_size; 526 l_map = imc->ch_l_map; 527 decode_addr(addr, hash, s_size, l_map, &idx, &sub_addr); 528 res->channel_idx = idx; 529 res->channel_addr = sub_addr; 530 531 /* Decode sub-channel/DIMM */ 532 hash = readl(imc->window + CHANNEL_EHASH_OFFSET); 533 s_size = imc->dimm_s_size[idx]; 534 l_map = imc->dimm_l_map[idx]; 535 decode_addr(res->channel_addr, hash, s_size, l_map, &idx, &sub_addr); 536 res->sub_channel_idx = idx; 537 res->sub_channel_addr = sub_addr; 538 539 return 0; 540 } 541 542 static void igen6_output_error(struct decoded_addr *res, 543 struct mem_ctl_info *mci, u64 ecclog) 544 { 545 enum hw_event_mc_err_type type = ecclog & ECC_ERROR_LOG_UE ? 546 HW_EVENT_ERR_UNCORRECTED : 547 HW_EVENT_ERR_CORRECTED; 548 549 edac_mc_handle_error(type, mci, 1, 550 res->sys_addr >> PAGE_SHIFT, 551 res->sys_addr & ~PAGE_MASK, 552 ECC_ERROR_LOG_SYND(ecclog), 553 res->channel_idx, res->sub_channel_idx, 554 -1, "", ""); 555 } 556 557 static struct gen_pool *ecclog_gen_pool_create(void) 558 { 559 struct gen_pool *pool; 560 561 pool = gen_pool_create(ilog2(sizeof(struct ecclog_node)), -1); 562 if (!pool) 563 return NULL; 564 565 if (gen_pool_add(pool, (unsigned long)ecclog_buf, ECCLOG_POOL_SIZE, -1)) { 566 gen_pool_destroy(pool); 567 return NULL; 568 } 569 570 return pool; 571 } 572 573 static int ecclog_gen_pool_add(int mc, u64 ecclog) 574 { 575 struct ecclog_node *node; 576 577 node = (void *)gen_pool_alloc(ecclog_pool, sizeof(*node)); 578 if (!node) 579 return -ENOMEM; 580 581 node->mc = mc; 582 node->ecclog = ecclog; 583 llist_add(&node->llnode, &ecclog_llist); 584 585 return 0; 586 } 587 588 /* 589 * Either the memory-mapped I/O status register ECC_ERROR_LOG or the PCI 590 * configuration space status register ERRSTS can indicate whether a 591 * correctable error or an uncorrectable error occurred. We only use the 592 * ECC_ERROR_LOG register to check error type, but need to clear both 593 * registers to enable future error events. 594 */ 595 static u64 ecclog_read_and_clear(struct igen6_imc *imc) 596 { 597 u64 ecclog = readq(imc->window + ECC_ERROR_LOG_OFFSET); 598 599 if (ecclog & (ECC_ERROR_LOG_CE | ECC_ERROR_LOG_UE)) { 600 /* Clear CE/UE bits by writing 1s */ 601 writeq(ecclog, imc->window + ECC_ERROR_LOG_OFFSET); 602 return ecclog; 603 } 604 605 return 0; 606 } 607 608 static void errsts_clear(struct igen6_imc *imc) 609 { 610 u16 errsts; 611 612 if (pci_read_config_word(imc->pdev, ERRSTS_OFFSET, &errsts)) { 613 igen6_printk(KERN_ERR, "Failed to read ERRSTS\n"); 614 return; 615 } 616 617 /* Clear CE/UE bits by writing 1s */ 618 if (errsts & (ERRSTS_CE | ERRSTS_UE)) 619 pci_write_config_word(imc->pdev, ERRSTS_OFFSET, errsts); 620 } 621 622 static int errcmd_enable_error_reporting(bool enable) 623 { 624 struct igen6_imc *imc = &igen6_pvt->imc[0]; 625 u16 errcmd; 626 int rc; 627 628 rc = pci_read_config_word(imc->pdev, ERRCMD_OFFSET, &errcmd); 629 if (rc) 630 return rc; 631 632 if (enable) 633 errcmd |= ERRCMD_CE | ERRSTS_UE; 634 else 635 errcmd &= ~(ERRCMD_CE | ERRSTS_UE); 636 637 rc = pci_write_config_word(imc->pdev, ERRCMD_OFFSET, errcmd); 638 if (rc) 639 return rc; 640 641 return 0; 642 } 643 644 static int ecclog_handler(void) 645 { 646 struct igen6_imc *imc; 647 int i, n = 0; 648 u64 ecclog; 649 650 for (i = 0; i < res_cfg->num_imc; i++) { 651 imc = &igen6_pvt->imc[i]; 652 653 /* errsts_clear() isn't NMI-safe. Delay it in the IRQ context */ 654 655 ecclog = ecclog_read_and_clear(imc); 656 if (!ecclog) 657 continue; 658 659 if (!ecclog_gen_pool_add(i, ecclog)) 660 irq_work_queue(&ecclog_irq_work); 661 662 n++; 663 } 664 665 return n; 666 } 667 668 static void ecclog_work_cb(struct work_struct *work) 669 { 670 struct ecclog_node *node, *tmp; 671 struct mem_ctl_info *mci; 672 struct llist_node *head; 673 struct decoded_addr res; 674 u64 eaddr; 675 676 head = llist_del_all(&ecclog_llist); 677 if (!head) 678 return; 679 680 llist_for_each_entry_safe(node, tmp, head, llnode) { 681 memset(&res, 0, sizeof(res)); 682 eaddr = ECC_ERROR_LOG_ADDR(node->ecclog) << 683 ECC_ERROR_LOG_ADDR_SHIFT; 684 res.mc = node->mc; 685 res.sys_addr = res_cfg->err_addr_to_sys_addr(eaddr, res.mc); 686 res.imc_addr = res_cfg->err_addr_to_imc_addr(eaddr, res.mc); 687 688 mci = igen6_pvt->imc[res.mc].mci; 689 690 edac_dbg(2, "MC %d, ecclog = 0x%llx\n", node->mc, node->ecclog); 691 igen6_mc_printk(mci, KERN_DEBUG, "HANDLING IBECC MEMORY ERROR\n"); 692 igen6_mc_printk(mci, KERN_DEBUG, "ADDR 0x%llx ", res.sys_addr); 693 694 if (!igen6_decode(&res)) 695 igen6_output_error(&res, mci, node->ecclog); 696 697 gen_pool_free(ecclog_pool, (unsigned long)node, sizeof(*node)); 698 } 699 } 700 701 static void ecclog_irq_work_cb(struct irq_work *irq_work) 702 { 703 int i; 704 705 for (i = 0; i < res_cfg->num_imc; i++) 706 errsts_clear(&igen6_pvt->imc[i]); 707 708 if (!llist_empty(&ecclog_llist)) 709 schedule_work(&ecclog_work); 710 } 711 712 static int ecclog_nmi_handler(unsigned int cmd, struct pt_regs *regs) 713 { 714 unsigned char reason; 715 716 if (!ecclog_handler()) 717 return NMI_DONE; 718 719 /* 720 * Both In-Band ECC correctable error and uncorrectable error are 721 * reported by SERR# NMI. The NMI generic code (see pci_serr_error()) 722 * doesn't clear the bit NMI_REASON_CLEAR_SERR (in port 0x61) to 723 * re-enable the SERR# NMI after NMI handling. So clear this bit here 724 * to re-enable SERR# NMI for receiving future In-Band ECC errors. 725 */ 726 reason = x86_platform.get_nmi_reason() & NMI_REASON_CLEAR_MASK; 727 reason |= NMI_REASON_CLEAR_SERR; 728 outb(reason, NMI_REASON_PORT); 729 reason &= ~NMI_REASON_CLEAR_SERR; 730 outb(reason, NMI_REASON_PORT); 731 732 return NMI_HANDLED; 733 } 734 735 static int ecclog_mce_handler(struct notifier_block *nb, unsigned long val, 736 void *data) 737 { 738 struct mce *mce = (struct mce *)data; 739 char *type; 740 741 if (mce->kflags & MCE_HANDLED_CEC) 742 return NOTIFY_DONE; 743 744 /* 745 * Ignore unless this is a memory related error. 746 * We don't check the bit MCI_STATUS_ADDRV of MCi_STATUS here, 747 * since this bit isn't set on some CPU (e.g., Tiger Lake UP3). 748 */ 749 if ((mce->status & 0xefff) >> 7 != 1) 750 return NOTIFY_DONE; 751 752 if (mce->mcgstatus & MCG_STATUS_MCIP) 753 type = "Exception"; 754 else 755 type = "Event"; 756 757 edac_dbg(0, "CPU %d: Machine Check %s: 0x%llx Bank %d: 0x%llx\n", 758 mce->extcpu, type, mce->mcgstatus, 759 mce->bank, mce->status); 760 edac_dbg(0, "TSC 0x%llx\n", mce->tsc); 761 edac_dbg(0, "ADDR 0x%llx\n", mce->addr); 762 edac_dbg(0, "MISC 0x%llx\n", mce->misc); 763 edac_dbg(0, "PROCESSOR %u:0x%x TIME %llu SOCKET %u APIC 0x%x\n", 764 mce->cpuvendor, mce->cpuid, mce->time, 765 mce->socketid, mce->apicid); 766 /* 767 * We just use the Machine Check for the memory error notification. 768 * Each memory controller is associated with an IBECC instance. 769 * Directly read and clear the error information(error address and 770 * error type) on all the IBECC instances so that we know on which 771 * memory controller the memory error(s) occurred. 772 */ 773 if (!ecclog_handler()) 774 return NOTIFY_DONE; 775 776 mce->kflags |= MCE_HANDLED_EDAC; 777 778 return NOTIFY_DONE; 779 } 780 781 static struct notifier_block ecclog_mce_dec = { 782 .notifier_call = ecclog_mce_handler, 783 .priority = MCE_PRIO_EDAC, 784 }; 785 786 static bool igen6_check_ecc(struct igen6_imc *imc) 787 { 788 u32 activate = readl(imc->window + IBECC_ACTIVATE_OFFSET); 789 790 return !!(activate & IBECC_ACTIVATE_EN); 791 } 792 793 static int igen6_get_dimm_config(struct mem_ctl_info *mci) 794 { 795 struct igen6_imc *imc = mci->pvt_info; 796 u32 mad_inter, mad_intra, mad_dimm; 797 int i, j, ndimms, mc = imc->mc; 798 struct dimm_info *dimm; 799 enum mem_type mtype; 800 enum dev_type dtype; 801 u64 dsize; 802 bool ecc; 803 804 edac_dbg(2, "\n"); 805 806 mad_inter = readl(imc->window + MAD_INTER_CHANNEL_OFFSET); 807 mtype = get_memory_type(mad_inter); 808 ecc = igen6_check_ecc(imc); 809 imc->ch_s_size = MAD_INTER_CHANNEL_CH_S_SIZE(mad_inter); 810 imc->ch_l_map = MAD_INTER_CHANNEL_CH_L_MAP(mad_inter); 811 812 for (i = 0; i < NUM_CHANNELS; i++) { 813 mad_intra = readl(imc->window + MAD_INTRA_CH0_OFFSET + i * 4); 814 mad_dimm = readl(imc->window + MAD_DIMM_CH0_OFFSET + i * 4); 815 816 imc->dimm_l_size[i] = MAD_DIMM_CH_DIMM_L_SIZE(mad_dimm); 817 imc->dimm_s_size[i] = MAD_DIMM_CH_DIMM_S_SIZE(mad_dimm); 818 imc->dimm_l_map[i] = MAD_INTRA_CH_DIMM_L_MAP(mad_intra); 819 imc->size += imc->dimm_s_size[i]; 820 imc->size += imc->dimm_l_size[i]; 821 ndimms = 0; 822 823 for (j = 0; j < NUM_DIMMS; j++) { 824 dimm = edac_get_dimm(mci, i, j, 0); 825 826 if (j ^ imc->dimm_l_map[i]) { 827 dtype = get_width(0, mad_dimm); 828 dsize = imc->dimm_s_size[i]; 829 } else { 830 dtype = get_width(1, mad_dimm); 831 dsize = imc->dimm_l_size[i]; 832 } 833 834 if (!dsize) 835 continue; 836 837 dimm->grain = 64; 838 dimm->mtype = mtype; 839 dimm->dtype = dtype; 840 dimm->nr_pages = MiB_TO_PAGES(dsize >> 20); 841 dimm->edac_mode = EDAC_SECDED; 842 snprintf(dimm->label, sizeof(dimm->label), 843 "MC#%d_Chan#%d_DIMM#%d", mc, i, j); 844 edac_dbg(0, "MC %d, Channel %d, DIMM %d, Size %llu MiB (%u pages)\n", 845 mc, i, j, dsize >> 20, dimm->nr_pages); 846 847 ndimms++; 848 } 849 850 if (ndimms && !ecc) { 851 igen6_printk(KERN_ERR, "MC%d In-Band ECC is disabled\n", mc); 852 return -ENODEV; 853 } 854 } 855 856 edac_dbg(0, "MC %d, total size %llu MiB\n", mc, imc->size >> 20); 857 858 return 0; 859 } 860 861 #ifdef CONFIG_EDAC_DEBUG 862 /* Top of upper usable DRAM */ 863 static u64 igen6_touud; 864 #define TOUUD_OFFSET 0xa8 865 866 static void igen6_reg_dump(struct igen6_imc *imc) 867 { 868 int i; 869 870 edac_dbg(2, "CHANNEL_HASH : 0x%x\n", 871 readl(imc->window + CHANNEL_HASH_OFFSET)); 872 edac_dbg(2, "CHANNEL_EHASH : 0x%x\n", 873 readl(imc->window + CHANNEL_EHASH_OFFSET)); 874 edac_dbg(2, "MAD_INTER_CHANNEL: 0x%x\n", 875 readl(imc->window + MAD_INTER_CHANNEL_OFFSET)); 876 edac_dbg(2, "ECC_ERROR_LOG : 0x%llx\n", 877 readq(imc->window + ECC_ERROR_LOG_OFFSET)); 878 879 for (i = 0; i < NUM_CHANNELS; i++) { 880 edac_dbg(2, "MAD_INTRA_CH%d : 0x%x\n", i, 881 readl(imc->window + MAD_INTRA_CH0_OFFSET + i * 4)); 882 edac_dbg(2, "MAD_DIMM_CH%d : 0x%x\n", i, 883 readl(imc->window + MAD_DIMM_CH0_OFFSET + i * 4)); 884 } 885 edac_dbg(2, "TOLUD : 0x%x", igen6_tolud); 886 edac_dbg(2, "TOUUD : 0x%llx", igen6_touud); 887 edac_dbg(2, "TOM : 0x%llx", igen6_tom); 888 } 889 890 static struct dentry *igen6_test; 891 892 static int debugfs_u64_set(void *data, u64 val) 893 { 894 u64 ecclog; 895 896 if ((val >= igen6_tolud && val < _4GB) || val >= igen6_touud) { 897 edac_dbg(0, "Address 0x%llx out of range\n", val); 898 return 0; 899 } 900 901 pr_warn_once("Fake error to 0x%llx injected via debugfs\n", val); 902 903 val >>= ECC_ERROR_LOG_ADDR_SHIFT; 904 ecclog = (val << ECC_ERROR_LOG_ADDR_SHIFT) | ECC_ERROR_LOG_CE; 905 906 if (!ecclog_gen_pool_add(0, ecclog)) 907 irq_work_queue(&ecclog_irq_work); 908 909 return 0; 910 } 911 DEFINE_SIMPLE_ATTRIBUTE(fops_u64_wo, NULL, debugfs_u64_set, "%llu\n"); 912 913 static void igen6_debug_setup(void) 914 { 915 igen6_test = edac_debugfs_create_dir("igen6_test"); 916 if (!igen6_test) 917 return; 918 919 if (!edac_debugfs_create_file("addr", 0200, igen6_test, 920 NULL, &fops_u64_wo)) { 921 debugfs_remove(igen6_test); 922 igen6_test = NULL; 923 } 924 } 925 926 static void igen6_debug_teardown(void) 927 { 928 debugfs_remove_recursive(igen6_test); 929 } 930 #else 931 static void igen6_reg_dump(struct igen6_imc *imc) {} 932 static void igen6_debug_setup(void) {} 933 static void igen6_debug_teardown(void) {} 934 #endif 935 936 static int igen6_pci_setup(struct pci_dev *pdev, u64 *mchbar) 937 { 938 union { 939 u64 v; 940 struct { 941 u32 v_lo; 942 u32 v_hi; 943 }; 944 } u; 945 946 edac_dbg(2, "\n"); 947 948 if (!res_cfg->ibecc_available(pdev)) { 949 edac_dbg(2, "No In-Band ECC IP\n"); 950 goto fail; 951 } 952 953 if (pci_read_config_dword(pdev, TOLUD_OFFSET, &igen6_tolud)) { 954 igen6_printk(KERN_ERR, "Failed to read TOLUD\n"); 955 goto fail; 956 } 957 958 igen6_tolud &= GENMASK(31, 20); 959 960 if (pci_read_config_dword(pdev, TOM_OFFSET, &u.v_lo)) { 961 igen6_printk(KERN_ERR, "Failed to read lower TOM\n"); 962 goto fail; 963 } 964 965 if (pci_read_config_dword(pdev, TOM_OFFSET + 4, &u.v_hi)) { 966 igen6_printk(KERN_ERR, "Failed to read upper TOM\n"); 967 goto fail; 968 } 969 970 igen6_tom = u.v & GENMASK_ULL(38, 20); 971 972 if (pci_read_config_dword(pdev, MCHBAR_OFFSET, &u.v_lo)) { 973 igen6_printk(KERN_ERR, "Failed to read lower MCHBAR\n"); 974 goto fail; 975 } 976 977 if (pci_read_config_dword(pdev, MCHBAR_OFFSET + 4, &u.v_hi)) { 978 igen6_printk(KERN_ERR, "Failed to read upper MCHBAR\n"); 979 goto fail; 980 } 981 982 if (!(u.v & MCHBAR_EN)) { 983 igen6_printk(KERN_ERR, "MCHBAR is disabled\n"); 984 goto fail; 985 } 986 987 *mchbar = MCHBAR_BASE(u.v); 988 989 #ifdef CONFIG_EDAC_DEBUG 990 if (pci_read_config_dword(pdev, TOUUD_OFFSET, &u.v_lo)) 991 edac_dbg(2, "Failed to read lower TOUUD\n"); 992 else if (pci_read_config_dword(pdev, TOUUD_OFFSET + 4, &u.v_hi)) 993 edac_dbg(2, "Failed to read upper TOUUD\n"); 994 else 995 igen6_touud = u.v & GENMASK_ULL(38, 20); 996 #endif 997 998 return 0; 999 fail: 1000 return -ENODEV; 1001 } 1002 1003 static int igen6_register_mci(int mc, u64 mchbar, struct pci_dev *pdev) 1004 { 1005 struct edac_mc_layer layers[2]; 1006 struct mem_ctl_info *mci; 1007 struct igen6_imc *imc; 1008 void __iomem *window; 1009 int rc; 1010 1011 edac_dbg(2, "\n"); 1012 1013 mchbar += mc * MCHBAR_SIZE; 1014 window = ioremap(mchbar, MCHBAR_SIZE); 1015 if (!window) { 1016 igen6_printk(KERN_ERR, "Failed to ioremap 0x%llx\n", mchbar); 1017 return -ENODEV; 1018 } 1019 1020 layers[0].type = EDAC_MC_LAYER_CHANNEL; 1021 layers[0].size = NUM_CHANNELS; 1022 layers[0].is_virt_csrow = false; 1023 layers[1].type = EDAC_MC_LAYER_SLOT; 1024 layers[1].size = NUM_DIMMS; 1025 layers[1].is_virt_csrow = true; 1026 1027 mci = edac_mc_alloc(mc, ARRAY_SIZE(layers), layers, 0); 1028 if (!mci) { 1029 rc = -ENOMEM; 1030 goto fail; 1031 } 1032 1033 mci->ctl_name = kasprintf(GFP_KERNEL, "Intel_client_SoC MC#%d", mc); 1034 if (!mci->ctl_name) { 1035 rc = -ENOMEM; 1036 goto fail2; 1037 } 1038 1039 mci->mtype_cap = MEM_FLAG_LPDDR4 | MEM_FLAG_DDR4; 1040 mci->edac_ctl_cap = EDAC_FLAG_SECDED; 1041 mci->edac_cap = EDAC_FLAG_SECDED; 1042 mci->mod_name = EDAC_MOD_STR; 1043 mci->dev_name = pci_name(pdev); 1044 mci->pvt_info = &igen6_pvt->imc[mc]; 1045 1046 imc = mci->pvt_info; 1047 device_initialize(&imc->dev); 1048 /* 1049 * EDAC core uses mci->pdev(pointer of structure device) as 1050 * memory controller ID. The client SoCs attach one or more 1051 * memory controllers to single pci_dev (single pci_dev->dev 1052 * can be for multiple memory controllers). 1053 * 1054 * To make mci->pdev unique, assign pci_dev->dev to mci->pdev 1055 * for the first memory controller and assign a unique imc->dev 1056 * to mci->pdev for each non-first memory controller. 1057 */ 1058 mci->pdev = mc ? &imc->dev : &pdev->dev; 1059 imc->mc = mc; 1060 imc->pdev = pdev; 1061 imc->window = window; 1062 1063 igen6_reg_dump(imc); 1064 1065 rc = igen6_get_dimm_config(mci); 1066 if (rc) 1067 goto fail3; 1068 1069 rc = edac_mc_add_mc(mci); 1070 if (rc) { 1071 igen6_printk(KERN_ERR, "Failed to register mci#%d\n", mc); 1072 goto fail3; 1073 } 1074 1075 imc->mci = mci; 1076 return 0; 1077 fail3: 1078 kfree(mci->ctl_name); 1079 fail2: 1080 edac_mc_free(mci); 1081 fail: 1082 iounmap(window); 1083 return rc; 1084 } 1085 1086 static void igen6_unregister_mcis(void) 1087 { 1088 struct mem_ctl_info *mci; 1089 struct igen6_imc *imc; 1090 int i; 1091 1092 edac_dbg(2, "\n"); 1093 1094 for (i = 0; i < res_cfg->num_imc; i++) { 1095 imc = &igen6_pvt->imc[i]; 1096 mci = imc->mci; 1097 if (!mci) 1098 continue; 1099 1100 edac_mc_del_mc(mci->pdev); 1101 kfree(mci->ctl_name); 1102 edac_mc_free(mci); 1103 iounmap(imc->window); 1104 } 1105 } 1106 1107 static int igen6_mem_slice_setup(u64 mchbar) 1108 { 1109 struct igen6_imc *imc = &igen6_pvt->imc[0]; 1110 u64 base = mchbar + res_cfg->cmf_base; 1111 u32 offset = res_cfg->ms_hash_offset; 1112 u32 size = res_cfg->cmf_size; 1113 u64 ms_s_size, ms_hash; 1114 void __iomem *cmf; 1115 int ms_l_map; 1116 1117 edac_dbg(2, "\n"); 1118 1119 if (imc[0].size < imc[1].size) { 1120 ms_s_size = imc[0].size; 1121 ms_l_map = 1; 1122 } else { 1123 ms_s_size = imc[1].size; 1124 ms_l_map = 0; 1125 } 1126 1127 igen6_pvt->ms_s_size = ms_s_size; 1128 igen6_pvt->ms_l_map = ms_l_map; 1129 1130 edac_dbg(0, "ms_s_size: %llu MiB, ms_l_map %d\n", 1131 ms_s_size >> 20, ms_l_map); 1132 1133 if (!size) 1134 return 0; 1135 1136 cmf = ioremap(base, size); 1137 if (!cmf) { 1138 igen6_printk(KERN_ERR, "Failed to ioremap cmf 0x%llx\n", base); 1139 return -ENODEV; 1140 } 1141 1142 ms_hash = readq(cmf + offset); 1143 igen6_pvt->ms_hash = ms_hash; 1144 1145 edac_dbg(0, "MEM_SLICE_HASH: 0x%llx\n", ms_hash); 1146 1147 iounmap(cmf); 1148 1149 return 0; 1150 } 1151 1152 static int register_err_handler(void) 1153 { 1154 int rc; 1155 1156 if (res_cfg->machine_check) { 1157 mce_register_decode_chain(&ecclog_mce_dec); 1158 return 0; 1159 } 1160 1161 rc = register_nmi_handler(NMI_SERR, ecclog_nmi_handler, 1162 0, IGEN6_NMI_NAME); 1163 if (rc) { 1164 igen6_printk(KERN_ERR, "Failed to register NMI handler\n"); 1165 return rc; 1166 } 1167 1168 return 0; 1169 } 1170 1171 static void unregister_err_handler(void) 1172 { 1173 if (res_cfg->machine_check) { 1174 mce_unregister_decode_chain(&ecclog_mce_dec); 1175 return; 1176 } 1177 1178 unregister_nmi_handler(NMI_SERR, IGEN6_NMI_NAME); 1179 } 1180 1181 static int igen6_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 1182 { 1183 u64 mchbar; 1184 int i, rc; 1185 1186 edac_dbg(2, "\n"); 1187 1188 igen6_pvt = kzalloc(sizeof(*igen6_pvt), GFP_KERNEL); 1189 if (!igen6_pvt) 1190 return -ENOMEM; 1191 1192 res_cfg = (struct res_config *)ent->driver_data; 1193 1194 rc = igen6_pci_setup(pdev, &mchbar); 1195 if (rc) 1196 goto fail; 1197 1198 for (i = 0; i < res_cfg->num_imc; i++) { 1199 rc = igen6_register_mci(i, mchbar, pdev); 1200 if (rc) 1201 goto fail2; 1202 } 1203 1204 if (res_cfg->num_imc > 1) { 1205 rc = igen6_mem_slice_setup(mchbar); 1206 if (rc) 1207 goto fail2; 1208 } 1209 1210 ecclog_pool = ecclog_gen_pool_create(); 1211 if (!ecclog_pool) { 1212 rc = -ENOMEM; 1213 goto fail2; 1214 } 1215 1216 INIT_WORK(&ecclog_work, ecclog_work_cb); 1217 init_irq_work(&ecclog_irq_work, ecclog_irq_work_cb); 1218 1219 /* Check if any pending errors before registering the NMI handler */ 1220 ecclog_handler(); 1221 1222 rc = register_err_handler(); 1223 if (rc) 1224 goto fail3; 1225 1226 /* Enable error reporting */ 1227 rc = errcmd_enable_error_reporting(true); 1228 if (rc) { 1229 igen6_printk(KERN_ERR, "Failed to enable error reporting\n"); 1230 goto fail4; 1231 } 1232 1233 igen6_debug_setup(); 1234 return 0; 1235 fail4: 1236 unregister_nmi_handler(NMI_SERR, IGEN6_NMI_NAME); 1237 fail3: 1238 gen_pool_destroy(ecclog_pool); 1239 fail2: 1240 igen6_unregister_mcis(); 1241 fail: 1242 kfree(igen6_pvt); 1243 return rc; 1244 } 1245 1246 static void igen6_remove(struct pci_dev *pdev) 1247 { 1248 edac_dbg(2, "\n"); 1249 1250 igen6_debug_teardown(); 1251 errcmd_enable_error_reporting(false); 1252 unregister_err_handler(); 1253 irq_work_sync(&ecclog_irq_work); 1254 flush_work(&ecclog_work); 1255 gen_pool_destroy(ecclog_pool); 1256 igen6_unregister_mcis(); 1257 kfree(igen6_pvt); 1258 } 1259 1260 static struct pci_driver igen6_driver = { 1261 .name = EDAC_MOD_STR, 1262 .probe = igen6_probe, 1263 .remove = igen6_remove, 1264 .id_table = igen6_pci_tbl, 1265 }; 1266 1267 static int __init igen6_init(void) 1268 { 1269 const char *owner; 1270 int rc; 1271 1272 edac_dbg(2, "\n"); 1273 1274 if (ghes_get_devices()) 1275 return -EBUSY; 1276 1277 owner = edac_get_owner(); 1278 if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR))) 1279 return -EBUSY; 1280 1281 edac_op_state = EDAC_OPSTATE_NMI; 1282 1283 rc = pci_register_driver(&igen6_driver); 1284 if (rc) 1285 return rc; 1286 1287 igen6_printk(KERN_INFO, "%s\n", IGEN6_REVISION); 1288 1289 return 0; 1290 } 1291 1292 static void __exit igen6_exit(void) 1293 { 1294 edac_dbg(2, "\n"); 1295 1296 pci_unregister_driver(&igen6_driver); 1297 } 1298 1299 module_init(igen6_init); 1300 module_exit(igen6_exit); 1301 1302 MODULE_LICENSE("GPL v2"); 1303 MODULE_AUTHOR("Qiuxu Zhuo"); 1304 MODULE_DESCRIPTION("MC Driver for Intel client SoC using In-Band ECC"); 1305