110590a9dSQiuxu Zhuo // SPDX-License-Identifier: GPL-2.0 210590a9dSQiuxu Zhuo /* 310590a9dSQiuxu Zhuo * Driver for Intel client SoC with integrated memory controller using IBECC 410590a9dSQiuxu Zhuo * 510590a9dSQiuxu Zhuo * Copyright (C) 2020 Intel Corporation 610590a9dSQiuxu Zhuo * 710590a9dSQiuxu Zhuo * The In-Band ECC (IBECC) IP provides ECC protection to all or specific 810590a9dSQiuxu Zhuo * regions of the physical memory space. It's used for memory controllers 910590a9dSQiuxu Zhuo * that don't support the out-of-band ECC which often needs an additional 1010590a9dSQiuxu Zhuo * storage device to each channel for storing ECC data. 1110590a9dSQiuxu Zhuo */ 1210590a9dSQiuxu Zhuo 1310590a9dSQiuxu Zhuo #include <linux/module.h> 1410590a9dSQiuxu Zhuo #include <linux/init.h> 1510590a9dSQiuxu Zhuo #include <linux/pci.h> 1610590a9dSQiuxu Zhuo #include <linux/slab.h> 1710590a9dSQiuxu Zhuo #include <linux/irq_work.h> 1810590a9dSQiuxu Zhuo #include <linux/llist.h> 1910590a9dSQiuxu Zhuo #include <linux/genalloc.h> 2010590a9dSQiuxu Zhuo #include <linux/edac.h> 2110590a9dSQiuxu Zhuo #include <linux/bits.h> 2210590a9dSQiuxu Zhuo #include <linux/io.h> 2310590a9dSQiuxu Zhuo #include <asm/mach_traps.h> 2410590a9dSQiuxu Zhuo #include <asm/nmi.h> 2510590a9dSQiuxu Zhuo 2610590a9dSQiuxu Zhuo #include "edac_mc.h" 2710590a9dSQiuxu Zhuo #include "edac_module.h" 2810590a9dSQiuxu Zhuo 2910590a9dSQiuxu Zhuo #define IGEN6_REVISION "v2.4" 3010590a9dSQiuxu Zhuo 3110590a9dSQiuxu Zhuo #define EDAC_MOD_STR "igen6_edac" 3210590a9dSQiuxu Zhuo #define IGEN6_NMI_NAME "igen6_ibecc" 3310590a9dSQiuxu Zhuo 3410590a9dSQiuxu Zhuo /* Debug macros */ 3510590a9dSQiuxu Zhuo #define igen6_printk(level, fmt, arg...) \ 3610590a9dSQiuxu Zhuo edac_printk(level, "igen6", fmt, ##arg) 3710590a9dSQiuxu Zhuo 3810590a9dSQiuxu Zhuo #define igen6_mc_printk(mci, level, fmt, arg...) \ 3910590a9dSQiuxu Zhuo edac_mc_chipset_printk(mci, level, "igen6", fmt, ##arg) 4010590a9dSQiuxu Zhuo 4110590a9dSQiuxu Zhuo #define GET_BITFIELD(v, lo, hi) (((v) & GENMASK_ULL(hi, lo)) >> (lo)) 4210590a9dSQiuxu Zhuo 4310590a9dSQiuxu Zhuo #define NUM_IMC 1 /* Max memory controllers */ 4410590a9dSQiuxu Zhuo #define NUM_CHANNELS 2 /* Max channels */ 4510590a9dSQiuxu Zhuo #define NUM_DIMMS 2 /* Max DIMMs per channel */ 4610590a9dSQiuxu Zhuo 4710590a9dSQiuxu Zhuo #define _4GB BIT_ULL(32) 4810590a9dSQiuxu Zhuo 4910590a9dSQiuxu Zhuo /* Size of physical memory */ 5010590a9dSQiuxu Zhuo #define TOM_OFFSET 0xa0 5110590a9dSQiuxu Zhuo /* Top of low usable DRAM */ 5210590a9dSQiuxu Zhuo #define TOLUD_OFFSET 0xbc 5310590a9dSQiuxu Zhuo /* Capability register C */ 5410590a9dSQiuxu Zhuo #define CAPID_C_OFFSET 0xec 5510590a9dSQiuxu Zhuo #define CAPID_C_IBECC BIT(15) 5610590a9dSQiuxu Zhuo 5710590a9dSQiuxu Zhuo /* Error Status */ 5810590a9dSQiuxu Zhuo #define ERRSTS_OFFSET 0xc8 5910590a9dSQiuxu Zhuo #define ERRSTS_CE BIT_ULL(6) 6010590a9dSQiuxu Zhuo #define ERRSTS_UE BIT_ULL(7) 6110590a9dSQiuxu Zhuo 6210590a9dSQiuxu Zhuo /* Error Command */ 6310590a9dSQiuxu Zhuo #define ERRCMD_OFFSET 0xca 6410590a9dSQiuxu Zhuo #define ERRCMD_CE BIT_ULL(6) 6510590a9dSQiuxu Zhuo #define ERRCMD_UE BIT_ULL(7) 6610590a9dSQiuxu Zhuo 6710590a9dSQiuxu Zhuo /* IBECC MMIO base address */ 6810590a9dSQiuxu Zhuo #define IBECC_BASE (res_cfg->ibecc_base) 6910590a9dSQiuxu Zhuo #define IBECC_ACTIVATE_OFFSET IBECC_BASE 7010590a9dSQiuxu Zhuo #define IBECC_ACTIVATE_EN BIT(0) 7110590a9dSQiuxu Zhuo 7210590a9dSQiuxu Zhuo /* IBECC error log */ 7310590a9dSQiuxu Zhuo #define ECC_ERROR_LOG_OFFSET (IBECC_BASE + 0x170) 7410590a9dSQiuxu Zhuo #define ECC_ERROR_LOG_CE BIT_ULL(62) 7510590a9dSQiuxu Zhuo #define ECC_ERROR_LOG_UE BIT_ULL(63) 7610590a9dSQiuxu Zhuo #define ECC_ERROR_LOG_ADDR_SHIFT 5 7710590a9dSQiuxu Zhuo #define ECC_ERROR_LOG_ADDR(v) GET_BITFIELD(v, 5, 38) 7810590a9dSQiuxu Zhuo #define ECC_ERROR_LOG_SYND(v) GET_BITFIELD(v, 46, 61) 7910590a9dSQiuxu Zhuo 8010590a9dSQiuxu Zhuo /* Host MMIO base address */ 8110590a9dSQiuxu Zhuo #define MCHBAR_OFFSET 0x48 8210590a9dSQiuxu Zhuo #define MCHBAR_EN BIT_ULL(0) 8310590a9dSQiuxu Zhuo #define MCHBAR_BASE(v) (GET_BITFIELD(v, 16, 38) << 16) 8410590a9dSQiuxu Zhuo #define MCHBAR_SIZE 0x10000 8510590a9dSQiuxu Zhuo 8610590a9dSQiuxu Zhuo /* Parameters for the channel decode stage */ 8710590a9dSQiuxu Zhuo #define MAD_INTER_CHANNEL_OFFSET 0x5000 8810590a9dSQiuxu Zhuo #define MAD_INTER_CHANNEL_DDR_TYPE(v) GET_BITFIELD(v, 0, 2) 8910590a9dSQiuxu Zhuo #define MAD_INTER_CHANNEL_ECHM(v) GET_BITFIELD(v, 3, 3) 9010590a9dSQiuxu Zhuo #define MAD_INTER_CHANNEL_CH_L_MAP(v) GET_BITFIELD(v, 4, 4) 9110590a9dSQiuxu Zhuo #define MAD_INTER_CHANNEL_CH_S_SIZE(v) ((u64)GET_BITFIELD(v, 12, 19) << 29) 9210590a9dSQiuxu Zhuo 9310590a9dSQiuxu Zhuo /* Parameters for DRAM decode stage */ 9410590a9dSQiuxu Zhuo #define MAD_INTRA_CH0_OFFSET 0x5004 9510590a9dSQiuxu Zhuo #define MAD_INTRA_CH_DIMM_L_MAP(v) GET_BITFIELD(v, 0, 0) 9610590a9dSQiuxu Zhuo 9710590a9dSQiuxu Zhuo /* DIMM characteristics */ 9810590a9dSQiuxu Zhuo #define MAD_DIMM_CH0_OFFSET 0x500c 9910590a9dSQiuxu Zhuo #define MAD_DIMM_CH_DIMM_L_SIZE(v) ((u64)GET_BITFIELD(v, 0, 6) << 29) 10010590a9dSQiuxu Zhuo #define MAD_DIMM_CH_DLW(v) GET_BITFIELD(v, 7, 8) 10110590a9dSQiuxu Zhuo #define MAD_DIMM_CH_DIMM_S_SIZE(v) ((u64)GET_BITFIELD(v, 16, 22) << 29) 10210590a9dSQiuxu Zhuo #define MAD_DIMM_CH_DSW(v) GET_BITFIELD(v, 24, 25) 10310590a9dSQiuxu Zhuo 10410590a9dSQiuxu Zhuo /* Hash for channel selection */ 10510590a9dSQiuxu Zhuo #define CHANNEL_HASH_OFFSET 0X5024 10610590a9dSQiuxu Zhuo /* Hash for enhanced channel selection */ 10710590a9dSQiuxu Zhuo #define CHANNEL_EHASH_OFFSET 0X5028 10810590a9dSQiuxu Zhuo #define CHANNEL_HASH_MASK(v) (GET_BITFIELD(v, 6, 19) << 6) 10910590a9dSQiuxu Zhuo #define CHANNEL_HASH_LSB_MASK_BIT(v) GET_BITFIELD(v, 24, 26) 11010590a9dSQiuxu Zhuo #define CHANNEL_HASH_MODE(v) GET_BITFIELD(v, 28, 28) 11110590a9dSQiuxu Zhuo 11210590a9dSQiuxu Zhuo static struct res_config { 11310590a9dSQiuxu Zhuo int num_imc; 11410590a9dSQiuxu Zhuo u32 ibecc_base; 11510590a9dSQiuxu Zhuo bool (*ibecc_available)(struct pci_dev *pdev); 11610590a9dSQiuxu Zhuo /* Convert error address logged in IBECC to system physical address */ 11710590a9dSQiuxu Zhuo u64 (*err_addr_to_sys_addr)(u64 eaddr); 11810590a9dSQiuxu Zhuo /* Convert error address logged in IBECC to integrated memory controller address */ 11910590a9dSQiuxu Zhuo u64 (*err_addr_to_imc_addr)(u64 eaddr); 12010590a9dSQiuxu Zhuo } *res_cfg; 12110590a9dSQiuxu Zhuo 12210590a9dSQiuxu Zhuo struct igen6_imc { 12310590a9dSQiuxu Zhuo int mc; 12410590a9dSQiuxu Zhuo struct mem_ctl_info *mci; 12510590a9dSQiuxu Zhuo struct pci_dev *pdev; 12610590a9dSQiuxu Zhuo struct device dev; 12710590a9dSQiuxu Zhuo void __iomem *window; 12810590a9dSQiuxu Zhuo u64 ch_s_size; 12910590a9dSQiuxu Zhuo int ch_l_map; 13010590a9dSQiuxu Zhuo u64 dimm_s_size[NUM_CHANNELS]; 13110590a9dSQiuxu Zhuo u64 dimm_l_size[NUM_CHANNELS]; 13210590a9dSQiuxu Zhuo int dimm_l_map[NUM_CHANNELS]; 13310590a9dSQiuxu Zhuo }; 13410590a9dSQiuxu Zhuo 13510590a9dSQiuxu Zhuo static struct igen6_pvt { 13610590a9dSQiuxu Zhuo struct igen6_imc imc[NUM_IMC]; 13710590a9dSQiuxu Zhuo } *igen6_pvt; 13810590a9dSQiuxu Zhuo 13910590a9dSQiuxu Zhuo /* The top of low usable DRAM */ 14010590a9dSQiuxu Zhuo static u32 igen6_tolud; 14110590a9dSQiuxu Zhuo /* The size of physical memory */ 14210590a9dSQiuxu Zhuo static u64 igen6_tom; 14310590a9dSQiuxu Zhuo 14410590a9dSQiuxu Zhuo struct decoded_addr { 14510590a9dSQiuxu Zhuo int mc; 14610590a9dSQiuxu Zhuo u64 imc_addr; 14710590a9dSQiuxu Zhuo u64 sys_addr; 14810590a9dSQiuxu Zhuo int channel_idx; 14910590a9dSQiuxu Zhuo u64 channel_addr; 15010590a9dSQiuxu Zhuo int sub_channel_idx; 15110590a9dSQiuxu Zhuo u64 sub_channel_addr; 15210590a9dSQiuxu Zhuo }; 15310590a9dSQiuxu Zhuo 15410590a9dSQiuxu Zhuo struct ecclog_node { 15510590a9dSQiuxu Zhuo struct llist_node llnode; 15610590a9dSQiuxu Zhuo int mc; 15710590a9dSQiuxu Zhuo u64 ecclog; 15810590a9dSQiuxu Zhuo }; 15910590a9dSQiuxu Zhuo 16010590a9dSQiuxu Zhuo /* 16110590a9dSQiuxu Zhuo * In the NMI handler, the driver uses the lock-less memory allocator 16210590a9dSQiuxu Zhuo * to allocate memory to store the IBECC error logs and links the logs 16310590a9dSQiuxu Zhuo * to the lock-less list. Delay printk() and the work of error reporting 16410590a9dSQiuxu Zhuo * to EDAC core in a worker. 16510590a9dSQiuxu Zhuo */ 16610590a9dSQiuxu Zhuo #define ECCLOG_POOL_SIZE PAGE_SIZE 167*77429eebSkernel test robot static LLIST_HEAD(ecclog_llist); 16810590a9dSQiuxu Zhuo static struct gen_pool *ecclog_pool; 16910590a9dSQiuxu Zhuo static char ecclog_buf[ECCLOG_POOL_SIZE]; 17010590a9dSQiuxu Zhuo static struct irq_work ecclog_irq_work; 17110590a9dSQiuxu Zhuo static struct work_struct ecclog_work; 17210590a9dSQiuxu Zhuo 17310590a9dSQiuxu Zhuo /* Compute die IDs for Elkhart Lake with IBECC */ 17410590a9dSQiuxu Zhuo #define DID_EHL_SKU5 0x4514 17510590a9dSQiuxu Zhuo #define DID_EHL_SKU6 0x4528 17610590a9dSQiuxu Zhuo #define DID_EHL_SKU7 0x452a 17710590a9dSQiuxu Zhuo #define DID_EHL_SKU8 0x4516 17810590a9dSQiuxu Zhuo #define DID_EHL_SKU9 0x452c 17910590a9dSQiuxu Zhuo #define DID_EHL_SKU10 0x452e 18010590a9dSQiuxu Zhuo #define DID_EHL_SKU11 0x4532 18110590a9dSQiuxu Zhuo #define DID_EHL_SKU12 0x4518 18210590a9dSQiuxu Zhuo #define DID_EHL_SKU13 0x451a 18310590a9dSQiuxu Zhuo #define DID_EHL_SKU14 0x4534 18410590a9dSQiuxu Zhuo #define DID_EHL_SKU15 0x4536 18510590a9dSQiuxu Zhuo 18610590a9dSQiuxu Zhuo static bool ehl_ibecc_available(struct pci_dev *pdev) 18710590a9dSQiuxu Zhuo { 18810590a9dSQiuxu Zhuo u32 v; 18910590a9dSQiuxu Zhuo 19010590a9dSQiuxu Zhuo if (pci_read_config_dword(pdev, CAPID_C_OFFSET, &v)) 19110590a9dSQiuxu Zhuo return false; 19210590a9dSQiuxu Zhuo 19310590a9dSQiuxu Zhuo return !!(CAPID_C_IBECC & v); 19410590a9dSQiuxu Zhuo } 19510590a9dSQiuxu Zhuo 19610590a9dSQiuxu Zhuo static u64 ehl_err_addr_to_sys_addr(u64 eaddr) 19710590a9dSQiuxu Zhuo { 19810590a9dSQiuxu Zhuo return eaddr; 19910590a9dSQiuxu Zhuo } 20010590a9dSQiuxu Zhuo 20110590a9dSQiuxu Zhuo static u64 ehl_err_addr_to_imc_addr(u64 eaddr) 20210590a9dSQiuxu Zhuo { 20310590a9dSQiuxu Zhuo if (eaddr < igen6_tolud) 20410590a9dSQiuxu Zhuo return eaddr; 20510590a9dSQiuxu Zhuo 20610590a9dSQiuxu Zhuo if (igen6_tom <= _4GB) 20710590a9dSQiuxu Zhuo return eaddr + igen6_tolud - _4GB; 20810590a9dSQiuxu Zhuo 20910590a9dSQiuxu Zhuo if (eaddr < _4GB) 21010590a9dSQiuxu Zhuo return eaddr + igen6_tolud - igen6_tom; 21110590a9dSQiuxu Zhuo 21210590a9dSQiuxu Zhuo return eaddr; 21310590a9dSQiuxu Zhuo } 21410590a9dSQiuxu Zhuo 21510590a9dSQiuxu Zhuo static struct res_config ehl_cfg = { 21610590a9dSQiuxu Zhuo .num_imc = 1, 21710590a9dSQiuxu Zhuo .ibecc_base = 0xdc00, 21810590a9dSQiuxu Zhuo .ibecc_available = ehl_ibecc_available, 21910590a9dSQiuxu Zhuo .err_addr_to_sys_addr = ehl_err_addr_to_sys_addr, 22010590a9dSQiuxu Zhuo .err_addr_to_imc_addr = ehl_err_addr_to_imc_addr, 22110590a9dSQiuxu Zhuo }; 22210590a9dSQiuxu Zhuo 22310590a9dSQiuxu Zhuo static const struct pci_device_id igen6_pci_tbl[] = { 22410590a9dSQiuxu Zhuo { PCI_VDEVICE(INTEL, DID_EHL_SKU5), (kernel_ulong_t)&ehl_cfg }, 22510590a9dSQiuxu Zhuo { PCI_VDEVICE(INTEL, DID_EHL_SKU6), (kernel_ulong_t)&ehl_cfg }, 22610590a9dSQiuxu Zhuo { PCI_VDEVICE(INTEL, DID_EHL_SKU7), (kernel_ulong_t)&ehl_cfg }, 22710590a9dSQiuxu Zhuo { PCI_VDEVICE(INTEL, DID_EHL_SKU8), (kernel_ulong_t)&ehl_cfg }, 22810590a9dSQiuxu Zhuo { PCI_VDEVICE(INTEL, DID_EHL_SKU9), (kernel_ulong_t)&ehl_cfg }, 22910590a9dSQiuxu Zhuo { PCI_VDEVICE(INTEL, DID_EHL_SKU10), (kernel_ulong_t)&ehl_cfg }, 23010590a9dSQiuxu Zhuo { PCI_VDEVICE(INTEL, DID_EHL_SKU11), (kernel_ulong_t)&ehl_cfg }, 23110590a9dSQiuxu Zhuo { PCI_VDEVICE(INTEL, DID_EHL_SKU12), (kernel_ulong_t)&ehl_cfg }, 23210590a9dSQiuxu Zhuo { PCI_VDEVICE(INTEL, DID_EHL_SKU13), (kernel_ulong_t)&ehl_cfg }, 23310590a9dSQiuxu Zhuo { PCI_VDEVICE(INTEL, DID_EHL_SKU14), (kernel_ulong_t)&ehl_cfg }, 23410590a9dSQiuxu Zhuo { PCI_VDEVICE(INTEL, DID_EHL_SKU15), (kernel_ulong_t)&ehl_cfg }, 23510590a9dSQiuxu Zhuo { }, 23610590a9dSQiuxu Zhuo }; 23710590a9dSQiuxu Zhuo MODULE_DEVICE_TABLE(pci, igen6_pci_tbl); 23810590a9dSQiuxu Zhuo 23910590a9dSQiuxu Zhuo static enum dev_type get_width(int dimm_l, u32 mad_dimm) 24010590a9dSQiuxu Zhuo { 24110590a9dSQiuxu Zhuo u32 w = dimm_l ? MAD_DIMM_CH_DLW(mad_dimm) : 24210590a9dSQiuxu Zhuo MAD_DIMM_CH_DSW(mad_dimm); 24310590a9dSQiuxu Zhuo 24410590a9dSQiuxu Zhuo switch (w) { 24510590a9dSQiuxu Zhuo case 0: 24610590a9dSQiuxu Zhuo return DEV_X8; 24710590a9dSQiuxu Zhuo case 1: 24810590a9dSQiuxu Zhuo return DEV_X16; 24910590a9dSQiuxu Zhuo case 2: 25010590a9dSQiuxu Zhuo return DEV_X32; 25110590a9dSQiuxu Zhuo default: 25210590a9dSQiuxu Zhuo return DEV_UNKNOWN; 25310590a9dSQiuxu Zhuo } 25410590a9dSQiuxu Zhuo } 25510590a9dSQiuxu Zhuo 25610590a9dSQiuxu Zhuo static enum mem_type get_memory_type(u32 mad_inter) 25710590a9dSQiuxu Zhuo { 25810590a9dSQiuxu Zhuo u32 t = MAD_INTER_CHANNEL_DDR_TYPE(mad_inter); 25910590a9dSQiuxu Zhuo 26010590a9dSQiuxu Zhuo switch (t) { 26110590a9dSQiuxu Zhuo case 0: 26210590a9dSQiuxu Zhuo return MEM_DDR4; 26310590a9dSQiuxu Zhuo case 1: 26410590a9dSQiuxu Zhuo return MEM_DDR3; 26510590a9dSQiuxu Zhuo case 2: 26610590a9dSQiuxu Zhuo return MEM_LPDDR3; 26710590a9dSQiuxu Zhuo case 3: 26810590a9dSQiuxu Zhuo return MEM_LPDDR4; 26910590a9dSQiuxu Zhuo case 4: 27010590a9dSQiuxu Zhuo return MEM_WIO2; 27110590a9dSQiuxu Zhuo default: 27210590a9dSQiuxu Zhuo return MEM_UNKNOWN; 27310590a9dSQiuxu Zhuo } 27410590a9dSQiuxu Zhuo } 27510590a9dSQiuxu Zhuo 27610590a9dSQiuxu Zhuo static int decode_chan_idx(u64 addr, u64 mask, int intlv_bit) 27710590a9dSQiuxu Zhuo { 27810590a9dSQiuxu Zhuo u64 hash_addr = addr & mask, hash = 0; 27910590a9dSQiuxu Zhuo u64 intlv = (addr >> intlv_bit) & 1; 28010590a9dSQiuxu Zhuo int i; 28110590a9dSQiuxu Zhuo 28210590a9dSQiuxu Zhuo for (i = 6; i < 20; i++) 28310590a9dSQiuxu Zhuo hash ^= (hash_addr >> i) & 1; 28410590a9dSQiuxu Zhuo 28510590a9dSQiuxu Zhuo return (int)hash ^ intlv; 28610590a9dSQiuxu Zhuo } 28710590a9dSQiuxu Zhuo 28810590a9dSQiuxu Zhuo static u64 decode_channel_addr(u64 addr, int intlv_bit) 28910590a9dSQiuxu Zhuo { 29010590a9dSQiuxu Zhuo u64 channel_addr; 29110590a9dSQiuxu Zhuo 29210590a9dSQiuxu Zhuo /* Remove the interleave bit and shift upper part down to fill gap */ 29310590a9dSQiuxu Zhuo channel_addr = GET_BITFIELD(addr, intlv_bit + 1, 63) << intlv_bit; 29410590a9dSQiuxu Zhuo channel_addr |= GET_BITFIELD(addr, 0, intlv_bit - 1); 29510590a9dSQiuxu Zhuo 29610590a9dSQiuxu Zhuo return channel_addr; 29710590a9dSQiuxu Zhuo } 29810590a9dSQiuxu Zhuo 29910590a9dSQiuxu Zhuo static void decode_addr(u64 addr, u32 hash, u64 s_size, int l_map, 30010590a9dSQiuxu Zhuo int *idx, u64 *sub_addr) 30110590a9dSQiuxu Zhuo { 30210590a9dSQiuxu Zhuo int intlv_bit = CHANNEL_HASH_LSB_MASK_BIT(hash) + 6; 30310590a9dSQiuxu Zhuo 30410590a9dSQiuxu Zhuo if (addr > 2 * s_size) { 30510590a9dSQiuxu Zhuo *sub_addr = addr - s_size; 30610590a9dSQiuxu Zhuo *idx = l_map; 30710590a9dSQiuxu Zhuo return; 30810590a9dSQiuxu Zhuo } 30910590a9dSQiuxu Zhuo 31010590a9dSQiuxu Zhuo if (CHANNEL_HASH_MODE(hash)) { 31110590a9dSQiuxu Zhuo *sub_addr = decode_channel_addr(addr, intlv_bit); 31210590a9dSQiuxu Zhuo *idx = decode_chan_idx(addr, CHANNEL_HASH_MASK(hash), intlv_bit); 31310590a9dSQiuxu Zhuo } else { 31410590a9dSQiuxu Zhuo *sub_addr = decode_channel_addr(addr, 6); 31510590a9dSQiuxu Zhuo *idx = GET_BITFIELD(addr, 6, 6); 31610590a9dSQiuxu Zhuo } 31710590a9dSQiuxu Zhuo } 31810590a9dSQiuxu Zhuo 31910590a9dSQiuxu Zhuo static int igen6_decode(struct decoded_addr *res) 32010590a9dSQiuxu Zhuo { 32110590a9dSQiuxu Zhuo struct igen6_imc *imc = &igen6_pvt->imc[res->mc]; 32210590a9dSQiuxu Zhuo u64 addr = res->imc_addr, sub_addr, s_size; 32310590a9dSQiuxu Zhuo int idx, l_map; 32410590a9dSQiuxu Zhuo u32 hash; 32510590a9dSQiuxu Zhuo 32610590a9dSQiuxu Zhuo if (addr >= igen6_tom) { 32710590a9dSQiuxu Zhuo edac_dbg(0, "Address 0x%llx out of range\n", addr); 32810590a9dSQiuxu Zhuo return -EINVAL; 32910590a9dSQiuxu Zhuo } 33010590a9dSQiuxu Zhuo 33110590a9dSQiuxu Zhuo /* Decode channel */ 33210590a9dSQiuxu Zhuo hash = readl(imc->window + CHANNEL_HASH_OFFSET); 33310590a9dSQiuxu Zhuo s_size = imc->ch_s_size; 33410590a9dSQiuxu Zhuo l_map = imc->ch_l_map; 33510590a9dSQiuxu Zhuo decode_addr(addr, hash, s_size, l_map, &idx, &sub_addr); 33610590a9dSQiuxu Zhuo res->channel_idx = idx; 33710590a9dSQiuxu Zhuo res->channel_addr = sub_addr; 33810590a9dSQiuxu Zhuo 33910590a9dSQiuxu Zhuo /* Decode sub-channel/DIMM */ 34010590a9dSQiuxu Zhuo hash = readl(imc->window + CHANNEL_EHASH_OFFSET); 34110590a9dSQiuxu Zhuo s_size = imc->dimm_s_size[idx]; 34210590a9dSQiuxu Zhuo l_map = imc->dimm_l_map[idx]; 34310590a9dSQiuxu Zhuo decode_addr(res->channel_addr, hash, s_size, l_map, &idx, &sub_addr); 34410590a9dSQiuxu Zhuo res->sub_channel_idx = idx; 34510590a9dSQiuxu Zhuo res->sub_channel_addr = sub_addr; 34610590a9dSQiuxu Zhuo 34710590a9dSQiuxu Zhuo return 0; 34810590a9dSQiuxu Zhuo } 34910590a9dSQiuxu Zhuo 35010590a9dSQiuxu Zhuo static void igen6_output_error(struct decoded_addr *res, 35110590a9dSQiuxu Zhuo struct mem_ctl_info *mci, u64 ecclog) 35210590a9dSQiuxu Zhuo { 35310590a9dSQiuxu Zhuo enum hw_event_mc_err_type type = ecclog & ECC_ERROR_LOG_UE ? 35410590a9dSQiuxu Zhuo HW_EVENT_ERR_UNCORRECTED : 35510590a9dSQiuxu Zhuo HW_EVENT_ERR_CORRECTED; 35610590a9dSQiuxu Zhuo 35710590a9dSQiuxu Zhuo edac_mc_handle_error(type, mci, 1, 35810590a9dSQiuxu Zhuo res->sys_addr >> PAGE_SHIFT, 35910590a9dSQiuxu Zhuo res->sys_addr & ~PAGE_MASK, 36010590a9dSQiuxu Zhuo ECC_ERROR_LOG_SYND(ecclog), 36110590a9dSQiuxu Zhuo res->channel_idx, res->sub_channel_idx, 36210590a9dSQiuxu Zhuo -1, "", ""); 36310590a9dSQiuxu Zhuo } 36410590a9dSQiuxu Zhuo 36510590a9dSQiuxu Zhuo static struct gen_pool *ecclog_gen_pool_create(void) 36610590a9dSQiuxu Zhuo { 36710590a9dSQiuxu Zhuo struct gen_pool *pool; 36810590a9dSQiuxu Zhuo 36910590a9dSQiuxu Zhuo pool = gen_pool_create(ilog2(sizeof(struct ecclog_node)), -1); 37010590a9dSQiuxu Zhuo if (!pool) 37110590a9dSQiuxu Zhuo return NULL; 37210590a9dSQiuxu Zhuo 37310590a9dSQiuxu Zhuo if (gen_pool_add(pool, (unsigned long)ecclog_buf, ECCLOG_POOL_SIZE, -1)) { 37410590a9dSQiuxu Zhuo gen_pool_destroy(pool); 37510590a9dSQiuxu Zhuo return NULL; 37610590a9dSQiuxu Zhuo } 37710590a9dSQiuxu Zhuo 37810590a9dSQiuxu Zhuo return pool; 37910590a9dSQiuxu Zhuo } 38010590a9dSQiuxu Zhuo 38110590a9dSQiuxu Zhuo static int ecclog_gen_pool_add(int mc, u64 ecclog) 38210590a9dSQiuxu Zhuo { 38310590a9dSQiuxu Zhuo struct ecclog_node *node; 38410590a9dSQiuxu Zhuo 38510590a9dSQiuxu Zhuo node = (void *)gen_pool_alloc(ecclog_pool, sizeof(*node)); 38610590a9dSQiuxu Zhuo if (!node) 38710590a9dSQiuxu Zhuo return -ENOMEM; 38810590a9dSQiuxu Zhuo 38910590a9dSQiuxu Zhuo node->mc = mc; 39010590a9dSQiuxu Zhuo node->ecclog = ecclog; 39110590a9dSQiuxu Zhuo llist_add(&node->llnode, &ecclog_llist); 39210590a9dSQiuxu Zhuo 39310590a9dSQiuxu Zhuo return 0; 39410590a9dSQiuxu Zhuo } 39510590a9dSQiuxu Zhuo 39610590a9dSQiuxu Zhuo /* 39710590a9dSQiuxu Zhuo * Either the memory-mapped I/O status register ECC_ERROR_LOG or the PCI 39810590a9dSQiuxu Zhuo * configuration space status register ERRSTS can indicate whether a 39910590a9dSQiuxu Zhuo * correctable error or an uncorrectable error occurred. We only use the 40010590a9dSQiuxu Zhuo * ECC_ERROR_LOG register to check error type, but need to clear both 40110590a9dSQiuxu Zhuo * registers to enable future error events. 40210590a9dSQiuxu Zhuo */ 40310590a9dSQiuxu Zhuo static u64 ecclog_read_and_clear(struct igen6_imc *imc) 40410590a9dSQiuxu Zhuo { 40510590a9dSQiuxu Zhuo u64 ecclog = readq(imc->window + ECC_ERROR_LOG_OFFSET); 40610590a9dSQiuxu Zhuo 40710590a9dSQiuxu Zhuo if (ecclog & (ECC_ERROR_LOG_CE | ECC_ERROR_LOG_UE)) { 40810590a9dSQiuxu Zhuo /* Clear CE/UE bits by writing 1s */ 40910590a9dSQiuxu Zhuo writeq(ecclog, imc->window + ECC_ERROR_LOG_OFFSET); 41010590a9dSQiuxu Zhuo return ecclog; 41110590a9dSQiuxu Zhuo } 41210590a9dSQiuxu Zhuo 41310590a9dSQiuxu Zhuo return 0; 41410590a9dSQiuxu Zhuo } 41510590a9dSQiuxu Zhuo 41610590a9dSQiuxu Zhuo static void errsts_clear(struct igen6_imc *imc) 41710590a9dSQiuxu Zhuo { 41810590a9dSQiuxu Zhuo u16 errsts; 41910590a9dSQiuxu Zhuo 42010590a9dSQiuxu Zhuo if (pci_read_config_word(imc->pdev, ERRSTS_OFFSET, &errsts)) { 42110590a9dSQiuxu Zhuo igen6_printk(KERN_ERR, "Failed to read ERRSTS\n"); 42210590a9dSQiuxu Zhuo return; 42310590a9dSQiuxu Zhuo } 42410590a9dSQiuxu Zhuo 42510590a9dSQiuxu Zhuo /* Clear CE/UE bits by writing 1s */ 42610590a9dSQiuxu Zhuo if (errsts & (ERRSTS_CE | ERRSTS_UE)) 42710590a9dSQiuxu Zhuo pci_write_config_word(imc->pdev, ERRSTS_OFFSET, errsts); 42810590a9dSQiuxu Zhuo } 42910590a9dSQiuxu Zhuo 43010590a9dSQiuxu Zhuo static int errcmd_enable_error_reporting(bool enable) 43110590a9dSQiuxu Zhuo { 43210590a9dSQiuxu Zhuo struct igen6_imc *imc = &igen6_pvt->imc[0]; 43310590a9dSQiuxu Zhuo u16 errcmd; 43410590a9dSQiuxu Zhuo int rc; 43510590a9dSQiuxu Zhuo 43610590a9dSQiuxu Zhuo rc = pci_read_config_word(imc->pdev, ERRCMD_OFFSET, &errcmd); 43710590a9dSQiuxu Zhuo if (rc) 43810590a9dSQiuxu Zhuo return rc; 43910590a9dSQiuxu Zhuo 44010590a9dSQiuxu Zhuo if (enable) 44110590a9dSQiuxu Zhuo errcmd |= ERRCMD_CE | ERRSTS_UE; 44210590a9dSQiuxu Zhuo else 44310590a9dSQiuxu Zhuo errcmd &= ~(ERRCMD_CE | ERRSTS_UE); 44410590a9dSQiuxu Zhuo 44510590a9dSQiuxu Zhuo rc = pci_write_config_word(imc->pdev, ERRCMD_OFFSET, errcmd); 44610590a9dSQiuxu Zhuo if (rc) 44710590a9dSQiuxu Zhuo return rc; 44810590a9dSQiuxu Zhuo 44910590a9dSQiuxu Zhuo return 0; 45010590a9dSQiuxu Zhuo } 45110590a9dSQiuxu Zhuo 45210590a9dSQiuxu Zhuo static int ecclog_handler(void) 45310590a9dSQiuxu Zhuo { 45410590a9dSQiuxu Zhuo struct igen6_imc *imc; 45510590a9dSQiuxu Zhuo int i, n = 0; 45610590a9dSQiuxu Zhuo u64 ecclog; 45710590a9dSQiuxu Zhuo 45810590a9dSQiuxu Zhuo for (i = 0; i < res_cfg->num_imc; i++) { 45910590a9dSQiuxu Zhuo imc = &igen6_pvt->imc[i]; 46010590a9dSQiuxu Zhuo 46110590a9dSQiuxu Zhuo /* errsts_clear() isn't NMI-safe. Delay it in the IRQ context */ 46210590a9dSQiuxu Zhuo 46310590a9dSQiuxu Zhuo ecclog = ecclog_read_and_clear(imc); 46410590a9dSQiuxu Zhuo if (!ecclog) 46510590a9dSQiuxu Zhuo continue; 46610590a9dSQiuxu Zhuo 46710590a9dSQiuxu Zhuo if (!ecclog_gen_pool_add(i, ecclog)) 46810590a9dSQiuxu Zhuo irq_work_queue(&ecclog_irq_work); 46910590a9dSQiuxu Zhuo 47010590a9dSQiuxu Zhuo n++; 47110590a9dSQiuxu Zhuo } 47210590a9dSQiuxu Zhuo 47310590a9dSQiuxu Zhuo return n; 47410590a9dSQiuxu Zhuo } 47510590a9dSQiuxu Zhuo 47610590a9dSQiuxu Zhuo static void ecclog_work_cb(struct work_struct *work) 47710590a9dSQiuxu Zhuo { 47810590a9dSQiuxu Zhuo struct ecclog_node *node, *tmp; 47910590a9dSQiuxu Zhuo struct mem_ctl_info *mci; 48010590a9dSQiuxu Zhuo struct llist_node *head; 48110590a9dSQiuxu Zhuo struct decoded_addr res; 48210590a9dSQiuxu Zhuo u64 eaddr; 48310590a9dSQiuxu Zhuo 48410590a9dSQiuxu Zhuo head = llist_del_all(&ecclog_llist); 48510590a9dSQiuxu Zhuo if (!head) 48610590a9dSQiuxu Zhuo return; 48710590a9dSQiuxu Zhuo 48810590a9dSQiuxu Zhuo llist_for_each_entry_safe(node, tmp, head, llnode) { 48910590a9dSQiuxu Zhuo memset(&res, 0, sizeof(res)); 49010590a9dSQiuxu Zhuo eaddr = ECC_ERROR_LOG_ADDR(node->ecclog) << 49110590a9dSQiuxu Zhuo ECC_ERROR_LOG_ADDR_SHIFT; 49210590a9dSQiuxu Zhuo res.mc = node->mc; 49310590a9dSQiuxu Zhuo res.sys_addr = res_cfg->err_addr_to_sys_addr(eaddr); 49410590a9dSQiuxu Zhuo res.imc_addr = res_cfg->err_addr_to_imc_addr(eaddr); 49510590a9dSQiuxu Zhuo 49610590a9dSQiuxu Zhuo mci = igen6_pvt->imc[res.mc].mci; 49710590a9dSQiuxu Zhuo 49810590a9dSQiuxu Zhuo edac_dbg(2, "MC %d, ecclog = 0x%llx\n", node->mc, node->ecclog); 49910590a9dSQiuxu Zhuo igen6_mc_printk(mci, KERN_DEBUG, "HANDLING IBECC MEMORY ERROR\n"); 50010590a9dSQiuxu Zhuo igen6_mc_printk(mci, KERN_DEBUG, "ADDR 0x%llx ", res.sys_addr); 50110590a9dSQiuxu Zhuo 50210590a9dSQiuxu Zhuo if (!igen6_decode(&res)) 50310590a9dSQiuxu Zhuo igen6_output_error(&res, mci, node->ecclog); 50410590a9dSQiuxu Zhuo 50510590a9dSQiuxu Zhuo gen_pool_free(ecclog_pool, (unsigned long)node, sizeof(*node)); 50610590a9dSQiuxu Zhuo } 50710590a9dSQiuxu Zhuo } 50810590a9dSQiuxu Zhuo 50910590a9dSQiuxu Zhuo static void ecclog_irq_work_cb(struct irq_work *irq_work) 51010590a9dSQiuxu Zhuo { 51110590a9dSQiuxu Zhuo int i; 51210590a9dSQiuxu Zhuo 51310590a9dSQiuxu Zhuo for (i = 0; i < res_cfg->num_imc; i++) 51410590a9dSQiuxu Zhuo errsts_clear(&igen6_pvt->imc[i]); 51510590a9dSQiuxu Zhuo 51610590a9dSQiuxu Zhuo if (!llist_empty(&ecclog_llist)) 51710590a9dSQiuxu Zhuo schedule_work(&ecclog_work); 51810590a9dSQiuxu Zhuo } 51910590a9dSQiuxu Zhuo 52010590a9dSQiuxu Zhuo static int ecclog_nmi_handler(unsigned int cmd, struct pt_regs *regs) 52110590a9dSQiuxu Zhuo { 52210590a9dSQiuxu Zhuo unsigned char reason; 52310590a9dSQiuxu Zhuo 52410590a9dSQiuxu Zhuo if (!ecclog_handler()) 52510590a9dSQiuxu Zhuo return NMI_DONE; 52610590a9dSQiuxu Zhuo 52710590a9dSQiuxu Zhuo /* 52810590a9dSQiuxu Zhuo * Both In-Band ECC correctable error and uncorrectable error are 52910590a9dSQiuxu Zhuo * reported by SERR# NMI. The NMI generic code (see pci_serr_error()) 53010590a9dSQiuxu Zhuo * doesn't clear the bit NMI_REASON_CLEAR_SERR (in port 0x61) to 53110590a9dSQiuxu Zhuo * re-enable the SERR# NMI after NMI handling. So clear this bit here 53210590a9dSQiuxu Zhuo * to re-enable SERR# NMI for receiving future In-Band ECC errors. 53310590a9dSQiuxu Zhuo */ 53410590a9dSQiuxu Zhuo reason = x86_platform.get_nmi_reason() & NMI_REASON_CLEAR_MASK; 53510590a9dSQiuxu Zhuo reason |= NMI_REASON_CLEAR_SERR; 53610590a9dSQiuxu Zhuo outb(reason, NMI_REASON_PORT); 53710590a9dSQiuxu Zhuo reason &= ~NMI_REASON_CLEAR_SERR; 53810590a9dSQiuxu Zhuo outb(reason, NMI_REASON_PORT); 53910590a9dSQiuxu Zhuo 54010590a9dSQiuxu Zhuo return NMI_HANDLED; 54110590a9dSQiuxu Zhuo } 54210590a9dSQiuxu Zhuo 54310590a9dSQiuxu Zhuo static bool igen6_check_ecc(struct igen6_imc *imc) 54410590a9dSQiuxu Zhuo { 54510590a9dSQiuxu Zhuo u32 activate = readl(imc->window + IBECC_ACTIVATE_OFFSET); 54610590a9dSQiuxu Zhuo 54710590a9dSQiuxu Zhuo return !!(activate & IBECC_ACTIVATE_EN); 54810590a9dSQiuxu Zhuo } 54910590a9dSQiuxu Zhuo 55010590a9dSQiuxu Zhuo static int igen6_get_dimm_config(struct mem_ctl_info *mci) 55110590a9dSQiuxu Zhuo { 55210590a9dSQiuxu Zhuo struct igen6_imc *imc = mci->pvt_info; 55310590a9dSQiuxu Zhuo u32 mad_inter, mad_intra, mad_dimm; 55410590a9dSQiuxu Zhuo int i, j, ndimms, mc = imc->mc; 55510590a9dSQiuxu Zhuo struct dimm_info *dimm; 55610590a9dSQiuxu Zhuo enum mem_type mtype; 55710590a9dSQiuxu Zhuo enum dev_type dtype; 55810590a9dSQiuxu Zhuo u64 dsize; 55910590a9dSQiuxu Zhuo bool ecc; 56010590a9dSQiuxu Zhuo 56110590a9dSQiuxu Zhuo edac_dbg(2, "\n"); 56210590a9dSQiuxu Zhuo 56310590a9dSQiuxu Zhuo mad_inter = readl(imc->window + MAD_INTER_CHANNEL_OFFSET); 56410590a9dSQiuxu Zhuo mtype = get_memory_type(mad_inter); 56510590a9dSQiuxu Zhuo ecc = igen6_check_ecc(imc); 56610590a9dSQiuxu Zhuo imc->ch_s_size = MAD_INTER_CHANNEL_CH_S_SIZE(mad_inter); 56710590a9dSQiuxu Zhuo imc->ch_l_map = MAD_INTER_CHANNEL_CH_L_MAP(mad_inter); 56810590a9dSQiuxu Zhuo 56910590a9dSQiuxu Zhuo for (i = 0; i < NUM_CHANNELS; i++) { 57010590a9dSQiuxu Zhuo mad_intra = readl(imc->window + MAD_INTRA_CH0_OFFSET + i * 4); 57110590a9dSQiuxu Zhuo mad_dimm = readl(imc->window + MAD_DIMM_CH0_OFFSET + i * 4); 57210590a9dSQiuxu Zhuo 57310590a9dSQiuxu Zhuo imc->dimm_l_size[i] = MAD_DIMM_CH_DIMM_L_SIZE(mad_dimm); 57410590a9dSQiuxu Zhuo imc->dimm_s_size[i] = MAD_DIMM_CH_DIMM_S_SIZE(mad_dimm); 57510590a9dSQiuxu Zhuo imc->dimm_l_map[i] = MAD_INTRA_CH_DIMM_L_MAP(mad_intra); 57610590a9dSQiuxu Zhuo ndimms = 0; 57710590a9dSQiuxu Zhuo 57810590a9dSQiuxu Zhuo for (j = 0; j < NUM_DIMMS; j++) { 57910590a9dSQiuxu Zhuo dimm = edac_get_dimm(mci, i, j, 0); 58010590a9dSQiuxu Zhuo 58110590a9dSQiuxu Zhuo if (j ^ imc->dimm_l_map[i]) { 58210590a9dSQiuxu Zhuo dtype = get_width(0, mad_dimm); 58310590a9dSQiuxu Zhuo dsize = imc->dimm_s_size[i]; 58410590a9dSQiuxu Zhuo } else { 58510590a9dSQiuxu Zhuo dtype = get_width(1, mad_dimm); 58610590a9dSQiuxu Zhuo dsize = imc->dimm_l_size[i]; 58710590a9dSQiuxu Zhuo } 58810590a9dSQiuxu Zhuo 58910590a9dSQiuxu Zhuo if (!dsize) 59010590a9dSQiuxu Zhuo continue; 59110590a9dSQiuxu Zhuo 59210590a9dSQiuxu Zhuo dimm->grain = 64; 59310590a9dSQiuxu Zhuo dimm->mtype = mtype; 59410590a9dSQiuxu Zhuo dimm->dtype = dtype; 59510590a9dSQiuxu Zhuo dimm->nr_pages = MiB_TO_PAGES(dsize >> 20); 59610590a9dSQiuxu Zhuo dimm->edac_mode = EDAC_SECDED; 59710590a9dSQiuxu Zhuo snprintf(dimm->label, sizeof(dimm->label), 59810590a9dSQiuxu Zhuo "MC#%d_Chan#%d_DIMM#%d", mc, i, j); 59910590a9dSQiuxu Zhuo edac_dbg(0, "MC %d, Channel %d, DIMM %d, Size %llu MiB (%u pages)\n", 60010590a9dSQiuxu Zhuo mc, i, j, dsize >> 20, dimm->nr_pages); 60110590a9dSQiuxu Zhuo 60210590a9dSQiuxu Zhuo ndimms++; 60310590a9dSQiuxu Zhuo } 60410590a9dSQiuxu Zhuo 60510590a9dSQiuxu Zhuo if (ndimms && !ecc) { 60610590a9dSQiuxu Zhuo igen6_printk(KERN_ERR, "MC%d In-Band ECC is disabled\n", mc); 60710590a9dSQiuxu Zhuo return -ENODEV; 60810590a9dSQiuxu Zhuo } 60910590a9dSQiuxu Zhuo } 61010590a9dSQiuxu Zhuo 61110590a9dSQiuxu Zhuo return 0; 61210590a9dSQiuxu Zhuo } 61310590a9dSQiuxu Zhuo 61410590a9dSQiuxu Zhuo #ifdef CONFIG_EDAC_DEBUG 6152223d8c7SQiuxu Zhuo /* Top of upper usable DRAM */ 6162223d8c7SQiuxu Zhuo static u64 igen6_touud; 6172223d8c7SQiuxu Zhuo #define TOUUD_OFFSET 0xa8 6182223d8c7SQiuxu Zhuo 61910590a9dSQiuxu Zhuo static void igen6_reg_dump(struct igen6_imc *imc) 62010590a9dSQiuxu Zhuo { 62110590a9dSQiuxu Zhuo int i; 62210590a9dSQiuxu Zhuo 62310590a9dSQiuxu Zhuo edac_dbg(2, "CHANNEL_HASH : 0x%x\n", 62410590a9dSQiuxu Zhuo readl(imc->window + CHANNEL_HASH_OFFSET)); 62510590a9dSQiuxu Zhuo edac_dbg(2, "CHANNEL_EHASH : 0x%x\n", 62610590a9dSQiuxu Zhuo readl(imc->window + CHANNEL_EHASH_OFFSET)); 62710590a9dSQiuxu Zhuo edac_dbg(2, "MAD_INTER_CHANNEL: 0x%x\n", 62810590a9dSQiuxu Zhuo readl(imc->window + MAD_INTER_CHANNEL_OFFSET)); 62910590a9dSQiuxu Zhuo edac_dbg(2, "ECC_ERROR_LOG : 0x%llx\n", 63010590a9dSQiuxu Zhuo readq(imc->window + ECC_ERROR_LOG_OFFSET)); 63110590a9dSQiuxu Zhuo 63210590a9dSQiuxu Zhuo for (i = 0; i < NUM_CHANNELS; i++) { 63310590a9dSQiuxu Zhuo edac_dbg(2, "MAD_INTRA_CH%d : 0x%x\n", i, 63410590a9dSQiuxu Zhuo readl(imc->window + MAD_INTRA_CH0_OFFSET + i * 4)); 63510590a9dSQiuxu Zhuo edac_dbg(2, "MAD_DIMM_CH%d : 0x%x\n", i, 63610590a9dSQiuxu Zhuo readl(imc->window + MAD_DIMM_CH0_OFFSET + i * 4)); 63710590a9dSQiuxu Zhuo } 63810590a9dSQiuxu Zhuo edac_dbg(2, "TOLUD : 0x%x", igen6_tolud); 6392223d8c7SQiuxu Zhuo edac_dbg(2, "TOUUD : 0x%llx", igen6_touud); 64010590a9dSQiuxu Zhuo edac_dbg(2, "TOM : 0x%llx", igen6_tom); 64110590a9dSQiuxu Zhuo } 6422223d8c7SQiuxu Zhuo 6432223d8c7SQiuxu Zhuo static struct dentry *igen6_test; 6442223d8c7SQiuxu Zhuo 6452223d8c7SQiuxu Zhuo static int debugfs_u64_set(void *data, u64 val) 6462223d8c7SQiuxu Zhuo { 6472223d8c7SQiuxu Zhuo u64 ecclog; 6482223d8c7SQiuxu Zhuo 6492223d8c7SQiuxu Zhuo if ((val >= igen6_tolud && val < _4GB) || val >= igen6_touud) { 6502223d8c7SQiuxu Zhuo edac_dbg(0, "Address 0x%llx out of range\n", val); 6512223d8c7SQiuxu Zhuo return 0; 6522223d8c7SQiuxu Zhuo } 6532223d8c7SQiuxu Zhuo 6542223d8c7SQiuxu Zhuo pr_warn_once("Fake error to 0x%llx injected via debugfs\n", val); 6552223d8c7SQiuxu Zhuo 6562223d8c7SQiuxu Zhuo val >>= ECC_ERROR_LOG_ADDR_SHIFT; 6572223d8c7SQiuxu Zhuo ecclog = (val << ECC_ERROR_LOG_ADDR_SHIFT) | ECC_ERROR_LOG_CE; 6582223d8c7SQiuxu Zhuo 6592223d8c7SQiuxu Zhuo if (!ecclog_gen_pool_add(0, ecclog)) 6602223d8c7SQiuxu Zhuo irq_work_queue(&ecclog_irq_work); 6612223d8c7SQiuxu Zhuo 6622223d8c7SQiuxu Zhuo return 0; 6632223d8c7SQiuxu Zhuo } 6642223d8c7SQiuxu Zhuo DEFINE_SIMPLE_ATTRIBUTE(fops_u64_wo, NULL, debugfs_u64_set, "%llu\n"); 6652223d8c7SQiuxu Zhuo 6662223d8c7SQiuxu Zhuo static void igen6_debug_setup(void) 6672223d8c7SQiuxu Zhuo { 6682223d8c7SQiuxu Zhuo igen6_test = edac_debugfs_create_dir("igen6_test"); 6692223d8c7SQiuxu Zhuo if (!igen6_test) 6702223d8c7SQiuxu Zhuo return; 6712223d8c7SQiuxu Zhuo 6722223d8c7SQiuxu Zhuo if (!edac_debugfs_create_file("addr", 0200, igen6_test, 6732223d8c7SQiuxu Zhuo NULL, &fops_u64_wo)) { 6742223d8c7SQiuxu Zhuo debugfs_remove(igen6_test); 6752223d8c7SQiuxu Zhuo igen6_test = NULL; 6762223d8c7SQiuxu Zhuo } 6772223d8c7SQiuxu Zhuo } 6782223d8c7SQiuxu Zhuo 6792223d8c7SQiuxu Zhuo static void igen6_debug_teardown(void) 6802223d8c7SQiuxu Zhuo { 6812223d8c7SQiuxu Zhuo debugfs_remove_recursive(igen6_test); 6822223d8c7SQiuxu Zhuo } 68310590a9dSQiuxu Zhuo #else 68410590a9dSQiuxu Zhuo static void igen6_reg_dump(struct igen6_imc *imc) {} 6852223d8c7SQiuxu Zhuo static void igen6_debug_setup(void) {} 6862223d8c7SQiuxu Zhuo static void igen6_debug_teardown(void) {} 68710590a9dSQiuxu Zhuo #endif 68810590a9dSQiuxu Zhuo 68910590a9dSQiuxu Zhuo static int igen6_pci_setup(struct pci_dev *pdev, u64 *mchbar) 69010590a9dSQiuxu Zhuo { 69110590a9dSQiuxu Zhuo union { 69210590a9dSQiuxu Zhuo u64 v; 69310590a9dSQiuxu Zhuo struct { 69410590a9dSQiuxu Zhuo u32 v_lo; 69510590a9dSQiuxu Zhuo u32 v_hi; 69610590a9dSQiuxu Zhuo }; 69710590a9dSQiuxu Zhuo } u; 69810590a9dSQiuxu Zhuo 69910590a9dSQiuxu Zhuo edac_dbg(2, "\n"); 70010590a9dSQiuxu Zhuo 70110590a9dSQiuxu Zhuo if (!res_cfg->ibecc_available(pdev)) { 70210590a9dSQiuxu Zhuo edac_dbg(2, "No In-Band ECC IP\n"); 70310590a9dSQiuxu Zhuo goto fail; 70410590a9dSQiuxu Zhuo } 70510590a9dSQiuxu Zhuo 70610590a9dSQiuxu Zhuo if (pci_read_config_dword(pdev, TOLUD_OFFSET, &igen6_tolud)) { 70710590a9dSQiuxu Zhuo igen6_printk(KERN_ERR, "Failed to read TOLUD\n"); 70810590a9dSQiuxu Zhuo goto fail; 70910590a9dSQiuxu Zhuo } 71010590a9dSQiuxu Zhuo 71110590a9dSQiuxu Zhuo igen6_tolud &= GENMASK(31, 20); 71210590a9dSQiuxu Zhuo 71310590a9dSQiuxu Zhuo if (pci_read_config_dword(pdev, TOM_OFFSET, &u.v_lo)) { 71410590a9dSQiuxu Zhuo igen6_printk(KERN_ERR, "Failed to read lower TOM\n"); 71510590a9dSQiuxu Zhuo goto fail; 71610590a9dSQiuxu Zhuo } 71710590a9dSQiuxu Zhuo 71810590a9dSQiuxu Zhuo if (pci_read_config_dword(pdev, TOM_OFFSET + 4, &u.v_hi)) { 71910590a9dSQiuxu Zhuo igen6_printk(KERN_ERR, "Failed to read upper TOM\n"); 72010590a9dSQiuxu Zhuo goto fail; 72110590a9dSQiuxu Zhuo } 72210590a9dSQiuxu Zhuo 72310590a9dSQiuxu Zhuo igen6_tom = u.v & GENMASK_ULL(38, 20); 72410590a9dSQiuxu Zhuo 72510590a9dSQiuxu Zhuo if (pci_read_config_dword(pdev, MCHBAR_OFFSET, &u.v_lo)) { 72610590a9dSQiuxu Zhuo igen6_printk(KERN_ERR, "Failed to read lower MCHBAR\n"); 72710590a9dSQiuxu Zhuo goto fail; 72810590a9dSQiuxu Zhuo } 72910590a9dSQiuxu Zhuo 73010590a9dSQiuxu Zhuo if (pci_read_config_dword(pdev, MCHBAR_OFFSET + 4, &u.v_hi)) { 73110590a9dSQiuxu Zhuo igen6_printk(KERN_ERR, "Failed to read upper MCHBAR\n"); 73210590a9dSQiuxu Zhuo goto fail; 73310590a9dSQiuxu Zhuo } 73410590a9dSQiuxu Zhuo 73510590a9dSQiuxu Zhuo if (!(u.v & MCHBAR_EN)) { 73610590a9dSQiuxu Zhuo igen6_printk(KERN_ERR, "MCHBAR is disabled\n"); 73710590a9dSQiuxu Zhuo goto fail; 73810590a9dSQiuxu Zhuo } 73910590a9dSQiuxu Zhuo 74010590a9dSQiuxu Zhuo *mchbar = MCHBAR_BASE(u.v); 74110590a9dSQiuxu Zhuo 7422223d8c7SQiuxu Zhuo #ifdef CONFIG_EDAC_DEBUG 7432223d8c7SQiuxu Zhuo if (pci_read_config_dword(pdev, TOUUD_OFFSET, &u.v_lo)) 7442223d8c7SQiuxu Zhuo edac_dbg(2, "Failed to read lower TOUUD\n"); 7452223d8c7SQiuxu Zhuo else if (pci_read_config_dword(pdev, TOUUD_OFFSET + 4, &u.v_hi)) 7462223d8c7SQiuxu Zhuo edac_dbg(2, "Failed to read upper TOUUD\n"); 7472223d8c7SQiuxu Zhuo else 7482223d8c7SQiuxu Zhuo igen6_touud = u.v & GENMASK_ULL(38, 20); 7492223d8c7SQiuxu Zhuo #endif 7502223d8c7SQiuxu Zhuo 75110590a9dSQiuxu Zhuo return 0; 75210590a9dSQiuxu Zhuo fail: 75310590a9dSQiuxu Zhuo return -ENODEV; 75410590a9dSQiuxu Zhuo } 75510590a9dSQiuxu Zhuo 75610590a9dSQiuxu Zhuo static int igen6_register_mci(int mc, u64 mchbar, struct pci_dev *pdev) 75710590a9dSQiuxu Zhuo { 75810590a9dSQiuxu Zhuo struct edac_mc_layer layers[2]; 75910590a9dSQiuxu Zhuo struct mem_ctl_info *mci; 76010590a9dSQiuxu Zhuo struct igen6_imc *imc; 76110590a9dSQiuxu Zhuo void __iomem *window; 76210590a9dSQiuxu Zhuo int rc; 76310590a9dSQiuxu Zhuo 76410590a9dSQiuxu Zhuo edac_dbg(2, "\n"); 76510590a9dSQiuxu Zhuo 76610590a9dSQiuxu Zhuo mchbar += mc * MCHBAR_SIZE; 76710590a9dSQiuxu Zhuo window = ioremap(mchbar, MCHBAR_SIZE); 76810590a9dSQiuxu Zhuo if (!window) { 76910590a9dSQiuxu Zhuo igen6_printk(KERN_ERR, "Failed to ioremap 0x%llx\n", mchbar); 77010590a9dSQiuxu Zhuo return -ENODEV; 77110590a9dSQiuxu Zhuo } 77210590a9dSQiuxu Zhuo 77310590a9dSQiuxu Zhuo layers[0].type = EDAC_MC_LAYER_CHANNEL; 77410590a9dSQiuxu Zhuo layers[0].size = NUM_CHANNELS; 77510590a9dSQiuxu Zhuo layers[0].is_virt_csrow = false; 77610590a9dSQiuxu Zhuo layers[1].type = EDAC_MC_LAYER_SLOT; 77710590a9dSQiuxu Zhuo layers[1].size = NUM_DIMMS; 77810590a9dSQiuxu Zhuo layers[1].is_virt_csrow = true; 77910590a9dSQiuxu Zhuo 78010590a9dSQiuxu Zhuo mci = edac_mc_alloc(mc, ARRAY_SIZE(layers), layers, 0); 78110590a9dSQiuxu Zhuo if (!mci) { 78210590a9dSQiuxu Zhuo rc = -ENOMEM; 78310590a9dSQiuxu Zhuo goto fail; 78410590a9dSQiuxu Zhuo } 78510590a9dSQiuxu Zhuo 78610590a9dSQiuxu Zhuo mci->ctl_name = kasprintf(GFP_KERNEL, "Intel_client_SoC MC#%d", mc); 78710590a9dSQiuxu Zhuo if (!mci->ctl_name) { 78810590a9dSQiuxu Zhuo rc = -ENOMEM; 78910590a9dSQiuxu Zhuo goto fail2; 79010590a9dSQiuxu Zhuo } 79110590a9dSQiuxu Zhuo 79210590a9dSQiuxu Zhuo mci->mtype_cap = MEM_FLAG_LPDDR4 | MEM_FLAG_DDR4; 79310590a9dSQiuxu Zhuo mci->edac_ctl_cap = EDAC_FLAG_SECDED; 79410590a9dSQiuxu Zhuo mci->edac_cap = EDAC_FLAG_SECDED; 79510590a9dSQiuxu Zhuo mci->mod_name = EDAC_MOD_STR; 79610590a9dSQiuxu Zhuo mci->dev_name = pci_name(pdev); 79710590a9dSQiuxu Zhuo mci->pvt_info = &igen6_pvt->imc[mc]; 79810590a9dSQiuxu Zhuo 79910590a9dSQiuxu Zhuo imc = mci->pvt_info; 80010590a9dSQiuxu Zhuo device_initialize(&imc->dev); 80110590a9dSQiuxu Zhuo /* 80210590a9dSQiuxu Zhuo * EDAC core uses mci->pdev(pointer of structure device) as 80310590a9dSQiuxu Zhuo * memory controller ID. The client SoCs attach one or more 80410590a9dSQiuxu Zhuo * memory controllers to single pci_dev (single pci_dev->dev 80510590a9dSQiuxu Zhuo * can be for multiple memory controllers). 80610590a9dSQiuxu Zhuo * 80710590a9dSQiuxu Zhuo * To make mci->pdev unique, assign pci_dev->dev to mci->pdev 80810590a9dSQiuxu Zhuo * for the first memory controller and assign a unique imc->dev 80910590a9dSQiuxu Zhuo * to mci->pdev for each non-first memory controller. 81010590a9dSQiuxu Zhuo */ 81110590a9dSQiuxu Zhuo mci->pdev = mc ? &imc->dev : &pdev->dev; 81210590a9dSQiuxu Zhuo imc->mc = mc; 81310590a9dSQiuxu Zhuo imc->pdev = pdev; 81410590a9dSQiuxu Zhuo imc->window = window; 81510590a9dSQiuxu Zhuo 81610590a9dSQiuxu Zhuo igen6_reg_dump(imc); 81710590a9dSQiuxu Zhuo 81810590a9dSQiuxu Zhuo rc = igen6_get_dimm_config(mci); 81910590a9dSQiuxu Zhuo if (rc) 82010590a9dSQiuxu Zhuo goto fail3; 82110590a9dSQiuxu Zhuo 82210590a9dSQiuxu Zhuo rc = edac_mc_add_mc(mci); 82310590a9dSQiuxu Zhuo if (rc) { 82410590a9dSQiuxu Zhuo igen6_printk(KERN_ERR, "Failed to register mci#%d\n", mc); 82510590a9dSQiuxu Zhuo goto fail3; 82610590a9dSQiuxu Zhuo } 82710590a9dSQiuxu Zhuo 82810590a9dSQiuxu Zhuo imc->mci = mci; 82910590a9dSQiuxu Zhuo return 0; 83010590a9dSQiuxu Zhuo fail3: 83110590a9dSQiuxu Zhuo kfree(mci->ctl_name); 83210590a9dSQiuxu Zhuo fail2: 83310590a9dSQiuxu Zhuo edac_mc_free(mci); 83410590a9dSQiuxu Zhuo fail: 83510590a9dSQiuxu Zhuo iounmap(window); 83610590a9dSQiuxu Zhuo return rc; 83710590a9dSQiuxu Zhuo } 83810590a9dSQiuxu Zhuo 83910590a9dSQiuxu Zhuo static void igen6_unregister_mcis(void) 84010590a9dSQiuxu Zhuo { 84110590a9dSQiuxu Zhuo struct mem_ctl_info *mci; 84210590a9dSQiuxu Zhuo struct igen6_imc *imc; 84310590a9dSQiuxu Zhuo int i; 84410590a9dSQiuxu Zhuo 84510590a9dSQiuxu Zhuo edac_dbg(2, "\n"); 84610590a9dSQiuxu Zhuo 84710590a9dSQiuxu Zhuo for (i = 0; i < res_cfg->num_imc; i++) { 84810590a9dSQiuxu Zhuo imc = &igen6_pvt->imc[i]; 84910590a9dSQiuxu Zhuo mci = imc->mci; 85010590a9dSQiuxu Zhuo if (!mci) 85110590a9dSQiuxu Zhuo continue; 85210590a9dSQiuxu Zhuo 85310590a9dSQiuxu Zhuo edac_mc_del_mc(mci->pdev); 85410590a9dSQiuxu Zhuo kfree(mci->ctl_name); 85510590a9dSQiuxu Zhuo edac_mc_free(mci); 85610590a9dSQiuxu Zhuo iounmap(imc->window); 85710590a9dSQiuxu Zhuo } 85810590a9dSQiuxu Zhuo } 85910590a9dSQiuxu Zhuo 86010590a9dSQiuxu Zhuo static int igen6_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 86110590a9dSQiuxu Zhuo { 86210590a9dSQiuxu Zhuo u64 mchbar; 86310590a9dSQiuxu Zhuo int i, rc; 86410590a9dSQiuxu Zhuo 86510590a9dSQiuxu Zhuo edac_dbg(2, "\n"); 86610590a9dSQiuxu Zhuo 86710590a9dSQiuxu Zhuo igen6_pvt = kzalloc(sizeof(*igen6_pvt), GFP_KERNEL); 86810590a9dSQiuxu Zhuo if (!igen6_pvt) 86910590a9dSQiuxu Zhuo return -ENOMEM; 87010590a9dSQiuxu Zhuo 87110590a9dSQiuxu Zhuo res_cfg = (struct res_config *)ent->driver_data; 87210590a9dSQiuxu Zhuo 87310590a9dSQiuxu Zhuo rc = igen6_pci_setup(pdev, &mchbar); 87410590a9dSQiuxu Zhuo if (rc) 87510590a9dSQiuxu Zhuo goto fail; 87610590a9dSQiuxu Zhuo 87710590a9dSQiuxu Zhuo for (i = 0; i < res_cfg->num_imc; i++) { 87810590a9dSQiuxu Zhuo rc = igen6_register_mci(i, mchbar, pdev); 87910590a9dSQiuxu Zhuo if (rc) 88010590a9dSQiuxu Zhuo goto fail2; 88110590a9dSQiuxu Zhuo } 88210590a9dSQiuxu Zhuo 88310590a9dSQiuxu Zhuo ecclog_pool = ecclog_gen_pool_create(); 88410590a9dSQiuxu Zhuo if (!ecclog_pool) { 88510590a9dSQiuxu Zhuo rc = -ENOMEM; 88610590a9dSQiuxu Zhuo goto fail2; 88710590a9dSQiuxu Zhuo } 88810590a9dSQiuxu Zhuo 88910590a9dSQiuxu Zhuo INIT_WORK(&ecclog_work, ecclog_work_cb); 89010590a9dSQiuxu Zhuo init_irq_work(&ecclog_irq_work, ecclog_irq_work_cb); 89110590a9dSQiuxu Zhuo 89210590a9dSQiuxu Zhuo /* Check if any pending errors before registering the NMI handler */ 89310590a9dSQiuxu Zhuo ecclog_handler(); 89410590a9dSQiuxu Zhuo 89510590a9dSQiuxu Zhuo rc = register_nmi_handler(NMI_SERR, ecclog_nmi_handler, 89610590a9dSQiuxu Zhuo 0, IGEN6_NMI_NAME); 89710590a9dSQiuxu Zhuo if (rc) { 89810590a9dSQiuxu Zhuo igen6_printk(KERN_ERR, "Failed to register NMI handler\n"); 89910590a9dSQiuxu Zhuo goto fail3; 90010590a9dSQiuxu Zhuo } 90110590a9dSQiuxu Zhuo 90210590a9dSQiuxu Zhuo /* Enable error reporting */ 90310590a9dSQiuxu Zhuo rc = errcmd_enable_error_reporting(true); 90410590a9dSQiuxu Zhuo if (rc) { 90510590a9dSQiuxu Zhuo igen6_printk(KERN_ERR, "Failed to enable error reporting\n"); 90610590a9dSQiuxu Zhuo goto fail4; 90710590a9dSQiuxu Zhuo } 90810590a9dSQiuxu Zhuo 9092223d8c7SQiuxu Zhuo igen6_debug_setup(); 91010590a9dSQiuxu Zhuo return 0; 91110590a9dSQiuxu Zhuo fail4: 91210590a9dSQiuxu Zhuo unregister_nmi_handler(NMI_SERR, IGEN6_NMI_NAME); 91310590a9dSQiuxu Zhuo fail3: 91410590a9dSQiuxu Zhuo gen_pool_destroy(ecclog_pool); 91510590a9dSQiuxu Zhuo fail2: 91610590a9dSQiuxu Zhuo igen6_unregister_mcis(); 91710590a9dSQiuxu Zhuo fail: 91810590a9dSQiuxu Zhuo kfree(igen6_pvt); 91910590a9dSQiuxu Zhuo return rc; 92010590a9dSQiuxu Zhuo } 92110590a9dSQiuxu Zhuo 92210590a9dSQiuxu Zhuo static void igen6_remove(struct pci_dev *pdev) 92310590a9dSQiuxu Zhuo { 92410590a9dSQiuxu Zhuo edac_dbg(2, "\n"); 92510590a9dSQiuxu Zhuo 9262223d8c7SQiuxu Zhuo igen6_debug_teardown(); 92710590a9dSQiuxu Zhuo errcmd_enable_error_reporting(false); 92810590a9dSQiuxu Zhuo unregister_nmi_handler(NMI_SERR, IGEN6_NMI_NAME); 92910590a9dSQiuxu Zhuo irq_work_sync(&ecclog_irq_work); 93010590a9dSQiuxu Zhuo flush_work(&ecclog_work); 93110590a9dSQiuxu Zhuo gen_pool_destroy(ecclog_pool); 93210590a9dSQiuxu Zhuo igen6_unregister_mcis(); 93310590a9dSQiuxu Zhuo kfree(igen6_pvt); 93410590a9dSQiuxu Zhuo } 93510590a9dSQiuxu Zhuo 93610590a9dSQiuxu Zhuo static struct pci_driver igen6_driver = { 93710590a9dSQiuxu Zhuo .name = EDAC_MOD_STR, 93810590a9dSQiuxu Zhuo .probe = igen6_probe, 93910590a9dSQiuxu Zhuo .remove = igen6_remove, 94010590a9dSQiuxu Zhuo .id_table = igen6_pci_tbl, 94110590a9dSQiuxu Zhuo }; 94210590a9dSQiuxu Zhuo 94310590a9dSQiuxu Zhuo static int __init igen6_init(void) 94410590a9dSQiuxu Zhuo { 94510590a9dSQiuxu Zhuo const char *owner; 94610590a9dSQiuxu Zhuo int rc; 94710590a9dSQiuxu Zhuo 94810590a9dSQiuxu Zhuo edac_dbg(2, "\n"); 94910590a9dSQiuxu Zhuo 95010590a9dSQiuxu Zhuo owner = edac_get_owner(); 95110590a9dSQiuxu Zhuo if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR))) 95210590a9dSQiuxu Zhuo return -ENODEV; 95310590a9dSQiuxu Zhuo 95410590a9dSQiuxu Zhuo edac_op_state = EDAC_OPSTATE_NMI; 95510590a9dSQiuxu Zhuo 95610590a9dSQiuxu Zhuo rc = pci_register_driver(&igen6_driver); 95710590a9dSQiuxu Zhuo if (rc) 95810590a9dSQiuxu Zhuo return rc; 95910590a9dSQiuxu Zhuo 96010590a9dSQiuxu Zhuo igen6_printk(KERN_INFO, "%s\n", IGEN6_REVISION); 96110590a9dSQiuxu Zhuo 96210590a9dSQiuxu Zhuo return 0; 96310590a9dSQiuxu Zhuo } 96410590a9dSQiuxu Zhuo 96510590a9dSQiuxu Zhuo static void __exit igen6_exit(void) 96610590a9dSQiuxu Zhuo { 96710590a9dSQiuxu Zhuo edac_dbg(2, "\n"); 96810590a9dSQiuxu Zhuo 96910590a9dSQiuxu Zhuo pci_unregister_driver(&igen6_driver); 97010590a9dSQiuxu Zhuo } 97110590a9dSQiuxu Zhuo 97210590a9dSQiuxu Zhuo module_init(igen6_init); 97310590a9dSQiuxu Zhuo module_exit(igen6_exit); 97410590a9dSQiuxu Zhuo 97510590a9dSQiuxu Zhuo MODULE_LICENSE("GPL v2"); 97610590a9dSQiuxu Zhuo MODULE_AUTHOR("Qiuxu Zhuo"); 97710590a9dSQiuxu Zhuo MODULE_DESCRIPTION("MC Driver for Intel client SoC using In-Band ECC"); 978