12bc65418SDoug Thompson #include "amd64_edac.h" 223ac4ae8SAndreas Herrmann #include <asm/amd_nb.h> 32bc65418SDoug Thompson 42bc65418SDoug Thompson static struct edac_pci_ctl_info *amd64_ctl_pci; 52bc65418SDoug Thompson 62bc65418SDoug Thompson static int report_gart_errors; 72bc65418SDoug Thompson module_param(report_gart_errors, int, 0644); 82bc65418SDoug Thompson 92bc65418SDoug Thompson /* 102bc65418SDoug Thompson * Set by command line parameter. If BIOS has enabled the ECC, this override is 112bc65418SDoug Thompson * cleared to prevent re-enabling the hardware by this driver. 122bc65418SDoug Thompson */ 132bc65418SDoug Thompson static int ecc_enable_override; 142bc65418SDoug Thompson module_param(ecc_enable_override, int, 0644); 152bc65418SDoug Thompson 16a29d8b8eSTejun Heo static struct msr __percpu *msrs; 1750542251SBorislav Petkov 18360b7f3cSBorislav Petkov /* 19360b7f3cSBorislav Petkov * count successfully initialized driver instances for setup_pci_device() 20360b7f3cSBorislav Petkov */ 21360b7f3cSBorislav Petkov static atomic_t drv_instances = ATOMIC_INIT(0); 22360b7f3cSBorislav Petkov 23cc4d8860SBorislav Petkov /* Per-node driver instances */ 24cc4d8860SBorislav Petkov static struct mem_ctl_info **mcis; 25ae7bb7c6SBorislav Petkov static struct ecc_settings **ecc_stngs; 262bc65418SDoug Thompson 272bc65418SDoug Thompson /* 28b70ef010SBorislav Petkov * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing 29b70ef010SBorislav Petkov * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching- 30b70ef010SBorislav Petkov * or higher value'. 31b70ef010SBorislav Petkov * 32b70ef010SBorislav Petkov *FIXME: Produce a better mapping/linearisation. 33b70ef010SBorislav Petkov */ 3439094443SBorislav Petkov struct scrubrate { 3539094443SBorislav Petkov u32 scrubval; /* bit pattern for scrub rate */ 3639094443SBorislav Petkov u32 bandwidth; /* bandwidth consumed (bytes/sec) */ 3739094443SBorislav Petkov } scrubrates[] = { 38b70ef010SBorislav Petkov { 0x01, 1600000000UL}, 39b70ef010SBorislav Petkov { 0x02, 800000000UL}, 40b70ef010SBorislav Petkov { 0x03, 400000000UL}, 41b70ef010SBorislav Petkov { 0x04, 200000000UL}, 42b70ef010SBorislav Petkov { 0x05, 100000000UL}, 43b70ef010SBorislav Petkov { 0x06, 50000000UL}, 44b70ef010SBorislav Petkov { 0x07, 25000000UL}, 45b70ef010SBorislav Petkov { 0x08, 12284069UL}, 46b70ef010SBorislav Petkov { 0x09, 6274509UL}, 47b70ef010SBorislav Petkov { 0x0A, 3121951UL}, 48b70ef010SBorislav Petkov { 0x0B, 1560975UL}, 49b70ef010SBorislav Petkov { 0x0C, 781440UL}, 50b70ef010SBorislav Petkov { 0x0D, 390720UL}, 51b70ef010SBorislav Petkov { 0x0E, 195300UL}, 52b70ef010SBorislav Petkov { 0x0F, 97650UL}, 53b70ef010SBorislav Petkov { 0x10, 48854UL}, 54b70ef010SBorislav Petkov { 0x11, 24427UL}, 55b70ef010SBorislav Petkov { 0x12, 12213UL}, 56b70ef010SBorislav Petkov { 0x13, 6101UL}, 57b70ef010SBorislav Petkov { 0x14, 3051UL}, 58b70ef010SBorislav Petkov { 0x15, 1523UL}, 59b70ef010SBorislav Petkov { 0x16, 761UL}, 60b70ef010SBorislav Petkov { 0x00, 0UL}, /* scrubbing off */ 61b70ef010SBorislav Petkov }; 62b70ef010SBorislav Petkov 63b2b0c605SBorislav Petkov static int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset, 64b2b0c605SBorislav Petkov u32 *val, const char *func) 65b2b0c605SBorislav Petkov { 66b2b0c605SBorislav Petkov int err = 0; 67b2b0c605SBorislav Petkov 68b2b0c605SBorislav Petkov err = pci_read_config_dword(pdev, offset, val); 69b2b0c605SBorislav Petkov if (err) 70b2b0c605SBorislav Petkov amd64_warn("%s: error reading F%dx%03x.\n", 71b2b0c605SBorislav Petkov func, PCI_FUNC(pdev->devfn), offset); 72b2b0c605SBorislav Petkov 73b2b0c605SBorislav Petkov return err; 74b2b0c605SBorislav Petkov } 75b2b0c605SBorislav Petkov 76b2b0c605SBorislav Petkov int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset, 77b2b0c605SBorislav Petkov u32 val, const char *func) 78b2b0c605SBorislav Petkov { 79b2b0c605SBorislav Petkov int err = 0; 80b2b0c605SBorislav Petkov 81b2b0c605SBorislav Petkov err = pci_write_config_dword(pdev, offset, val); 82b2b0c605SBorislav Petkov if (err) 83b2b0c605SBorislav Petkov amd64_warn("%s: error writing to F%dx%03x.\n", 84b2b0c605SBorislav Petkov func, PCI_FUNC(pdev->devfn), offset); 85b2b0c605SBorislav Petkov 86b2b0c605SBorislav Petkov return err; 87b2b0c605SBorislav Petkov } 88b2b0c605SBorislav Petkov 89b2b0c605SBorislav Petkov /* 90b2b0c605SBorislav Petkov * 91b2b0c605SBorislav Petkov * Depending on the family, F2 DCT reads need special handling: 92b2b0c605SBorislav Petkov * 93b2b0c605SBorislav Petkov * K8: has a single DCT only 94b2b0c605SBorislav Petkov * 95b2b0c605SBorislav Petkov * F10h: each DCT has its own set of regs 96b2b0c605SBorislav Petkov * DCT0 -> F2x040.. 97b2b0c605SBorislav Petkov * DCT1 -> F2x140.. 98b2b0c605SBorislav Petkov * 99b2b0c605SBorislav Petkov * F15h: we select which DCT we access using F1x10C[DctCfgSel] 100b2b0c605SBorislav Petkov * 101b2b0c605SBorislav Petkov */ 102b2b0c605SBorislav Petkov static int k8_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val, 103b2b0c605SBorislav Petkov const char *func) 104b2b0c605SBorislav Petkov { 105b2b0c605SBorislav Petkov if (addr >= 0x100) 106b2b0c605SBorislav Petkov return -EINVAL; 107b2b0c605SBorislav Petkov 108b2b0c605SBorislav Petkov return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func); 109b2b0c605SBorislav Petkov } 110b2b0c605SBorislav Petkov 111b2b0c605SBorislav Petkov static int f10_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val, 112b2b0c605SBorislav Petkov const char *func) 113b2b0c605SBorislav Petkov { 114b2b0c605SBorislav Petkov return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func); 115b2b0c605SBorislav Petkov } 116b2b0c605SBorislav Petkov 11773ba8593SBorislav Petkov /* 11873ba8593SBorislav Petkov * Select DCT to which PCI cfg accesses are routed 11973ba8593SBorislav Petkov */ 12073ba8593SBorislav Petkov static void f15h_select_dct(struct amd64_pvt *pvt, u8 dct) 12173ba8593SBorislav Petkov { 12273ba8593SBorislav Petkov u32 reg = 0; 12373ba8593SBorislav Petkov 12473ba8593SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, ®); 12573ba8593SBorislav Petkov reg &= 0xfffffffe; 12673ba8593SBorislav Petkov reg |= dct; 12773ba8593SBorislav Petkov amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg); 12873ba8593SBorislav Petkov } 12973ba8593SBorislav Petkov 130b2b0c605SBorislav Petkov static int f15_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val, 131b2b0c605SBorislav Petkov const char *func) 132b2b0c605SBorislav Petkov { 133b2b0c605SBorislav Petkov u8 dct = 0; 134b2b0c605SBorislav Petkov 135b2b0c605SBorislav Petkov if (addr >= 0x140 && addr <= 0x1a0) { 136b2b0c605SBorislav Petkov dct = 1; 137b2b0c605SBorislav Petkov addr -= 0x100; 138b2b0c605SBorislav Petkov } 139b2b0c605SBorislav Petkov 14073ba8593SBorislav Petkov f15h_select_dct(pvt, dct); 141b2b0c605SBorislav Petkov 142b2b0c605SBorislav Petkov return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func); 143b2b0c605SBorislav Petkov } 144b2b0c605SBorislav Petkov 145b70ef010SBorislav Petkov /* 1462bc65418SDoug Thompson * Memory scrubber control interface. For K8, memory scrubbing is handled by 1472bc65418SDoug Thompson * hardware and can involve L2 cache, dcache as well as the main memory. With 1482bc65418SDoug Thompson * F10, this is extended to L3 cache scrubbing on CPU models sporting that 1492bc65418SDoug Thompson * functionality. 1502bc65418SDoug Thompson * 1512bc65418SDoug Thompson * This causes the "units" for the scrubbing speed to vary from 64 byte blocks 1522bc65418SDoug Thompson * (dram) over to cache lines. This is nasty, so we will use bandwidth in 1532bc65418SDoug Thompson * bytes/sec for the setting. 1542bc65418SDoug Thompson * 1552bc65418SDoug Thompson * Currently, we only do dram scrubbing. If the scrubbing is done in software on 1562bc65418SDoug Thompson * other archs, we might not have access to the caches directly. 1572bc65418SDoug Thompson */ 1582bc65418SDoug Thompson 1592bc65418SDoug Thompson /* 1602bc65418SDoug Thompson * scan the scrub rate mapping table for a close or matching bandwidth value to 1612bc65418SDoug Thompson * issue. If requested is too big, then use last maximum value found. 1622bc65418SDoug Thompson */ 163395ae783SBorislav Petkov static int __amd64_set_scrub_rate(struct pci_dev *ctl, u32 new_bw, u32 min_rate) 1642bc65418SDoug Thompson { 1652bc65418SDoug Thompson u32 scrubval; 1662bc65418SDoug Thompson int i; 1672bc65418SDoug Thompson 1682bc65418SDoug Thompson /* 1692bc65418SDoug Thompson * map the configured rate (new_bw) to a value specific to the AMD64 1702bc65418SDoug Thompson * memory controller and apply to register. Search for the first 1712bc65418SDoug Thompson * bandwidth entry that is greater or equal than the setting requested 1722bc65418SDoug Thompson * and program that. If at last entry, turn off DRAM scrubbing. 1732bc65418SDoug Thompson */ 1742bc65418SDoug Thompson for (i = 0; i < ARRAY_SIZE(scrubrates); i++) { 1752bc65418SDoug Thompson /* 1762bc65418SDoug Thompson * skip scrub rates which aren't recommended 1772bc65418SDoug Thompson * (see F10 BKDG, F3x58) 1782bc65418SDoug Thompson */ 179395ae783SBorislav Petkov if (scrubrates[i].scrubval < min_rate) 1802bc65418SDoug Thompson continue; 1812bc65418SDoug Thompson 1822bc65418SDoug Thompson if (scrubrates[i].bandwidth <= new_bw) 1832bc65418SDoug Thompson break; 1842bc65418SDoug Thompson 1852bc65418SDoug Thompson /* 1862bc65418SDoug Thompson * if no suitable bandwidth found, turn off DRAM scrubbing 1872bc65418SDoug Thompson * entirely by falling back to the last element in the 1882bc65418SDoug Thompson * scrubrates array. 1892bc65418SDoug Thompson */ 1902bc65418SDoug Thompson } 1912bc65418SDoug Thompson 1922bc65418SDoug Thompson scrubval = scrubrates[i].scrubval; 1932bc65418SDoug Thompson 1945980bb9cSBorislav Petkov pci_write_bits32(ctl, SCRCTRL, scrubval, 0x001F); 1952bc65418SDoug Thompson 19639094443SBorislav Petkov if (scrubval) 19739094443SBorislav Petkov return scrubrates[i].bandwidth; 19839094443SBorislav Petkov 1992bc65418SDoug Thompson return 0; 2002bc65418SDoug Thompson } 2012bc65418SDoug Thompson 202395ae783SBorislav Petkov static int amd64_set_scrub_rate(struct mem_ctl_info *mci, u32 bw) 2032bc65418SDoug Thompson { 2042bc65418SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 20587b3e0e6SBorislav Petkov u32 min_scrubrate = 0x5; 2062bc65418SDoug Thompson 20787b3e0e6SBorislav Petkov if (boot_cpu_data.x86 == 0xf) 20887b3e0e6SBorislav Petkov min_scrubrate = 0x0; 20987b3e0e6SBorislav Petkov 21073ba8593SBorislav Petkov /* F15h Erratum #505 */ 21173ba8593SBorislav Petkov if (boot_cpu_data.x86 == 0x15) 21273ba8593SBorislav Petkov f15h_select_dct(pvt, 0); 21373ba8593SBorislav Petkov 21487b3e0e6SBorislav Petkov return __amd64_set_scrub_rate(pvt->F3, bw, min_scrubrate); 2152bc65418SDoug Thompson } 2162bc65418SDoug Thompson 21739094443SBorislav Petkov static int amd64_get_scrub_rate(struct mem_ctl_info *mci) 2182bc65418SDoug Thompson { 2192bc65418SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 2202bc65418SDoug Thompson u32 scrubval = 0; 22139094443SBorislav Petkov int i, retval = -EINVAL; 2222bc65418SDoug Thompson 22373ba8593SBorislav Petkov /* F15h Erratum #505 */ 22473ba8593SBorislav Petkov if (boot_cpu_data.x86 == 0x15) 22573ba8593SBorislav Petkov f15h_select_dct(pvt, 0); 22673ba8593SBorislav Petkov 2275980bb9cSBorislav Petkov amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval); 2282bc65418SDoug Thompson 2292bc65418SDoug Thompson scrubval = scrubval & 0x001F; 2302bc65418SDoug Thompson 231926311fdSRoel Kluin for (i = 0; i < ARRAY_SIZE(scrubrates); i++) { 2322bc65418SDoug Thompson if (scrubrates[i].scrubval == scrubval) { 23339094443SBorislav Petkov retval = scrubrates[i].bandwidth; 2342bc65418SDoug Thompson break; 2352bc65418SDoug Thompson } 2362bc65418SDoug Thompson } 23739094443SBorislav Petkov return retval; 2382bc65418SDoug Thompson } 2392bc65418SDoug Thompson 2406775763aSDoug Thompson /* 2417f19bf75SBorislav Petkov * returns true if the SysAddr given by sys_addr matches the 2427f19bf75SBorislav Petkov * DRAM base/limit associated with node_id 2436775763aSDoug Thompson */ 244b487c33eSBorislav Petkov static bool amd64_base_limit_match(struct amd64_pvt *pvt, u64 sys_addr, 245b487c33eSBorislav Petkov unsigned nid) 2466775763aSDoug Thompson { 2477f19bf75SBorislav Petkov u64 addr; 2486775763aSDoug Thompson 2496775763aSDoug Thompson /* The K8 treats this as a 40-bit value. However, bits 63-40 will be 2506775763aSDoug Thompson * all ones if the most significant implemented address bit is 1. 2516775763aSDoug Thompson * Here we discard bits 63-40. See section 3.4.2 of AMD publication 2526775763aSDoug Thompson * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1 2536775763aSDoug Thompson * Application Programming. 2546775763aSDoug Thompson */ 2556775763aSDoug Thompson addr = sys_addr & 0x000000ffffffffffull; 2566775763aSDoug Thompson 2577f19bf75SBorislav Petkov return ((addr >= get_dram_base(pvt, nid)) && 2587f19bf75SBorislav Petkov (addr <= get_dram_limit(pvt, nid))); 2596775763aSDoug Thompson } 2606775763aSDoug Thompson 2616775763aSDoug Thompson /* 2626775763aSDoug Thompson * Attempt to map a SysAddr to a node. On success, return a pointer to the 2636775763aSDoug Thompson * mem_ctl_info structure for the node that the SysAddr maps to. 2646775763aSDoug Thompson * 2656775763aSDoug Thompson * On failure, return NULL. 2666775763aSDoug Thompson */ 2676775763aSDoug Thompson static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci, 2686775763aSDoug Thompson u64 sys_addr) 2696775763aSDoug Thompson { 2706775763aSDoug Thompson struct amd64_pvt *pvt; 271b487c33eSBorislav Petkov unsigned node_id; 2726775763aSDoug Thompson u32 intlv_en, bits; 2736775763aSDoug Thompson 2746775763aSDoug Thompson /* 2756775763aSDoug Thompson * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section 2766775763aSDoug Thompson * 3.4.4.2) registers to map the SysAddr to a node ID. 2776775763aSDoug Thompson */ 2786775763aSDoug Thompson pvt = mci->pvt_info; 2796775763aSDoug Thompson 2806775763aSDoug Thompson /* 2816775763aSDoug Thompson * The value of this field should be the same for all DRAM Base 2826775763aSDoug Thompson * registers. Therefore we arbitrarily choose to read it from the 2836775763aSDoug Thompson * register for node 0. 2846775763aSDoug Thompson */ 2857f19bf75SBorislav Petkov intlv_en = dram_intlv_en(pvt, 0); 2866775763aSDoug Thompson 2876775763aSDoug Thompson if (intlv_en == 0) { 2887f19bf75SBorislav Petkov for (node_id = 0; node_id < DRAM_RANGES; node_id++) { 2896775763aSDoug Thompson if (amd64_base_limit_match(pvt, sys_addr, node_id)) 2906775763aSDoug Thompson goto found; 2916775763aSDoug Thompson } 2928edc5445SBorislav Petkov goto err_no_match; 2938edc5445SBorislav Petkov } 2946775763aSDoug Thompson 29572f158feSBorislav Petkov if (unlikely((intlv_en != 0x01) && 29672f158feSBorislav Petkov (intlv_en != 0x03) && 29772f158feSBorislav Petkov (intlv_en != 0x07))) { 29824f9a7feSBorislav Petkov amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en); 2996775763aSDoug Thompson return NULL; 3006775763aSDoug Thompson } 3016775763aSDoug Thompson 3026775763aSDoug Thompson bits = (((u32) sys_addr) >> 12) & intlv_en; 3036775763aSDoug Thompson 3046775763aSDoug Thompson for (node_id = 0; ; ) { 3057f19bf75SBorislav Petkov if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits) 3066775763aSDoug Thompson break; /* intlv_sel field matches */ 3076775763aSDoug Thompson 3087f19bf75SBorislav Petkov if (++node_id >= DRAM_RANGES) 3096775763aSDoug Thompson goto err_no_match; 3106775763aSDoug Thompson } 3116775763aSDoug Thompson 3126775763aSDoug Thompson /* sanity test for sys_addr */ 3136775763aSDoug Thompson if (unlikely(!amd64_base_limit_match(pvt, sys_addr, node_id))) { 31424f9a7feSBorislav Petkov amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address" 31524f9a7feSBorislav Petkov "range for node %d with node interleaving enabled.\n", 3168edc5445SBorislav Petkov __func__, sys_addr, node_id); 3176775763aSDoug Thompson return NULL; 3186775763aSDoug Thompson } 3196775763aSDoug Thompson 3206775763aSDoug Thompson found: 321b487c33eSBorislav Petkov return edac_mc_find((int)node_id); 3226775763aSDoug Thompson 3236775763aSDoug Thompson err_no_match: 3246775763aSDoug Thompson debugf2("sys_addr 0x%lx doesn't match any node\n", 3256775763aSDoug Thompson (unsigned long)sys_addr); 3266775763aSDoug Thompson 3276775763aSDoug Thompson return NULL; 3286775763aSDoug Thompson } 329e2ce7255SDoug Thompson 330e2ce7255SDoug Thompson /* 33111c75eadSBorislav Petkov * compute the CS base address of the @csrow on the DRAM controller @dct. 33211c75eadSBorislav Petkov * For details see F2x[5C:40] in the processor's BKDG 333e2ce7255SDoug Thompson */ 33411c75eadSBorislav Petkov static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct, 33511c75eadSBorislav Petkov u64 *base, u64 *mask) 336e2ce7255SDoug Thompson { 33711c75eadSBorislav Petkov u64 csbase, csmask, base_bits, mask_bits; 33811c75eadSBorislav Petkov u8 addr_shift; 33911c75eadSBorislav Petkov 34011c75eadSBorislav Petkov if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) { 34111c75eadSBorislav Petkov csbase = pvt->csels[dct].csbases[csrow]; 34211c75eadSBorislav Petkov csmask = pvt->csels[dct].csmasks[csrow]; 34311c75eadSBorislav Petkov base_bits = GENMASK(21, 31) | GENMASK(9, 15); 34411c75eadSBorislav Petkov mask_bits = GENMASK(21, 29) | GENMASK(9, 15); 34511c75eadSBorislav Petkov addr_shift = 4; 34611c75eadSBorislav Petkov } else { 34711c75eadSBorislav Petkov csbase = pvt->csels[dct].csbases[csrow]; 34811c75eadSBorislav Petkov csmask = pvt->csels[dct].csmasks[csrow >> 1]; 34911c75eadSBorislav Petkov addr_shift = 8; 35011c75eadSBorislav Petkov 35111c75eadSBorislav Petkov if (boot_cpu_data.x86 == 0x15) 35211c75eadSBorislav Petkov base_bits = mask_bits = GENMASK(19,30) | GENMASK(5,13); 35311c75eadSBorislav Petkov else 35411c75eadSBorislav Petkov base_bits = mask_bits = GENMASK(19,28) | GENMASK(5,13); 355e2ce7255SDoug Thompson } 356e2ce7255SDoug Thompson 35711c75eadSBorislav Petkov *base = (csbase & base_bits) << addr_shift; 358e2ce7255SDoug Thompson 35911c75eadSBorislav Petkov *mask = ~0ULL; 36011c75eadSBorislav Petkov /* poke holes for the csmask */ 36111c75eadSBorislav Petkov *mask &= ~(mask_bits << addr_shift); 36211c75eadSBorislav Petkov /* OR them in */ 36311c75eadSBorislav Petkov *mask |= (csmask & mask_bits) << addr_shift; 364e2ce7255SDoug Thompson } 365e2ce7255SDoug Thompson 36611c75eadSBorislav Petkov #define for_each_chip_select(i, dct, pvt) \ 36711c75eadSBorislav Petkov for (i = 0; i < pvt->csels[dct].b_cnt; i++) 36811c75eadSBorislav Petkov 369614ec9d8SBorislav Petkov #define chip_select_base(i, dct, pvt) \ 370614ec9d8SBorislav Petkov pvt->csels[dct].csbases[i] 371614ec9d8SBorislav Petkov 37211c75eadSBorislav Petkov #define for_each_chip_select_mask(i, dct, pvt) \ 37311c75eadSBorislav Petkov for (i = 0; i < pvt->csels[dct].m_cnt; i++) 37411c75eadSBorislav Petkov 375e2ce7255SDoug Thompson /* 376e2ce7255SDoug Thompson * @input_addr is an InputAddr associated with the node given by mci. Return the 377e2ce7255SDoug Thompson * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr). 378e2ce7255SDoug Thompson */ 379e2ce7255SDoug Thompson static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr) 380e2ce7255SDoug Thompson { 381e2ce7255SDoug Thompson struct amd64_pvt *pvt; 382e2ce7255SDoug Thompson int csrow; 383e2ce7255SDoug Thompson u64 base, mask; 384e2ce7255SDoug Thompson 385e2ce7255SDoug Thompson pvt = mci->pvt_info; 386e2ce7255SDoug Thompson 38711c75eadSBorislav Petkov for_each_chip_select(csrow, 0, pvt) { 38811c75eadSBorislav Petkov if (!csrow_enabled(csrow, 0, pvt)) 389e2ce7255SDoug Thompson continue; 390e2ce7255SDoug Thompson 39111c75eadSBorislav Petkov get_cs_base_and_mask(pvt, csrow, 0, &base, &mask); 39211c75eadSBorislav Petkov 39311c75eadSBorislav Petkov mask = ~mask; 394e2ce7255SDoug Thompson 395e2ce7255SDoug Thompson if ((input_addr & mask) == (base & mask)) { 396e2ce7255SDoug Thompson debugf2("InputAddr 0x%lx matches csrow %d (node %d)\n", 397e2ce7255SDoug Thompson (unsigned long)input_addr, csrow, 398e2ce7255SDoug Thompson pvt->mc_node_id); 399e2ce7255SDoug Thompson 400e2ce7255SDoug Thompson return csrow; 401e2ce7255SDoug Thompson } 402e2ce7255SDoug Thompson } 403e2ce7255SDoug Thompson debugf2("no matching csrow for InputAddr 0x%lx (MC node %d)\n", 404e2ce7255SDoug Thompson (unsigned long)input_addr, pvt->mc_node_id); 405e2ce7255SDoug Thompson 406e2ce7255SDoug Thompson return -1; 407e2ce7255SDoug Thompson } 408e2ce7255SDoug Thompson 409e2ce7255SDoug Thompson /* 410e2ce7255SDoug Thompson * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094) 411e2ce7255SDoug Thompson * for the node represented by mci. Info is passed back in *hole_base, 412e2ce7255SDoug Thompson * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if 413e2ce7255SDoug Thompson * info is invalid. Info may be invalid for either of the following reasons: 414e2ce7255SDoug Thompson * 415e2ce7255SDoug Thompson * - The revision of the node is not E or greater. In this case, the DRAM Hole 416e2ce7255SDoug Thompson * Address Register does not exist. 417e2ce7255SDoug Thompson * 418e2ce7255SDoug Thompson * - The DramHoleValid bit is cleared in the DRAM Hole Address Register, 419e2ce7255SDoug Thompson * indicating that its contents are not valid. 420e2ce7255SDoug Thompson * 421e2ce7255SDoug Thompson * The values passed back in *hole_base, *hole_offset, and *hole_size are 422e2ce7255SDoug Thompson * complete 32-bit values despite the fact that the bitfields in the DHAR 423e2ce7255SDoug Thompson * only represent bits 31-24 of the base and offset values. 424e2ce7255SDoug Thompson */ 425e2ce7255SDoug Thompson int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base, 426e2ce7255SDoug Thompson u64 *hole_offset, u64 *hole_size) 427e2ce7255SDoug Thompson { 428e2ce7255SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 429e2ce7255SDoug Thompson u64 base; 430e2ce7255SDoug Thompson 431e2ce7255SDoug Thompson /* only revE and later have the DRAM Hole Address Register */ 4321433eb99SBorislav Petkov if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_E) { 433e2ce7255SDoug Thompson debugf1(" revision %d for node %d does not support DHAR\n", 434e2ce7255SDoug Thompson pvt->ext_model, pvt->mc_node_id); 435e2ce7255SDoug Thompson return 1; 436e2ce7255SDoug Thompson } 437e2ce7255SDoug Thompson 438bc21fa57SBorislav Petkov /* valid for Fam10h and above */ 439c8e518d5SBorislav Petkov if (boot_cpu_data.x86 >= 0x10 && !dhar_mem_hoist_valid(pvt)) { 440e2ce7255SDoug Thompson debugf1(" Dram Memory Hoisting is DISABLED on this system\n"); 441e2ce7255SDoug Thompson return 1; 442e2ce7255SDoug Thompson } 443e2ce7255SDoug Thompson 444c8e518d5SBorislav Petkov if (!dhar_valid(pvt)) { 445e2ce7255SDoug Thompson debugf1(" Dram Memory Hoisting is DISABLED on this node %d\n", 446e2ce7255SDoug Thompson pvt->mc_node_id); 447e2ce7255SDoug Thompson return 1; 448e2ce7255SDoug Thompson } 449e2ce7255SDoug Thompson 450e2ce7255SDoug Thompson /* This node has Memory Hoisting */ 451e2ce7255SDoug Thompson 452e2ce7255SDoug Thompson /* +------------------+--------------------+--------------------+----- 453e2ce7255SDoug Thompson * | memory | DRAM hole | relocated | 454e2ce7255SDoug Thompson * | [0, (x - 1)] | [x, 0xffffffff] | addresses from | 455e2ce7255SDoug Thompson * | | | DRAM hole | 456e2ce7255SDoug Thompson * | | | [0x100000000, | 457e2ce7255SDoug Thompson * | | | (0x100000000+ | 458e2ce7255SDoug Thompson * | | | (0xffffffff-x))] | 459e2ce7255SDoug Thompson * +------------------+--------------------+--------------------+----- 460e2ce7255SDoug Thompson * 461e2ce7255SDoug Thompson * Above is a diagram of physical memory showing the DRAM hole and the 462e2ce7255SDoug Thompson * relocated addresses from the DRAM hole. As shown, the DRAM hole 463e2ce7255SDoug Thompson * starts at address x (the base address) and extends through address 464e2ce7255SDoug Thompson * 0xffffffff. The DRAM Hole Address Register (DHAR) relocates the 465e2ce7255SDoug Thompson * addresses in the hole so that they start at 0x100000000. 466e2ce7255SDoug Thompson */ 467e2ce7255SDoug Thompson 468bc21fa57SBorislav Petkov base = dhar_base(pvt); 469e2ce7255SDoug Thompson 470e2ce7255SDoug Thompson *hole_base = base; 471e2ce7255SDoug Thompson *hole_size = (0x1ull << 32) - base; 472e2ce7255SDoug Thompson 473e2ce7255SDoug Thompson if (boot_cpu_data.x86 > 0xf) 474bc21fa57SBorislav Petkov *hole_offset = f10_dhar_offset(pvt); 475e2ce7255SDoug Thompson else 476bc21fa57SBorislav Petkov *hole_offset = k8_dhar_offset(pvt); 477e2ce7255SDoug Thompson 478e2ce7255SDoug Thompson debugf1(" DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n", 479e2ce7255SDoug Thompson pvt->mc_node_id, (unsigned long)*hole_base, 480e2ce7255SDoug Thompson (unsigned long)*hole_offset, (unsigned long)*hole_size); 481e2ce7255SDoug Thompson 482e2ce7255SDoug Thompson return 0; 483e2ce7255SDoug Thompson } 484e2ce7255SDoug Thompson EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info); 485e2ce7255SDoug Thompson 48693c2df58SDoug Thompson /* 48793c2df58SDoug Thompson * Return the DramAddr that the SysAddr given by @sys_addr maps to. It is 48893c2df58SDoug Thompson * assumed that sys_addr maps to the node given by mci. 48993c2df58SDoug Thompson * 49093c2df58SDoug Thompson * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section 49193c2df58SDoug Thompson * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a 49293c2df58SDoug Thompson * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled, 49393c2df58SDoug Thompson * then it is also involved in translating a SysAddr to a DramAddr. Sections 49493c2df58SDoug Thompson * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting. 49593c2df58SDoug Thompson * These parts of the documentation are unclear. I interpret them as follows: 49693c2df58SDoug Thompson * 49793c2df58SDoug Thompson * When node n receives a SysAddr, it processes the SysAddr as follows: 49893c2df58SDoug Thompson * 49993c2df58SDoug Thompson * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM 50093c2df58SDoug Thompson * Limit registers for node n. If the SysAddr is not within the range 50193c2df58SDoug Thompson * specified by the base and limit values, then node n ignores the Sysaddr 50293c2df58SDoug Thompson * (since it does not map to node n). Otherwise continue to step 2 below. 50393c2df58SDoug Thompson * 50493c2df58SDoug Thompson * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is 50593c2df58SDoug Thompson * disabled so skip to step 3 below. Otherwise see if the SysAddr is within 50693c2df58SDoug Thompson * the range of relocated addresses (starting at 0x100000000) from the DRAM 50793c2df58SDoug Thompson * hole. If not, skip to step 3 below. Else get the value of the 50893c2df58SDoug Thompson * DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the 50993c2df58SDoug Thompson * offset defined by this value from the SysAddr. 51093c2df58SDoug Thompson * 51193c2df58SDoug Thompson * 3. Obtain the base address for node n from the DRAMBase field of the DRAM 51293c2df58SDoug Thompson * Base register for node n. To obtain the DramAddr, subtract the base 51393c2df58SDoug Thompson * address from the SysAddr, as shown near the start of section 3.4.4 (p.70). 51493c2df58SDoug Thompson */ 51593c2df58SDoug Thompson static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr) 51693c2df58SDoug Thompson { 5177f19bf75SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 51893c2df58SDoug Thompson u64 dram_base, hole_base, hole_offset, hole_size, dram_addr; 51993c2df58SDoug Thompson int ret = 0; 52093c2df58SDoug Thompson 5217f19bf75SBorislav Petkov dram_base = get_dram_base(pvt, pvt->mc_node_id); 52293c2df58SDoug Thompson 52393c2df58SDoug Thompson ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset, 52493c2df58SDoug Thompson &hole_size); 52593c2df58SDoug Thompson if (!ret) { 52693c2df58SDoug Thompson if ((sys_addr >= (1ull << 32)) && 52793c2df58SDoug Thompson (sys_addr < ((1ull << 32) + hole_size))) { 52893c2df58SDoug Thompson /* use DHAR to translate SysAddr to DramAddr */ 52993c2df58SDoug Thompson dram_addr = sys_addr - hole_offset; 53093c2df58SDoug Thompson 53193c2df58SDoug Thompson debugf2("using DHAR to translate SysAddr 0x%lx to " 53293c2df58SDoug Thompson "DramAddr 0x%lx\n", 53393c2df58SDoug Thompson (unsigned long)sys_addr, 53493c2df58SDoug Thompson (unsigned long)dram_addr); 53593c2df58SDoug Thompson 53693c2df58SDoug Thompson return dram_addr; 53793c2df58SDoug Thompson } 53893c2df58SDoug Thompson } 53993c2df58SDoug Thompson 54093c2df58SDoug Thompson /* 54193c2df58SDoug Thompson * Translate the SysAddr to a DramAddr as shown near the start of 54293c2df58SDoug Thompson * section 3.4.4 (p. 70). Although sys_addr is a 64-bit value, the k8 54393c2df58SDoug Thompson * only deals with 40-bit values. Therefore we discard bits 63-40 of 54493c2df58SDoug Thompson * sys_addr below. If bit 39 of sys_addr is 1 then the bits we 54593c2df58SDoug Thompson * discard are all 1s. Otherwise the bits we discard are all 0s. See 54693c2df58SDoug Thompson * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture 54793c2df58SDoug Thompson * Programmer's Manual Volume 1 Application Programming. 54893c2df58SDoug Thompson */ 549f678b8ccSBorislav Petkov dram_addr = (sys_addr & GENMASK(0, 39)) - dram_base; 55093c2df58SDoug Thompson 55193c2df58SDoug Thompson debugf2("using DRAM Base register to translate SysAddr 0x%lx to " 55293c2df58SDoug Thompson "DramAddr 0x%lx\n", (unsigned long)sys_addr, 55393c2df58SDoug Thompson (unsigned long)dram_addr); 55493c2df58SDoug Thompson return dram_addr; 55593c2df58SDoug Thompson } 55693c2df58SDoug Thompson 55793c2df58SDoug Thompson /* 55893c2df58SDoug Thompson * @intlv_en is the value of the IntlvEn field from a DRAM Base register 55993c2df58SDoug Thompson * (section 3.4.4.1). Return the number of bits from a SysAddr that are used 56093c2df58SDoug Thompson * for node interleaving. 56193c2df58SDoug Thompson */ 56293c2df58SDoug Thompson static int num_node_interleave_bits(unsigned intlv_en) 56393c2df58SDoug Thompson { 56493c2df58SDoug Thompson static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 }; 56593c2df58SDoug Thompson int n; 56693c2df58SDoug Thompson 56793c2df58SDoug Thompson BUG_ON(intlv_en > 7); 56893c2df58SDoug Thompson n = intlv_shift_table[intlv_en]; 56993c2df58SDoug Thompson return n; 57093c2df58SDoug Thompson } 57193c2df58SDoug Thompson 57293c2df58SDoug Thompson /* Translate the DramAddr given by @dram_addr to an InputAddr. */ 57393c2df58SDoug Thompson static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr) 57493c2df58SDoug Thompson { 57593c2df58SDoug Thompson struct amd64_pvt *pvt; 57693c2df58SDoug Thompson int intlv_shift; 57793c2df58SDoug Thompson u64 input_addr; 57893c2df58SDoug Thompson 57993c2df58SDoug Thompson pvt = mci->pvt_info; 58093c2df58SDoug Thompson 58193c2df58SDoug Thompson /* 58293c2df58SDoug Thompson * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E) 58393c2df58SDoug Thompson * concerning translating a DramAddr to an InputAddr. 58493c2df58SDoug Thompson */ 5857f19bf75SBorislav Petkov intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0)); 586f678b8ccSBorislav Petkov input_addr = ((dram_addr >> intlv_shift) & GENMASK(12, 35)) + 58793c2df58SDoug Thompson (dram_addr & 0xfff); 58893c2df58SDoug Thompson 58993c2df58SDoug Thompson debugf2(" Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n", 59093c2df58SDoug Thompson intlv_shift, (unsigned long)dram_addr, 59193c2df58SDoug Thompson (unsigned long)input_addr); 59293c2df58SDoug Thompson 59393c2df58SDoug Thompson return input_addr; 59493c2df58SDoug Thompson } 59593c2df58SDoug Thompson 59693c2df58SDoug Thompson /* 59793c2df58SDoug Thompson * Translate the SysAddr represented by @sys_addr to an InputAddr. It is 59893c2df58SDoug Thompson * assumed that @sys_addr maps to the node given by mci. 59993c2df58SDoug Thompson */ 60093c2df58SDoug Thompson static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr) 60193c2df58SDoug Thompson { 60293c2df58SDoug Thompson u64 input_addr; 60393c2df58SDoug Thompson 60493c2df58SDoug Thompson input_addr = 60593c2df58SDoug Thompson dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr)); 60693c2df58SDoug Thompson 60793c2df58SDoug Thompson debugf2("SysAdddr 0x%lx translates to InputAddr 0x%lx\n", 60893c2df58SDoug Thompson (unsigned long)sys_addr, (unsigned long)input_addr); 60993c2df58SDoug Thompson 61093c2df58SDoug Thompson return input_addr; 61193c2df58SDoug Thompson } 61293c2df58SDoug Thompson 61393c2df58SDoug Thompson 61493c2df58SDoug Thompson /* 61593c2df58SDoug Thompson * @input_addr is an InputAddr associated with the node represented by mci. 61693c2df58SDoug Thompson * Translate @input_addr to a DramAddr and return the result. 61793c2df58SDoug Thompson */ 61893c2df58SDoug Thompson static u64 input_addr_to_dram_addr(struct mem_ctl_info *mci, u64 input_addr) 61993c2df58SDoug Thompson { 62093c2df58SDoug Thompson struct amd64_pvt *pvt; 621b487c33eSBorislav Petkov unsigned node_id, intlv_shift; 62293c2df58SDoug Thompson u64 bits, dram_addr; 62393c2df58SDoug Thompson u32 intlv_sel; 62493c2df58SDoug Thompson 62593c2df58SDoug Thompson /* 62693c2df58SDoug Thompson * Near the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E) 62793c2df58SDoug Thompson * shows how to translate a DramAddr to an InputAddr. Here we reverse 62893c2df58SDoug Thompson * this procedure. When translating from a DramAddr to an InputAddr, the 62993c2df58SDoug Thompson * bits used for node interleaving are discarded. Here we recover these 63093c2df58SDoug Thompson * bits from the IntlvSel field of the DRAM Limit register (section 63193c2df58SDoug Thompson * 3.4.4.2) for the node that input_addr is associated with. 63293c2df58SDoug Thompson */ 63393c2df58SDoug Thompson pvt = mci->pvt_info; 63493c2df58SDoug Thompson node_id = pvt->mc_node_id; 635b487c33eSBorislav Petkov 636b487c33eSBorislav Petkov BUG_ON(node_id > 7); 63793c2df58SDoug Thompson 6387f19bf75SBorislav Petkov intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0)); 63993c2df58SDoug Thompson if (intlv_shift == 0) { 64093c2df58SDoug Thompson debugf1(" InputAddr 0x%lx translates to DramAddr of " 64193c2df58SDoug Thompson "same value\n", (unsigned long)input_addr); 64293c2df58SDoug Thompson 64393c2df58SDoug Thompson return input_addr; 64493c2df58SDoug Thompson } 64593c2df58SDoug Thompson 646f678b8ccSBorislav Petkov bits = ((input_addr & GENMASK(12, 35)) << intlv_shift) + 64793c2df58SDoug Thompson (input_addr & 0xfff); 64893c2df58SDoug Thompson 6497f19bf75SBorislav Petkov intlv_sel = dram_intlv_sel(pvt, node_id) & ((1 << intlv_shift) - 1); 65093c2df58SDoug Thompson dram_addr = bits + (intlv_sel << 12); 65193c2df58SDoug Thompson 65293c2df58SDoug Thompson debugf1("InputAddr 0x%lx translates to DramAddr 0x%lx " 65393c2df58SDoug Thompson "(%d node interleave bits)\n", (unsigned long)input_addr, 65493c2df58SDoug Thompson (unsigned long)dram_addr, intlv_shift); 65593c2df58SDoug Thompson 65693c2df58SDoug Thompson return dram_addr; 65793c2df58SDoug Thompson } 65893c2df58SDoug Thompson 65993c2df58SDoug Thompson /* 66093c2df58SDoug Thompson * @dram_addr is a DramAddr that maps to the node represented by mci. Convert 66193c2df58SDoug Thompson * @dram_addr to a SysAddr. 66293c2df58SDoug Thompson */ 66393c2df58SDoug Thompson static u64 dram_addr_to_sys_addr(struct mem_ctl_info *mci, u64 dram_addr) 66493c2df58SDoug Thompson { 66593c2df58SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 6667f19bf75SBorislav Petkov u64 hole_base, hole_offset, hole_size, base, sys_addr; 66793c2df58SDoug Thompson int ret = 0; 66893c2df58SDoug Thompson 66993c2df58SDoug Thompson ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset, 67093c2df58SDoug Thompson &hole_size); 67193c2df58SDoug Thompson if (!ret) { 67293c2df58SDoug Thompson if ((dram_addr >= hole_base) && 67393c2df58SDoug Thompson (dram_addr < (hole_base + hole_size))) { 67493c2df58SDoug Thompson sys_addr = dram_addr + hole_offset; 67593c2df58SDoug Thompson 67693c2df58SDoug Thompson debugf1("using DHAR to translate DramAddr 0x%lx to " 67793c2df58SDoug Thompson "SysAddr 0x%lx\n", (unsigned long)dram_addr, 67893c2df58SDoug Thompson (unsigned long)sys_addr); 67993c2df58SDoug Thompson 68093c2df58SDoug Thompson return sys_addr; 68193c2df58SDoug Thompson } 68293c2df58SDoug Thompson } 68393c2df58SDoug Thompson 6847f19bf75SBorislav Petkov base = get_dram_base(pvt, pvt->mc_node_id); 68593c2df58SDoug Thompson sys_addr = dram_addr + base; 68693c2df58SDoug Thompson 68793c2df58SDoug Thompson /* 68893c2df58SDoug Thompson * The sys_addr we have computed up to this point is a 40-bit value 68993c2df58SDoug Thompson * because the k8 deals with 40-bit values. However, the value we are 69093c2df58SDoug Thompson * supposed to return is a full 64-bit physical address. The AMD 69193c2df58SDoug Thompson * x86-64 architecture specifies that the most significant implemented 69293c2df58SDoug Thompson * address bit through bit 63 of a physical address must be either all 69393c2df58SDoug Thompson * 0s or all 1s. Therefore we sign-extend the 40-bit sys_addr to a 69493c2df58SDoug Thompson * 64-bit value below. See section 3.4.2 of AMD publication 24592: 69593c2df58SDoug Thompson * AMD x86-64 Architecture Programmer's Manual Volume 1 Application 69693c2df58SDoug Thompson * Programming. 69793c2df58SDoug Thompson */ 69893c2df58SDoug Thompson sys_addr |= ~((sys_addr & (1ull << 39)) - 1); 69993c2df58SDoug Thompson 70093c2df58SDoug Thompson debugf1(" Node %d, DramAddr 0x%lx to SysAddr 0x%lx\n", 70193c2df58SDoug Thompson pvt->mc_node_id, (unsigned long)dram_addr, 70293c2df58SDoug Thompson (unsigned long)sys_addr); 70393c2df58SDoug Thompson 70493c2df58SDoug Thompson return sys_addr; 70593c2df58SDoug Thompson } 70693c2df58SDoug Thompson 70793c2df58SDoug Thompson /* 70893c2df58SDoug Thompson * @input_addr is an InputAddr associated with the node given by mci. Translate 70993c2df58SDoug Thompson * @input_addr to a SysAddr. 71093c2df58SDoug Thompson */ 71193c2df58SDoug Thompson static inline u64 input_addr_to_sys_addr(struct mem_ctl_info *mci, 71293c2df58SDoug Thompson u64 input_addr) 71393c2df58SDoug Thompson { 71493c2df58SDoug Thompson return dram_addr_to_sys_addr(mci, 71593c2df58SDoug Thompson input_addr_to_dram_addr(mci, input_addr)); 71693c2df58SDoug Thompson } 71793c2df58SDoug Thompson 71893c2df58SDoug Thompson /* 71993c2df58SDoug Thompson * Find the minimum and maximum InputAddr values that map to the given @csrow. 72093c2df58SDoug Thompson * Pass back these values in *input_addr_min and *input_addr_max. 72193c2df58SDoug Thompson */ 72293c2df58SDoug Thompson static void find_csrow_limits(struct mem_ctl_info *mci, int csrow, 72393c2df58SDoug Thompson u64 *input_addr_min, u64 *input_addr_max) 72493c2df58SDoug Thompson { 72593c2df58SDoug Thompson struct amd64_pvt *pvt; 72693c2df58SDoug Thompson u64 base, mask; 72793c2df58SDoug Thompson 72893c2df58SDoug Thompson pvt = mci->pvt_info; 72911c75eadSBorislav Petkov BUG_ON((csrow < 0) || (csrow >= pvt->csels[0].b_cnt)); 73093c2df58SDoug Thompson 73111c75eadSBorislav Petkov get_cs_base_and_mask(pvt, csrow, 0, &base, &mask); 73293c2df58SDoug Thompson 73393c2df58SDoug Thompson *input_addr_min = base & ~mask; 73411c75eadSBorislav Petkov *input_addr_max = base | mask; 73593c2df58SDoug Thompson } 73693c2df58SDoug Thompson 73793c2df58SDoug Thompson /* Map the Error address to a PAGE and PAGE OFFSET. */ 73893c2df58SDoug Thompson static inline void error_address_to_page_and_offset(u64 error_address, 73993c2df58SDoug Thompson u32 *page, u32 *offset) 74093c2df58SDoug Thompson { 74193c2df58SDoug Thompson *page = (u32) (error_address >> PAGE_SHIFT); 74293c2df58SDoug Thompson *offset = ((u32) error_address) & ~PAGE_MASK; 74393c2df58SDoug Thompson } 74493c2df58SDoug Thompson 74593c2df58SDoug Thompson /* 74693c2df58SDoug Thompson * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address 74793c2df58SDoug Thompson * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers 74893c2df58SDoug Thompson * of a node that detected an ECC memory error. mci represents the node that 74993c2df58SDoug Thompson * the error address maps to (possibly different from the node that detected 75093c2df58SDoug Thompson * the error). Return the number of the csrow that sys_addr maps to, or -1 on 75193c2df58SDoug Thompson * error. 75293c2df58SDoug Thompson */ 75393c2df58SDoug Thompson static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr) 75493c2df58SDoug Thompson { 75593c2df58SDoug Thompson int csrow; 75693c2df58SDoug Thompson 75793c2df58SDoug Thompson csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr)); 75893c2df58SDoug Thompson 75993c2df58SDoug Thompson if (csrow == -1) 76024f9a7feSBorislav Petkov amd64_mc_err(mci, "Failed to translate InputAddr to csrow for " 76193c2df58SDoug Thompson "address 0x%lx\n", (unsigned long)sys_addr); 76293c2df58SDoug Thompson return csrow; 76393c2df58SDoug Thompson } 764e2ce7255SDoug Thompson 765bfc04aecSBorislav Petkov static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16); 7662da11654SDoug Thompson 7672da11654SDoug Thompson /* 7682da11654SDoug Thompson * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs 7692da11654SDoug Thompson * are ECC capable. 7702da11654SDoug Thompson */ 7711f6189edSDan Carpenter static unsigned long amd64_determine_edac_cap(struct amd64_pvt *pvt) 7722da11654SDoug Thompson { 773cb328507SBorislav Petkov u8 bit; 7741f6189edSDan Carpenter unsigned long edac_cap = EDAC_FLAG_NONE; 7752da11654SDoug Thompson 7761433eb99SBorislav Petkov bit = (boot_cpu_data.x86 > 0xf || pvt->ext_model >= K8_REV_F) 7772da11654SDoug Thompson ? 19 7782da11654SDoug Thompson : 17; 7792da11654SDoug Thompson 780584fcff4SBorislav Petkov if (pvt->dclr0 & BIT(bit)) 7812da11654SDoug Thompson edac_cap = EDAC_FLAG_SECDED; 7822da11654SDoug Thompson 7832da11654SDoug Thompson return edac_cap; 7842da11654SDoug Thompson } 7852da11654SDoug Thompson 7868c671751SBorislav Petkov static void amd64_debug_display_dimm_sizes(struct amd64_pvt *, u8); 7872da11654SDoug Thompson 78868798e17SBorislav Petkov static void amd64_dump_dramcfg_low(u32 dclr, int chan) 78968798e17SBorislav Petkov { 79068798e17SBorislav Petkov debugf1("F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr); 79168798e17SBorislav Petkov 79268798e17SBorislav Petkov debugf1(" DIMM type: %sbuffered; all DIMMs support ECC: %s\n", 79368798e17SBorislav Petkov (dclr & BIT(16)) ? "un" : "", 79468798e17SBorislav Petkov (dclr & BIT(19)) ? "yes" : "no"); 79568798e17SBorislav Petkov 79668798e17SBorislav Petkov debugf1(" PAR/ERR parity: %s\n", 79768798e17SBorislav Petkov (dclr & BIT(8)) ? "enabled" : "disabled"); 79868798e17SBorislav Petkov 799cb328507SBorislav Petkov if (boot_cpu_data.x86 == 0x10) 80068798e17SBorislav Petkov debugf1(" DCT 128bit mode width: %s\n", 80168798e17SBorislav Petkov (dclr & BIT(11)) ? "128b" : "64b"); 80268798e17SBorislav Petkov 80368798e17SBorislav Petkov debugf1(" x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n", 80468798e17SBorislav Petkov (dclr & BIT(12)) ? "yes" : "no", 80568798e17SBorislav Petkov (dclr & BIT(13)) ? "yes" : "no", 80668798e17SBorislav Petkov (dclr & BIT(14)) ? "yes" : "no", 80768798e17SBorislav Petkov (dclr & BIT(15)) ? "yes" : "no"); 80868798e17SBorislav Petkov } 80968798e17SBorislav Petkov 8102da11654SDoug Thompson /* Display and decode various NB registers for debug purposes. */ 811b2b0c605SBorislav Petkov static void dump_misc_regs(struct amd64_pvt *pvt) 8122da11654SDoug Thompson { 81368798e17SBorislav Petkov debugf1("F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap); 8142da11654SDoug Thompson 81568798e17SBorislav Petkov debugf1(" NB two channel DRAM capable: %s\n", 8165980bb9cSBorislav Petkov (pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no"); 81768798e17SBorislav Petkov 81868798e17SBorislav Petkov debugf1(" ECC capable: %s, ChipKill ECC capable: %s\n", 8195980bb9cSBorislav Petkov (pvt->nbcap & NBCAP_SECDED) ? "yes" : "no", 8205980bb9cSBorislav Petkov (pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no"); 82168798e17SBorislav Petkov 82268798e17SBorislav Petkov amd64_dump_dramcfg_low(pvt->dclr0, 0); 8232da11654SDoug Thompson 8248de1d91eSBorislav Petkov debugf1("F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare); 8252da11654SDoug Thompson 8268de1d91eSBorislav Petkov debugf1("F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, " 8278de1d91eSBorislav Petkov "offset: 0x%08x\n", 828bc21fa57SBorislav Petkov pvt->dhar, dhar_base(pvt), 829bc21fa57SBorislav Petkov (boot_cpu_data.x86 == 0xf) ? k8_dhar_offset(pvt) 830bc21fa57SBorislav Petkov : f10_dhar_offset(pvt)); 8312da11654SDoug Thompson 832c8e518d5SBorislav Petkov debugf1(" DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no"); 8332da11654SDoug Thompson 8348c671751SBorislav Petkov amd64_debug_display_dimm_sizes(pvt, 0); 8354d796364SBorislav Petkov 8364d796364SBorislav Petkov /* everything below this point is Fam10h and above */ 8374d796364SBorislav Petkov if (boot_cpu_data.x86 == 0xf) 8382da11654SDoug Thompson return; 8394d796364SBorislav Petkov 8408c671751SBorislav Petkov amd64_debug_display_dimm_sizes(pvt, 1); 8412da11654SDoug Thompson 842a3b7db09SBorislav Petkov amd64_info("using %s syndromes.\n", ((pvt->ecc_sym_sz == 8) ? "x8" : "x4")); 843ad6a32e9SBorislav Petkov 8448de1d91eSBorislav Petkov /* Only if NOT ganged does dclr1 have valid info */ 84568798e17SBorislav Petkov if (!dct_ganging_enabled(pvt)) 84668798e17SBorislav Petkov amd64_dump_dramcfg_low(pvt->dclr1, 1); 8472da11654SDoug Thompson } 8482da11654SDoug Thompson 84994be4bffSDoug Thompson /* 85011c75eadSBorislav Petkov * see BKDG, F2x[1,0][5C:40], F2[1,0][6C:60] 85194be4bffSDoug Thompson */ 85211c75eadSBorislav Petkov static void prep_chip_selects(struct amd64_pvt *pvt) 85394be4bffSDoug Thompson { 8541433eb99SBorislav Petkov if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) { 85511c75eadSBorislav Petkov pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8; 85611c75eadSBorislav Petkov pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8; 8579d858bb1SBorislav Petkov } else { 85811c75eadSBorislav Petkov pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8; 85911c75eadSBorislav Petkov pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4; 8609d858bb1SBorislav Petkov } 86194be4bffSDoug Thompson } 86294be4bffSDoug Thompson 86394be4bffSDoug Thompson /* 86411c75eadSBorislav Petkov * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers 86594be4bffSDoug Thompson */ 866b2b0c605SBorislav Petkov static void read_dct_base_mask(struct amd64_pvt *pvt) 86794be4bffSDoug Thompson { 86811c75eadSBorislav Petkov int cs; 86994be4bffSDoug Thompson 87011c75eadSBorislav Petkov prep_chip_selects(pvt); 87194be4bffSDoug Thompson 87211c75eadSBorislav Petkov for_each_chip_select(cs, 0, pvt) { 87371d2a32eSBorislav Petkov int reg0 = DCSB0 + (cs * 4); 87471d2a32eSBorislav Petkov int reg1 = DCSB1 + (cs * 4); 87511c75eadSBorislav Petkov u32 *base0 = &pvt->csels[0].csbases[cs]; 87611c75eadSBorislav Petkov u32 *base1 = &pvt->csels[1].csbases[cs]; 877b2b0c605SBorislav Petkov 87811c75eadSBorislav Petkov if (!amd64_read_dct_pci_cfg(pvt, reg0, base0)) 87994be4bffSDoug Thompson debugf0(" DCSB0[%d]=0x%08x reg: F2x%x\n", 88011c75eadSBorislav Petkov cs, *base0, reg0); 88194be4bffSDoug Thompson 88211c75eadSBorislav Petkov if (boot_cpu_data.x86 == 0xf || dct_ganging_enabled(pvt)) 88311c75eadSBorislav Petkov continue; 884b2b0c605SBorislav Petkov 88511c75eadSBorislav Petkov if (!amd64_read_dct_pci_cfg(pvt, reg1, base1)) 88694be4bffSDoug Thompson debugf0(" DCSB1[%d]=0x%08x reg: F2x%x\n", 88711c75eadSBorislav Petkov cs, *base1, reg1); 88894be4bffSDoug Thompson } 88994be4bffSDoug Thompson 89011c75eadSBorislav Petkov for_each_chip_select_mask(cs, 0, pvt) { 89171d2a32eSBorislav Petkov int reg0 = DCSM0 + (cs * 4); 89271d2a32eSBorislav Petkov int reg1 = DCSM1 + (cs * 4); 89311c75eadSBorislav Petkov u32 *mask0 = &pvt->csels[0].csmasks[cs]; 89411c75eadSBorislav Petkov u32 *mask1 = &pvt->csels[1].csmasks[cs]; 895b2b0c605SBorislav Petkov 89611c75eadSBorislav Petkov if (!amd64_read_dct_pci_cfg(pvt, reg0, mask0)) 89794be4bffSDoug Thompson debugf0(" DCSM0[%d]=0x%08x reg: F2x%x\n", 89811c75eadSBorislav Petkov cs, *mask0, reg0); 89994be4bffSDoug Thompson 90011c75eadSBorislav Petkov if (boot_cpu_data.x86 == 0xf || dct_ganging_enabled(pvt)) 90111c75eadSBorislav Petkov continue; 902b2b0c605SBorislav Petkov 90311c75eadSBorislav Petkov if (!amd64_read_dct_pci_cfg(pvt, reg1, mask1)) 90494be4bffSDoug Thompson debugf0(" DCSM1[%d]=0x%08x reg: F2x%x\n", 90511c75eadSBorislav Petkov cs, *mask1, reg1); 90694be4bffSDoug Thompson } 9076ba5dcdcSBorislav Petkov } 90894be4bffSDoug Thompson 90924f9a7feSBorislav Petkov static enum mem_type amd64_determine_memory_type(struct amd64_pvt *pvt, int cs) 91094be4bffSDoug Thompson { 91194be4bffSDoug Thompson enum mem_type type; 91294be4bffSDoug Thompson 913cb328507SBorislav Petkov /* F15h supports only DDR3 */ 914cb328507SBorislav Petkov if (boot_cpu_data.x86 >= 0x15) 915cb328507SBorislav Petkov type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3; 916cb328507SBorislav Petkov else if (boot_cpu_data.x86 == 0x10 || pvt->ext_model >= K8_REV_F) { 9176b4c0bdeSBorislav Petkov if (pvt->dchr0 & DDR3_MODE) 9186b4c0bdeSBorislav Petkov type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3; 9196b4c0bdeSBorislav Petkov else 92094be4bffSDoug Thompson type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2; 92194be4bffSDoug Thompson } else { 92294be4bffSDoug Thompson type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR; 92394be4bffSDoug Thompson } 92494be4bffSDoug Thompson 92524f9a7feSBorislav Petkov amd64_info("CS%d: %s\n", cs, edac_mem_types[type]); 92694be4bffSDoug Thompson 92794be4bffSDoug Thompson return type; 92894be4bffSDoug Thompson } 92994be4bffSDoug Thompson 930cb328507SBorislav Petkov /* Get the number of DCT channels the memory controller is using. */ 931ddff876dSDoug Thompson static int k8_early_channel_count(struct amd64_pvt *pvt) 932ddff876dSDoug Thompson { 933cb328507SBorislav Petkov int flag; 934ddff876dSDoug Thompson 9359f56da0eSBorislav Petkov if (pvt->ext_model >= K8_REV_F) 936ddff876dSDoug Thompson /* RevF (NPT) and later */ 93741d8bfabSBorislav Petkov flag = pvt->dclr0 & WIDTH_128; 9389f56da0eSBorislav Petkov else 939ddff876dSDoug Thompson /* RevE and earlier */ 940ddff876dSDoug Thompson flag = pvt->dclr0 & REVE_WIDTH_128; 941ddff876dSDoug Thompson 942ddff876dSDoug Thompson /* not used */ 943ddff876dSDoug Thompson pvt->dclr1 = 0; 944ddff876dSDoug Thompson 945ddff876dSDoug Thompson return (flag) ? 2 : 1; 946ddff876dSDoug Thompson } 947ddff876dSDoug Thompson 94870046624SBorislav Petkov /* On F10h and later ErrAddr is MC4_ADDR[47:1] */ 94970046624SBorislav Petkov static u64 get_error_address(struct mce *m) 950ddff876dSDoug Thompson { 951c1ae6830SBorislav Petkov struct cpuinfo_x86 *c = &boot_cpu_data; 952c1ae6830SBorislav Petkov u64 addr; 95370046624SBorislav Petkov u8 start_bit = 1; 95470046624SBorislav Petkov u8 end_bit = 47; 95570046624SBorislav Petkov 956c1ae6830SBorislav Petkov if (c->x86 == 0xf) { 95770046624SBorislav Petkov start_bit = 3; 95870046624SBorislav Petkov end_bit = 39; 95970046624SBorislav Petkov } 96070046624SBorislav Petkov 961c1ae6830SBorislav Petkov addr = m->addr & GENMASK(start_bit, end_bit); 962c1ae6830SBorislav Petkov 963c1ae6830SBorislav Petkov /* 964c1ae6830SBorislav Petkov * Erratum 637 workaround 965c1ae6830SBorislav Petkov */ 966c1ae6830SBorislav Petkov if (c->x86 == 0x15) { 967c1ae6830SBorislav Petkov struct amd64_pvt *pvt; 968c1ae6830SBorislav Petkov u64 cc6_base, tmp_addr; 969c1ae6830SBorislav Petkov u32 tmp; 970c1ae6830SBorislav Petkov u8 mce_nid, intlv_en; 971c1ae6830SBorislav Petkov 972c1ae6830SBorislav Petkov if ((addr & GENMASK(24, 47)) >> 24 != 0x00fdf7) 973c1ae6830SBorislav Petkov return addr; 974c1ae6830SBorislav Petkov 975c1ae6830SBorislav Petkov mce_nid = amd_get_nb_id(m->extcpu); 976c1ae6830SBorislav Petkov pvt = mcis[mce_nid]->pvt_info; 977c1ae6830SBorislav Petkov 978c1ae6830SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_LIM, &tmp); 979c1ae6830SBorislav Petkov intlv_en = tmp >> 21 & 0x7; 980c1ae6830SBorislav Petkov 981c1ae6830SBorislav Petkov /* add [47:27] + 3 trailing bits */ 982c1ae6830SBorislav Petkov cc6_base = (tmp & GENMASK(0, 20)) << 3; 983c1ae6830SBorislav Petkov 984c1ae6830SBorislav Petkov /* reverse and add DramIntlvEn */ 985c1ae6830SBorislav Petkov cc6_base |= intlv_en ^ 0x7; 986c1ae6830SBorislav Petkov 987c1ae6830SBorislav Petkov /* pin at [47:24] */ 988c1ae6830SBorislav Petkov cc6_base <<= 24; 989c1ae6830SBorislav Petkov 990c1ae6830SBorislav Petkov if (!intlv_en) 991c1ae6830SBorislav Petkov return cc6_base | (addr & GENMASK(0, 23)); 992c1ae6830SBorislav Petkov 993c1ae6830SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_BASE, &tmp); 994c1ae6830SBorislav Petkov 995c1ae6830SBorislav Petkov /* faster log2 */ 996c1ae6830SBorislav Petkov tmp_addr = (addr & GENMASK(12, 23)) << __fls(intlv_en + 1); 997c1ae6830SBorislav Petkov 998c1ae6830SBorislav Petkov /* OR DramIntlvSel into bits [14:12] */ 999c1ae6830SBorislav Petkov tmp_addr |= (tmp & GENMASK(21, 23)) >> 9; 1000c1ae6830SBorislav Petkov 1001c1ae6830SBorislav Petkov /* add remaining [11:0] bits from original MC4_ADDR */ 1002c1ae6830SBorislav Petkov tmp_addr |= addr & GENMASK(0, 11); 1003c1ae6830SBorislav Petkov 1004c1ae6830SBorislav Petkov return cc6_base | tmp_addr; 1005c1ae6830SBorislav Petkov } 1006c1ae6830SBorislav Petkov 1007c1ae6830SBorislav Petkov return addr; 1008ddff876dSDoug Thompson } 1009ddff876dSDoug Thompson 10107f19bf75SBorislav Petkov static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range) 1011ddff876dSDoug Thompson { 1012f08e457cSBorislav Petkov struct cpuinfo_x86 *c = &boot_cpu_data; 101371d2a32eSBorislav Petkov int off = range << 3; 1014ddff876dSDoug Thompson 10157f19bf75SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off, &pvt->ranges[range].base.lo); 10167f19bf75SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo); 1017ddff876dSDoug Thompson 1018f08e457cSBorislav Petkov if (c->x86 == 0xf) 10197f19bf75SBorislav Petkov return; 1020ddff876dSDoug Thompson 10217f19bf75SBorislav Petkov if (!dram_rw(pvt, range)) 10227f19bf75SBorislav Petkov return; 1023ddff876dSDoug Thompson 10247f19bf75SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off, &pvt->ranges[range].base.hi); 10257f19bf75SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi); 1026f08e457cSBorislav Petkov 1027f08e457cSBorislav Petkov /* Factor in CC6 save area by reading dst node's limit reg */ 1028f08e457cSBorislav Petkov if (c->x86 == 0x15) { 1029f08e457cSBorislav Petkov struct pci_dev *f1 = NULL; 1030f08e457cSBorislav Petkov u8 nid = dram_dst_node(pvt, range); 1031f08e457cSBorislav Petkov u32 llim; 1032f08e457cSBorislav Petkov 1033f08e457cSBorislav Petkov f1 = pci_get_domain_bus_and_slot(0, 0, PCI_DEVFN(0x18 + nid, 1)); 1034f08e457cSBorislav Petkov if (WARN_ON(!f1)) 1035f08e457cSBorislav Petkov return; 1036f08e457cSBorislav Petkov 1037f08e457cSBorislav Petkov amd64_read_pci_cfg(f1, DRAM_LOCAL_NODE_LIM, &llim); 1038f08e457cSBorislav Petkov 1039f08e457cSBorislav Petkov pvt->ranges[range].lim.lo &= GENMASK(0, 15); 1040f08e457cSBorislav Petkov 1041f08e457cSBorislav Petkov /* {[39:27],111b} */ 1042f08e457cSBorislav Petkov pvt->ranges[range].lim.lo |= ((llim & 0x1fff) << 3 | 0x7) << 16; 1043f08e457cSBorislav Petkov 1044f08e457cSBorislav Petkov pvt->ranges[range].lim.hi &= GENMASK(0, 7); 1045f08e457cSBorislav Petkov 1046f08e457cSBorislav Petkov /* [47:40] */ 1047f08e457cSBorislav Petkov pvt->ranges[range].lim.hi |= llim >> 13; 1048f08e457cSBorislav Petkov 1049f08e457cSBorislav Petkov pci_dev_put(f1); 1050f08e457cSBorislav Petkov } 1051ddff876dSDoug Thompson } 1052ddff876dSDoug Thompson 1053f192c7b1SBorislav Petkov static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr, 1054f192c7b1SBorislav Petkov u16 syndrome) 1055ddff876dSDoug Thompson { 1056ddff876dSDoug Thompson struct mem_ctl_info *src_mci; 1057f192c7b1SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 1058ddff876dSDoug Thompson int channel, csrow; 1059ddff876dSDoug Thompson u32 page, offset; 1060ddff876dSDoug Thompson 1061ddff876dSDoug Thompson /* CHIPKILL enabled */ 1062f192c7b1SBorislav Petkov if (pvt->nbcfg & NBCFG_CHIPKILL) { 1063bfc04aecSBorislav Petkov channel = get_channel_from_ecc_syndrome(mci, syndrome); 1064ddff876dSDoug Thompson if (channel < 0) { 1065ddff876dSDoug Thompson /* 1066ddff876dSDoug Thompson * Syndrome didn't map, so we don't know which of the 1067ddff876dSDoug Thompson * 2 DIMMs is in error. So we need to ID 'both' of them 1068ddff876dSDoug Thompson * as suspect. 1069ddff876dSDoug Thompson */ 107024f9a7feSBorislav Petkov amd64_mc_warn(mci, "unknown syndrome 0x%04x - possible " 1071ad6a32e9SBorislav Petkov "error reporting race\n", syndrome); 1072ddff876dSDoug Thompson edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR); 1073ddff876dSDoug Thompson return; 1074ddff876dSDoug Thompson } 1075ddff876dSDoug Thompson } else { 1076ddff876dSDoug Thompson /* 1077ddff876dSDoug Thompson * non-chipkill ecc mode 1078ddff876dSDoug Thompson * 1079ddff876dSDoug Thompson * The k8 documentation is unclear about how to determine the 1080ddff876dSDoug Thompson * channel number when using non-chipkill memory. This method 1081ddff876dSDoug Thompson * was obtained from email communication with someone at AMD. 1082ddff876dSDoug Thompson * (Wish the email was placed in this comment - norsk) 1083ddff876dSDoug Thompson */ 108444e9e2eeSBorislav Petkov channel = ((sys_addr & BIT(3)) != 0); 1085ddff876dSDoug Thompson } 1086ddff876dSDoug Thompson 1087ddff876dSDoug Thompson /* 1088ddff876dSDoug Thompson * Find out which node the error address belongs to. This may be 1089ddff876dSDoug Thompson * different from the node that detected the error. 1090ddff876dSDoug Thompson */ 109144e9e2eeSBorislav Petkov src_mci = find_mc_by_sys_addr(mci, sys_addr); 10922cff18c2SKeith Mannthey if (!src_mci) { 109324f9a7feSBorislav Petkov amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n", 109444e9e2eeSBorislav Petkov (unsigned long)sys_addr); 1095ddff876dSDoug Thompson edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR); 1096ddff876dSDoug Thompson return; 1097ddff876dSDoug Thompson } 1098ddff876dSDoug Thompson 109944e9e2eeSBorislav Petkov /* Now map the sys_addr to a CSROW */ 110044e9e2eeSBorislav Petkov csrow = sys_addr_to_csrow(src_mci, sys_addr); 1101ddff876dSDoug Thompson if (csrow < 0) { 1102ddff876dSDoug Thompson edac_mc_handle_ce_no_info(src_mci, EDAC_MOD_STR); 1103ddff876dSDoug Thompson } else { 110444e9e2eeSBorislav Petkov error_address_to_page_and_offset(sys_addr, &page, &offset); 1105ddff876dSDoug Thompson 1106ddff876dSDoug Thompson edac_mc_handle_ce(src_mci, page, offset, syndrome, csrow, 1107ddff876dSDoug Thompson channel, EDAC_MOD_STR); 1108ddff876dSDoug Thompson } 1109ddff876dSDoug Thompson } 1110ddff876dSDoug Thompson 111141d8bfabSBorislav Petkov static int ddr2_cs_size(unsigned i, bool dct_width) 1112ddff876dSDoug Thompson { 111341d8bfabSBorislav Petkov unsigned shift = 0; 1114ddff876dSDoug Thompson 111541d8bfabSBorislav Petkov if (i <= 2) 111641d8bfabSBorislav Petkov shift = i; 111741d8bfabSBorislav Petkov else if (!(i & 0x1)) 111841d8bfabSBorislav Petkov shift = i >> 1; 11191433eb99SBorislav Petkov else 112041d8bfabSBorislav Petkov shift = (i + 1) >> 1; 1121ddff876dSDoug Thompson 112241d8bfabSBorislav Petkov return 128 << (shift + !!dct_width); 112341d8bfabSBorislav Petkov } 112441d8bfabSBorislav Petkov 112541d8bfabSBorislav Petkov static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, 112641d8bfabSBorislav Petkov unsigned cs_mode) 112741d8bfabSBorislav Petkov { 112841d8bfabSBorislav Petkov u32 dclr = dct ? pvt->dclr1 : pvt->dclr0; 112941d8bfabSBorislav Petkov 113041d8bfabSBorislav Petkov if (pvt->ext_model >= K8_REV_F) { 113141d8bfabSBorislav Petkov WARN_ON(cs_mode > 11); 113241d8bfabSBorislav Petkov return ddr2_cs_size(cs_mode, dclr & WIDTH_128); 113341d8bfabSBorislav Petkov } 113441d8bfabSBorislav Petkov else if (pvt->ext_model >= K8_REV_D) { 113541d8bfabSBorislav Petkov WARN_ON(cs_mode > 10); 113641d8bfabSBorislav Petkov 113741d8bfabSBorislav Petkov if (cs_mode == 3 || cs_mode == 8) 113841d8bfabSBorislav Petkov return 32 << (cs_mode - 1); 113941d8bfabSBorislav Petkov else 114041d8bfabSBorislav Petkov return 32 << cs_mode; 114141d8bfabSBorislav Petkov } 114241d8bfabSBorislav Petkov else { 114341d8bfabSBorislav Petkov WARN_ON(cs_mode > 6); 114441d8bfabSBorislav Petkov return 32 << cs_mode; 114541d8bfabSBorislav Petkov } 1146ddff876dSDoug Thompson } 1147ddff876dSDoug Thompson 11481afd3c98SDoug Thompson /* 11491afd3c98SDoug Thompson * Get the number of DCT channels in use. 11501afd3c98SDoug Thompson * 11511afd3c98SDoug Thompson * Return: 11521afd3c98SDoug Thompson * number of Memory Channels in operation 11531afd3c98SDoug Thompson * Pass back: 11541afd3c98SDoug Thompson * contents of the DCL0_LOW register 11551afd3c98SDoug Thompson */ 11567d20d14dSBorislav Petkov static int f1x_early_channel_count(struct amd64_pvt *pvt) 11571afd3c98SDoug Thompson { 11586ba5dcdcSBorislav Petkov int i, j, channels = 0; 1159ddff876dSDoug Thompson 11607d20d14dSBorislav Petkov /* On F10h, if we are in 128 bit mode, then we are using 2 channels */ 116141d8bfabSBorislav Petkov if (boot_cpu_data.x86 == 0x10 && (pvt->dclr0 & WIDTH_128)) 11627d20d14dSBorislav Petkov return 2; 11631afd3c98SDoug Thompson 11641afd3c98SDoug Thompson /* 1165d16149e8SBorislav Petkov * Need to check if in unganged mode: In such, there are 2 channels, 1166d16149e8SBorislav Petkov * but they are not in 128 bit mode and thus the above 'dclr0' status 1167d16149e8SBorislav Petkov * bit will be OFF. 11681afd3c98SDoug Thompson * 11691afd3c98SDoug Thompson * Need to check DCT0[0] and DCT1[0] to see if only one of them has 11701afd3c98SDoug Thompson * their CSEnable bit on. If so, then SINGLE DIMM case. 11711afd3c98SDoug Thompson */ 1172d16149e8SBorislav Petkov debugf0("Data width is not 128 bits - need more decoding\n"); 11731afd3c98SDoug Thompson 11741afd3c98SDoug Thompson /* 11751afd3c98SDoug Thompson * Check DRAM Bank Address Mapping values for each DIMM to see if there 11761afd3c98SDoug Thompson * is more than just one DIMM present in unganged mode. Need to check 11771afd3c98SDoug Thompson * both controllers since DIMMs can be placed in either one. 11781afd3c98SDoug Thompson */ 1179525a1b20SBorislav Petkov for (i = 0; i < 2; i++) { 1180525a1b20SBorislav Petkov u32 dbam = (i ? pvt->dbam1 : pvt->dbam0); 11811afd3c98SDoug Thompson 118257a30854SWan Wei for (j = 0; j < 4; j++) { 118357a30854SWan Wei if (DBAM_DIMM(j, dbam) > 0) { 11841afd3c98SDoug Thompson channels++; 118557a30854SWan Wei break; 11861afd3c98SDoug Thompson } 118757a30854SWan Wei } 118857a30854SWan Wei } 11891afd3c98SDoug Thompson 1190d16149e8SBorislav Petkov if (channels > 2) 1191d16149e8SBorislav Petkov channels = 2; 1192d16149e8SBorislav Petkov 119324f9a7feSBorislav Petkov amd64_info("MCT channel count: %d\n", channels); 11941afd3c98SDoug Thompson 11951afd3c98SDoug Thompson return channels; 11961afd3c98SDoug Thompson } 11971afd3c98SDoug Thompson 119841d8bfabSBorislav Petkov static int ddr3_cs_size(unsigned i, bool dct_width) 11991afd3c98SDoug Thompson { 120041d8bfabSBorislav Petkov unsigned shift = 0; 120141d8bfabSBorislav Petkov int cs_size = 0; 120241d8bfabSBorislav Petkov 120341d8bfabSBorislav Petkov if (i == 0 || i == 3 || i == 4) 120441d8bfabSBorislav Petkov cs_size = -1; 120541d8bfabSBorislav Petkov else if (i <= 2) 120641d8bfabSBorislav Petkov shift = i; 120741d8bfabSBorislav Petkov else if (i == 12) 120841d8bfabSBorislav Petkov shift = 7; 120941d8bfabSBorislav Petkov else if (!(i & 0x1)) 121041d8bfabSBorislav Petkov shift = i >> 1; 121141d8bfabSBorislav Petkov else 121241d8bfabSBorislav Petkov shift = (i + 1) >> 1; 121341d8bfabSBorislav Petkov 121441d8bfabSBorislav Petkov if (cs_size != -1) 121541d8bfabSBorislav Petkov cs_size = (128 * (1 << !!dct_width)) << shift; 121641d8bfabSBorislav Petkov 121741d8bfabSBorislav Petkov return cs_size; 121841d8bfabSBorislav Petkov } 121941d8bfabSBorislav Petkov 122041d8bfabSBorislav Petkov static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, 122141d8bfabSBorislav Petkov unsigned cs_mode) 122241d8bfabSBorislav Petkov { 122341d8bfabSBorislav Petkov u32 dclr = dct ? pvt->dclr1 : pvt->dclr0; 122441d8bfabSBorislav Petkov 122541d8bfabSBorislav Petkov WARN_ON(cs_mode > 11); 12261433eb99SBorislav Petkov 12271433eb99SBorislav Petkov if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE) 122841d8bfabSBorislav Petkov return ddr3_cs_size(cs_mode, dclr & WIDTH_128); 12291433eb99SBorislav Petkov else 123041d8bfabSBorislav Petkov return ddr2_cs_size(cs_mode, dclr & WIDTH_128); 123141d8bfabSBorislav Petkov } 12321433eb99SBorislav Petkov 123341d8bfabSBorislav Petkov /* 123441d8bfabSBorislav Petkov * F15h supports only 64bit DCT interfaces 123541d8bfabSBorislav Petkov */ 123641d8bfabSBorislav Petkov static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, 123741d8bfabSBorislav Petkov unsigned cs_mode) 123841d8bfabSBorislav Petkov { 123941d8bfabSBorislav Petkov WARN_ON(cs_mode > 12); 124041d8bfabSBorislav Petkov 124141d8bfabSBorislav Petkov return ddr3_cs_size(cs_mode, false); 12421afd3c98SDoug Thompson } 12431afd3c98SDoug Thompson 12445a5d2371SBorislav Petkov static void read_dram_ctl_register(struct amd64_pvt *pvt) 12456163b5d4SDoug Thompson { 12466163b5d4SDoug Thompson 12475a5d2371SBorislav Petkov if (boot_cpu_data.x86 == 0xf) 12485a5d2371SBorislav Petkov return; 12495a5d2371SBorislav Petkov 125078da121eSBorislav Petkov if (!amd64_read_dct_pci_cfg(pvt, DCT_SEL_LO, &pvt->dct_sel_lo)) { 125178da121eSBorislav Petkov debugf0("F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n", 125278da121eSBorislav Petkov pvt->dct_sel_lo, dct_sel_baseaddr(pvt)); 12536163b5d4SDoug Thompson 12545a5d2371SBorislav Petkov debugf0(" DCTs operate in %s mode.\n", 12555a5d2371SBorislav Petkov (dct_ganging_enabled(pvt) ? "ganged" : "unganged")); 12566163b5d4SDoug Thompson 125772381bd5SBorislav Petkov if (!dct_ganging_enabled(pvt)) 125872381bd5SBorislav Petkov debugf0(" Address range split per DCT: %s\n", 125972381bd5SBorislav Petkov (dct_high_range_enabled(pvt) ? "yes" : "no")); 126072381bd5SBorislav Petkov 126178da121eSBorislav Petkov debugf0(" data interleave for ECC: %s, " 126272381bd5SBorislav Petkov "DRAM cleared since last warm reset: %s\n", 126372381bd5SBorislav Petkov (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"), 126472381bd5SBorislav Petkov (dct_memory_cleared(pvt) ? "yes" : "no")); 126572381bd5SBorislav Petkov 126678da121eSBorislav Petkov debugf0(" channel interleave: %s, " 126778da121eSBorislav Petkov "interleave bits selector: 0x%x\n", 126872381bd5SBorislav Petkov (dct_interleave_enabled(pvt) ? "enabled" : "disabled"), 12696163b5d4SDoug Thompson dct_sel_interleave_addr(pvt)); 12706163b5d4SDoug Thompson } 12716163b5d4SDoug Thompson 127278da121eSBorislav Petkov amd64_read_dct_pci_cfg(pvt, DCT_SEL_HI, &pvt->dct_sel_hi); 12736163b5d4SDoug Thompson } 12746163b5d4SDoug Thompson 1275f71d0a05SDoug Thompson /* 1276229a7a11SBorislav Petkov * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory 1277f71d0a05SDoug Thompson * Interleaving Modes. 1278f71d0a05SDoug Thompson */ 1279b15f0fcaSBorislav Petkov static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr, 1280229a7a11SBorislav Petkov bool hi_range_sel, u8 intlv_en) 12816163b5d4SDoug Thompson { 1282151fa71cSBorislav Petkov u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1; 12836163b5d4SDoug Thompson 12846163b5d4SDoug Thompson if (dct_ganging_enabled(pvt)) 1285229a7a11SBorislav Petkov return 0; 1286229a7a11SBorislav Petkov 1287229a7a11SBorislav Petkov if (hi_range_sel) 1288229a7a11SBorislav Petkov return dct_sel_high; 1289229a7a11SBorislav Petkov 1290f71d0a05SDoug Thompson /* 1291f71d0a05SDoug Thompson * see F2x110[DctSelIntLvAddr] - channel interleave mode 1292f71d0a05SDoug Thompson */ 1293229a7a11SBorislav Petkov if (dct_interleave_enabled(pvt)) { 1294229a7a11SBorislav Petkov u8 intlv_addr = dct_sel_interleave_addr(pvt); 12956163b5d4SDoug Thompson 1296229a7a11SBorislav Petkov /* return DCT select function: 0=DCT0, 1=DCT1 */ 1297229a7a11SBorislav Petkov if (!intlv_addr) 1298229a7a11SBorislav Petkov return sys_addr >> 6 & 1; 12996163b5d4SDoug Thompson 1300229a7a11SBorislav Petkov if (intlv_addr & 0x2) { 1301229a7a11SBorislav Petkov u8 shift = intlv_addr & 0x1 ? 9 : 6; 1302229a7a11SBorislav Petkov u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) % 2; 1303229a7a11SBorislav Petkov 1304229a7a11SBorislav Petkov return ((sys_addr >> shift) & 1) ^ temp; 13056163b5d4SDoug Thompson } 13066163b5d4SDoug Thompson 1307229a7a11SBorislav Petkov return (sys_addr >> (12 + hweight8(intlv_en))) & 1; 1308229a7a11SBorislav Petkov } 1309229a7a11SBorislav Petkov 1310229a7a11SBorislav Petkov if (dct_high_range_enabled(pvt)) 1311229a7a11SBorislav Petkov return ~dct_sel_high & 1; 13126163b5d4SDoug Thompson 13136163b5d4SDoug Thompson return 0; 13146163b5d4SDoug Thompson } 13156163b5d4SDoug Thompson 1316c8e518d5SBorislav Petkov /* Convert the sys_addr to the normalized DCT address */ 1317e761359aSBorislav Petkov static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, unsigned range, 1318c8e518d5SBorislav Petkov u64 sys_addr, bool hi_rng, 1319c8e518d5SBorislav Petkov u32 dct_sel_base_addr) 13206163b5d4SDoug Thompson { 13216163b5d4SDoug Thompson u64 chan_off; 1322c8e518d5SBorislav Petkov u64 dram_base = get_dram_base(pvt, range); 1323c8e518d5SBorislav Petkov u64 hole_off = f10_dhar_offset(pvt); 1324c8e518d5SBorislav Petkov u64 dct_sel_base_off = (pvt->dct_sel_hi & 0xFFFFFC00) << 16; 13256163b5d4SDoug Thompson 1326c8e518d5SBorislav Petkov if (hi_rng) { 1327c8e518d5SBorislav Petkov /* 1328c8e518d5SBorislav Petkov * if 1329c8e518d5SBorislav Petkov * base address of high range is below 4Gb 1330c8e518d5SBorislav Petkov * (bits [47:27] at [31:11]) 1331c8e518d5SBorislav Petkov * DRAM address space on this DCT is hoisted above 4Gb && 1332c8e518d5SBorislav Petkov * sys_addr > 4Gb 1333c8e518d5SBorislav Petkov * 1334c8e518d5SBorislav Petkov * remove hole offset from sys_addr 1335c8e518d5SBorislav Petkov * else 1336c8e518d5SBorislav Petkov * remove high range offset from sys_addr 1337c8e518d5SBorislav Petkov */ 1338c8e518d5SBorislav Petkov if ((!(dct_sel_base_addr >> 16) || 1339c8e518d5SBorislav Petkov dct_sel_base_addr < dhar_base(pvt)) && 1340972ea17aSBorislav Petkov dhar_valid(pvt) && 1341c8e518d5SBorislav Petkov (sys_addr >= BIT_64(32))) 1342bc21fa57SBorislav Petkov chan_off = hole_off; 13436163b5d4SDoug Thompson else 13446163b5d4SDoug Thompson chan_off = dct_sel_base_off; 13456163b5d4SDoug Thompson } else { 1346c8e518d5SBorislav Petkov /* 1347c8e518d5SBorislav Petkov * if 1348c8e518d5SBorislav Petkov * we have a valid hole && 1349c8e518d5SBorislav Petkov * sys_addr > 4Gb 1350c8e518d5SBorislav Petkov * 1351c8e518d5SBorislav Petkov * remove hole 1352c8e518d5SBorislav Petkov * else 1353c8e518d5SBorislav Petkov * remove dram base to normalize to DCT address 1354c8e518d5SBorislav Petkov */ 1355972ea17aSBorislav Petkov if (dhar_valid(pvt) && (sys_addr >= BIT_64(32))) 1356bc21fa57SBorislav Petkov chan_off = hole_off; 13576163b5d4SDoug Thompson else 1358c8e518d5SBorislav Petkov chan_off = dram_base; 13596163b5d4SDoug Thompson } 13606163b5d4SDoug Thompson 1361c8e518d5SBorislav Petkov return (sys_addr & GENMASK(6,47)) - (chan_off & GENMASK(23,47)); 13626163b5d4SDoug Thompson } 13636163b5d4SDoug Thompson 13646163b5d4SDoug Thompson /* 13656163b5d4SDoug Thompson * checks if the csrow passed in is marked as SPARED, if so returns the new 13666163b5d4SDoug Thompson * spare row 13676163b5d4SDoug Thompson */ 136811c75eadSBorislav Petkov static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow) 13696163b5d4SDoug Thompson { 1370614ec9d8SBorislav Petkov int tmp_cs; 13716163b5d4SDoug Thompson 1372614ec9d8SBorislav Petkov if (online_spare_swap_done(pvt, dct) && 1373614ec9d8SBorislav Petkov csrow == online_spare_bad_dramcs(pvt, dct)) { 1374614ec9d8SBorislav Petkov 1375614ec9d8SBorislav Petkov for_each_chip_select(tmp_cs, dct, pvt) { 1376614ec9d8SBorislav Petkov if (chip_select_base(tmp_cs, dct, pvt) & 0x2) { 1377614ec9d8SBorislav Petkov csrow = tmp_cs; 1378614ec9d8SBorislav Petkov break; 1379614ec9d8SBorislav Petkov } 1380614ec9d8SBorislav Petkov } 13816163b5d4SDoug Thompson } 13826163b5d4SDoug Thompson return csrow; 13836163b5d4SDoug Thompson } 13846163b5d4SDoug Thompson 13856163b5d4SDoug Thompson /* 13866163b5d4SDoug Thompson * Iterate over the DRAM DCT "base" and "mask" registers looking for a 13876163b5d4SDoug Thompson * SystemAddr match on the specified 'ChannelSelect' and 'NodeID' 13886163b5d4SDoug Thompson * 13896163b5d4SDoug Thompson * Return: 13906163b5d4SDoug Thompson * -EINVAL: NOT FOUND 13916163b5d4SDoug Thompson * 0..csrow = Chip-Select Row 13926163b5d4SDoug Thompson */ 1393b15f0fcaSBorislav Petkov static int f1x_lookup_addr_in_dct(u64 in_addr, u32 nid, u8 dct) 13946163b5d4SDoug Thompson { 13956163b5d4SDoug Thompson struct mem_ctl_info *mci; 13966163b5d4SDoug Thompson struct amd64_pvt *pvt; 139711c75eadSBorislav Petkov u64 cs_base, cs_mask; 13986163b5d4SDoug Thompson int cs_found = -EINVAL; 13996163b5d4SDoug Thompson int csrow; 14006163b5d4SDoug Thompson 1401cc4d8860SBorislav Petkov mci = mcis[nid]; 14026163b5d4SDoug Thompson if (!mci) 14036163b5d4SDoug Thompson return cs_found; 14046163b5d4SDoug Thompson 14056163b5d4SDoug Thompson pvt = mci->pvt_info; 14066163b5d4SDoug Thompson 140711c75eadSBorislav Petkov debugf1("input addr: 0x%llx, DCT: %d\n", in_addr, dct); 14086163b5d4SDoug Thompson 140911c75eadSBorislav Petkov for_each_chip_select(csrow, dct, pvt) { 141011c75eadSBorislav Petkov if (!csrow_enabled(csrow, dct, pvt)) 14116163b5d4SDoug Thompson continue; 14126163b5d4SDoug Thompson 141311c75eadSBorislav Petkov get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask); 14146163b5d4SDoug Thompson 141511c75eadSBorislav Petkov debugf1(" CSROW=%d CSBase=0x%llx CSMask=0x%llx\n", 14166163b5d4SDoug Thompson csrow, cs_base, cs_mask); 14176163b5d4SDoug Thompson 141811c75eadSBorislav Petkov cs_mask = ~cs_mask; 14196163b5d4SDoug Thompson 142011c75eadSBorislav Petkov debugf1(" (InputAddr & ~CSMask)=0x%llx " 142111c75eadSBorislav Petkov "(CSBase & ~CSMask)=0x%llx\n", 142211c75eadSBorislav Petkov (in_addr & cs_mask), (cs_base & cs_mask)); 14236163b5d4SDoug Thompson 142411c75eadSBorislav Petkov if ((in_addr & cs_mask) == (cs_base & cs_mask)) { 142511c75eadSBorislav Petkov cs_found = f10_process_possible_spare(pvt, dct, csrow); 14266163b5d4SDoug Thompson 14276163b5d4SDoug Thompson debugf1(" MATCH csrow=%d\n", cs_found); 14286163b5d4SDoug Thompson break; 14296163b5d4SDoug Thompson } 14306163b5d4SDoug Thompson } 14316163b5d4SDoug Thompson return cs_found; 14326163b5d4SDoug Thompson } 14336163b5d4SDoug Thompson 143495b0ef55SBorislav Petkov /* 143595b0ef55SBorislav Petkov * See F2x10C. Non-interleaved graphics framebuffer memory under the 16G is 143695b0ef55SBorislav Petkov * swapped with a region located at the bottom of memory so that the GPU can use 143795b0ef55SBorislav Petkov * the interleaved region and thus two channels. 143895b0ef55SBorislav Petkov */ 1439b15f0fcaSBorislav Petkov static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr) 144095b0ef55SBorislav Petkov { 144195b0ef55SBorislav Petkov u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr; 144295b0ef55SBorislav Petkov 144395b0ef55SBorislav Petkov if (boot_cpu_data.x86 == 0x10) { 144495b0ef55SBorislav Petkov /* only revC3 and revE have that feature */ 144595b0ef55SBorislav Petkov if (boot_cpu_data.x86_model < 4 || 144695b0ef55SBorislav Petkov (boot_cpu_data.x86_model < 0xa && 144795b0ef55SBorislav Petkov boot_cpu_data.x86_mask < 3)) 144895b0ef55SBorislav Petkov return sys_addr; 144995b0ef55SBorislav Petkov } 145095b0ef55SBorislav Petkov 145195b0ef55SBorislav Petkov amd64_read_dct_pci_cfg(pvt, SWAP_INTLV_REG, &swap_reg); 145295b0ef55SBorislav Petkov 145395b0ef55SBorislav Petkov if (!(swap_reg & 0x1)) 145495b0ef55SBorislav Petkov return sys_addr; 145595b0ef55SBorislav Petkov 145695b0ef55SBorislav Petkov swap_base = (swap_reg >> 3) & 0x7f; 145795b0ef55SBorislav Petkov swap_limit = (swap_reg >> 11) & 0x7f; 145895b0ef55SBorislav Petkov rgn_size = (swap_reg >> 20) & 0x7f; 145995b0ef55SBorislav Petkov tmp_addr = sys_addr >> 27; 146095b0ef55SBorislav Petkov 146195b0ef55SBorislav Petkov if (!(sys_addr >> 34) && 146295b0ef55SBorislav Petkov (((tmp_addr >= swap_base) && 146395b0ef55SBorislav Petkov (tmp_addr <= swap_limit)) || 146495b0ef55SBorislav Petkov (tmp_addr < rgn_size))) 146595b0ef55SBorislav Petkov return sys_addr ^ (u64)swap_base << 27; 146695b0ef55SBorislav Petkov 146795b0ef55SBorislav Petkov return sys_addr; 146895b0ef55SBorislav Petkov } 146995b0ef55SBorislav Petkov 1470f71d0a05SDoug Thompson /* For a given @dram_range, check if @sys_addr falls within it. */ 1471e761359aSBorislav Petkov static int f1x_match_to_this_node(struct amd64_pvt *pvt, unsigned range, 1472f71d0a05SDoug Thompson u64 sys_addr, int *nid, int *chan_sel) 1473f71d0a05SDoug Thompson { 1474229a7a11SBorislav Petkov int cs_found = -EINVAL; 1475c8e518d5SBorislav Petkov u64 chan_addr; 14765d4b58e8SBorislav Petkov u32 dct_sel_base; 147711c75eadSBorislav Petkov u8 channel; 1478229a7a11SBorislav Petkov bool high_range = false; 1479f71d0a05SDoug Thompson 14807f19bf75SBorislav Petkov u8 node_id = dram_dst_node(pvt, range); 1481229a7a11SBorislav Petkov u8 intlv_en = dram_intlv_en(pvt, range); 14827f19bf75SBorislav Petkov u32 intlv_sel = dram_intlv_sel(pvt, range); 1483f71d0a05SDoug Thompson 1484c8e518d5SBorislav Petkov debugf1("(range %d) SystemAddr= 0x%llx Limit=0x%llx\n", 1485c8e518d5SBorislav Petkov range, sys_addr, get_dram_limit(pvt, range)); 1486f71d0a05SDoug Thompson 1487355fba60SBorislav Petkov if (dhar_valid(pvt) && 1488355fba60SBorislav Petkov dhar_base(pvt) <= sys_addr && 1489355fba60SBorislav Petkov sys_addr < BIT_64(32)) { 1490355fba60SBorislav Petkov amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n", 1491355fba60SBorislav Petkov sys_addr); 1492f71d0a05SDoug Thompson return -EINVAL; 1493355fba60SBorislav Petkov } 1494355fba60SBorislav Petkov 1495f030ddfbSBorislav Petkov if (intlv_en && (intlv_sel != ((sys_addr >> 12) & intlv_en))) 1496355fba60SBorislav Petkov return -EINVAL; 1497f71d0a05SDoug Thompson 1498b15f0fcaSBorislav Petkov sys_addr = f1x_swap_interleaved_region(pvt, sys_addr); 149995b0ef55SBorislav Petkov 1500f71d0a05SDoug Thompson dct_sel_base = dct_sel_baseaddr(pvt); 1501f71d0a05SDoug Thompson 1502f71d0a05SDoug Thompson /* 1503f71d0a05SDoug Thompson * check whether addresses >= DctSelBaseAddr[47:27] are to be used to 1504f71d0a05SDoug Thompson * select between DCT0 and DCT1. 1505f71d0a05SDoug Thompson */ 1506f71d0a05SDoug Thompson if (dct_high_range_enabled(pvt) && 1507f71d0a05SDoug Thompson !dct_ganging_enabled(pvt) && 1508f71d0a05SDoug Thompson ((sys_addr >> 27) >= (dct_sel_base >> 11))) 1509229a7a11SBorislav Petkov high_range = true; 1510f71d0a05SDoug Thompson 1511b15f0fcaSBorislav Petkov channel = f1x_determine_channel(pvt, sys_addr, high_range, intlv_en); 1512f71d0a05SDoug Thompson 1513b15f0fcaSBorislav Petkov chan_addr = f1x_get_norm_dct_addr(pvt, range, sys_addr, 1514c8e518d5SBorislav Petkov high_range, dct_sel_base); 1515f71d0a05SDoug Thompson 1516e2f79dbdSBorislav Petkov /* Remove node interleaving, see F1x120 */ 1517e2f79dbdSBorislav Petkov if (intlv_en) 1518e2f79dbdSBorislav Petkov chan_addr = ((chan_addr >> (12 + hweight8(intlv_en))) << 12) | 1519e2f79dbdSBorislav Petkov (chan_addr & 0xfff); 1520f71d0a05SDoug Thompson 15215d4b58e8SBorislav Petkov /* remove channel interleave */ 1522f71d0a05SDoug Thompson if (dct_interleave_enabled(pvt) && 1523f71d0a05SDoug Thompson !dct_high_range_enabled(pvt) && 1524f71d0a05SDoug Thompson !dct_ganging_enabled(pvt)) { 15255d4b58e8SBorislav Petkov 15265d4b58e8SBorislav Petkov if (dct_sel_interleave_addr(pvt) != 1) { 15275d4b58e8SBorislav Petkov if (dct_sel_interleave_addr(pvt) == 0x3) 15285d4b58e8SBorislav Petkov /* hash 9 */ 15295d4b58e8SBorislav Petkov chan_addr = ((chan_addr >> 10) << 9) | 15305d4b58e8SBorislav Petkov (chan_addr & 0x1ff); 15315d4b58e8SBorislav Petkov else 15325d4b58e8SBorislav Petkov /* A[6] or hash 6 */ 15335d4b58e8SBorislav Petkov chan_addr = ((chan_addr >> 7) << 6) | 15345d4b58e8SBorislav Petkov (chan_addr & 0x3f); 15355d4b58e8SBorislav Petkov } else 15365d4b58e8SBorislav Petkov /* A[12] */ 15375d4b58e8SBorislav Petkov chan_addr = ((chan_addr >> 13) << 12) | 15385d4b58e8SBorislav Petkov (chan_addr & 0xfff); 1539f71d0a05SDoug Thompson } 1540f71d0a05SDoug Thompson 15415d4b58e8SBorislav Petkov debugf1(" Normalized DCT addr: 0x%llx\n", chan_addr); 1542f71d0a05SDoug Thompson 1543b15f0fcaSBorislav Petkov cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, channel); 1544f71d0a05SDoug Thompson 1545f71d0a05SDoug Thompson if (cs_found >= 0) { 1546f71d0a05SDoug Thompson *nid = node_id; 1547f71d0a05SDoug Thompson *chan_sel = channel; 1548f71d0a05SDoug Thompson } 1549f71d0a05SDoug Thompson return cs_found; 1550f71d0a05SDoug Thompson } 1551f71d0a05SDoug Thompson 1552b15f0fcaSBorislav Petkov static int f1x_translate_sysaddr_to_cs(struct amd64_pvt *pvt, u64 sys_addr, 1553f71d0a05SDoug Thompson int *node, int *chan_sel) 1554f71d0a05SDoug Thompson { 1555e761359aSBorislav Petkov int cs_found = -EINVAL; 1556e761359aSBorislav Petkov unsigned range; 1557f71d0a05SDoug Thompson 15587f19bf75SBorislav Petkov for (range = 0; range < DRAM_RANGES; range++) { 1559f71d0a05SDoug Thompson 15607f19bf75SBorislav Petkov if (!dram_rw(pvt, range)) 1561f71d0a05SDoug Thompson continue; 1562f71d0a05SDoug Thompson 15637f19bf75SBorislav Petkov if ((get_dram_base(pvt, range) <= sys_addr) && 15647f19bf75SBorislav Petkov (get_dram_limit(pvt, range) >= sys_addr)) { 1565f71d0a05SDoug Thompson 1566b15f0fcaSBorislav Petkov cs_found = f1x_match_to_this_node(pvt, range, 1567f71d0a05SDoug Thompson sys_addr, node, 1568f71d0a05SDoug Thompson chan_sel); 1569f71d0a05SDoug Thompson if (cs_found >= 0) 1570f71d0a05SDoug Thompson break; 1571f71d0a05SDoug Thompson } 1572f71d0a05SDoug Thompson } 1573f71d0a05SDoug Thompson return cs_found; 1574f71d0a05SDoug Thompson } 1575f71d0a05SDoug Thompson 1576f71d0a05SDoug Thompson /* 1577bdc30a0cSBorislav Petkov * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps 1578bdc30a0cSBorislav Petkov * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW). 1579f71d0a05SDoug Thompson * 1580bdc30a0cSBorislav Petkov * The @sys_addr is usually an error address received from the hardware 1581bdc30a0cSBorislav Petkov * (MCX_ADDR). 1582f71d0a05SDoug Thompson */ 1583b15f0fcaSBorislav Petkov static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr, 1584f192c7b1SBorislav Petkov u16 syndrome) 1585f71d0a05SDoug Thompson { 1586f71d0a05SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 1587f71d0a05SDoug Thompson u32 page, offset; 1588f71d0a05SDoug Thompson int nid, csrow, chan = 0; 1589f71d0a05SDoug Thompson 1590b15f0fcaSBorislav Petkov csrow = f1x_translate_sysaddr_to_cs(pvt, sys_addr, &nid, &chan); 1591f71d0a05SDoug Thompson 1592bdc30a0cSBorislav Petkov if (csrow < 0) { 1593bdc30a0cSBorislav Petkov edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR); 1594bdc30a0cSBorislav Petkov return; 1595bdc30a0cSBorislav Petkov } 1596bdc30a0cSBorislav Petkov 1597f71d0a05SDoug Thompson error_address_to_page_and_offset(sys_addr, &page, &offset); 1598f71d0a05SDoug Thompson 1599f71d0a05SDoug Thompson /* 1600bdc30a0cSBorislav Petkov * We need the syndromes for channel detection only when we're 1601bdc30a0cSBorislav Petkov * ganged. Otherwise @chan should already contain the channel at 1602bdc30a0cSBorislav Petkov * this point. 1603f71d0a05SDoug Thompson */ 1604a97fa68eSBorislav Petkov if (dct_ganging_enabled(pvt)) 1605bfc04aecSBorislav Petkov chan = get_channel_from_ecc_syndrome(mci, syndrome); 1606f71d0a05SDoug Thompson 1607bdc30a0cSBorislav Petkov if (chan >= 0) 1608bdc30a0cSBorislav Petkov edac_mc_handle_ce(mci, page, offset, syndrome, csrow, chan, 1609bdc30a0cSBorislav Petkov EDAC_MOD_STR); 1610bdc30a0cSBorislav Petkov else 1611bdc30a0cSBorislav Petkov /* 1612bdc30a0cSBorislav Petkov * Channel unknown, report all channels on this CSROW as failed. 1613bdc30a0cSBorislav Petkov */ 1614bdc30a0cSBorislav Petkov for (chan = 0; chan < mci->csrows[csrow].nr_channels; chan++) 1615f71d0a05SDoug Thompson edac_mc_handle_ce(mci, page, offset, syndrome, 1616f71d0a05SDoug Thompson csrow, chan, EDAC_MOD_STR); 1617f71d0a05SDoug Thompson } 1618f71d0a05SDoug Thompson 1619f71d0a05SDoug Thompson /* 16208566c4dfSBorislav Petkov * debug routine to display the memory sizes of all logical DIMMs and its 1621cb328507SBorislav Petkov * CSROWs 1622f71d0a05SDoug Thompson */ 16238c671751SBorislav Petkov static void amd64_debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl) 1624f71d0a05SDoug Thompson { 1625603adaf6SBorislav Petkov int dimm, size0, size1, factor = 0; 1626525a1b20SBorislav Petkov u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases; 1627525a1b20SBorislav Petkov u32 dbam = ctrl ? pvt->dbam1 : pvt->dbam0; 1628f71d0a05SDoug Thompson 16298566c4dfSBorislav Petkov if (boot_cpu_data.x86 == 0xf) { 163041d8bfabSBorislav Petkov if (pvt->dclr0 & WIDTH_128) 1631603adaf6SBorislav Petkov factor = 1; 1632603adaf6SBorislav Petkov 16338566c4dfSBorislav Petkov /* K8 families < revF not supported yet */ 16341433eb99SBorislav Petkov if (pvt->ext_model < K8_REV_F) 16358566c4dfSBorislav Petkov return; 16368566c4dfSBorislav Petkov else 16378566c4dfSBorislav Petkov WARN_ON(ctrl != 0); 16388566c4dfSBorislav Petkov } 16398566c4dfSBorislav Petkov 16404d796364SBorislav Petkov dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1 : pvt->dbam0; 164111c75eadSBorislav Petkov dcsb = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->csels[1].csbases 164211c75eadSBorislav Petkov : pvt->csels[0].csbases; 1643f71d0a05SDoug Thompson 16444d796364SBorislav Petkov debugf1("F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n", ctrl, dbam); 1645f71d0a05SDoug Thompson 16468566c4dfSBorislav Petkov edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl); 16478566c4dfSBorislav Petkov 1648f71d0a05SDoug Thompson /* Dump memory sizes for DIMM and its CSROWs */ 1649f71d0a05SDoug Thompson for (dimm = 0; dimm < 4; dimm++) { 1650f71d0a05SDoug Thompson 1651f71d0a05SDoug Thompson size0 = 0; 165211c75eadSBorislav Petkov if (dcsb[dimm*2] & DCSB_CS_ENABLE) 165341d8bfabSBorislav Petkov size0 = pvt->ops->dbam_to_cs(pvt, ctrl, 165441d8bfabSBorislav Petkov DBAM_DIMM(dimm, dbam)); 1655f71d0a05SDoug Thompson 1656f71d0a05SDoug Thompson size1 = 0; 165711c75eadSBorislav Petkov if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE) 165841d8bfabSBorislav Petkov size1 = pvt->ops->dbam_to_cs(pvt, ctrl, 165941d8bfabSBorislav Petkov DBAM_DIMM(dimm, dbam)); 1660f71d0a05SDoug Thompson 166124f9a7feSBorislav Petkov amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n", 1662603adaf6SBorislav Petkov dimm * 2, size0 << factor, 1663603adaf6SBorislav Petkov dimm * 2 + 1, size1 << factor); 1664f71d0a05SDoug Thompson } 1665f71d0a05SDoug Thompson } 1666f71d0a05SDoug Thompson 16674d37607aSDoug Thompson static struct amd64_family_type amd64_family_types[] = { 16684d37607aSDoug Thompson [K8_CPUS] = { 16690092b20dSBorislav Petkov .ctl_name = "K8", 16708d5b5d9cSBorislav Petkov .f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP, 16718d5b5d9cSBorislav Petkov .f3_id = PCI_DEVICE_ID_AMD_K8_NB_MISC, 16724d37607aSDoug Thompson .ops = { 16734d37607aSDoug Thompson .early_channel_count = k8_early_channel_count, 16744d37607aSDoug Thompson .map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow, 16751433eb99SBorislav Petkov .dbam_to_cs = k8_dbam_to_chip_select, 1676b2b0c605SBorislav Petkov .read_dct_pci_cfg = k8_read_dct_pci_cfg, 16774d37607aSDoug Thompson } 16784d37607aSDoug Thompson }, 16794d37607aSDoug Thompson [F10_CPUS] = { 16800092b20dSBorislav Petkov .ctl_name = "F10h", 16818d5b5d9cSBorislav Petkov .f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP, 16828d5b5d9cSBorislav Petkov .f3_id = PCI_DEVICE_ID_AMD_10H_NB_MISC, 16834d37607aSDoug Thompson .ops = { 16847d20d14dSBorislav Petkov .early_channel_count = f1x_early_channel_count, 1685b15f0fcaSBorislav Petkov .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, 16861433eb99SBorislav Petkov .dbam_to_cs = f10_dbam_to_chip_select, 1687b2b0c605SBorislav Petkov .read_dct_pci_cfg = f10_read_dct_pci_cfg, 1688b2b0c605SBorislav Petkov } 1689b2b0c605SBorislav Petkov }, 1690b2b0c605SBorislav Petkov [F15_CPUS] = { 1691b2b0c605SBorislav Petkov .ctl_name = "F15h", 1692df71a053SBorislav Petkov .f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1, 1693df71a053SBorislav Petkov .f3_id = PCI_DEVICE_ID_AMD_15H_NB_F3, 1694b2b0c605SBorislav Petkov .ops = { 16957d20d14dSBorislav Petkov .early_channel_count = f1x_early_channel_count, 1696b15f0fcaSBorislav Petkov .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, 169741d8bfabSBorislav Petkov .dbam_to_cs = f15_dbam_to_chip_select, 1698b2b0c605SBorislav Petkov .read_dct_pci_cfg = f15_read_dct_pci_cfg, 16994d37607aSDoug Thompson } 17004d37607aSDoug Thompson }, 17014d37607aSDoug Thompson }; 17024d37607aSDoug Thompson 17034d37607aSDoug Thompson static struct pci_dev *pci_get_related_function(unsigned int vendor, 17044d37607aSDoug Thompson unsigned int device, 17054d37607aSDoug Thompson struct pci_dev *related) 17064d37607aSDoug Thompson { 17074d37607aSDoug Thompson struct pci_dev *dev = NULL; 17084d37607aSDoug Thompson 17094d37607aSDoug Thompson dev = pci_get_device(vendor, device, dev); 17104d37607aSDoug Thompson while (dev) { 17114d37607aSDoug Thompson if ((dev->bus->number == related->bus->number) && 17124d37607aSDoug Thompson (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn))) 17134d37607aSDoug Thompson break; 17144d37607aSDoug Thompson dev = pci_get_device(vendor, device, dev); 17154d37607aSDoug Thompson } 17164d37607aSDoug Thompson 17174d37607aSDoug Thompson return dev; 17184d37607aSDoug Thompson } 17194d37607aSDoug Thompson 1720b1289d6fSDoug Thompson /* 1721bfc04aecSBorislav Petkov * These are tables of eigenvectors (one per line) which can be used for the 1722bfc04aecSBorislav Petkov * construction of the syndrome tables. The modified syndrome search algorithm 1723bfc04aecSBorislav Petkov * uses those to find the symbol in error and thus the DIMM. 1724b1289d6fSDoug Thompson * 1725bfc04aecSBorislav Petkov * Algorithm courtesy of Ross LaFetra from AMD. 1726b1289d6fSDoug Thompson */ 1727bfc04aecSBorislav Petkov static u16 x4_vectors[] = { 1728bfc04aecSBorislav Petkov 0x2f57, 0x1afe, 0x66cc, 0xdd88, 1729bfc04aecSBorislav Petkov 0x11eb, 0x3396, 0x7f4c, 0xeac8, 1730bfc04aecSBorislav Petkov 0x0001, 0x0002, 0x0004, 0x0008, 1731bfc04aecSBorislav Petkov 0x1013, 0x3032, 0x4044, 0x8088, 1732bfc04aecSBorislav Petkov 0x106b, 0x30d6, 0x70fc, 0xe0a8, 1733bfc04aecSBorislav Petkov 0x4857, 0xc4fe, 0x13cc, 0x3288, 1734bfc04aecSBorislav Petkov 0x1ac5, 0x2f4a, 0x5394, 0xa1e8, 1735bfc04aecSBorislav Petkov 0x1f39, 0x251e, 0xbd6c, 0x6bd8, 1736bfc04aecSBorislav Petkov 0x15c1, 0x2a42, 0x89ac, 0x4758, 1737bfc04aecSBorislav Petkov 0x2b03, 0x1602, 0x4f0c, 0xca08, 1738bfc04aecSBorislav Petkov 0x1f07, 0x3a0e, 0x6b04, 0xbd08, 1739bfc04aecSBorislav Petkov 0x8ba7, 0x465e, 0x244c, 0x1cc8, 1740bfc04aecSBorislav Petkov 0x2b87, 0x164e, 0x642c, 0xdc18, 1741bfc04aecSBorislav Petkov 0x40b9, 0x80de, 0x1094, 0x20e8, 1742bfc04aecSBorislav Petkov 0x27db, 0x1eb6, 0x9dac, 0x7b58, 1743bfc04aecSBorislav Petkov 0x11c1, 0x2242, 0x84ac, 0x4c58, 1744bfc04aecSBorislav Petkov 0x1be5, 0x2d7a, 0x5e34, 0xa718, 1745bfc04aecSBorislav Petkov 0x4b39, 0x8d1e, 0x14b4, 0x28d8, 1746bfc04aecSBorislav Petkov 0x4c97, 0xc87e, 0x11fc, 0x33a8, 1747bfc04aecSBorislav Petkov 0x8e97, 0x497e, 0x2ffc, 0x1aa8, 1748bfc04aecSBorislav Petkov 0x16b3, 0x3d62, 0x4f34, 0x8518, 1749bfc04aecSBorislav Petkov 0x1e2f, 0x391a, 0x5cac, 0xf858, 1750bfc04aecSBorislav Petkov 0x1d9f, 0x3b7a, 0x572c, 0xfe18, 1751bfc04aecSBorislav Petkov 0x15f5, 0x2a5a, 0x5264, 0xa3b8, 1752bfc04aecSBorislav Petkov 0x1dbb, 0x3b66, 0x715c, 0xe3f8, 1753bfc04aecSBorislav Petkov 0x4397, 0xc27e, 0x17fc, 0x3ea8, 1754bfc04aecSBorislav Petkov 0x1617, 0x3d3e, 0x6464, 0xb8b8, 1755bfc04aecSBorislav Petkov 0x23ff, 0x12aa, 0xab6c, 0x56d8, 1756bfc04aecSBorislav Petkov 0x2dfb, 0x1ba6, 0x913c, 0x7328, 1757bfc04aecSBorislav Petkov 0x185d, 0x2ca6, 0x7914, 0x9e28, 1758bfc04aecSBorislav Petkov 0x171b, 0x3e36, 0x7d7c, 0xebe8, 1759bfc04aecSBorislav Petkov 0x4199, 0x82ee, 0x19f4, 0x2e58, 1760bfc04aecSBorislav Petkov 0x4807, 0xc40e, 0x130c, 0x3208, 1761bfc04aecSBorislav Petkov 0x1905, 0x2e0a, 0x5804, 0xac08, 1762bfc04aecSBorislav Petkov 0x213f, 0x132a, 0xadfc, 0x5ba8, 1763bfc04aecSBorislav Petkov 0x19a9, 0x2efe, 0xb5cc, 0x6f88, 1764b1289d6fSDoug Thompson }; 1765b1289d6fSDoug Thompson 1766bfc04aecSBorislav Petkov static u16 x8_vectors[] = { 1767bfc04aecSBorislav Petkov 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480, 1768bfc04aecSBorislav Petkov 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80, 1769bfc04aecSBorislav Petkov 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80, 1770bfc04aecSBorislav Petkov 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80, 1771bfc04aecSBorislav Petkov 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780, 1772bfc04aecSBorislav Petkov 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080, 1773bfc04aecSBorislav Petkov 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080, 1774bfc04aecSBorislav Petkov 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080, 1775bfc04aecSBorislav Petkov 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80, 1776bfc04aecSBorislav Petkov 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580, 1777bfc04aecSBorislav Petkov 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880, 1778bfc04aecSBorislav Petkov 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280, 1779bfc04aecSBorislav Petkov 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180, 1780bfc04aecSBorislav Petkov 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580, 1781bfc04aecSBorislav Petkov 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280, 1782bfc04aecSBorislav Petkov 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180, 1783bfc04aecSBorislav Petkov 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080, 1784bfc04aecSBorislav Petkov 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080, 1785bfc04aecSBorislav Petkov 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000, 1786bfc04aecSBorislav Petkov }; 1787bfc04aecSBorislav Petkov 1788d34a6ecdSBorislav Petkov static int decode_syndrome(u16 syndrome, u16 *vectors, unsigned num_vecs, 1789d34a6ecdSBorislav Petkov unsigned v_dim) 1790b1289d6fSDoug Thompson { 1791bfc04aecSBorislav Petkov unsigned int i, err_sym; 1792b1289d6fSDoug Thompson 1793bfc04aecSBorislav Petkov for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) { 1794bfc04aecSBorislav Petkov u16 s = syndrome; 1795d34a6ecdSBorislav Petkov unsigned v_idx = err_sym * v_dim; 1796d34a6ecdSBorislav Petkov unsigned v_end = (err_sym + 1) * v_dim; 1797b1289d6fSDoug Thompson 1798bfc04aecSBorislav Petkov /* walk over all 16 bits of the syndrome */ 1799bfc04aecSBorislav Petkov for (i = 1; i < (1U << 16); i <<= 1) { 1800bfc04aecSBorislav Petkov 1801bfc04aecSBorislav Petkov /* if bit is set in that eigenvector... */ 1802bfc04aecSBorislav Petkov if (v_idx < v_end && vectors[v_idx] & i) { 1803bfc04aecSBorislav Petkov u16 ev_comp = vectors[v_idx++]; 1804bfc04aecSBorislav Petkov 1805bfc04aecSBorislav Petkov /* ... and bit set in the modified syndrome, */ 1806bfc04aecSBorislav Petkov if (s & i) { 1807bfc04aecSBorislav Petkov /* remove it. */ 1808bfc04aecSBorislav Petkov s ^= ev_comp; 1809bfc04aecSBorislav Petkov 1810bfc04aecSBorislav Petkov if (!s) 1811bfc04aecSBorislav Petkov return err_sym; 1812bfc04aecSBorislav Petkov } 1813bfc04aecSBorislav Petkov 1814bfc04aecSBorislav Petkov } else if (s & i) 1815bfc04aecSBorislav Petkov /* can't get to zero, move to next symbol */ 1816bfc04aecSBorislav Petkov break; 1817bfc04aecSBorislav Petkov } 1818b1289d6fSDoug Thompson } 1819b1289d6fSDoug Thompson 1820b1289d6fSDoug Thompson debugf0("syndrome(%x) not found\n", syndrome); 1821b1289d6fSDoug Thompson return -1; 1822b1289d6fSDoug Thompson } 1823d27bf6faSDoug Thompson 1824bfc04aecSBorislav Petkov static int map_err_sym_to_channel(int err_sym, int sym_size) 1825bfc04aecSBorislav Petkov { 1826bfc04aecSBorislav Petkov if (sym_size == 4) 1827bfc04aecSBorislav Petkov switch (err_sym) { 1828bfc04aecSBorislav Petkov case 0x20: 1829bfc04aecSBorislav Petkov case 0x21: 1830bfc04aecSBorislav Petkov return 0; 1831bfc04aecSBorislav Petkov break; 1832bfc04aecSBorislav Petkov case 0x22: 1833bfc04aecSBorislav Petkov case 0x23: 1834bfc04aecSBorislav Petkov return 1; 1835bfc04aecSBorislav Petkov break; 1836bfc04aecSBorislav Petkov default: 1837bfc04aecSBorislav Petkov return err_sym >> 4; 1838bfc04aecSBorislav Petkov break; 1839bfc04aecSBorislav Petkov } 1840bfc04aecSBorislav Petkov /* x8 symbols */ 1841bfc04aecSBorislav Petkov else 1842bfc04aecSBorislav Petkov switch (err_sym) { 1843bfc04aecSBorislav Petkov /* imaginary bits not in a DIMM */ 1844bfc04aecSBorislav Petkov case 0x10: 1845bfc04aecSBorislav Petkov WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n", 1846bfc04aecSBorislav Petkov err_sym); 1847bfc04aecSBorislav Petkov return -1; 1848bfc04aecSBorislav Petkov break; 1849bfc04aecSBorislav Petkov 1850bfc04aecSBorislav Petkov case 0x11: 1851bfc04aecSBorislav Petkov return 0; 1852bfc04aecSBorislav Petkov break; 1853bfc04aecSBorislav Petkov case 0x12: 1854bfc04aecSBorislav Petkov return 1; 1855bfc04aecSBorislav Petkov break; 1856bfc04aecSBorislav Petkov default: 1857bfc04aecSBorislav Petkov return err_sym >> 3; 1858bfc04aecSBorislav Petkov break; 1859bfc04aecSBorislav Petkov } 1860bfc04aecSBorislav Petkov return -1; 1861bfc04aecSBorislav Petkov } 1862bfc04aecSBorislav Petkov 1863bfc04aecSBorislav Petkov static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome) 1864bfc04aecSBorislav Petkov { 1865bfc04aecSBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 1866ad6a32e9SBorislav Petkov int err_sym = -1; 1867bfc04aecSBorislav Petkov 1868a3b7db09SBorislav Petkov if (pvt->ecc_sym_sz == 8) 1869bfc04aecSBorislav Petkov err_sym = decode_syndrome(syndrome, x8_vectors, 1870ad6a32e9SBorislav Petkov ARRAY_SIZE(x8_vectors), 1871a3b7db09SBorislav Petkov pvt->ecc_sym_sz); 1872a3b7db09SBorislav Petkov else if (pvt->ecc_sym_sz == 4) 1873ad6a32e9SBorislav Petkov err_sym = decode_syndrome(syndrome, x4_vectors, 1874ad6a32e9SBorislav Petkov ARRAY_SIZE(x4_vectors), 1875a3b7db09SBorislav Petkov pvt->ecc_sym_sz); 1876ad6a32e9SBorislav Petkov else { 1877a3b7db09SBorislav Petkov amd64_warn("Illegal syndrome type: %u\n", pvt->ecc_sym_sz); 1878ad6a32e9SBorislav Petkov return err_sym; 1879bfc04aecSBorislav Petkov } 1880ad6a32e9SBorislav Petkov 1881a3b7db09SBorislav Petkov return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz); 188241c31044SBorislav Petkov } 1883bfc04aecSBorislav Petkov 1884d27bf6faSDoug Thompson /* 1885d27bf6faSDoug Thompson * Handle any Correctable Errors (CEs) that have occurred. Check for valid ERROR 1886d27bf6faSDoug Thompson * ADDRESS and process. 1887d27bf6faSDoug Thompson */ 1888f192c7b1SBorislav Petkov static void amd64_handle_ce(struct mem_ctl_info *mci, struct mce *m) 1889d27bf6faSDoug Thompson { 1890d27bf6faSDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 189144e9e2eeSBorislav Petkov u64 sys_addr; 1892f192c7b1SBorislav Petkov u16 syndrome; 1893d27bf6faSDoug Thompson 1894d27bf6faSDoug Thompson /* Ensure that the Error Address is VALID */ 1895f192c7b1SBorislav Petkov if (!(m->status & MCI_STATUS_ADDRV)) { 189624f9a7feSBorislav Petkov amd64_mc_err(mci, "HW has no ERROR_ADDRESS available\n"); 1897d27bf6faSDoug Thompson edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR); 1898d27bf6faSDoug Thompson return; 1899d27bf6faSDoug Thompson } 1900d27bf6faSDoug Thompson 190170046624SBorislav Petkov sys_addr = get_error_address(m); 1902f192c7b1SBorislav Petkov syndrome = extract_syndrome(m->status); 1903d27bf6faSDoug Thompson 190424f9a7feSBorislav Petkov amd64_mc_err(mci, "CE ERROR_ADDRESS= 0x%llx\n", sys_addr); 1905d27bf6faSDoug Thompson 1906f192c7b1SBorislav Petkov pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, syndrome); 1907d27bf6faSDoug Thompson } 1908d27bf6faSDoug Thompson 1909d27bf6faSDoug Thompson /* Handle any Un-correctable Errors (UEs) */ 1910f192c7b1SBorislav Petkov static void amd64_handle_ue(struct mem_ctl_info *mci, struct mce *m) 1911d27bf6faSDoug Thompson { 19121f6bcee7SBorislav Petkov struct mem_ctl_info *log_mci, *src_mci = NULL; 1913d27bf6faSDoug Thompson int csrow; 191444e9e2eeSBorislav Petkov u64 sys_addr; 1915d27bf6faSDoug Thompson u32 page, offset; 1916d27bf6faSDoug Thompson 1917d27bf6faSDoug Thompson log_mci = mci; 1918d27bf6faSDoug Thompson 1919f192c7b1SBorislav Petkov if (!(m->status & MCI_STATUS_ADDRV)) { 192024f9a7feSBorislav Petkov amd64_mc_err(mci, "HW has no ERROR_ADDRESS available\n"); 1921d27bf6faSDoug Thompson edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR); 1922d27bf6faSDoug Thompson return; 1923d27bf6faSDoug Thompson } 1924d27bf6faSDoug Thompson 192570046624SBorislav Petkov sys_addr = get_error_address(m); 1926d27bf6faSDoug Thompson 1927d27bf6faSDoug Thompson /* 1928d27bf6faSDoug Thompson * Find out which node the error address belongs to. This may be 1929d27bf6faSDoug Thompson * different from the node that detected the error. 1930d27bf6faSDoug Thompson */ 193144e9e2eeSBorislav Petkov src_mci = find_mc_by_sys_addr(mci, sys_addr); 1932d27bf6faSDoug Thompson if (!src_mci) { 193324f9a7feSBorislav Petkov amd64_mc_err(mci, "ERROR ADDRESS (0x%lx) NOT mapped to a MC\n", 193444e9e2eeSBorislav Petkov (unsigned long)sys_addr); 1935d27bf6faSDoug Thompson edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR); 1936d27bf6faSDoug Thompson return; 1937d27bf6faSDoug Thompson } 1938d27bf6faSDoug Thompson 1939d27bf6faSDoug Thompson log_mci = src_mci; 1940d27bf6faSDoug Thompson 194144e9e2eeSBorislav Petkov csrow = sys_addr_to_csrow(log_mci, sys_addr); 1942d27bf6faSDoug Thompson if (csrow < 0) { 194324f9a7feSBorislav Petkov amd64_mc_err(mci, "ERROR_ADDRESS (0x%lx) NOT mapped to CS\n", 194444e9e2eeSBorislav Petkov (unsigned long)sys_addr); 1945d27bf6faSDoug Thompson edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR); 1946d27bf6faSDoug Thompson } else { 194744e9e2eeSBorislav Petkov error_address_to_page_and_offset(sys_addr, &page, &offset); 1948d27bf6faSDoug Thompson edac_mc_handle_ue(log_mci, page, offset, csrow, EDAC_MOD_STR); 1949d27bf6faSDoug Thompson } 1950d27bf6faSDoug Thompson } 1951d27bf6faSDoug Thompson 1952549d042dSBorislav Petkov static inline void __amd64_decode_bus_error(struct mem_ctl_info *mci, 1953f192c7b1SBorislav Petkov struct mce *m) 1954d27bf6faSDoug Thompson { 1955f192c7b1SBorislav Petkov u16 ec = EC(m->status); 1956f192c7b1SBorislav Petkov u8 xec = XEC(m->status, 0x1f); 1957f192c7b1SBorislav Petkov u8 ecc_type = (m->status >> 45) & 0x3; 1958d27bf6faSDoug Thompson 1959b70ef010SBorislav Petkov /* Bail early out if this was an 'observed' error */ 19605980bb9cSBorislav Petkov if (PP(ec) == NBSL_PP_OBS) 1961b70ef010SBorislav Petkov return; 1962d27bf6faSDoug Thompson 1963ecaf5606SBorislav Petkov /* Do only ECC errors */ 1964ecaf5606SBorislav Petkov if (xec && xec != F10_NBSL_EXT_ERR_ECC) 1965d27bf6faSDoug Thompson return; 1966d27bf6faSDoug Thompson 1967ecaf5606SBorislav Petkov if (ecc_type == 2) 1968f192c7b1SBorislav Petkov amd64_handle_ce(mci, m); 1969ecaf5606SBorislav Petkov else if (ecc_type == 1) 1970f192c7b1SBorislav Petkov amd64_handle_ue(mci, m); 1971d27bf6faSDoug Thompson } 1972d27bf6faSDoug Thompson 1973b0b07a2bSBorislav Petkov void amd64_decode_bus_error(int node_id, struct mce *m) 1974d27bf6faSDoug Thompson { 1975b0b07a2bSBorislav Petkov __amd64_decode_bus_error(mcis[node_id], m); 1976d27bf6faSDoug Thompson } 1977d27bf6faSDoug Thompson 19780ec449eeSDoug Thompson /* 19798d5b5d9cSBorislav Petkov * Use pvt->F2 which contains the F2 CPU PCI device to get the related 1980bbd0c1f6SBorislav Petkov * F1 (AddrMap) and F3 (Misc) devices. Return negative value on error. 19810ec449eeSDoug Thompson */ 1982360b7f3cSBorislav Petkov static int reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 f1_id, u16 f3_id) 19830ec449eeSDoug Thompson { 19840ec449eeSDoug Thompson /* Reserve the ADDRESS MAP Device */ 19858d5b5d9cSBorislav Petkov pvt->F1 = pci_get_related_function(pvt->F2->vendor, f1_id, pvt->F2); 19868d5b5d9cSBorislav Petkov if (!pvt->F1) { 198724f9a7feSBorislav Petkov amd64_err("error address map device not found: " 19880ec449eeSDoug Thompson "vendor %x device 0x%x (broken BIOS?)\n", 1989bbd0c1f6SBorislav Petkov PCI_VENDOR_ID_AMD, f1_id); 1990bbd0c1f6SBorislav Petkov return -ENODEV; 19910ec449eeSDoug Thompson } 19920ec449eeSDoug Thompson 19930ec449eeSDoug Thompson /* Reserve the MISC Device */ 19948d5b5d9cSBorislav Petkov pvt->F3 = pci_get_related_function(pvt->F2->vendor, f3_id, pvt->F2); 19958d5b5d9cSBorislav Petkov if (!pvt->F3) { 19968d5b5d9cSBorislav Petkov pci_dev_put(pvt->F1); 19978d5b5d9cSBorislav Petkov pvt->F1 = NULL; 19980ec449eeSDoug Thompson 199924f9a7feSBorislav Petkov amd64_err("error F3 device not found: " 20000ec449eeSDoug Thompson "vendor %x device 0x%x (broken BIOS?)\n", 2001bbd0c1f6SBorislav Petkov PCI_VENDOR_ID_AMD, f3_id); 20028d5b5d9cSBorislav Petkov 2003bbd0c1f6SBorislav Petkov return -ENODEV; 20040ec449eeSDoug Thompson } 20058d5b5d9cSBorislav Petkov debugf1("F1: %s\n", pci_name(pvt->F1)); 20068d5b5d9cSBorislav Petkov debugf1("F2: %s\n", pci_name(pvt->F2)); 20078d5b5d9cSBorislav Petkov debugf1("F3: %s\n", pci_name(pvt->F3)); 20080ec449eeSDoug Thompson 20090ec449eeSDoug Thompson return 0; 20100ec449eeSDoug Thompson } 20110ec449eeSDoug Thompson 2012360b7f3cSBorislav Petkov static void free_mc_sibling_devs(struct amd64_pvt *pvt) 20130ec449eeSDoug Thompson { 20148d5b5d9cSBorislav Petkov pci_dev_put(pvt->F1); 20158d5b5d9cSBorislav Petkov pci_dev_put(pvt->F3); 20160ec449eeSDoug Thompson } 20170ec449eeSDoug Thompson 20180ec449eeSDoug Thompson /* 20190ec449eeSDoug Thompson * Retrieve the hardware registers of the memory controller (this includes the 20200ec449eeSDoug Thompson * 'Address Map' and 'Misc' device regs) 20210ec449eeSDoug Thompson */ 2022360b7f3cSBorislav Petkov static void read_mc_regs(struct amd64_pvt *pvt) 20230ec449eeSDoug Thompson { 2024a3b7db09SBorislav Petkov struct cpuinfo_x86 *c = &boot_cpu_data; 20250ec449eeSDoug Thompson u64 msr_val; 2026ad6a32e9SBorislav Petkov u32 tmp; 2027e761359aSBorislav Petkov unsigned range; 20280ec449eeSDoug Thompson 20290ec449eeSDoug Thompson /* 20300ec449eeSDoug Thompson * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since 20310ec449eeSDoug Thompson * those are Read-As-Zero 20320ec449eeSDoug Thompson */ 2033e97f8bb8SBorislav Petkov rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem); 2034e97f8bb8SBorislav Petkov debugf0(" TOP_MEM: 0x%016llx\n", pvt->top_mem); 20350ec449eeSDoug Thompson 20360ec449eeSDoug Thompson /* check first whether TOP_MEM2 is enabled */ 20370ec449eeSDoug Thompson rdmsrl(MSR_K8_SYSCFG, msr_val); 20380ec449eeSDoug Thompson if (msr_val & (1U << 21)) { 2039e97f8bb8SBorislav Petkov rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2); 2040e97f8bb8SBorislav Petkov debugf0(" TOP_MEM2: 0x%016llx\n", pvt->top_mem2); 20410ec449eeSDoug Thompson } else 20420ec449eeSDoug Thompson debugf0(" TOP_MEM2 disabled.\n"); 20430ec449eeSDoug Thompson 20445980bb9cSBorislav Petkov amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap); 20450ec449eeSDoug Thompson 20465a5d2371SBorislav Petkov read_dram_ctl_register(pvt); 20470ec449eeSDoug Thompson 20487f19bf75SBorislav Petkov for (range = 0; range < DRAM_RANGES; range++) { 20497f19bf75SBorislav Petkov u8 rw; 20500ec449eeSDoug Thompson 20517f19bf75SBorislav Petkov /* read settings for this DRAM range */ 20527f19bf75SBorislav Petkov read_dram_base_limit_regs(pvt, range); 2053e97f8bb8SBorislav Petkov 20547f19bf75SBorislav Petkov rw = dram_rw(pvt, range); 20557f19bf75SBorislav Petkov if (!rw) 20567f19bf75SBorislav Petkov continue; 20577f19bf75SBorislav Petkov 20587f19bf75SBorislav Petkov debugf1(" DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n", 20597f19bf75SBorislav Petkov range, 20607f19bf75SBorislav Petkov get_dram_base(pvt, range), 20617f19bf75SBorislav Petkov get_dram_limit(pvt, range)); 20627f19bf75SBorislav Petkov 20637f19bf75SBorislav Petkov debugf1(" IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n", 20647f19bf75SBorislav Petkov dram_intlv_en(pvt, range) ? "Enabled" : "Disabled", 20657f19bf75SBorislav Petkov (rw & 0x1) ? "R" : "-", 20667f19bf75SBorislav Petkov (rw & 0x2) ? "W" : "-", 20677f19bf75SBorislav Petkov dram_intlv_sel(pvt, range), 20687f19bf75SBorislav Petkov dram_dst_node(pvt, range)); 20690ec449eeSDoug Thompson } 20700ec449eeSDoug Thompson 2071b2b0c605SBorislav Petkov read_dct_base_mask(pvt); 20720ec449eeSDoug Thompson 2073bc21fa57SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar); 2074525a1b20SBorislav Petkov amd64_read_dct_pci_cfg(pvt, DBAM0, &pvt->dbam0); 20750ec449eeSDoug Thompson 20768d5b5d9cSBorislav Petkov amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare); 20770ec449eeSDoug Thompson 2078cb328507SBorislav Petkov amd64_read_dct_pci_cfg(pvt, DCLR0, &pvt->dclr0); 2079cb328507SBorislav Petkov amd64_read_dct_pci_cfg(pvt, DCHR0, &pvt->dchr0); 20800ec449eeSDoug Thompson 208178da121eSBorislav Petkov if (!dct_ganging_enabled(pvt)) { 2082cb328507SBorislav Petkov amd64_read_dct_pci_cfg(pvt, DCLR1, &pvt->dclr1); 2083cb328507SBorislav Petkov amd64_read_dct_pci_cfg(pvt, DCHR1, &pvt->dchr1); 20840ec449eeSDoug Thompson } 2085b2b0c605SBorislav Petkov 2086a3b7db09SBorislav Petkov pvt->ecc_sym_sz = 4; 2087a3b7db09SBorislav Petkov 2088a3b7db09SBorislav Petkov if (c->x86 >= 0x10) { 20898d5b5d9cSBorislav Petkov amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp); 2090525a1b20SBorislav Petkov amd64_read_dct_pci_cfg(pvt, DBAM1, &pvt->dbam1); 2091a3b7db09SBorislav Petkov 2092a3b7db09SBorislav Petkov /* F10h, revD and later can do x8 ECC too */ 2093a3b7db09SBorislav Petkov if ((c->x86 > 0x10 || c->x86_model > 7) && tmp & BIT(25)) 2094a3b7db09SBorislav Petkov pvt->ecc_sym_sz = 8; 2095525a1b20SBorislav Petkov } 2096b2b0c605SBorislav Petkov dump_misc_regs(pvt); 20970ec449eeSDoug Thompson } 20980ec449eeSDoug Thompson 20990ec449eeSDoug Thompson /* 21000ec449eeSDoug Thompson * NOTE: CPU Revision Dependent code 21010ec449eeSDoug Thompson * 21020ec449eeSDoug Thompson * Input: 210311c75eadSBorislav Petkov * @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1) 21040ec449eeSDoug Thompson * k8 private pointer to --> 21050ec449eeSDoug Thompson * DRAM Bank Address mapping register 21060ec449eeSDoug Thompson * node_id 21070ec449eeSDoug Thompson * DCL register where dual_channel_active is 21080ec449eeSDoug Thompson * 21090ec449eeSDoug Thompson * The DBAM register consists of 4 sets of 4 bits each definitions: 21100ec449eeSDoug Thompson * 21110ec449eeSDoug Thompson * Bits: CSROWs 21120ec449eeSDoug Thompson * 0-3 CSROWs 0 and 1 21130ec449eeSDoug Thompson * 4-7 CSROWs 2 and 3 21140ec449eeSDoug Thompson * 8-11 CSROWs 4 and 5 21150ec449eeSDoug Thompson * 12-15 CSROWs 6 and 7 21160ec449eeSDoug Thompson * 21170ec449eeSDoug Thompson * Values range from: 0 to 15 21180ec449eeSDoug Thompson * The meaning of the values depends on CPU revision and dual-channel state, 21190ec449eeSDoug Thompson * see relevant BKDG more info. 21200ec449eeSDoug Thompson * 21210ec449eeSDoug Thompson * The memory controller provides for total of only 8 CSROWs in its current 21220ec449eeSDoug Thompson * architecture. Each "pair" of CSROWs normally represents just one DIMM in 21230ec449eeSDoug Thompson * single channel or two (2) DIMMs in dual channel mode. 21240ec449eeSDoug Thompson * 21250ec449eeSDoug Thompson * The following code logic collapses the various tables for CSROW based on CPU 21260ec449eeSDoug Thompson * revision. 21270ec449eeSDoug Thompson * 21280ec449eeSDoug Thompson * Returns: 21290ec449eeSDoug Thompson * The number of PAGE_SIZE pages on the specified CSROW number it 21300ec449eeSDoug Thompson * encompasses 21310ec449eeSDoug Thompson * 21320ec449eeSDoug Thompson */ 213341d8bfabSBorislav Petkov static u32 amd64_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr) 21340ec449eeSDoug Thompson { 21351433eb99SBorislav Petkov u32 cs_mode, nr_pages; 21360ec449eeSDoug Thompson 21370ec449eeSDoug Thompson /* 21380ec449eeSDoug Thompson * The math on this doesn't look right on the surface because x/2*4 can 21390ec449eeSDoug Thompson * be simplified to x*2 but this expression makes use of the fact that 21400ec449eeSDoug Thompson * it is integral math where 1/2=0. This intermediate value becomes the 21410ec449eeSDoug Thompson * number of bits to shift the DBAM register to extract the proper CSROW 21420ec449eeSDoug Thompson * field. 21430ec449eeSDoug Thompson */ 21441433eb99SBorislav Petkov cs_mode = (pvt->dbam0 >> ((csrow_nr / 2) * 4)) & 0xF; 21450ec449eeSDoug Thompson 214641d8bfabSBorislav Petkov nr_pages = pvt->ops->dbam_to_cs(pvt, dct, cs_mode) << (20 - PAGE_SHIFT); 21470ec449eeSDoug Thompson 21480ec449eeSDoug Thompson /* 21490ec449eeSDoug Thompson * If dual channel then double the memory size of single channel. 21500ec449eeSDoug Thompson * Channel count is 1 or 2 21510ec449eeSDoug Thompson */ 21520ec449eeSDoug Thompson nr_pages <<= (pvt->channel_count - 1); 21530ec449eeSDoug Thompson 21541433eb99SBorislav Petkov debugf0(" (csrow=%d) DBAM map index= %d\n", csrow_nr, cs_mode); 21550ec449eeSDoug Thompson debugf0(" nr_pages= %u channel-count = %d\n", 21560ec449eeSDoug Thompson nr_pages, pvt->channel_count); 21570ec449eeSDoug Thompson 21580ec449eeSDoug Thompson return nr_pages; 21590ec449eeSDoug Thompson } 21600ec449eeSDoug Thompson 21610ec449eeSDoug Thompson /* 21620ec449eeSDoug Thompson * Initialize the array of csrow attribute instances, based on the values 21630ec449eeSDoug Thompson * from pci config hardware registers. 21640ec449eeSDoug Thompson */ 2165360b7f3cSBorislav Petkov static int init_csrows(struct mem_ctl_info *mci) 21660ec449eeSDoug Thompson { 21670ec449eeSDoug Thompson struct csrow_info *csrow; 21682299ef71SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 216911c75eadSBorislav Petkov u64 input_addr_min, input_addr_max, sys_addr, base, mask; 21702299ef71SBorislav Petkov u32 val; 21716ba5dcdcSBorislav Petkov int i, empty = 1; 21720ec449eeSDoug Thompson 2173a97fa68eSBorislav Petkov amd64_read_pci_cfg(pvt->F3, NBCFG, &val); 21740ec449eeSDoug Thompson 21752299ef71SBorislav Petkov pvt->nbcfg = val; 21760ec449eeSDoug Thompson 21772299ef71SBorislav Petkov debugf0("node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n", 21782299ef71SBorislav Petkov pvt->mc_node_id, val, 2179a97fa68eSBorislav Petkov !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE)); 21800ec449eeSDoug Thompson 218111c75eadSBorislav Petkov for_each_chip_select(i, 0, pvt) { 21820ec449eeSDoug Thompson csrow = &mci->csrows[i]; 21830ec449eeSDoug Thompson 218411c75eadSBorislav Petkov if (!csrow_enabled(i, 0, pvt)) { 21850ec449eeSDoug Thompson debugf1("----CSROW %d EMPTY for node %d\n", i, 21860ec449eeSDoug Thompson pvt->mc_node_id); 21870ec449eeSDoug Thompson continue; 21880ec449eeSDoug Thompson } 21890ec449eeSDoug Thompson 21900ec449eeSDoug Thompson debugf1("----CSROW %d VALID for MC node %d\n", 21910ec449eeSDoug Thompson i, pvt->mc_node_id); 21920ec449eeSDoug Thompson 21930ec449eeSDoug Thompson empty = 0; 219441d8bfabSBorislav Petkov csrow->nr_pages = amd64_csrow_nr_pages(pvt, 0, i); 21950ec449eeSDoug Thompson find_csrow_limits(mci, i, &input_addr_min, &input_addr_max); 21960ec449eeSDoug Thompson sys_addr = input_addr_to_sys_addr(mci, input_addr_min); 21970ec449eeSDoug Thompson csrow->first_page = (u32) (sys_addr >> PAGE_SHIFT); 21980ec449eeSDoug Thompson sys_addr = input_addr_to_sys_addr(mci, input_addr_max); 21990ec449eeSDoug Thompson csrow->last_page = (u32) (sys_addr >> PAGE_SHIFT); 220011c75eadSBorislav Petkov 220111c75eadSBorislav Petkov get_cs_base_and_mask(pvt, i, 0, &base, &mask); 220211c75eadSBorislav Petkov csrow->page_mask = ~mask; 22030ec449eeSDoug Thompson /* 8 bytes of resolution */ 22040ec449eeSDoug Thompson 220524f9a7feSBorislav Petkov csrow->mtype = amd64_determine_memory_type(pvt, i); 22060ec449eeSDoug Thompson 22070ec449eeSDoug Thompson debugf1(" for MC node %d csrow %d:\n", pvt->mc_node_id, i); 22080ec449eeSDoug Thompson debugf1(" input_addr_min: 0x%lx input_addr_max: 0x%lx\n", 22090ec449eeSDoug Thompson (unsigned long)input_addr_min, 22100ec449eeSDoug Thompson (unsigned long)input_addr_max); 22110ec449eeSDoug Thompson debugf1(" sys_addr: 0x%lx page_mask: 0x%lx\n", 22120ec449eeSDoug Thompson (unsigned long)sys_addr, csrow->page_mask); 22130ec449eeSDoug Thompson debugf1(" nr_pages: %u first_page: 0x%lx " 22140ec449eeSDoug Thompson "last_page: 0x%lx\n", 22150ec449eeSDoug Thompson (unsigned)csrow->nr_pages, 22160ec449eeSDoug Thompson csrow->first_page, csrow->last_page); 22170ec449eeSDoug Thompson 22180ec449eeSDoug Thompson /* 22190ec449eeSDoug Thompson * determine whether CHIPKILL or JUST ECC or NO ECC is operating 22200ec449eeSDoug Thompson */ 2221a97fa68eSBorislav Petkov if (pvt->nbcfg & NBCFG_ECC_ENABLE) 22220ec449eeSDoug Thompson csrow->edac_mode = 2223a97fa68eSBorislav Petkov (pvt->nbcfg & NBCFG_CHIPKILL) ? 22240ec449eeSDoug Thompson EDAC_S4ECD4ED : EDAC_SECDED; 22250ec449eeSDoug Thompson else 22260ec449eeSDoug Thompson csrow->edac_mode = EDAC_NONE; 22270ec449eeSDoug Thompson } 22280ec449eeSDoug Thompson 22290ec449eeSDoug Thompson return empty; 22300ec449eeSDoug Thompson } 2231d27bf6faSDoug Thompson 223206724535SBorislav Petkov /* get all cores on this DCT */ 2233b487c33eSBorislav Petkov static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, unsigned nid) 2234f9431992SDoug Thompson { 223506724535SBorislav Petkov int cpu; 2236f9431992SDoug Thompson 223706724535SBorislav Petkov for_each_online_cpu(cpu) 223806724535SBorislav Petkov if (amd_get_nb_id(cpu) == nid) 223906724535SBorislav Petkov cpumask_set_cpu(cpu, mask); 2240f9431992SDoug Thompson } 2241f9431992SDoug Thompson 2242f9431992SDoug Thompson /* check MCG_CTL on all the cpus on this node */ 2243b487c33eSBorislav Petkov static bool amd64_nb_mce_bank_enabled_on_node(unsigned nid) 2244f9431992SDoug Thompson { 2245ba578cb3SRusty Russell cpumask_var_t mask; 224650542251SBorislav Petkov int cpu, nbe; 224706724535SBorislav Petkov bool ret = false; 2248f9431992SDoug Thompson 2249ba578cb3SRusty Russell if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) { 225024f9a7feSBorislav Petkov amd64_warn("%s: Error allocating mask\n", __func__); 225106724535SBorislav Petkov return false; 225206724535SBorislav Petkov } 225306724535SBorislav Petkov 2254ba578cb3SRusty Russell get_cpus_on_this_dct_cpumask(mask, nid); 225506724535SBorislav Petkov 2256ba578cb3SRusty Russell rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs); 2257ba578cb3SRusty Russell 2258ba578cb3SRusty Russell for_each_cpu(cpu, mask) { 225950542251SBorislav Petkov struct msr *reg = per_cpu_ptr(msrs, cpu); 22605980bb9cSBorislav Petkov nbe = reg->l & MSR_MCGCTL_NBE; 226106724535SBorislav Petkov 226206724535SBorislav Petkov debugf0("core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n", 226350542251SBorislav Petkov cpu, reg->q, 226406724535SBorislav Petkov (nbe ? "enabled" : "disabled")); 226506724535SBorislav Petkov 226606724535SBorislav Petkov if (!nbe) 226706724535SBorislav Petkov goto out; 226806724535SBorislav Petkov } 226906724535SBorislav Petkov ret = true; 227006724535SBorislav Petkov 227106724535SBorislav Petkov out: 2272ba578cb3SRusty Russell free_cpumask_var(mask); 2273f9431992SDoug Thompson return ret; 2274f9431992SDoug Thompson } 2275f9431992SDoug Thompson 22762299ef71SBorislav Petkov static int toggle_ecc_err_reporting(struct ecc_settings *s, u8 nid, bool on) 2277f6d6ae96SBorislav Petkov { 2278f6d6ae96SBorislav Petkov cpumask_var_t cmask; 227950542251SBorislav Petkov int cpu; 2280f6d6ae96SBorislav Petkov 2281f6d6ae96SBorislav Petkov if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) { 228224f9a7feSBorislav Petkov amd64_warn("%s: error allocating mask\n", __func__); 2283f6d6ae96SBorislav Petkov return false; 2284f6d6ae96SBorislav Petkov } 2285f6d6ae96SBorislav Petkov 2286ae7bb7c6SBorislav Petkov get_cpus_on_this_dct_cpumask(cmask, nid); 2287f6d6ae96SBorislav Petkov 2288f6d6ae96SBorislav Petkov rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs); 2289f6d6ae96SBorislav Petkov 2290f6d6ae96SBorislav Petkov for_each_cpu(cpu, cmask) { 2291f6d6ae96SBorislav Petkov 229250542251SBorislav Petkov struct msr *reg = per_cpu_ptr(msrs, cpu); 229350542251SBorislav Petkov 2294f6d6ae96SBorislav Petkov if (on) { 22955980bb9cSBorislav Petkov if (reg->l & MSR_MCGCTL_NBE) 2296ae7bb7c6SBorislav Petkov s->flags.nb_mce_enable = 1; 2297f6d6ae96SBorislav Petkov 22985980bb9cSBorislav Petkov reg->l |= MSR_MCGCTL_NBE; 2299f6d6ae96SBorislav Petkov } else { 2300f6d6ae96SBorislav Petkov /* 2301d95cf4deSBorislav Petkov * Turn off NB MCE reporting only when it was off before 2302f6d6ae96SBorislav Petkov */ 2303ae7bb7c6SBorislav Petkov if (!s->flags.nb_mce_enable) 23045980bb9cSBorislav Petkov reg->l &= ~MSR_MCGCTL_NBE; 2305f6d6ae96SBorislav Petkov } 2306f6d6ae96SBorislav Petkov } 2307f6d6ae96SBorislav Petkov wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs); 2308f6d6ae96SBorislav Petkov 2309f6d6ae96SBorislav Petkov free_cpumask_var(cmask); 2310f6d6ae96SBorislav Petkov 2311f6d6ae96SBorislav Petkov return 0; 2312f6d6ae96SBorislav Petkov } 2313f6d6ae96SBorislav Petkov 23142299ef71SBorislav Petkov static bool enable_ecc_error_reporting(struct ecc_settings *s, u8 nid, 23152299ef71SBorislav Petkov struct pci_dev *F3) 2316f6d6ae96SBorislav Petkov { 23172299ef71SBorislav Petkov bool ret = true; 2318c9f4f26eSBorislav Petkov u32 value, mask = 0x3; /* UECC/CECC enable */ 2319f6d6ae96SBorislav Petkov 23202299ef71SBorislav Petkov if (toggle_ecc_err_reporting(s, nid, ON)) { 23212299ef71SBorislav Petkov amd64_warn("Error enabling ECC reporting over MCGCTL!\n"); 23222299ef71SBorislav Petkov return false; 23232299ef71SBorislav Petkov } 23242299ef71SBorislav Petkov 2325c9f4f26eSBorislav Petkov amd64_read_pci_cfg(F3, NBCTL, &value); 2326f6d6ae96SBorislav Petkov 2327ae7bb7c6SBorislav Petkov s->old_nbctl = value & mask; 2328ae7bb7c6SBorislav Petkov s->nbctl_valid = true; 2329f6d6ae96SBorislav Petkov 2330f6d6ae96SBorislav Petkov value |= mask; 2331c9f4f26eSBorislav Petkov amd64_write_pci_cfg(F3, NBCTL, value); 2332f6d6ae96SBorislav Petkov 2333a97fa68eSBorislav Petkov amd64_read_pci_cfg(F3, NBCFG, &value); 2334f6d6ae96SBorislav Petkov 2335a97fa68eSBorislav Petkov debugf0("1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n", 2336a97fa68eSBorislav Petkov nid, value, !!(value & NBCFG_ECC_ENABLE)); 2337f6d6ae96SBorislav Petkov 2338a97fa68eSBorislav Petkov if (!(value & NBCFG_ECC_ENABLE)) { 233924f9a7feSBorislav Petkov amd64_warn("DRAM ECC disabled on this node, enabling...\n"); 2340f6d6ae96SBorislav Petkov 2341ae7bb7c6SBorislav Petkov s->flags.nb_ecc_prev = 0; 2342d95cf4deSBorislav Petkov 2343f6d6ae96SBorislav Petkov /* Attempt to turn on DRAM ECC Enable */ 2344a97fa68eSBorislav Petkov value |= NBCFG_ECC_ENABLE; 2345a97fa68eSBorislav Petkov amd64_write_pci_cfg(F3, NBCFG, value); 2346f6d6ae96SBorislav Petkov 2347a97fa68eSBorislav Petkov amd64_read_pci_cfg(F3, NBCFG, &value); 2348f6d6ae96SBorislav Petkov 2349a97fa68eSBorislav Petkov if (!(value & NBCFG_ECC_ENABLE)) { 235024f9a7feSBorislav Petkov amd64_warn("Hardware rejected DRAM ECC enable," 235124f9a7feSBorislav Petkov "check memory DIMM configuration.\n"); 23522299ef71SBorislav Petkov ret = false; 2353f6d6ae96SBorislav Petkov } else { 235424f9a7feSBorislav Petkov amd64_info("Hardware accepted DRAM ECC Enable\n"); 2355f6d6ae96SBorislav Petkov } 2356d95cf4deSBorislav Petkov } else { 2357ae7bb7c6SBorislav Petkov s->flags.nb_ecc_prev = 1; 2358f6d6ae96SBorislav Petkov } 2359d95cf4deSBorislav Petkov 2360a97fa68eSBorislav Petkov debugf0("2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n", 2361a97fa68eSBorislav Petkov nid, value, !!(value & NBCFG_ECC_ENABLE)); 2362f6d6ae96SBorislav Petkov 23632299ef71SBorislav Petkov return ret; 2364f6d6ae96SBorislav Petkov } 2365f6d6ae96SBorislav Petkov 2366360b7f3cSBorislav Petkov static void restore_ecc_error_reporting(struct ecc_settings *s, u8 nid, 2367360b7f3cSBorislav Petkov struct pci_dev *F3) 2368f6d6ae96SBorislav Petkov { 2369c9f4f26eSBorislav Petkov u32 value, mask = 0x3; /* UECC/CECC enable */ 2370c9f4f26eSBorislav Petkov 2371f6d6ae96SBorislav Petkov 2372ae7bb7c6SBorislav Petkov if (!s->nbctl_valid) 2373f6d6ae96SBorislav Petkov return; 2374f6d6ae96SBorislav Petkov 2375c9f4f26eSBorislav Petkov amd64_read_pci_cfg(F3, NBCTL, &value); 2376f6d6ae96SBorislav Petkov value &= ~mask; 2377ae7bb7c6SBorislav Petkov value |= s->old_nbctl; 2378f6d6ae96SBorislav Petkov 2379c9f4f26eSBorislav Petkov amd64_write_pci_cfg(F3, NBCTL, value); 2380f6d6ae96SBorislav Petkov 2381ae7bb7c6SBorislav Petkov /* restore previous BIOS DRAM ECC "off" setting we force-enabled */ 2382ae7bb7c6SBorislav Petkov if (!s->flags.nb_ecc_prev) { 2383a97fa68eSBorislav Petkov amd64_read_pci_cfg(F3, NBCFG, &value); 2384a97fa68eSBorislav Petkov value &= ~NBCFG_ECC_ENABLE; 2385a97fa68eSBorislav Petkov amd64_write_pci_cfg(F3, NBCFG, value); 2386d95cf4deSBorislav Petkov } 2387d95cf4deSBorislav Petkov 2388d95cf4deSBorislav Petkov /* restore the NB Enable MCGCTL bit */ 23892299ef71SBorislav Petkov if (toggle_ecc_err_reporting(s, nid, OFF)) 239024f9a7feSBorislav Petkov amd64_warn("Error restoring NB MCGCTL settings!\n"); 2391f6d6ae96SBorislav Petkov } 2392f6d6ae96SBorislav Petkov 2393f9431992SDoug Thompson /* 23942299ef71SBorislav Petkov * EDAC requires that the BIOS have ECC enabled before 23952299ef71SBorislav Petkov * taking over the processing of ECC errors. A command line 23962299ef71SBorislav Petkov * option allows to force-enable hardware ECC later in 23972299ef71SBorislav Petkov * enable_ecc_error_reporting(). 2398f9431992SDoug Thompson */ 2399cab4d277SBorislav Petkov static const char *ecc_msg = 2400cab4d277SBorislav Petkov "ECC disabled in the BIOS or no ECC capability, module will not load.\n" 2401cab4d277SBorislav Petkov " Either enable ECC checking or force module loading by setting " 2402cab4d277SBorislav Petkov "'ecc_enable_override'.\n" 2403cab4d277SBorislav Petkov " (Note that use of the override may cause unknown side effects.)\n"; 2404be3468e8SBorislav Petkov 24052299ef71SBorislav Petkov static bool ecc_enabled(struct pci_dev *F3, u8 nid) 2406f9431992SDoug Thompson { 2407f9431992SDoug Thompson u32 value; 24082299ef71SBorislav Petkov u8 ecc_en = 0; 240906724535SBorislav Petkov bool nb_mce_en = false; 2410f9431992SDoug Thompson 2411a97fa68eSBorislav Petkov amd64_read_pci_cfg(F3, NBCFG, &value); 2412f9431992SDoug Thompson 2413a97fa68eSBorislav Petkov ecc_en = !!(value & NBCFG_ECC_ENABLE); 24142299ef71SBorislav Petkov amd64_info("DRAM ECC %s.\n", (ecc_en ? "enabled" : "disabled")); 2415be3468e8SBorislav Petkov 24162299ef71SBorislav Petkov nb_mce_en = amd64_nb_mce_bank_enabled_on_node(nid); 241706724535SBorislav Petkov if (!nb_mce_en) 24182299ef71SBorislav Petkov amd64_notice("NB MCE bank disabled, set MSR " 24192299ef71SBorislav Petkov "0x%08x[4] on node %d to enable.\n", 24202299ef71SBorislav Petkov MSR_IA32_MCG_CTL, nid); 2421be3468e8SBorislav Petkov 24222299ef71SBorislav Petkov if (!ecc_en || !nb_mce_en) { 242324f9a7feSBorislav Petkov amd64_notice("%s", ecc_msg); 24242299ef71SBorislav Petkov return false; 2425be3468e8SBorislav Petkov } 24262299ef71SBorislav Petkov return true; 2427f9431992SDoug Thompson } 2428f9431992SDoug Thompson 24297d6034d3SDoug Thompson struct mcidev_sysfs_attribute sysfs_attrs[ARRAY_SIZE(amd64_dbg_attrs) + 24307d6034d3SDoug Thompson ARRAY_SIZE(amd64_inj_attrs) + 24317d6034d3SDoug Thompson 1]; 24327d6034d3SDoug Thompson 24337d6034d3SDoug Thompson struct mcidev_sysfs_attribute terminator = { .attr = { .name = NULL } }; 24347d6034d3SDoug Thompson 2435360b7f3cSBorislav Petkov static void set_mc_sysfs_attrs(struct mem_ctl_info *mci) 24367d6034d3SDoug Thompson { 24377d6034d3SDoug Thompson unsigned int i = 0, j = 0; 24387d6034d3SDoug Thompson 24397d6034d3SDoug Thompson for (; i < ARRAY_SIZE(amd64_dbg_attrs); i++) 24407d6034d3SDoug Thompson sysfs_attrs[i] = amd64_dbg_attrs[i]; 24417d6034d3SDoug Thompson 2442a135cef7SBorislav Petkov if (boot_cpu_data.x86 >= 0x10) 24437d6034d3SDoug Thompson for (j = 0; j < ARRAY_SIZE(amd64_inj_attrs); j++, i++) 24447d6034d3SDoug Thompson sysfs_attrs[i] = amd64_inj_attrs[j]; 24457d6034d3SDoug Thompson 24467d6034d3SDoug Thompson sysfs_attrs[i] = terminator; 24477d6034d3SDoug Thompson 24487d6034d3SDoug Thompson mci->mc_driver_sysfs_attributes = sysfs_attrs; 24497d6034d3SDoug Thompson } 24507d6034d3SDoug Thompson 2451df71a053SBorislav Petkov static void setup_mci_misc_attrs(struct mem_ctl_info *mci, 2452df71a053SBorislav Petkov struct amd64_family_type *fam) 24537d6034d3SDoug Thompson { 24547d6034d3SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 24557d6034d3SDoug Thompson 24567d6034d3SDoug Thompson mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2; 24577d6034d3SDoug Thompson mci->edac_ctl_cap = EDAC_FLAG_NONE; 24587d6034d3SDoug Thompson 24595980bb9cSBorislav Petkov if (pvt->nbcap & NBCAP_SECDED) 24607d6034d3SDoug Thompson mci->edac_ctl_cap |= EDAC_FLAG_SECDED; 24617d6034d3SDoug Thompson 24625980bb9cSBorislav Petkov if (pvt->nbcap & NBCAP_CHIPKILL) 24637d6034d3SDoug Thompson mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED; 24647d6034d3SDoug Thompson 24657d6034d3SDoug Thompson mci->edac_cap = amd64_determine_edac_cap(pvt); 24667d6034d3SDoug Thompson mci->mod_name = EDAC_MOD_STR; 24677d6034d3SDoug Thompson mci->mod_ver = EDAC_AMD64_VERSION; 2468df71a053SBorislav Petkov mci->ctl_name = fam->ctl_name; 24698d5b5d9cSBorislav Petkov mci->dev_name = pci_name(pvt->F2); 24707d6034d3SDoug Thompson mci->ctl_page_to_phys = NULL; 24717d6034d3SDoug Thompson 24727d6034d3SDoug Thompson /* memory scrubber interface */ 24737d6034d3SDoug Thompson mci->set_sdram_scrub_rate = amd64_set_scrub_rate; 24747d6034d3SDoug Thompson mci->get_sdram_scrub_rate = amd64_get_scrub_rate; 24757d6034d3SDoug Thompson } 24767d6034d3SDoug Thompson 24770092b20dSBorislav Petkov /* 24780092b20dSBorislav Petkov * returns a pointer to the family descriptor on success, NULL otherwise. 24790092b20dSBorislav Petkov */ 24800092b20dSBorislav Petkov static struct amd64_family_type *amd64_per_family_init(struct amd64_pvt *pvt) 2481395ae783SBorislav Petkov { 24820092b20dSBorislav Petkov u8 fam = boot_cpu_data.x86; 24830092b20dSBorislav Petkov struct amd64_family_type *fam_type = NULL; 24840092b20dSBorislav Petkov 24850092b20dSBorislav Petkov switch (fam) { 2486395ae783SBorislav Petkov case 0xf: 24870092b20dSBorislav Petkov fam_type = &amd64_family_types[K8_CPUS]; 2488b8cfa02fSBorislav Petkov pvt->ops = &amd64_family_types[K8_CPUS].ops; 2489395ae783SBorislav Petkov break; 2490df71a053SBorislav Petkov 2491395ae783SBorislav Petkov case 0x10: 24920092b20dSBorislav Petkov fam_type = &amd64_family_types[F10_CPUS]; 2493b8cfa02fSBorislav Petkov pvt->ops = &amd64_family_types[F10_CPUS].ops; 2494df71a053SBorislav Petkov break; 2495df71a053SBorislav Petkov 2496df71a053SBorislav Petkov case 0x15: 2497df71a053SBorislav Petkov fam_type = &amd64_family_types[F15_CPUS]; 2498df71a053SBorislav Petkov pvt->ops = &amd64_family_types[F15_CPUS].ops; 2499395ae783SBorislav Petkov break; 2500395ae783SBorislav Petkov 2501395ae783SBorislav Petkov default: 250224f9a7feSBorislav Petkov amd64_err("Unsupported family!\n"); 25030092b20dSBorislav Petkov return NULL; 2504395ae783SBorislav Petkov } 25050092b20dSBorislav Petkov 2506b8cfa02fSBorislav Petkov pvt->ext_model = boot_cpu_data.x86_model >> 4; 2507b8cfa02fSBorislav Petkov 2508df71a053SBorislav Petkov amd64_info("%s %sdetected (node %d).\n", fam_type->ctl_name, 25090092b20dSBorislav Petkov (fam == 0xf ? 25100092b20dSBorislav Petkov (pvt->ext_model >= K8_REV_F ? "revF or later " 25110092b20dSBorislav Petkov : "revE or earlier ") 251224f9a7feSBorislav Petkov : ""), pvt->mc_node_id); 25130092b20dSBorislav Petkov return fam_type; 2514395ae783SBorislav Petkov } 2515395ae783SBorislav Petkov 25162299ef71SBorislav Petkov static int amd64_init_one_instance(struct pci_dev *F2) 25177d6034d3SDoug Thompson { 25187d6034d3SDoug Thompson struct amd64_pvt *pvt = NULL; 25190092b20dSBorislav Petkov struct amd64_family_type *fam_type = NULL; 2520360b7f3cSBorislav Petkov struct mem_ctl_info *mci = NULL; 25217d6034d3SDoug Thompson int err = 0, ret; 2522360b7f3cSBorislav Petkov u8 nid = get_node_id(F2); 25237d6034d3SDoug Thompson 25247d6034d3SDoug Thompson ret = -ENOMEM; 25257d6034d3SDoug Thompson pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL); 25267d6034d3SDoug Thompson if (!pvt) 2527360b7f3cSBorislav Petkov goto err_ret; 25287d6034d3SDoug Thompson 2529360b7f3cSBorislav Petkov pvt->mc_node_id = nid; 25308d5b5d9cSBorislav Petkov pvt->F2 = F2; 25317d6034d3SDoug Thompson 2532395ae783SBorislav Petkov ret = -EINVAL; 25330092b20dSBorislav Petkov fam_type = amd64_per_family_init(pvt); 25340092b20dSBorislav Petkov if (!fam_type) 2535395ae783SBorislav Petkov goto err_free; 2536395ae783SBorislav Petkov 25377d6034d3SDoug Thompson ret = -ENODEV; 2538360b7f3cSBorislav Petkov err = reserve_mc_sibling_devs(pvt, fam_type->f1_id, fam_type->f3_id); 25397d6034d3SDoug Thompson if (err) 25407d6034d3SDoug Thompson goto err_free; 25417d6034d3SDoug Thompson 2542360b7f3cSBorislav Petkov read_mc_regs(pvt); 25437d6034d3SDoug Thompson 25447d6034d3SDoug Thompson /* 25457d6034d3SDoug Thompson * We need to determine how many memory channels there are. Then use 25467d6034d3SDoug Thompson * that information for calculating the size of the dynamic instance 2547360b7f3cSBorislav Petkov * tables in the 'mci' structure. 25487d6034d3SDoug Thompson */ 2549360b7f3cSBorislav Petkov ret = -EINVAL; 25507d6034d3SDoug Thompson pvt->channel_count = pvt->ops->early_channel_count(pvt); 25517d6034d3SDoug Thompson if (pvt->channel_count < 0) 2552360b7f3cSBorislav Petkov goto err_siblings; 25537d6034d3SDoug Thompson 25547d6034d3SDoug Thompson ret = -ENOMEM; 255511c75eadSBorislav Petkov mci = edac_mc_alloc(0, pvt->csels[0].b_cnt, pvt->channel_count, nid); 25567d6034d3SDoug Thompson if (!mci) 2557360b7f3cSBorislav Petkov goto err_siblings; 25587d6034d3SDoug Thompson 25597d6034d3SDoug Thompson mci->pvt_info = pvt; 25608d5b5d9cSBorislav Petkov mci->dev = &pvt->F2->dev; 25617d6034d3SDoug Thompson 2562df71a053SBorislav Petkov setup_mci_misc_attrs(mci, fam_type); 2563360b7f3cSBorislav Petkov 2564360b7f3cSBorislav Petkov if (init_csrows(mci)) 25657d6034d3SDoug Thompson mci->edac_cap = EDAC_FLAG_NONE; 25667d6034d3SDoug Thompson 2567360b7f3cSBorislav Petkov set_mc_sysfs_attrs(mci); 25687d6034d3SDoug Thompson 25697d6034d3SDoug Thompson ret = -ENODEV; 25707d6034d3SDoug Thompson if (edac_mc_add_mc(mci)) { 25717d6034d3SDoug Thompson debugf1("failed edac_mc_add_mc()\n"); 25727d6034d3SDoug Thompson goto err_add_mc; 25737d6034d3SDoug Thompson } 25747d6034d3SDoug Thompson 2575549d042dSBorislav Petkov /* register stuff with EDAC MCE */ 2576549d042dSBorislav Petkov if (report_gart_errors) 2577549d042dSBorislav Petkov amd_report_gart_errors(true); 2578549d042dSBorislav Petkov 2579549d042dSBorislav Petkov amd_register_ecc_decoder(amd64_decode_bus_error); 2580549d042dSBorislav Petkov 2581360b7f3cSBorislav Petkov mcis[nid] = mci; 2582360b7f3cSBorislav Petkov 2583360b7f3cSBorislav Petkov atomic_inc(&drv_instances); 2584360b7f3cSBorislav Petkov 25857d6034d3SDoug Thompson return 0; 25867d6034d3SDoug Thompson 25877d6034d3SDoug Thompson err_add_mc: 25887d6034d3SDoug Thompson edac_mc_free(mci); 25897d6034d3SDoug Thompson 2590360b7f3cSBorislav Petkov err_siblings: 2591360b7f3cSBorislav Petkov free_mc_sibling_devs(pvt); 25927d6034d3SDoug Thompson 2593360b7f3cSBorislav Petkov err_free: 2594360b7f3cSBorislav Petkov kfree(pvt); 25957d6034d3SDoug Thompson 2596360b7f3cSBorislav Petkov err_ret: 25977d6034d3SDoug Thompson return ret; 25987d6034d3SDoug Thompson } 25997d6034d3SDoug Thompson 26002299ef71SBorislav Petkov static int __devinit amd64_probe_one_instance(struct pci_dev *pdev, 26017d6034d3SDoug Thompson const struct pci_device_id *mc_type) 26027d6034d3SDoug Thompson { 2603ae7bb7c6SBorislav Petkov u8 nid = get_node_id(pdev); 26042299ef71SBorislav Petkov struct pci_dev *F3 = node_to_amd_nb(nid)->misc; 2605ae7bb7c6SBorislav Petkov struct ecc_settings *s; 26062299ef71SBorislav Petkov int ret = 0; 26077d6034d3SDoug Thompson 26087d6034d3SDoug Thompson ret = pci_enable_device(pdev); 2609b8cfa02fSBorislav Petkov if (ret < 0) { 26107d6034d3SDoug Thompson debugf0("ret=%d\n", ret); 2611b8cfa02fSBorislav Petkov return -EIO; 2612b8cfa02fSBorislav Petkov } 2613b8cfa02fSBorislav Petkov 2614ae7bb7c6SBorislav Petkov ret = -ENOMEM; 2615ae7bb7c6SBorislav Petkov s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL); 2616ae7bb7c6SBorislav Petkov if (!s) 26172299ef71SBorislav Petkov goto err_out; 2618ae7bb7c6SBorislav Petkov 2619ae7bb7c6SBorislav Petkov ecc_stngs[nid] = s; 2620ae7bb7c6SBorislav Petkov 26212299ef71SBorislav Petkov if (!ecc_enabled(F3, nid)) { 26222299ef71SBorislav Petkov ret = -ENODEV; 26232299ef71SBorislav Petkov 26242299ef71SBorislav Petkov if (!ecc_enable_override) 26252299ef71SBorislav Petkov goto err_enable; 26262299ef71SBorislav Petkov 26272299ef71SBorislav Petkov amd64_warn("Forcing ECC on!\n"); 26282299ef71SBorislav Petkov 26292299ef71SBorislav Petkov if (!enable_ecc_error_reporting(s, nid, F3)) 26302299ef71SBorislav Petkov goto err_enable; 26312299ef71SBorislav Petkov } 26322299ef71SBorislav Petkov 26332299ef71SBorislav Petkov ret = amd64_init_one_instance(pdev); 2634360b7f3cSBorislav Petkov if (ret < 0) { 2635ae7bb7c6SBorislav Petkov amd64_err("Error probing instance: %d\n", nid); 2636360b7f3cSBorislav Petkov restore_ecc_error_reporting(s, nid, F3); 2637360b7f3cSBorislav Petkov } 26387d6034d3SDoug Thompson 26397d6034d3SDoug Thompson return ret; 26402299ef71SBorislav Petkov 26412299ef71SBorislav Petkov err_enable: 26422299ef71SBorislav Petkov kfree(s); 26432299ef71SBorislav Petkov ecc_stngs[nid] = NULL; 26442299ef71SBorislav Petkov 26452299ef71SBorislav Petkov err_out: 26462299ef71SBorislav Petkov return ret; 26477d6034d3SDoug Thompson } 26487d6034d3SDoug Thompson 26497d6034d3SDoug Thompson static void __devexit amd64_remove_one_instance(struct pci_dev *pdev) 26507d6034d3SDoug Thompson { 26517d6034d3SDoug Thompson struct mem_ctl_info *mci; 26527d6034d3SDoug Thompson struct amd64_pvt *pvt; 2653360b7f3cSBorislav Petkov u8 nid = get_node_id(pdev); 2654360b7f3cSBorislav Petkov struct pci_dev *F3 = node_to_amd_nb(nid)->misc; 2655360b7f3cSBorislav Petkov struct ecc_settings *s = ecc_stngs[nid]; 26567d6034d3SDoug Thompson 26577d6034d3SDoug Thompson /* Remove from EDAC CORE tracking list */ 26587d6034d3SDoug Thompson mci = edac_mc_del_mc(&pdev->dev); 26597d6034d3SDoug Thompson if (!mci) 26607d6034d3SDoug Thompson return; 26617d6034d3SDoug Thompson 26627d6034d3SDoug Thompson pvt = mci->pvt_info; 26637d6034d3SDoug Thompson 2664360b7f3cSBorislav Petkov restore_ecc_error_reporting(s, nid, F3); 26657d6034d3SDoug Thompson 2666360b7f3cSBorislav Petkov free_mc_sibling_devs(pvt); 26677d6034d3SDoug Thompson 2668549d042dSBorislav Petkov /* unregister from EDAC MCE */ 2669549d042dSBorislav Petkov amd_report_gart_errors(false); 2670549d042dSBorislav Petkov amd_unregister_ecc_decoder(amd64_decode_bus_error); 2671549d042dSBorislav Petkov 2672360b7f3cSBorislav Petkov kfree(ecc_stngs[nid]); 2673360b7f3cSBorislav Petkov ecc_stngs[nid] = NULL; 2674ae7bb7c6SBorislav Petkov 26757d6034d3SDoug Thompson /* Free the EDAC CORE resources */ 26768f68ed97SBorislav Petkov mci->pvt_info = NULL; 2677360b7f3cSBorislav Petkov mcis[nid] = NULL; 26788f68ed97SBorislav Petkov 26798f68ed97SBorislav Petkov kfree(pvt); 26807d6034d3SDoug Thompson edac_mc_free(mci); 26817d6034d3SDoug Thompson } 26827d6034d3SDoug Thompson 26837d6034d3SDoug Thompson /* 26847d6034d3SDoug Thompson * This table is part of the interface for loading drivers for PCI devices. The 26857d6034d3SDoug Thompson * PCI core identifies what devices are on a system during boot, and then 26867d6034d3SDoug Thompson * inquiry this table to see if this driver is for a given device found. 26877d6034d3SDoug Thompson */ 26887d6034d3SDoug Thompson static const struct pci_device_id amd64_pci_table[] __devinitdata = { 26897d6034d3SDoug Thompson { 26907d6034d3SDoug Thompson .vendor = PCI_VENDOR_ID_AMD, 26917d6034d3SDoug Thompson .device = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL, 26927d6034d3SDoug Thompson .subvendor = PCI_ANY_ID, 26937d6034d3SDoug Thompson .subdevice = PCI_ANY_ID, 26947d6034d3SDoug Thompson .class = 0, 26957d6034d3SDoug Thompson .class_mask = 0, 26967d6034d3SDoug Thompson }, 26977d6034d3SDoug Thompson { 26987d6034d3SDoug Thompson .vendor = PCI_VENDOR_ID_AMD, 26997d6034d3SDoug Thompson .device = PCI_DEVICE_ID_AMD_10H_NB_DRAM, 27007d6034d3SDoug Thompson .subvendor = PCI_ANY_ID, 27017d6034d3SDoug Thompson .subdevice = PCI_ANY_ID, 27027d6034d3SDoug Thompson .class = 0, 27037d6034d3SDoug Thompson .class_mask = 0, 27047d6034d3SDoug Thompson }, 2705df71a053SBorislav Petkov { 2706df71a053SBorislav Petkov .vendor = PCI_VENDOR_ID_AMD, 2707df71a053SBorislav Petkov .device = PCI_DEVICE_ID_AMD_15H_NB_F2, 2708df71a053SBorislav Petkov .subvendor = PCI_ANY_ID, 2709df71a053SBorislav Petkov .subdevice = PCI_ANY_ID, 2710df71a053SBorislav Petkov .class = 0, 2711df71a053SBorislav Petkov .class_mask = 0, 2712df71a053SBorislav Petkov }, 2713df71a053SBorislav Petkov 27147d6034d3SDoug Thompson {0, } 27157d6034d3SDoug Thompson }; 27167d6034d3SDoug Thompson MODULE_DEVICE_TABLE(pci, amd64_pci_table); 27177d6034d3SDoug Thompson 27187d6034d3SDoug Thompson static struct pci_driver amd64_pci_driver = { 27197d6034d3SDoug Thompson .name = EDAC_MOD_STR, 27202299ef71SBorislav Petkov .probe = amd64_probe_one_instance, 27217d6034d3SDoug Thompson .remove = __devexit_p(amd64_remove_one_instance), 27227d6034d3SDoug Thompson .id_table = amd64_pci_table, 27237d6034d3SDoug Thompson }; 27247d6034d3SDoug Thompson 2725360b7f3cSBorislav Petkov static void setup_pci_device(void) 27267d6034d3SDoug Thompson { 27277d6034d3SDoug Thompson struct mem_ctl_info *mci; 27287d6034d3SDoug Thompson struct amd64_pvt *pvt; 27297d6034d3SDoug Thompson 27307d6034d3SDoug Thompson if (amd64_ctl_pci) 27317d6034d3SDoug Thompson return; 27327d6034d3SDoug Thompson 2733cc4d8860SBorislav Petkov mci = mcis[0]; 27347d6034d3SDoug Thompson if (mci) { 27357d6034d3SDoug Thompson 27367d6034d3SDoug Thompson pvt = mci->pvt_info; 27377d6034d3SDoug Thompson amd64_ctl_pci = 27388d5b5d9cSBorislav Petkov edac_pci_create_generic_ctl(&pvt->F2->dev, EDAC_MOD_STR); 27397d6034d3SDoug Thompson 27407d6034d3SDoug Thompson if (!amd64_ctl_pci) { 27417d6034d3SDoug Thompson pr_warning("%s(): Unable to create PCI control\n", 27427d6034d3SDoug Thompson __func__); 27437d6034d3SDoug Thompson 27447d6034d3SDoug Thompson pr_warning("%s(): PCI error report via EDAC not set\n", 27457d6034d3SDoug Thompson __func__); 27467d6034d3SDoug Thompson } 27477d6034d3SDoug Thompson } 27487d6034d3SDoug Thompson } 27497d6034d3SDoug Thompson 27507d6034d3SDoug Thompson static int __init amd64_edac_init(void) 27517d6034d3SDoug Thompson { 2752360b7f3cSBorislav Petkov int err = -ENODEV; 27537d6034d3SDoug Thompson 2754df71a053SBorislav Petkov printk(KERN_INFO "AMD64 EDAC driver v%s\n", EDAC_AMD64_VERSION); 27557d6034d3SDoug Thompson 27567d6034d3SDoug Thompson opstate_init(); 27577d6034d3SDoug Thompson 27589653a5c7SHans Rosenfeld if (amd_cache_northbridges() < 0) 275956b34b91SBorislav Petkov goto err_ret; 27607d6034d3SDoug Thompson 2761cc4d8860SBorislav Petkov err = -ENOMEM; 2762cc4d8860SBorislav Petkov mcis = kzalloc(amd_nb_num() * sizeof(mcis[0]), GFP_KERNEL); 2763ae7bb7c6SBorislav Petkov ecc_stngs = kzalloc(amd_nb_num() * sizeof(ecc_stngs[0]), GFP_KERNEL); 2764360b7f3cSBorislav Petkov if (!(mcis && ecc_stngs)) 2765a9f0fbe2SBorislav Petkov goto err_free; 2766cc4d8860SBorislav Petkov 276750542251SBorislav Petkov msrs = msrs_alloc(); 276856b34b91SBorislav Petkov if (!msrs) 2769360b7f3cSBorislav Petkov goto err_free; 277050542251SBorislav Petkov 27717d6034d3SDoug Thompson err = pci_register_driver(&amd64_pci_driver); 27727d6034d3SDoug Thompson if (err) 277356b34b91SBorislav Petkov goto err_pci; 27747d6034d3SDoug Thompson 277556b34b91SBorislav Petkov err = -ENODEV; 2776360b7f3cSBorislav Petkov if (!atomic_read(&drv_instances)) 2777360b7f3cSBorislav Petkov goto err_no_instances; 27787d6034d3SDoug Thompson 2779360b7f3cSBorislav Petkov setup_pci_device(); 27807d6034d3SDoug Thompson return 0; 27817d6034d3SDoug Thompson 2782360b7f3cSBorislav Petkov err_no_instances: 27837d6034d3SDoug Thompson pci_unregister_driver(&amd64_pci_driver); 2784cc4d8860SBorislav Petkov 278556b34b91SBorislav Petkov err_pci: 278656b34b91SBorislav Petkov msrs_free(msrs); 278756b34b91SBorislav Petkov msrs = NULL; 2788cc4d8860SBorislav Petkov 2789360b7f3cSBorislav Petkov err_free: 2790360b7f3cSBorislav Petkov kfree(mcis); 2791360b7f3cSBorislav Petkov mcis = NULL; 2792360b7f3cSBorislav Petkov 2793360b7f3cSBorislav Petkov kfree(ecc_stngs); 2794360b7f3cSBorislav Petkov ecc_stngs = NULL; 2795360b7f3cSBorislav Petkov 279656b34b91SBorislav Petkov err_ret: 27977d6034d3SDoug Thompson return err; 27987d6034d3SDoug Thompson } 27997d6034d3SDoug Thompson 28007d6034d3SDoug Thompson static void __exit amd64_edac_exit(void) 28017d6034d3SDoug Thompson { 28027d6034d3SDoug Thompson if (amd64_ctl_pci) 28037d6034d3SDoug Thompson edac_pci_release_generic_ctl(amd64_ctl_pci); 28047d6034d3SDoug Thompson 28057d6034d3SDoug Thompson pci_unregister_driver(&amd64_pci_driver); 280650542251SBorislav Petkov 2807ae7bb7c6SBorislav Petkov kfree(ecc_stngs); 2808ae7bb7c6SBorislav Petkov ecc_stngs = NULL; 2809ae7bb7c6SBorislav Petkov 2810cc4d8860SBorislav Petkov kfree(mcis); 2811cc4d8860SBorislav Petkov mcis = NULL; 2812cc4d8860SBorislav Petkov 281350542251SBorislav Petkov msrs_free(msrs); 281450542251SBorislav Petkov msrs = NULL; 28157d6034d3SDoug Thompson } 28167d6034d3SDoug Thompson 28177d6034d3SDoug Thompson module_init(amd64_edac_init); 28187d6034d3SDoug Thompson module_exit(amd64_edac_exit); 28197d6034d3SDoug Thompson 28207d6034d3SDoug Thompson MODULE_LICENSE("GPL"); 28217d6034d3SDoug Thompson MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, " 28227d6034d3SDoug Thompson "Dave Peterson, Thayne Harbaugh"); 28237d6034d3SDoug Thompson MODULE_DESCRIPTION("MC support for AMD64 memory controllers - " 28247d6034d3SDoug Thompson EDAC_AMD64_VERSION); 28257d6034d3SDoug Thompson 28267d6034d3SDoug Thompson module_param(edac_op_state, int, 0444); 28277d6034d3SDoug Thompson MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI"); 2828