12bc65418SDoug Thompson #include "amd64_edac.h" 223ac4ae8SAndreas Herrmann #include <asm/amd_nb.h> 32bc65418SDoug Thompson 42bc65418SDoug Thompson static struct edac_pci_ctl_info *amd64_ctl_pci; 52bc65418SDoug Thompson 62bc65418SDoug Thompson static int report_gart_errors; 72bc65418SDoug Thompson module_param(report_gart_errors, int, 0644); 82bc65418SDoug Thompson 92bc65418SDoug Thompson /* 102bc65418SDoug Thompson * Set by command line parameter. If BIOS has enabled the ECC, this override is 112bc65418SDoug Thompson * cleared to prevent re-enabling the hardware by this driver. 122bc65418SDoug Thompson */ 132bc65418SDoug Thompson static int ecc_enable_override; 142bc65418SDoug Thompson module_param(ecc_enable_override, int, 0644); 152bc65418SDoug Thompson 16a29d8b8eSTejun Heo static struct msr __percpu *msrs; 1750542251SBorislav Petkov 18360b7f3cSBorislav Petkov /* 19360b7f3cSBorislav Petkov * count successfully initialized driver instances for setup_pci_device() 20360b7f3cSBorislav Petkov */ 21360b7f3cSBorislav Petkov static atomic_t drv_instances = ATOMIC_INIT(0); 22360b7f3cSBorislav Petkov 23cc4d8860SBorislav Petkov /* Per-node driver instances */ 24cc4d8860SBorislav Petkov static struct mem_ctl_info **mcis; 25ae7bb7c6SBorislav Petkov static struct ecc_settings **ecc_stngs; 262bc65418SDoug Thompson 272bc65418SDoug Thompson /* 28b70ef010SBorislav Petkov * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing 29b70ef010SBorislav Petkov * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching- 30b70ef010SBorislav Petkov * or higher value'. 31b70ef010SBorislav Petkov * 32b70ef010SBorislav Petkov *FIXME: Produce a better mapping/linearisation. 33b70ef010SBorislav Petkov */ 3439094443SBorislav Petkov struct scrubrate { 3539094443SBorislav Petkov u32 scrubval; /* bit pattern for scrub rate */ 3639094443SBorislav Petkov u32 bandwidth; /* bandwidth consumed (bytes/sec) */ 3739094443SBorislav Petkov } scrubrates[] = { 38b70ef010SBorislav Petkov { 0x01, 1600000000UL}, 39b70ef010SBorislav Petkov { 0x02, 800000000UL}, 40b70ef010SBorislav Petkov { 0x03, 400000000UL}, 41b70ef010SBorislav Petkov { 0x04, 200000000UL}, 42b70ef010SBorislav Petkov { 0x05, 100000000UL}, 43b70ef010SBorislav Petkov { 0x06, 50000000UL}, 44b70ef010SBorislav Petkov { 0x07, 25000000UL}, 45b70ef010SBorislav Petkov { 0x08, 12284069UL}, 46b70ef010SBorislav Petkov { 0x09, 6274509UL}, 47b70ef010SBorislav Petkov { 0x0A, 3121951UL}, 48b70ef010SBorislav Petkov { 0x0B, 1560975UL}, 49b70ef010SBorislav Petkov { 0x0C, 781440UL}, 50b70ef010SBorislav Petkov { 0x0D, 390720UL}, 51b70ef010SBorislav Petkov { 0x0E, 195300UL}, 52b70ef010SBorislav Petkov { 0x0F, 97650UL}, 53b70ef010SBorislav Petkov { 0x10, 48854UL}, 54b70ef010SBorislav Petkov { 0x11, 24427UL}, 55b70ef010SBorislav Petkov { 0x12, 12213UL}, 56b70ef010SBorislav Petkov { 0x13, 6101UL}, 57b70ef010SBorislav Petkov { 0x14, 3051UL}, 58b70ef010SBorislav Petkov { 0x15, 1523UL}, 59b70ef010SBorislav Petkov { 0x16, 761UL}, 60b70ef010SBorislav Petkov { 0x00, 0UL}, /* scrubbing off */ 61b70ef010SBorislav Petkov }; 62b70ef010SBorislav Petkov 63b2b0c605SBorislav Petkov static int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset, 64b2b0c605SBorislav Petkov u32 *val, const char *func) 65b2b0c605SBorislav Petkov { 66b2b0c605SBorislav Petkov int err = 0; 67b2b0c605SBorislav Petkov 68b2b0c605SBorislav Petkov err = pci_read_config_dword(pdev, offset, val); 69b2b0c605SBorislav Petkov if (err) 70b2b0c605SBorislav Petkov amd64_warn("%s: error reading F%dx%03x.\n", 71b2b0c605SBorislav Petkov func, PCI_FUNC(pdev->devfn), offset); 72b2b0c605SBorislav Petkov 73b2b0c605SBorislav Petkov return err; 74b2b0c605SBorislav Petkov } 75b2b0c605SBorislav Petkov 76b2b0c605SBorislav Petkov int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset, 77b2b0c605SBorislav Petkov u32 val, const char *func) 78b2b0c605SBorislav Petkov { 79b2b0c605SBorislav Petkov int err = 0; 80b2b0c605SBorislav Petkov 81b2b0c605SBorislav Petkov err = pci_write_config_dword(pdev, offset, val); 82b2b0c605SBorislav Petkov if (err) 83b2b0c605SBorislav Petkov amd64_warn("%s: error writing to F%dx%03x.\n", 84b2b0c605SBorislav Petkov func, PCI_FUNC(pdev->devfn), offset); 85b2b0c605SBorislav Petkov 86b2b0c605SBorislav Petkov return err; 87b2b0c605SBorislav Petkov } 88b2b0c605SBorislav Petkov 89b2b0c605SBorislav Petkov /* 90b2b0c605SBorislav Petkov * 91b2b0c605SBorislav Petkov * Depending on the family, F2 DCT reads need special handling: 92b2b0c605SBorislav Petkov * 93b2b0c605SBorislav Petkov * K8: has a single DCT only 94b2b0c605SBorislav Petkov * 95b2b0c605SBorislav Petkov * F10h: each DCT has its own set of regs 96b2b0c605SBorislav Petkov * DCT0 -> F2x040.. 97b2b0c605SBorislav Petkov * DCT1 -> F2x140.. 98b2b0c605SBorislav Petkov * 99b2b0c605SBorislav Petkov * F15h: we select which DCT we access using F1x10C[DctCfgSel] 100b2b0c605SBorislav Petkov * 101b2b0c605SBorislav Petkov */ 102b2b0c605SBorislav Petkov static int k8_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val, 103b2b0c605SBorislav Petkov const char *func) 104b2b0c605SBorislav Petkov { 105b2b0c605SBorislav Petkov if (addr >= 0x100) 106b2b0c605SBorislav Petkov return -EINVAL; 107b2b0c605SBorislav Petkov 108b2b0c605SBorislav Petkov return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func); 109b2b0c605SBorislav Petkov } 110b2b0c605SBorislav Petkov 111b2b0c605SBorislav Petkov static int f10_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val, 112b2b0c605SBorislav Petkov const char *func) 113b2b0c605SBorislav Petkov { 114b2b0c605SBorislav Petkov return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func); 115b2b0c605SBorislav Petkov } 116b2b0c605SBorislav Petkov 117b2b0c605SBorislav Petkov static int f15_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val, 118b2b0c605SBorislav Petkov const char *func) 119b2b0c605SBorislav Petkov { 120b2b0c605SBorislav Petkov u32 reg = 0; 121b2b0c605SBorislav Petkov u8 dct = 0; 122b2b0c605SBorislav Petkov 123b2b0c605SBorislav Petkov if (addr >= 0x140 && addr <= 0x1a0) { 124b2b0c605SBorislav Petkov dct = 1; 125b2b0c605SBorislav Petkov addr -= 0x100; 126b2b0c605SBorislav Petkov } 127b2b0c605SBorislav Petkov 128b2b0c605SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, ®); 129b2b0c605SBorislav Petkov reg &= 0xfffffffe; 130b2b0c605SBorislav Petkov reg |= dct; 131b2b0c605SBorislav Petkov amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg); 132b2b0c605SBorislav Petkov 133b2b0c605SBorislav Petkov return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func); 134b2b0c605SBorislav Petkov } 135b2b0c605SBorislav Petkov 136b70ef010SBorislav Petkov /* 1372bc65418SDoug Thompson * Memory scrubber control interface. For K8, memory scrubbing is handled by 1382bc65418SDoug Thompson * hardware and can involve L2 cache, dcache as well as the main memory. With 1392bc65418SDoug Thompson * F10, this is extended to L3 cache scrubbing on CPU models sporting that 1402bc65418SDoug Thompson * functionality. 1412bc65418SDoug Thompson * 1422bc65418SDoug Thompson * This causes the "units" for the scrubbing speed to vary from 64 byte blocks 1432bc65418SDoug Thompson * (dram) over to cache lines. This is nasty, so we will use bandwidth in 1442bc65418SDoug Thompson * bytes/sec for the setting. 1452bc65418SDoug Thompson * 1462bc65418SDoug Thompson * Currently, we only do dram scrubbing. If the scrubbing is done in software on 1472bc65418SDoug Thompson * other archs, we might not have access to the caches directly. 1482bc65418SDoug Thompson */ 1492bc65418SDoug Thompson 1502bc65418SDoug Thompson /* 1512bc65418SDoug Thompson * scan the scrub rate mapping table for a close or matching bandwidth value to 1522bc65418SDoug Thompson * issue. If requested is too big, then use last maximum value found. 1532bc65418SDoug Thompson */ 154395ae783SBorislav Petkov static int __amd64_set_scrub_rate(struct pci_dev *ctl, u32 new_bw, u32 min_rate) 1552bc65418SDoug Thompson { 1562bc65418SDoug Thompson u32 scrubval; 1572bc65418SDoug Thompson int i; 1582bc65418SDoug Thompson 1592bc65418SDoug Thompson /* 1602bc65418SDoug Thompson * map the configured rate (new_bw) to a value specific to the AMD64 1612bc65418SDoug Thompson * memory controller and apply to register. Search for the first 1622bc65418SDoug Thompson * bandwidth entry that is greater or equal than the setting requested 1632bc65418SDoug Thompson * and program that. If at last entry, turn off DRAM scrubbing. 1642bc65418SDoug Thompson */ 1652bc65418SDoug Thompson for (i = 0; i < ARRAY_SIZE(scrubrates); i++) { 1662bc65418SDoug Thompson /* 1672bc65418SDoug Thompson * skip scrub rates which aren't recommended 1682bc65418SDoug Thompson * (see F10 BKDG, F3x58) 1692bc65418SDoug Thompson */ 170395ae783SBorislav Petkov if (scrubrates[i].scrubval < min_rate) 1712bc65418SDoug Thompson continue; 1722bc65418SDoug Thompson 1732bc65418SDoug Thompson if (scrubrates[i].bandwidth <= new_bw) 1742bc65418SDoug Thompson break; 1752bc65418SDoug Thompson 1762bc65418SDoug Thompson /* 1772bc65418SDoug Thompson * if no suitable bandwidth found, turn off DRAM scrubbing 1782bc65418SDoug Thompson * entirely by falling back to the last element in the 1792bc65418SDoug Thompson * scrubrates array. 1802bc65418SDoug Thompson */ 1812bc65418SDoug Thompson } 1822bc65418SDoug Thompson 1832bc65418SDoug Thompson scrubval = scrubrates[i].scrubval; 1842bc65418SDoug Thompson 1855980bb9cSBorislav Petkov pci_write_bits32(ctl, SCRCTRL, scrubval, 0x001F); 1862bc65418SDoug Thompson 18739094443SBorislav Petkov if (scrubval) 18839094443SBorislav Petkov return scrubrates[i].bandwidth; 18939094443SBorislav Petkov 1902bc65418SDoug Thompson return 0; 1912bc65418SDoug Thompson } 1922bc65418SDoug Thompson 193395ae783SBorislav Petkov static int amd64_set_scrub_rate(struct mem_ctl_info *mci, u32 bw) 1942bc65418SDoug Thompson { 1952bc65418SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 19687b3e0e6SBorislav Petkov u32 min_scrubrate = 0x5; 1972bc65418SDoug Thompson 19887b3e0e6SBorislav Petkov if (boot_cpu_data.x86 == 0xf) 19987b3e0e6SBorislav Petkov min_scrubrate = 0x0; 20087b3e0e6SBorislav Petkov 20187b3e0e6SBorislav Petkov return __amd64_set_scrub_rate(pvt->F3, bw, min_scrubrate); 2022bc65418SDoug Thompson } 2032bc65418SDoug Thompson 20439094443SBorislav Petkov static int amd64_get_scrub_rate(struct mem_ctl_info *mci) 2052bc65418SDoug Thompson { 2062bc65418SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 2072bc65418SDoug Thompson u32 scrubval = 0; 20839094443SBorislav Petkov int i, retval = -EINVAL; 2092bc65418SDoug Thompson 2105980bb9cSBorislav Petkov amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval); 2112bc65418SDoug Thompson 2122bc65418SDoug Thompson scrubval = scrubval & 0x001F; 2132bc65418SDoug Thompson 214926311fdSRoel Kluin for (i = 0; i < ARRAY_SIZE(scrubrates); i++) { 2152bc65418SDoug Thompson if (scrubrates[i].scrubval == scrubval) { 21639094443SBorislav Petkov retval = scrubrates[i].bandwidth; 2172bc65418SDoug Thompson break; 2182bc65418SDoug Thompson } 2192bc65418SDoug Thompson } 22039094443SBorislav Petkov return retval; 2212bc65418SDoug Thompson } 2222bc65418SDoug Thompson 2236775763aSDoug Thompson /* 2247f19bf75SBorislav Petkov * returns true if the SysAddr given by sys_addr matches the 2257f19bf75SBorislav Petkov * DRAM base/limit associated with node_id 2266775763aSDoug Thompson */ 227b487c33eSBorislav Petkov static bool amd64_base_limit_match(struct amd64_pvt *pvt, u64 sys_addr, 228b487c33eSBorislav Petkov unsigned nid) 2296775763aSDoug Thompson { 2307f19bf75SBorislav Petkov u64 addr; 2316775763aSDoug Thompson 2326775763aSDoug Thompson /* The K8 treats this as a 40-bit value. However, bits 63-40 will be 2336775763aSDoug Thompson * all ones if the most significant implemented address bit is 1. 2346775763aSDoug Thompson * Here we discard bits 63-40. See section 3.4.2 of AMD publication 2356775763aSDoug Thompson * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1 2366775763aSDoug Thompson * Application Programming. 2376775763aSDoug Thompson */ 2386775763aSDoug Thompson addr = sys_addr & 0x000000ffffffffffull; 2396775763aSDoug Thompson 2407f19bf75SBorislav Petkov return ((addr >= get_dram_base(pvt, nid)) && 2417f19bf75SBorislav Petkov (addr <= get_dram_limit(pvt, nid))); 2426775763aSDoug Thompson } 2436775763aSDoug Thompson 2446775763aSDoug Thompson /* 2456775763aSDoug Thompson * Attempt to map a SysAddr to a node. On success, return a pointer to the 2466775763aSDoug Thompson * mem_ctl_info structure for the node that the SysAddr maps to. 2476775763aSDoug Thompson * 2486775763aSDoug Thompson * On failure, return NULL. 2496775763aSDoug Thompson */ 2506775763aSDoug Thompson static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci, 2516775763aSDoug Thompson u64 sys_addr) 2526775763aSDoug Thompson { 2536775763aSDoug Thompson struct amd64_pvt *pvt; 254b487c33eSBorislav Petkov unsigned node_id; 2556775763aSDoug Thompson u32 intlv_en, bits; 2566775763aSDoug Thompson 2576775763aSDoug Thompson /* 2586775763aSDoug Thompson * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section 2596775763aSDoug Thompson * 3.4.4.2) registers to map the SysAddr to a node ID. 2606775763aSDoug Thompson */ 2616775763aSDoug Thompson pvt = mci->pvt_info; 2626775763aSDoug Thompson 2636775763aSDoug Thompson /* 2646775763aSDoug Thompson * The value of this field should be the same for all DRAM Base 2656775763aSDoug Thompson * registers. Therefore we arbitrarily choose to read it from the 2666775763aSDoug Thompson * register for node 0. 2676775763aSDoug Thompson */ 2687f19bf75SBorislav Petkov intlv_en = dram_intlv_en(pvt, 0); 2696775763aSDoug Thompson 2706775763aSDoug Thompson if (intlv_en == 0) { 2717f19bf75SBorislav Petkov for (node_id = 0; node_id < DRAM_RANGES; node_id++) { 2726775763aSDoug Thompson if (amd64_base_limit_match(pvt, sys_addr, node_id)) 2736775763aSDoug Thompson goto found; 2746775763aSDoug Thompson } 2758edc5445SBorislav Petkov goto err_no_match; 2768edc5445SBorislav Petkov } 2776775763aSDoug Thompson 27872f158feSBorislav Petkov if (unlikely((intlv_en != 0x01) && 27972f158feSBorislav Petkov (intlv_en != 0x03) && 28072f158feSBorislav Petkov (intlv_en != 0x07))) { 28124f9a7feSBorislav Petkov amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en); 2826775763aSDoug Thompson return NULL; 2836775763aSDoug Thompson } 2846775763aSDoug Thompson 2856775763aSDoug Thompson bits = (((u32) sys_addr) >> 12) & intlv_en; 2866775763aSDoug Thompson 2876775763aSDoug Thompson for (node_id = 0; ; ) { 2887f19bf75SBorislav Petkov if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits) 2896775763aSDoug Thompson break; /* intlv_sel field matches */ 2906775763aSDoug Thompson 2917f19bf75SBorislav Petkov if (++node_id >= DRAM_RANGES) 2926775763aSDoug Thompson goto err_no_match; 2936775763aSDoug Thompson } 2946775763aSDoug Thompson 2956775763aSDoug Thompson /* sanity test for sys_addr */ 2966775763aSDoug Thompson if (unlikely(!amd64_base_limit_match(pvt, sys_addr, node_id))) { 29724f9a7feSBorislav Petkov amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address" 29824f9a7feSBorislav Petkov "range for node %d with node interleaving enabled.\n", 2998edc5445SBorislav Petkov __func__, sys_addr, node_id); 3006775763aSDoug Thompson return NULL; 3016775763aSDoug Thompson } 3026775763aSDoug Thompson 3036775763aSDoug Thompson found: 304b487c33eSBorislav Petkov return edac_mc_find((int)node_id); 3056775763aSDoug Thompson 3066775763aSDoug Thompson err_no_match: 3076775763aSDoug Thompson debugf2("sys_addr 0x%lx doesn't match any node\n", 3086775763aSDoug Thompson (unsigned long)sys_addr); 3096775763aSDoug Thompson 3106775763aSDoug Thompson return NULL; 3116775763aSDoug Thompson } 312e2ce7255SDoug Thompson 313e2ce7255SDoug Thompson /* 31411c75eadSBorislav Petkov * compute the CS base address of the @csrow on the DRAM controller @dct. 31511c75eadSBorislav Petkov * For details see F2x[5C:40] in the processor's BKDG 316e2ce7255SDoug Thompson */ 31711c75eadSBorislav Petkov static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct, 31811c75eadSBorislav Petkov u64 *base, u64 *mask) 319e2ce7255SDoug Thompson { 32011c75eadSBorislav Petkov u64 csbase, csmask, base_bits, mask_bits; 32111c75eadSBorislav Petkov u8 addr_shift; 32211c75eadSBorislav Petkov 32311c75eadSBorislav Petkov if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) { 32411c75eadSBorislav Petkov csbase = pvt->csels[dct].csbases[csrow]; 32511c75eadSBorislav Petkov csmask = pvt->csels[dct].csmasks[csrow]; 32611c75eadSBorislav Petkov base_bits = GENMASK(21, 31) | GENMASK(9, 15); 32711c75eadSBorislav Petkov mask_bits = GENMASK(21, 29) | GENMASK(9, 15); 32811c75eadSBorislav Petkov addr_shift = 4; 32911c75eadSBorislav Petkov } else { 33011c75eadSBorislav Petkov csbase = pvt->csels[dct].csbases[csrow]; 33111c75eadSBorislav Petkov csmask = pvt->csels[dct].csmasks[csrow >> 1]; 33211c75eadSBorislav Petkov addr_shift = 8; 33311c75eadSBorislav Petkov 33411c75eadSBorislav Petkov if (boot_cpu_data.x86 == 0x15) 33511c75eadSBorislav Petkov base_bits = mask_bits = GENMASK(19,30) | GENMASK(5,13); 33611c75eadSBorislav Petkov else 33711c75eadSBorislav Petkov base_bits = mask_bits = GENMASK(19,28) | GENMASK(5,13); 338e2ce7255SDoug Thompson } 339e2ce7255SDoug Thompson 34011c75eadSBorislav Petkov *base = (csbase & base_bits) << addr_shift; 341e2ce7255SDoug Thompson 34211c75eadSBorislav Petkov *mask = ~0ULL; 34311c75eadSBorislav Petkov /* poke holes for the csmask */ 34411c75eadSBorislav Petkov *mask &= ~(mask_bits << addr_shift); 34511c75eadSBorislav Petkov /* OR them in */ 34611c75eadSBorislav Petkov *mask |= (csmask & mask_bits) << addr_shift; 347e2ce7255SDoug Thompson } 348e2ce7255SDoug Thompson 34911c75eadSBorislav Petkov #define for_each_chip_select(i, dct, pvt) \ 35011c75eadSBorislav Petkov for (i = 0; i < pvt->csels[dct].b_cnt; i++) 35111c75eadSBorislav Petkov 352614ec9d8SBorislav Petkov #define chip_select_base(i, dct, pvt) \ 353614ec9d8SBorislav Petkov pvt->csels[dct].csbases[i] 354614ec9d8SBorislav Petkov 35511c75eadSBorislav Petkov #define for_each_chip_select_mask(i, dct, pvt) \ 35611c75eadSBorislav Petkov for (i = 0; i < pvt->csels[dct].m_cnt; i++) 35711c75eadSBorislav Petkov 358e2ce7255SDoug Thompson /* 359e2ce7255SDoug Thompson * @input_addr is an InputAddr associated with the node given by mci. Return the 360e2ce7255SDoug Thompson * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr). 361e2ce7255SDoug Thompson */ 362e2ce7255SDoug Thompson static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr) 363e2ce7255SDoug Thompson { 364e2ce7255SDoug Thompson struct amd64_pvt *pvt; 365e2ce7255SDoug Thompson int csrow; 366e2ce7255SDoug Thompson u64 base, mask; 367e2ce7255SDoug Thompson 368e2ce7255SDoug Thompson pvt = mci->pvt_info; 369e2ce7255SDoug Thompson 37011c75eadSBorislav Petkov for_each_chip_select(csrow, 0, pvt) { 37111c75eadSBorislav Petkov if (!csrow_enabled(csrow, 0, pvt)) 372e2ce7255SDoug Thompson continue; 373e2ce7255SDoug Thompson 37411c75eadSBorislav Petkov get_cs_base_and_mask(pvt, csrow, 0, &base, &mask); 37511c75eadSBorislav Petkov 37611c75eadSBorislav Petkov mask = ~mask; 377e2ce7255SDoug Thompson 378e2ce7255SDoug Thompson if ((input_addr & mask) == (base & mask)) { 379e2ce7255SDoug Thompson debugf2("InputAddr 0x%lx matches csrow %d (node %d)\n", 380e2ce7255SDoug Thompson (unsigned long)input_addr, csrow, 381e2ce7255SDoug Thompson pvt->mc_node_id); 382e2ce7255SDoug Thompson 383e2ce7255SDoug Thompson return csrow; 384e2ce7255SDoug Thompson } 385e2ce7255SDoug Thompson } 386e2ce7255SDoug Thompson debugf2("no matching csrow for InputAddr 0x%lx (MC node %d)\n", 387e2ce7255SDoug Thompson (unsigned long)input_addr, pvt->mc_node_id); 388e2ce7255SDoug Thompson 389e2ce7255SDoug Thompson return -1; 390e2ce7255SDoug Thompson } 391e2ce7255SDoug Thompson 392e2ce7255SDoug Thompson /* 393e2ce7255SDoug Thompson * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094) 394e2ce7255SDoug Thompson * for the node represented by mci. Info is passed back in *hole_base, 395e2ce7255SDoug Thompson * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if 396e2ce7255SDoug Thompson * info is invalid. Info may be invalid for either of the following reasons: 397e2ce7255SDoug Thompson * 398e2ce7255SDoug Thompson * - The revision of the node is not E or greater. In this case, the DRAM Hole 399e2ce7255SDoug Thompson * Address Register does not exist. 400e2ce7255SDoug Thompson * 401e2ce7255SDoug Thompson * - The DramHoleValid bit is cleared in the DRAM Hole Address Register, 402e2ce7255SDoug Thompson * indicating that its contents are not valid. 403e2ce7255SDoug Thompson * 404e2ce7255SDoug Thompson * The values passed back in *hole_base, *hole_offset, and *hole_size are 405e2ce7255SDoug Thompson * complete 32-bit values despite the fact that the bitfields in the DHAR 406e2ce7255SDoug Thompson * only represent bits 31-24 of the base and offset values. 407e2ce7255SDoug Thompson */ 408e2ce7255SDoug Thompson int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base, 409e2ce7255SDoug Thompson u64 *hole_offset, u64 *hole_size) 410e2ce7255SDoug Thompson { 411e2ce7255SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 412e2ce7255SDoug Thompson u64 base; 413e2ce7255SDoug Thompson 414e2ce7255SDoug Thompson /* only revE and later have the DRAM Hole Address Register */ 4151433eb99SBorislav Petkov if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_E) { 416e2ce7255SDoug Thompson debugf1(" revision %d for node %d does not support DHAR\n", 417e2ce7255SDoug Thompson pvt->ext_model, pvt->mc_node_id); 418e2ce7255SDoug Thompson return 1; 419e2ce7255SDoug Thompson } 420e2ce7255SDoug Thompson 421bc21fa57SBorislav Petkov /* valid for Fam10h and above */ 422c8e518d5SBorislav Petkov if (boot_cpu_data.x86 >= 0x10 && !dhar_mem_hoist_valid(pvt)) { 423e2ce7255SDoug Thompson debugf1(" Dram Memory Hoisting is DISABLED on this system\n"); 424e2ce7255SDoug Thompson return 1; 425e2ce7255SDoug Thompson } 426e2ce7255SDoug Thompson 427c8e518d5SBorislav Petkov if (!dhar_valid(pvt)) { 428e2ce7255SDoug Thompson debugf1(" Dram Memory Hoisting is DISABLED on this node %d\n", 429e2ce7255SDoug Thompson pvt->mc_node_id); 430e2ce7255SDoug Thompson return 1; 431e2ce7255SDoug Thompson } 432e2ce7255SDoug Thompson 433e2ce7255SDoug Thompson /* This node has Memory Hoisting */ 434e2ce7255SDoug Thompson 435e2ce7255SDoug Thompson /* +------------------+--------------------+--------------------+----- 436e2ce7255SDoug Thompson * | memory | DRAM hole | relocated | 437e2ce7255SDoug Thompson * | [0, (x - 1)] | [x, 0xffffffff] | addresses from | 438e2ce7255SDoug Thompson * | | | DRAM hole | 439e2ce7255SDoug Thompson * | | | [0x100000000, | 440e2ce7255SDoug Thompson * | | | (0x100000000+ | 441e2ce7255SDoug Thompson * | | | (0xffffffff-x))] | 442e2ce7255SDoug Thompson * +------------------+--------------------+--------------------+----- 443e2ce7255SDoug Thompson * 444e2ce7255SDoug Thompson * Above is a diagram of physical memory showing the DRAM hole and the 445e2ce7255SDoug Thompson * relocated addresses from the DRAM hole. As shown, the DRAM hole 446e2ce7255SDoug Thompson * starts at address x (the base address) and extends through address 447e2ce7255SDoug Thompson * 0xffffffff. The DRAM Hole Address Register (DHAR) relocates the 448e2ce7255SDoug Thompson * addresses in the hole so that they start at 0x100000000. 449e2ce7255SDoug Thompson */ 450e2ce7255SDoug Thompson 451bc21fa57SBorislav Petkov base = dhar_base(pvt); 452e2ce7255SDoug Thompson 453e2ce7255SDoug Thompson *hole_base = base; 454e2ce7255SDoug Thompson *hole_size = (0x1ull << 32) - base; 455e2ce7255SDoug Thompson 456e2ce7255SDoug Thompson if (boot_cpu_data.x86 > 0xf) 457bc21fa57SBorislav Petkov *hole_offset = f10_dhar_offset(pvt); 458e2ce7255SDoug Thompson else 459bc21fa57SBorislav Petkov *hole_offset = k8_dhar_offset(pvt); 460e2ce7255SDoug Thompson 461e2ce7255SDoug Thompson debugf1(" DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n", 462e2ce7255SDoug Thompson pvt->mc_node_id, (unsigned long)*hole_base, 463e2ce7255SDoug Thompson (unsigned long)*hole_offset, (unsigned long)*hole_size); 464e2ce7255SDoug Thompson 465e2ce7255SDoug Thompson return 0; 466e2ce7255SDoug Thompson } 467e2ce7255SDoug Thompson EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info); 468e2ce7255SDoug Thompson 46993c2df58SDoug Thompson /* 47093c2df58SDoug Thompson * Return the DramAddr that the SysAddr given by @sys_addr maps to. It is 47193c2df58SDoug Thompson * assumed that sys_addr maps to the node given by mci. 47293c2df58SDoug Thompson * 47393c2df58SDoug Thompson * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section 47493c2df58SDoug Thompson * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a 47593c2df58SDoug Thompson * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled, 47693c2df58SDoug Thompson * then it is also involved in translating a SysAddr to a DramAddr. Sections 47793c2df58SDoug Thompson * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting. 47893c2df58SDoug Thompson * These parts of the documentation are unclear. I interpret them as follows: 47993c2df58SDoug Thompson * 48093c2df58SDoug Thompson * When node n receives a SysAddr, it processes the SysAddr as follows: 48193c2df58SDoug Thompson * 48293c2df58SDoug Thompson * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM 48393c2df58SDoug Thompson * Limit registers for node n. If the SysAddr is not within the range 48493c2df58SDoug Thompson * specified by the base and limit values, then node n ignores the Sysaddr 48593c2df58SDoug Thompson * (since it does not map to node n). Otherwise continue to step 2 below. 48693c2df58SDoug Thompson * 48793c2df58SDoug Thompson * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is 48893c2df58SDoug Thompson * disabled so skip to step 3 below. Otherwise see if the SysAddr is within 48993c2df58SDoug Thompson * the range of relocated addresses (starting at 0x100000000) from the DRAM 49093c2df58SDoug Thompson * hole. If not, skip to step 3 below. Else get the value of the 49193c2df58SDoug Thompson * DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the 49293c2df58SDoug Thompson * offset defined by this value from the SysAddr. 49393c2df58SDoug Thompson * 49493c2df58SDoug Thompson * 3. Obtain the base address for node n from the DRAMBase field of the DRAM 49593c2df58SDoug Thompson * Base register for node n. To obtain the DramAddr, subtract the base 49693c2df58SDoug Thompson * address from the SysAddr, as shown near the start of section 3.4.4 (p.70). 49793c2df58SDoug Thompson */ 49893c2df58SDoug Thompson static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr) 49993c2df58SDoug Thompson { 5007f19bf75SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 50193c2df58SDoug Thompson u64 dram_base, hole_base, hole_offset, hole_size, dram_addr; 50293c2df58SDoug Thompson int ret = 0; 50393c2df58SDoug Thompson 5047f19bf75SBorislav Petkov dram_base = get_dram_base(pvt, pvt->mc_node_id); 50593c2df58SDoug Thompson 50693c2df58SDoug Thompson ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset, 50793c2df58SDoug Thompson &hole_size); 50893c2df58SDoug Thompson if (!ret) { 50993c2df58SDoug Thompson if ((sys_addr >= (1ull << 32)) && 51093c2df58SDoug Thompson (sys_addr < ((1ull << 32) + hole_size))) { 51193c2df58SDoug Thompson /* use DHAR to translate SysAddr to DramAddr */ 51293c2df58SDoug Thompson dram_addr = sys_addr - hole_offset; 51393c2df58SDoug Thompson 51493c2df58SDoug Thompson debugf2("using DHAR to translate SysAddr 0x%lx to " 51593c2df58SDoug Thompson "DramAddr 0x%lx\n", 51693c2df58SDoug Thompson (unsigned long)sys_addr, 51793c2df58SDoug Thompson (unsigned long)dram_addr); 51893c2df58SDoug Thompson 51993c2df58SDoug Thompson return dram_addr; 52093c2df58SDoug Thompson } 52193c2df58SDoug Thompson } 52293c2df58SDoug Thompson 52393c2df58SDoug Thompson /* 52493c2df58SDoug Thompson * Translate the SysAddr to a DramAddr as shown near the start of 52593c2df58SDoug Thompson * section 3.4.4 (p. 70). Although sys_addr is a 64-bit value, the k8 52693c2df58SDoug Thompson * only deals with 40-bit values. Therefore we discard bits 63-40 of 52793c2df58SDoug Thompson * sys_addr below. If bit 39 of sys_addr is 1 then the bits we 52893c2df58SDoug Thompson * discard are all 1s. Otherwise the bits we discard are all 0s. See 52993c2df58SDoug Thompson * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture 53093c2df58SDoug Thompson * Programmer's Manual Volume 1 Application Programming. 53193c2df58SDoug Thompson */ 532f678b8ccSBorislav Petkov dram_addr = (sys_addr & GENMASK(0, 39)) - dram_base; 53393c2df58SDoug Thompson 53493c2df58SDoug Thompson debugf2("using DRAM Base register to translate SysAddr 0x%lx to " 53593c2df58SDoug Thompson "DramAddr 0x%lx\n", (unsigned long)sys_addr, 53693c2df58SDoug Thompson (unsigned long)dram_addr); 53793c2df58SDoug Thompson return dram_addr; 53893c2df58SDoug Thompson } 53993c2df58SDoug Thompson 54093c2df58SDoug Thompson /* 54193c2df58SDoug Thompson * @intlv_en is the value of the IntlvEn field from a DRAM Base register 54293c2df58SDoug Thompson * (section 3.4.4.1). Return the number of bits from a SysAddr that are used 54393c2df58SDoug Thompson * for node interleaving. 54493c2df58SDoug Thompson */ 54593c2df58SDoug Thompson static int num_node_interleave_bits(unsigned intlv_en) 54693c2df58SDoug Thompson { 54793c2df58SDoug Thompson static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 }; 54893c2df58SDoug Thompson int n; 54993c2df58SDoug Thompson 55093c2df58SDoug Thompson BUG_ON(intlv_en > 7); 55193c2df58SDoug Thompson n = intlv_shift_table[intlv_en]; 55293c2df58SDoug Thompson return n; 55393c2df58SDoug Thompson } 55493c2df58SDoug Thompson 55593c2df58SDoug Thompson /* Translate the DramAddr given by @dram_addr to an InputAddr. */ 55693c2df58SDoug Thompson static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr) 55793c2df58SDoug Thompson { 55893c2df58SDoug Thompson struct amd64_pvt *pvt; 55993c2df58SDoug Thompson int intlv_shift; 56093c2df58SDoug Thompson u64 input_addr; 56193c2df58SDoug Thompson 56293c2df58SDoug Thompson pvt = mci->pvt_info; 56393c2df58SDoug Thompson 56493c2df58SDoug Thompson /* 56593c2df58SDoug Thompson * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E) 56693c2df58SDoug Thompson * concerning translating a DramAddr to an InputAddr. 56793c2df58SDoug Thompson */ 5687f19bf75SBorislav Petkov intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0)); 569f678b8ccSBorislav Petkov input_addr = ((dram_addr >> intlv_shift) & GENMASK(12, 35)) + 57093c2df58SDoug Thompson (dram_addr & 0xfff); 57193c2df58SDoug Thompson 57293c2df58SDoug Thompson debugf2(" Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n", 57393c2df58SDoug Thompson intlv_shift, (unsigned long)dram_addr, 57493c2df58SDoug Thompson (unsigned long)input_addr); 57593c2df58SDoug Thompson 57693c2df58SDoug Thompson return input_addr; 57793c2df58SDoug Thompson } 57893c2df58SDoug Thompson 57993c2df58SDoug Thompson /* 58093c2df58SDoug Thompson * Translate the SysAddr represented by @sys_addr to an InputAddr. It is 58193c2df58SDoug Thompson * assumed that @sys_addr maps to the node given by mci. 58293c2df58SDoug Thompson */ 58393c2df58SDoug Thompson static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr) 58493c2df58SDoug Thompson { 58593c2df58SDoug Thompson u64 input_addr; 58693c2df58SDoug Thompson 58793c2df58SDoug Thompson input_addr = 58893c2df58SDoug Thompson dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr)); 58993c2df58SDoug Thompson 59093c2df58SDoug Thompson debugf2("SysAdddr 0x%lx translates to InputAddr 0x%lx\n", 59193c2df58SDoug Thompson (unsigned long)sys_addr, (unsigned long)input_addr); 59293c2df58SDoug Thompson 59393c2df58SDoug Thompson return input_addr; 59493c2df58SDoug Thompson } 59593c2df58SDoug Thompson 59693c2df58SDoug Thompson 59793c2df58SDoug Thompson /* 59893c2df58SDoug Thompson * @input_addr is an InputAddr associated with the node represented by mci. 59993c2df58SDoug Thompson * Translate @input_addr to a DramAddr and return the result. 60093c2df58SDoug Thompson */ 60193c2df58SDoug Thompson static u64 input_addr_to_dram_addr(struct mem_ctl_info *mci, u64 input_addr) 60293c2df58SDoug Thompson { 60393c2df58SDoug Thompson struct amd64_pvt *pvt; 604b487c33eSBorislav Petkov unsigned node_id, intlv_shift; 60593c2df58SDoug Thompson u64 bits, dram_addr; 60693c2df58SDoug Thompson u32 intlv_sel; 60793c2df58SDoug Thompson 60893c2df58SDoug Thompson /* 60993c2df58SDoug Thompson * Near the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E) 61093c2df58SDoug Thompson * shows how to translate a DramAddr to an InputAddr. Here we reverse 61193c2df58SDoug Thompson * this procedure. When translating from a DramAddr to an InputAddr, the 61293c2df58SDoug Thompson * bits used for node interleaving are discarded. Here we recover these 61393c2df58SDoug Thompson * bits from the IntlvSel field of the DRAM Limit register (section 61493c2df58SDoug Thompson * 3.4.4.2) for the node that input_addr is associated with. 61593c2df58SDoug Thompson */ 61693c2df58SDoug Thompson pvt = mci->pvt_info; 61793c2df58SDoug Thompson node_id = pvt->mc_node_id; 618b487c33eSBorislav Petkov 619b487c33eSBorislav Petkov BUG_ON(node_id > 7); 62093c2df58SDoug Thompson 6217f19bf75SBorislav Petkov intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0)); 62293c2df58SDoug Thompson if (intlv_shift == 0) { 62393c2df58SDoug Thompson debugf1(" InputAddr 0x%lx translates to DramAddr of " 62493c2df58SDoug Thompson "same value\n", (unsigned long)input_addr); 62593c2df58SDoug Thompson 62693c2df58SDoug Thompson return input_addr; 62793c2df58SDoug Thompson } 62893c2df58SDoug Thompson 629f678b8ccSBorislav Petkov bits = ((input_addr & GENMASK(12, 35)) << intlv_shift) + 63093c2df58SDoug Thompson (input_addr & 0xfff); 63193c2df58SDoug Thompson 6327f19bf75SBorislav Petkov intlv_sel = dram_intlv_sel(pvt, node_id) & ((1 << intlv_shift) - 1); 63393c2df58SDoug Thompson dram_addr = bits + (intlv_sel << 12); 63493c2df58SDoug Thompson 63593c2df58SDoug Thompson debugf1("InputAddr 0x%lx translates to DramAddr 0x%lx " 63693c2df58SDoug Thompson "(%d node interleave bits)\n", (unsigned long)input_addr, 63793c2df58SDoug Thompson (unsigned long)dram_addr, intlv_shift); 63893c2df58SDoug Thompson 63993c2df58SDoug Thompson return dram_addr; 64093c2df58SDoug Thompson } 64193c2df58SDoug Thompson 64293c2df58SDoug Thompson /* 64393c2df58SDoug Thompson * @dram_addr is a DramAddr that maps to the node represented by mci. Convert 64493c2df58SDoug Thompson * @dram_addr to a SysAddr. 64593c2df58SDoug Thompson */ 64693c2df58SDoug Thompson static u64 dram_addr_to_sys_addr(struct mem_ctl_info *mci, u64 dram_addr) 64793c2df58SDoug Thompson { 64893c2df58SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 6497f19bf75SBorislav Petkov u64 hole_base, hole_offset, hole_size, base, sys_addr; 65093c2df58SDoug Thompson int ret = 0; 65193c2df58SDoug Thompson 65293c2df58SDoug Thompson ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset, 65393c2df58SDoug Thompson &hole_size); 65493c2df58SDoug Thompson if (!ret) { 65593c2df58SDoug Thompson if ((dram_addr >= hole_base) && 65693c2df58SDoug Thompson (dram_addr < (hole_base + hole_size))) { 65793c2df58SDoug Thompson sys_addr = dram_addr + hole_offset; 65893c2df58SDoug Thompson 65993c2df58SDoug Thompson debugf1("using DHAR to translate DramAddr 0x%lx to " 66093c2df58SDoug Thompson "SysAddr 0x%lx\n", (unsigned long)dram_addr, 66193c2df58SDoug Thompson (unsigned long)sys_addr); 66293c2df58SDoug Thompson 66393c2df58SDoug Thompson return sys_addr; 66493c2df58SDoug Thompson } 66593c2df58SDoug Thompson } 66693c2df58SDoug Thompson 6677f19bf75SBorislav Petkov base = get_dram_base(pvt, pvt->mc_node_id); 66893c2df58SDoug Thompson sys_addr = dram_addr + base; 66993c2df58SDoug Thompson 67093c2df58SDoug Thompson /* 67193c2df58SDoug Thompson * The sys_addr we have computed up to this point is a 40-bit value 67293c2df58SDoug Thompson * because the k8 deals with 40-bit values. However, the value we are 67393c2df58SDoug Thompson * supposed to return is a full 64-bit physical address. The AMD 67493c2df58SDoug Thompson * x86-64 architecture specifies that the most significant implemented 67593c2df58SDoug Thompson * address bit through bit 63 of a physical address must be either all 67693c2df58SDoug Thompson * 0s or all 1s. Therefore we sign-extend the 40-bit sys_addr to a 67793c2df58SDoug Thompson * 64-bit value below. See section 3.4.2 of AMD publication 24592: 67893c2df58SDoug Thompson * AMD x86-64 Architecture Programmer's Manual Volume 1 Application 67993c2df58SDoug Thompson * Programming. 68093c2df58SDoug Thompson */ 68193c2df58SDoug Thompson sys_addr |= ~((sys_addr & (1ull << 39)) - 1); 68293c2df58SDoug Thompson 68393c2df58SDoug Thompson debugf1(" Node %d, DramAddr 0x%lx to SysAddr 0x%lx\n", 68493c2df58SDoug Thompson pvt->mc_node_id, (unsigned long)dram_addr, 68593c2df58SDoug Thompson (unsigned long)sys_addr); 68693c2df58SDoug Thompson 68793c2df58SDoug Thompson return sys_addr; 68893c2df58SDoug Thompson } 68993c2df58SDoug Thompson 69093c2df58SDoug Thompson /* 69193c2df58SDoug Thompson * @input_addr is an InputAddr associated with the node given by mci. Translate 69293c2df58SDoug Thompson * @input_addr to a SysAddr. 69393c2df58SDoug Thompson */ 69493c2df58SDoug Thompson static inline u64 input_addr_to_sys_addr(struct mem_ctl_info *mci, 69593c2df58SDoug Thompson u64 input_addr) 69693c2df58SDoug Thompson { 69793c2df58SDoug Thompson return dram_addr_to_sys_addr(mci, 69893c2df58SDoug Thompson input_addr_to_dram_addr(mci, input_addr)); 69993c2df58SDoug Thompson } 70093c2df58SDoug Thompson 70193c2df58SDoug Thompson /* 70293c2df58SDoug Thompson * Find the minimum and maximum InputAddr values that map to the given @csrow. 70393c2df58SDoug Thompson * Pass back these values in *input_addr_min and *input_addr_max. 70493c2df58SDoug Thompson */ 70593c2df58SDoug Thompson static void find_csrow_limits(struct mem_ctl_info *mci, int csrow, 70693c2df58SDoug Thompson u64 *input_addr_min, u64 *input_addr_max) 70793c2df58SDoug Thompson { 70893c2df58SDoug Thompson struct amd64_pvt *pvt; 70993c2df58SDoug Thompson u64 base, mask; 71093c2df58SDoug Thompson 71193c2df58SDoug Thompson pvt = mci->pvt_info; 71211c75eadSBorislav Petkov BUG_ON((csrow < 0) || (csrow >= pvt->csels[0].b_cnt)); 71393c2df58SDoug Thompson 71411c75eadSBorislav Petkov get_cs_base_and_mask(pvt, csrow, 0, &base, &mask); 71593c2df58SDoug Thompson 71693c2df58SDoug Thompson *input_addr_min = base & ~mask; 71711c75eadSBorislav Petkov *input_addr_max = base | mask; 71893c2df58SDoug Thompson } 71993c2df58SDoug Thompson 72093c2df58SDoug Thompson /* Map the Error address to a PAGE and PAGE OFFSET. */ 72193c2df58SDoug Thompson static inline void error_address_to_page_and_offset(u64 error_address, 72293c2df58SDoug Thompson u32 *page, u32 *offset) 72393c2df58SDoug Thompson { 72493c2df58SDoug Thompson *page = (u32) (error_address >> PAGE_SHIFT); 72593c2df58SDoug Thompson *offset = ((u32) error_address) & ~PAGE_MASK; 72693c2df58SDoug Thompson } 72793c2df58SDoug Thompson 72893c2df58SDoug Thompson /* 72993c2df58SDoug Thompson * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address 73093c2df58SDoug Thompson * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers 73193c2df58SDoug Thompson * of a node that detected an ECC memory error. mci represents the node that 73293c2df58SDoug Thompson * the error address maps to (possibly different from the node that detected 73393c2df58SDoug Thompson * the error). Return the number of the csrow that sys_addr maps to, or -1 on 73493c2df58SDoug Thompson * error. 73593c2df58SDoug Thompson */ 73693c2df58SDoug Thompson static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr) 73793c2df58SDoug Thompson { 73893c2df58SDoug Thompson int csrow; 73993c2df58SDoug Thompson 74093c2df58SDoug Thompson csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr)); 74193c2df58SDoug Thompson 74293c2df58SDoug Thompson if (csrow == -1) 74324f9a7feSBorislav Petkov amd64_mc_err(mci, "Failed to translate InputAddr to csrow for " 74493c2df58SDoug Thompson "address 0x%lx\n", (unsigned long)sys_addr); 74593c2df58SDoug Thompson return csrow; 74693c2df58SDoug Thompson } 747e2ce7255SDoug Thompson 748bfc04aecSBorislav Petkov static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16); 7492da11654SDoug Thompson 7502da11654SDoug Thompson /* 7512da11654SDoug Thompson * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs 7522da11654SDoug Thompson * are ECC capable. 7532da11654SDoug Thompson */ 7542da11654SDoug Thompson static enum edac_type amd64_determine_edac_cap(struct amd64_pvt *pvt) 7552da11654SDoug Thompson { 756cb328507SBorislav Petkov u8 bit; 757584fcff4SBorislav Petkov enum dev_type edac_cap = EDAC_FLAG_NONE; 7582da11654SDoug Thompson 7591433eb99SBorislav Petkov bit = (boot_cpu_data.x86 > 0xf || pvt->ext_model >= K8_REV_F) 7602da11654SDoug Thompson ? 19 7612da11654SDoug Thompson : 17; 7622da11654SDoug Thompson 763584fcff4SBorislav Petkov if (pvt->dclr0 & BIT(bit)) 7642da11654SDoug Thompson edac_cap = EDAC_FLAG_SECDED; 7652da11654SDoug Thompson 7662da11654SDoug Thompson return edac_cap; 7672da11654SDoug Thompson } 7682da11654SDoug Thompson 7698c671751SBorislav Petkov static void amd64_debug_display_dimm_sizes(struct amd64_pvt *, u8); 7702da11654SDoug Thompson 77168798e17SBorislav Petkov static void amd64_dump_dramcfg_low(u32 dclr, int chan) 77268798e17SBorislav Petkov { 77368798e17SBorislav Petkov debugf1("F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr); 77468798e17SBorislav Petkov 77568798e17SBorislav Petkov debugf1(" DIMM type: %sbuffered; all DIMMs support ECC: %s\n", 77668798e17SBorislav Petkov (dclr & BIT(16)) ? "un" : "", 77768798e17SBorislav Petkov (dclr & BIT(19)) ? "yes" : "no"); 77868798e17SBorislav Petkov 77968798e17SBorislav Petkov debugf1(" PAR/ERR parity: %s\n", 78068798e17SBorislav Petkov (dclr & BIT(8)) ? "enabled" : "disabled"); 78168798e17SBorislav Petkov 782cb328507SBorislav Petkov if (boot_cpu_data.x86 == 0x10) 78368798e17SBorislav Petkov debugf1(" DCT 128bit mode width: %s\n", 78468798e17SBorislav Petkov (dclr & BIT(11)) ? "128b" : "64b"); 78568798e17SBorislav Petkov 78668798e17SBorislav Petkov debugf1(" x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n", 78768798e17SBorislav Petkov (dclr & BIT(12)) ? "yes" : "no", 78868798e17SBorislav Petkov (dclr & BIT(13)) ? "yes" : "no", 78968798e17SBorislav Petkov (dclr & BIT(14)) ? "yes" : "no", 79068798e17SBorislav Petkov (dclr & BIT(15)) ? "yes" : "no"); 79168798e17SBorislav Petkov } 79268798e17SBorislav Petkov 7932da11654SDoug Thompson /* Display and decode various NB registers for debug purposes. */ 794b2b0c605SBorislav Petkov static void dump_misc_regs(struct amd64_pvt *pvt) 7952da11654SDoug Thompson { 79668798e17SBorislav Petkov debugf1("F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap); 7972da11654SDoug Thompson 79868798e17SBorislav Petkov debugf1(" NB two channel DRAM capable: %s\n", 7995980bb9cSBorislav Petkov (pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no"); 80068798e17SBorislav Petkov 80168798e17SBorislav Petkov debugf1(" ECC capable: %s, ChipKill ECC capable: %s\n", 8025980bb9cSBorislav Petkov (pvt->nbcap & NBCAP_SECDED) ? "yes" : "no", 8035980bb9cSBorislav Petkov (pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no"); 80468798e17SBorislav Petkov 80568798e17SBorislav Petkov amd64_dump_dramcfg_low(pvt->dclr0, 0); 8062da11654SDoug Thompson 8078de1d91eSBorislav Petkov debugf1("F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare); 8082da11654SDoug Thompson 8098de1d91eSBorislav Petkov debugf1("F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, " 8108de1d91eSBorislav Petkov "offset: 0x%08x\n", 811bc21fa57SBorislav Petkov pvt->dhar, dhar_base(pvt), 812bc21fa57SBorislav Petkov (boot_cpu_data.x86 == 0xf) ? k8_dhar_offset(pvt) 813bc21fa57SBorislav Petkov : f10_dhar_offset(pvt)); 8142da11654SDoug Thompson 815c8e518d5SBorislav Petkov debugf1(" DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no"); 8162da11654SDoug Thompson 8178c671751SBorislav Petkov amd64_debug_display_dimm_sizes(pvt, 0); 8184d796364SBorislav Petkov 8194d796364SBorislav Petkov /* everything below this point is Fam10h and above */ 8204d796364SBorislav Petkov if (boot_cpu_data.x86 == 0xf) 8212da11654SDoug Thompson return; 8224d796364SBorislav Petkov 8238c671751SBorislav Petkov amd64_debug_display_dimm_sizes(pvt, 1); 8242da11654SDoug Thompson 825a3b7db09SBorislav Petkov amd64_info("using %s syndromes.\n", ((pvt->ecc_sym_sz == 8) ? "x8" : "x4")); 826ad6a32e9SBorislav Petkov 8278de1d91eSBorislav Petkov /* Only if NOT ganged does dclr1 have valid info */ 82868798e17SBorislav Petkov if (!dct_ganging_enabled(pvt)) 82968798e17SBorislav Petkov amd64_dump_dramcfg_low(pvt->dclr1, 1); 8302da11654SDoug Thompson } 8312da11654SDoug Thompson 83294be4bffSDoug Thompson /* 83311c75eadSBorislav Petkov * see BKDG, F2x[1,0][5C:40], F2[1,0][6C:60] 83494be4bffSDoug Thompson */ 83511c75eadSBorislav Petkov static void prep_chip_selects(struct amd64_pvt *pvt) 83694be4bffSDoug Thompson { 8371433eb99SBorislav Petkov if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) { 83811c75eadSBorislav Petkov pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8; 83911c75eadSBorislav Petkov pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8; 8409d858bb1SBorislav Petkov } else { 84111c75eadSBorislav Petkov pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8; 84211c75eadSBorislav Petkov pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4; 8439d858bb1SBorislav Petkov } 84494be4bffSDoug Thompson } 84594be4bffSDoug Thompson 84694be4bffSDoug Thompson /* 84711c75eadSBorislav Petkov * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers 84894be4bffSDoug Thompson */ 849b2b0c605SBorislav Petkov static void read_dct_base_mask(struct amd64_pvt *pvt) 85094be4bffSDoug Thompson { 85111c75eadSBorislav Petkov int cs; 85294be4bffSDoug Thompson 85311c75eadSBorislav Petkov prep_chip_selects(pvt); 85494be4bffSDoug Thompson 85511c75eadSBorislav Petkov for_each_chip_select(cs, 0, pvt) { 85671d2a32eSBorislav Petkov int reg0 = DCSB0 + (cs * 4); 85771d2a32eSBorislav Petkov int reg1 = DCSB1 + (cs * 4); 85811c75eadSBorislav Petkov u32 *base0 = &pvt->csels[0].csbases[cs]; 85911c75eadSBorislav Petkov u32 *base1 = &pvt->csels[1].csbases[cs]; 860b2b0c605SBorislav Petkov 86111c75eadSBorislav Petkov if (!amd64_read_dct_pci_cfg(pvt, reg0, base0)) 86294be4bffSDoug Thompson debugf0(" DCSB0[%d]=0x%08x reg: F2x%x\n", 86311c75eadSBorislav Petkov cs, *base0, reg0); 86494be4bffSDoug Thompson 86511c75eadSBorislav Petkov if (boot_cpu_data.x86 == 0xf || dct_ganging_enabled(pvt)) 86611c75eadSBorislav Petkov continue; 867b2b0c605SBorislav Petkov 86811c75eadSBorislav Petkov if (!amd64_read_dct_pci_cfg(pvt, reg1, base1)) 86994be4bffSDoug Thompson debugf0(" DCSB1[%d]=0x%08x reg: F2x%x\n", 87011c75eadSBorislav Petkov cs, *base1, reg1); 87194be4bffSDoug Thompson } 87294be4bffSDoug Thompson 87311c75eadSBorislav Petkov for_each_chip_select_mask(cs, 0, pvt) { 87471d2a32eSBorislav Petkov int reg0 = DCSM0 + (cs * 4); 87571d2a32eSBorislav Petkov int reg1 = DCSM1 + (cs * 4); 87611c75eadSBorislav Petkov u32 *mask0 = &pvt->csels[0].csmasks[cs]; 87711c75eadSBorislav Petkov u32 *mask1 = &pvt->csels[1].csmasks[cs]; 878b2b0c605SBorislav Petkov 87911c75eadSBorislav Petkov if (!amd64_read_dct_pci_cfg(pvt, reg0, mask0)) 88094be4bffSDoug Thompson debugf0(" DCSM0[%d]=0x%08x reg: F2x%x\n", 88111c75eadSBorislav Petkov cs, *mask0, reg0); 88294be4bffSDoug Thompson 88311c75eadSBorislav Petkov if (boot_cpu_data.x86 == 0xf || dct_ganging_enabled(pvt)) 88411c75eadSBorislav Petkov continue; 885b2b0c605SBorislav Petkov 88611c75eadSBorislav Petkov if (!amd64_read_dct_pci_cfg(pvt, reg1, mask1)) 88794be4bffSDoug Thompson debugf0(" DCSM1[%d]=0x%08x reg: F2x%x\n", 88811c75eadSBorislav Petkov cs, *mask1, reg1); 88994be4bffSDoug Thompson } 8906ba5dcdcSBorislav Petkov } 89194be4bffSDoug Thompson 89224f9a7feSBorislav Petkov static enum mem_type amd64_determine_memory_type(struct amd64_pvt *pvt, int cs) 89394be4bffSDoug Thompson { 89494be4bffSDoug Thompson enum mem_type type; 89594be4bffSDoug Thompson 896cb328507SBorislav Petkov /* F15h supports only DDR3 */ 897cb328507SBorislav Petkov if (boot_cpu_data.x86 >= 0x15) 898cb328507SBorislav Petkov type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3; 899cb328507SBorislav Petkov else if (boot_cpu_data.x86 == 0x10 || pvt->ext_model >= K8_REV_F) { 9006b4c0bdeSBorislav Petkov if (pvt->dchr0 & DDR3_MODE) 9016b4c0bdeSBorislav Petkov type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3; 9026b4c0bdeSBorislav Petkov else 90394be4bffSDoug Thompson type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2; 90494be4bffSDoug Thompson } else { 90594be4bffSDoug Thompson type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR; 90694be4bffSDoug Thompson } 90794be4bffSDoug Thompson 90824f9a7feSBorislav Petkov amd64_info("CS%d: %s\n", cs, edac_mem_types[type]); 90994be4bffSDoug Thompson 91094be4bffSDoug Thompson return type; 91194be4bffSDoug Thompson } 91294be4bffSDoug Thompson 913cb328507SBorislav Petkov /* Get the number of DCT channels the memory controller is using. */ 914ddff876dSDoug Thompson static int k8_early_channel_count(struct amd64_pvt *pvt) 915ddff876dSDoug Thompson { 916cb328507SBorislav Petkov int flag; 917ddff876dSDoug Thompson 9189f56da0eSBorislav Petkov if (pvt->ext_model >= K8_REV_F) 919ddff876dSDoug Thompson /* RevF (NPT) and later */ 92041d8bfabSBorislav Petkov flag = pvt->dclr0 & WIDTH_128; 9219f56da0eSBorislav Petkov else 922ddff876dSDoug Thompson /* RevE and earlier */ 923ddff876dSDoug Thompson flag = pvt->dclr0 & REVE_WIDTH_128; 924ddff876dSDoug Thompson 925ddff876dSDoug Thompson /* not used */ 926ddff876dSDoug Thompson pvt->dclr1 = 0; 927ddff876dSDoug Thompson 928ddff876dSDoug Thompson return (flag) ? 2 : 1; 929ddff876dSDoug Thompson } 930ddff876dSDoug Thompson 93170046624SBorislav Petkov /* On F10h and later ErrAddr is MC4_ADDR[47:1] */ 93270046624SBorislav Petkov static u64 get_error_address(struct mce *m) 933ddff876dSDoug Thompson { 93470046624SBorislav Petkov u8 start_bit = 1; 93570046624SBorislav Petkov u8 end_bit = 47; 93670046624SBorislav Petkov 93770046624SBorislav Petkov if (boot_cpu_data.x86 == 0xf) { 93870046624SBorislav Petkov start_bit = 3; 93970046624SBorislav Petkov end_bit = 39; 94070046624SBorislav Petkov } 94170046624SBorislav Petkov 94270046624SBorislav Petkov return m->addr & GENMASK(start_bit, end_bit); 943ddff876dSDoug Thompson } 944ddff876dSDoug Thompson 9457f19bf75SBorislav Petkov static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range) 946ddff876dSDoug Thompson { 947f08e457cSBorislav Petkov struct cpuinfo_x86 *c = &boot_cpu_data; 94871d2a32eSBorislav Petkov int off = range << 3; 949ddff876dSDoug Thompson 9507f19bf75SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off, &pvt->ranges[range].base.lo); 9517f19bf75SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo); 952ddff876dSDoug Thompson 953f08e457cSBorislav Petkov if (c->x86 == 0xf) 9547f19bf75SBorislav Petkov return; 955ddff876dSDoug Thompson 9567f19bf75SBorislav Petkov if (!dram_rw(pvt, range)) 9577f19bf75SBorislav Petkov return; 958ddff876dSDoug Thompson 9597f19bf75SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off, &pvt->ranges[range].base.hi); 9607f19bf75SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi); 961f08e457cSBorislav Petkov 962f08e457cSBorislav Petkov /* Factor in CC6 save area by reading dst node's limit reg */ 963f08e457cSBorislav Petkov if (c->x86 == 0x15) { 964f08e457cSBorislav Petkov struct pci_dev *f1 = NULL; 965f08e457cSBorislav Petkov u8 nid = dram_dst_node(pvt, range); 966f08e457cSBorislav Petkov u32 llim; 967f08e457cSBorislav Petkov 968f08e457cSBorislav Petkov f1 = pci_get_domain_bus_and_slot(0, 0, PCI_DEVFN(0x18 + nid, 1)); 969f08e457cSBorislav Petkov if (WARN_ON(!f1)) 970f08e457cSBorislav Petkov return; 971f08e457cSBorislav Petkov 972f08e457cSBorislav Petkov amd64_read_pci_cfg(f1, DRAM_LOCAL_NODE_LIM, &llim); 973f08e457cSBorislav Petkov 974f08e457cSBorislav Petkov pvt->ranges[range].lim.lo &= GENMASK(0, 15); 975f08e457cSBorislav Petkov 976f08e457cSBorislav Petkov /* {[39:27],111b} */ 977f08e457cSBorislav Petkov pvt->ranges[range].lim.lo |= ((llim & 0x1fff) << 3 | 0x7) << 16; 978f08e457cSBorislav Petkov 979f08e457cSBorislav Petkov pvt->ranges[range].lim.hi &= GENMASK(0, 7); 980f08e457cSBorislav Petkov 981f08e457cSBorislav Petkov /* [47:40] */ 982f08e457cSBorislav Petkov pvt->ranges[range].lim.hi |= llim >> 13; 983f08e457cSBorislav Petkov 984f08e457cSBorislav Petkov pci_dev_put(f1); 985f08e457cSBorislav Petkov } 986ddff876dSDoug Thompson } 987ddff876dSDoug Thompson 988f192c7b1SBorislav Petkov static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr, 989f192c7b1SBorislav Petkov u16 syndrome) 990ddff876dSDoug Thompson { 991ddff876dSDoug Thompson struct mem_ctl_info *src_mci; 992f192c7b1SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 993ddff876dSDoug Thompson int channel, csrow; 994ddff876dSDoug Thompson u32 page, offset; 995ddff876dSDoug Thompson 996ddff876dSDoug Thompson /* CHIPKILL enabled */ 997f192c7b1SBorislav Petkov if (pvt->nbcfg & NBCFG_CHIPKILL) { 998bfc04aecSBorislav Petkov channel = get_channel_from_ecc_syndrome(mci, syndrome); 999ddff876dSDoug Thompson if (channel < 0) { 1000ddff876dSDoug Thompson /* 1001ddff876dSDoug Thompson * Syndrome didn't map, so we don't know which of the 1002ddff876dSDoug Thompson * 2 DIMMs is in error. So we need to ID 'both' of them 1003ddff876dSDoug Thompson * as suspect. 1004ddff876dSDoug Thompson */ 100524f9a7feSBorislav Petkov amd64_mc_warn(mci, "unknown syndrome 0x%04x - possible " 1006ad6a32e9SBorislav Petkov "error reporting race\n", syndrome); 1007ddff876dSDoug Thompson edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR); 1008ddff876dSDoug Thompson return; 1009ddff876dSDoug Thompson } 1010ddff876dSDoug Thompson } else { 1011ddff876dSDoug Thompson /* 1012ddff876dSDoug Thompson * non-chipkill ecc mode 1013ddff876dSDoug Thompson * 1014ddff876dSDoug Thompson * The k8 documentation is unclear about how to determine the 1015ddff876dSDoug Thompson * channel number when using non-chipkill memory. This method 1016ddff876dSDoug Thompson * was obtained from email communication with someone at AMD. 1017ddff876dSDoug Thompson * (Wish the email was placed in this comment - norsk) 1018ddff876dSDoug Thompson */ 101944e9e2eeSBorislav Petkov channel = ((sys_addr & BIT(3)) != 0); 1020ddff876dSDoug Thompson } 1021ddff876dSDoug Thompson 1022ddff876dSDoug Thompson /* 1023ddff876dSDoug Thompson * Find out which node the error address belongs to. This may be 1024ddff876dSDoug Thompson * different from the node that detected the error. 1025ddff876dSDoug Thompson */ 102644e9e2eeSBorislav Petkov src_mci = find_mc_by_sys_addr(mci, sys_addr); 10272cff18c2SKeith Mannthey if (!src_mci) { 102824f9a7feSBorislav Petkov amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n", 102944e9e2eeSBorislav Petkov (unsigned long)sys_addr); 1030ddff876dSDoug Thompson edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR); 1031ddff876dSDoug Thompson return; 1032ddff876dSDoug Thompson } 1033ddff876dSDoug Thompson 103444e9e2eeSBorislav Petkov /* Now map the sys_addr to a CSROW */ 103544e9e2eeSBorislav Petkov csrow = sys_addr_to_csrow(src_mci, sys_addr); 1036ddff876dSDoug Thompson if (csrow < 0) { 1037ddff876dSDoug Thompson edac_mc_handle_ce_no_info(src_mci, EDAC_MOD_STR); 1038ddff876dSDoug Thompson } else { 103944e9e2eeSBorislav Petkov error_address_to_page_and_offset(sys_addr, &page, &offset); 1040ddff876dSDoug Thompson 1041ddff876dSDoug Thompson edac_mc_handle_ce(src_mci, page, offset, syndrome, csrow, 1042ddff876dSDoug Thompson channel, EDAC_MOD_STR); 1043ddff876dSDoug Thompson } 1044ddff876dSDoug Thompson } 1045ddff876dSDoug Thompson 104641d8bfabSBorislav Petkov static int ddr2_cs_size(unsigned i, bool dct_width) 1047ddff876dSDoug Thompson { 104841d8bfabSBorislav Petkov unsigned shift = 0; 1049ddff876dSDoug Thompson 105041d8bfabSBorislav Petkov if (i <= 2) 105141d8bfabSBorislav Petkov shift = i; 105241d8bfabSBorislav Petkov else if (!(i & 0x1)) 105341d8bfabSBorislav Petkov shift = i >> 1; 10541433eb99SBorislav Petkov else 105541d8bfabSBorislav Petkov shift = (i + 1) >> 1; 1056ddff876dSDoug Thompson 105741d8bfabSBorislav Petkov return 128 << (shift + !!dct_width); 105841d8bfabSBorislav Petkov } 105941d8bfabSBorislav Petkov 106041d8bfabSBorislav Petkov static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, 106141d8bfabSBorislav Petkov unsigned cs_mode) 106241d8bfabSBorislav Petkov { 106341d8bfabSBorislav Petkov u32 dclr = dct ? pvt->dclr1 : pvt->dclr0; 106441d8bfabSBorislav Petkov 106541d8bfabSBorislav Petkov if (pvt->ext_model >= K8_REV_F) { 106641d8bfabSBorislav Petkov WARN_ON(cs_mode > 11); 106741d8bfabSBorislav Petkov return ddr2_cs_size(cs_mode, dclr & WIDTH_128); 106841d8bfabSBorislav Petkov } 106941d8bfabSBorislav Petkov else if (pvt->ext_model >= K8_REV_D) { 107041d8bfabSBorislav Petkov WARN_ON(cs_mode > 10); 107141d8bfabSBorislav Petkov 107241d8bfabSBorislav Petkov if (cs_mode == 3 || cs_mode == 8) 107341d8bfabSBorislav Petkov return 32 << (cs_mode - 1); 107441d8bfabSBorislav Petkov else 107541d8bfabSBorislav Petkov return 32 << cs_mode; 107641d8bfabSBorislav Petkov } 107741d8bfabSBorislav Petkov else { 107841d8bfabSBorislav Petkov WARN_ON(cs_mode > 6); 107941d8bfabSBorislav Petkov return 32 << cs_mode; 108041d8bfabSBorislav Petkov } 1081ddff876dSDoug Thompson } 1082ddff876dSDoug Thompson 10831afd3c98SDoug Thompson /* 10841afd3c98SDoug Thompson * Get the number of DCT channels in use. 10851afd3c98SDoug Thompson * 10861afd3c98SDoug Thompson * Return: 10871afd3c98SDoug Thompson * number of Memory Channels in operation 10881afd3c98SDoug Thompson * Pass back: 10891afd3c98SDoug Thompson * contents of the DCL0_LOW register 10901afd3c98SDoug Thompson */ 10917d20d14dSBorislav Petkov static int f1x_early_channel_count(struct amd64_pvt *pvt) 10921afd3c98SDoug Thompson { 10936ba5dcdcSBorislav Petkov int i, j, channels = 0; 1094ddff876dSDoug Thompson 10957d20d14dSBorislav Petkov /* On F10h, if we are in 128 bit mode, then we are using 2 channels */ 109641d8bfabSBorislav Petkov if (boot_cpu_data.x86 == 0x10 && (pvt->dclr0 & WIDTH_128)) 10977d20d14dSBorislav Petkov return 2; 10981afd3c98SDoug Thompson 10991afd3c98SDoug Thompson /* 1100d16149e8SBorislav Petkov * Need to check if in unganged mode: In such, there are 2 channels, 1101d16149e8SBorislav Petkov * but they are not in 128 bit mode and thus the above 'dclr0' status 1102d16149e8SBorislav Petkov * bit will be OFF. 11031afd3c98SDoug Thompson * 11041afd3c98SDoug Thompson * Need to check DCT0[0] and DCT1[0] to see if only one of them has 11051afd3c98SDoug Thompson * their CSEnable bit on. If so, then SINGLE DIMM case. 11061afd3c98SDoug Thompson */ 1107d16149e8SBorislav Petkov debugf0("Data width is not 128 bits - need more decoding\n"); 11081afd3c98SDoug Thompson 11091afd3c98SDoug Thompson /* 11101afd3c98SDoug Thompson * Check DRAM Bank Address Mapping values for each DIMM to see if there 11111afd3c98SDoug Thompson * is more than just one DIMM present in unganged mode. Need to check 11121afd3c98SDoug Thompson * both controllers since DIMMs can be placed in either one. 11131afd3c98SDoug Thompson */ 1114525a1b20SBorislav Petkov for (i = 0; i < 2; i++) { 1115525a1b20SBorislav Petkov u32 dbam = (i ? pvt->dbam1 : pvt->dbam0); 11161afd3c98SDoug Thompson 111757a30854SWan Wei for (j = 0; j < 4; j++) { 111857a30854SWan Wei if (DBAM_DIMM(j, dbam) > 0) { 11191afd3c98SDoug Thompson channels++; 112057a30854SWan Wei break; 11211afd3c98SDoug Thompson } 112257a30854SWan Wei } 112357a30854SWan Wei } 11241afd3c98SDoug Thompson 1125d16149e8SBorislav Petkov if (channels > 2) 1126d16149e8SBorislav Petkov channels = 2; 1127d16149e8SBorislav Petkov 112824f9a7feSBorislav Petkov amd64_info("MCT channel count: %d\n", channels); 11291afd3c98SDoug Thompson 11301afd3c98SDoug Thompson return channels; 11311afd3c98SDoug Thompson } 11321afd3c98SDoug Thompson 113341d8bfabSBorislav Petkov static int ddr3_cs_size(unsigned i, bool dct_width) 11341afd3c98SDoug Thompson { 113541d8bfabSBorislav Petkov unsigned shift = 0; 113641d8bfabSBorislav Petkov int cs_size = 0; 113741d8bfabSBorislav Petkov 113841d8bfabSBorislav Petkov if (i == 0 || i == 3 || i == 4) 113941d8bfabSBorislav Petkov cs_size = -1; 114041d8bfabSBorislav Petkov else if (i <= 2) 114141d8bfabSBorislav Petkov shift = i; 114241d8bfabSBorislav Petkov else if (i == 12) 114341d8bfabSBorislav Petkov shift = 7; 114441d8bfabSBorislav Petkov else if (!(i & 0x1)) 114541d8bfabSBorislav Petkov shift = i >> 1; 114641d8bfabSBorislav Petkov else 114741d8bfabSBorislav Petkov shift = (i + 1) >> 1; 114841d8bfabSBorislav Petkov 114941d8bfabSBorislav Petkov if (cs_size != -1) 115041d8bfabSBorislav Petkov cs_size = (128 * (1 << !!dct_width)) << shift; 115141d8bfabSBorislav Petkov 115241d8bfabSBorislav Petkov return cs_size; 115341d8bfabSBorislav Petkov } 115441d8bfabSBorislav Petkov 115541d8bfabSBorislav Petkov static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, 115641d8bfabSBorislav Petkov unsigned cs_mode) 115741d8bfabSBorislav Petkov { 115841d8bfabSBorislav Petkov u32 dclr = dct ? pvt->dclr1 : pvt->dclr0; 115941d8bfabSBorislav Petkov 116041d8bfabSBorislav Petkov WARN_ON(cs_mode > 11); 11611433eb99SBorislav Petkov 11621433eb99SBorislav Petkov if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE) 116341d8bfabSBorislav Petkov return ddr3_cs_size(cs_mode, dclr & WIDTH_128); 11641433eb99SBorislav Petkov else 116541d8bfabSBorislav Petkov return ddr2_cs_size(cs_mode, dclr & WIDTH_128); 116641d8bfabSBorislav Petkov } 11671433eb99SBorislav Petkov 116841d8bfabSBorislav Petkov /* 116941d8bfabSBorislav Petkov * F15h supports only 64bit DCT interfaces 117041d8bfabSBorislav Petkov */ 117141d8bfabSBorislav Petkov static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, 117241d8bfabSBorislav Petkov unsigned cs_mode) 117341d8bfabSBorislav Petkov { 117441d8bfabSBorislav Petkov WARN_ON(cs_mode > 12); 117541d8bfabSBorislav Petkov 117641d8bfabSBorislav Petkov return ddr3_cs_size(cs_mode, false); 11771afd3c98SDoug Thompson } 11781afd3c98SDoug Thompson 11795a5d2371SBorislav Petkov static void read_dram_ctl_register(struct amd64_pvt *pvt) 11806163b5d4SDoug Thompson { 11816163b5d4SDoug Thompson 11825a5d2371SBorislav Petkov if (boot_cpu_data.x86 == 0xf) 11835a5d2371SBorislav Petkov return; 11845a5d2371SBorislav Petkov 118578da121eSBorislav Petkov if (!amd64_read_dct_pci_cfg(pvt, DCT_SEL_LO, &pvt->dct_sel_lo)) { 118678da121eSBorislav Petkov debugf0("F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n", 118778da121eSBorislav Petkov pvt->dct_sel_lo, dct_sel_baseaddr(pvt)); 11886163b5d4SDoug Thompson 11895a5d2371SBorislav Petkov debugf0(" DCTs operate in %s mode.\n", 11905a5d2371SBorislav Petkov (dct_ganging_enabled(pvt) ? "ganged" : "unganged")); 11916163b5d4SDoug Thompson 119272381bd5SBorislav Petkov if (!dct_ganging_enabled(pvt)) 119372381bd5SBorislav Petkov debugf0(" Address range split per DCT: %s\n", 119472381bd5SBorislav Petkov (dct_high_range_enabled(pvt) ? "yes" : "no")); 119572381bd5SBorislav Petkov 119678da121eSBorislav Petkov debugf0(" data interleave for ECC: %s, " 119772381bd5SBorislav Petkov "DRAM cleared since last warm reset: %s\n", 119872381bd5SBorislav Petkov (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"), 119972381bd5SBorislav Petkov (dct_memory_cleared(pvt) ? "yes" : "no")); 120072381bd5SBorislav Petkov 120178da121eSBorislav Petkov debugf0(" channel interleave: %s, " 120278da121eSBorislav Petkov "interleave bits selector: 0x%x\n", 120372381bd5SBorislav Petkov (dct_interleave_enabled(pvt) ? "enabled" : "disabled"), 12046163b5d4SDoug Thompson dct_sel_interleave_addr(pvt)); 12056163b5d4SDoug Thompson } 12066163b5d4SDoug Thompson 120778da121eSBorislav Petkov amd64_read_dct_pci_cfg(pvt, DCT_SEL_HI, &pvt->dct_sel_hi); 12086163b5d4SDoug Thompson } 12096163b5d4SDoug Thompson 1210f71d0a05SDoug Thompson /* 1211229a7a11SBorislav Petkov * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory 1212f71d0a05SDoug Thompson * Interleaving Modes. 1213f71d0a05SDoug Thompson */ 1214b15f0fcaSBorislav Petkov static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr, 1215229a7a11SBorislav Petkov bool hi_range_sel, u8 intlv_en) 12166163b5d4SDoug Thompson { 1217151fa71cSBorislav Petkov u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1; 12186163b5d4SDoug Thompson 12196163b5d4SDoug Thompson if (dct_ganging_enabled(pvt)) 1220229a7a11SBorislav Petkov return 0; 1221229a7a11SBorislav Petkov 1222229a7a11SBorislav Petkov if (hi_range_sel) 1223229a7a11SBorislav Petkov return dct_sel_high; 1224229a7a11SBorislav Petkov 1225f71d0a05SDoug Thompson /* 1226f71d0a05SDoug Thompson * see F2x110[DctSelIntLvAddr] - channel interleave mode 1227f71d0a05SDoug Thompson */ 1228229a7a11SBorislav Petkov if (dct_interleave_enabled(pvt)) { 1229229a7a11SBorislav Petkov u8 intlv_addr = dct_sel_interleave_addr(pvt); 12306163b5d4SDoug Thompson 1231229a7a11SBorislav Petkov /* return DCT select function: 0=DCT0, 1=DCT1 */ 1232229a7a11SBorislav Petkov if (!intlv_addr) 1233229a7a11SBorislav Petkov return sys_addr >> 6 & 1; 12346163b5d4SDoug Thompson 1235229a7a11SBorislav Petkov if (intlv_addr & 0x2) { 1236229a7a11SBorislav Petkov u8 shift = intlv_addr & 0x1 ? 9 : 6; 1237229a7a11SBorislav Petkov u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) % 2; 1238229a7a11SBorislav Petkov 1239229a7a11SBorislav Petkov return ((sys_addr >> shift) & 1) ^ temp; 12406163b5d4SDoug Thompson } 12416163b5d4SDoug Thompson 1242229a7a11SBorislav Petkov return (sys_addr >> (12 + hweight8(intlv_en))) & 1; 1243229a7a11SBorislav Petkov } 1244229a7a11SBorislav Petkov 1245229a7a11SBorislav Petkov if (dct_high_range_enabled(pvt)) 1246229a7a11SBorislav Petkov return ~dct_sel_high & 1; 12476163b5d4SDoug Thompson 12486163b5d4SDoug Thompson return 0; 12496163b5d4SDoug Thompson } 12506163b5d4SDoug Thompson 1251c8e518d5SBorislav Petkov /* Convert the sys_addr to the normalized DCT address */ 1252e761359aSBorislav Petkov static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, unsigned range, 1253c8e518d5SBorislav Petkov u64 sys_addr, bool hi_rng, 1254c8e518d5SBorislav Petkov u32 dct_sel_base_addr) 12556163b5d4SDoug Thompson { 12566163b5d4SDoug Thompson u64 chan_off; 1257c8e518d5SBorislav Petkov u64 dram_base = get_dram_base(pvt, range); 1258c8e518d5SBorislav Petkov u64 hole_off = f10_dhar_offset(pvt); 1259c8e518d5SBorislav Petkov u64 dct_sel_base_off = (pvt->dct_sel_hi & 0xFFFFFC00) << 16; 12606163b5d4SDoug Thompson 1261c8e518d5SBorislav Petkov if (hi_rng) { 1262c8e518d5SBorislav Petkov /* 1263c8e518d5SBorislav Petkov * if 1264c8e518d5SBorislav Petkov * base address of high range is below 4Gb 1265c8e518d5SBorislav Petkov * (bits [47:27] at [31:11]) 1266c8e518d5SBorislav Petkov * DRAM address space on this DCT is hoisted above 4Gb && 1267c8e518d5SBorislav Petkov * sys_addr > 4Gb 1268c8e518d5SBorislav Petkov * 1269c8e518d5SBorislav Petkov * remove hole offset from sys_addr 1270c8e518d5SBorislav Petkov * else 1271c8e518d5SBorislav Petkov * remove high range offset from sys_addr 1272c8e518d5SBorislav Petkov */ 1273c8e518d5SBorislav Petkov if ((!(dct_sel_base_addr >> 16) || 1274c8e518d5SBorislav Petkov dct_sel_base_addr < dhar_base(pvt)) && 1275972ea17aSBorislav Petkov dhar_valid(pvt) && 1276c8e518d5SBorislav Petkov (sys_addr >= BIT_64(32))) 1277bc21fa57SBorislav Petkov chan_off = hole_off; 12786163b5d4SDoug Thompson else 12796163b5d4SDoug Thompson chan_off = dct_sel_base_off; 12806163b5d4SDoug Thompson } else { 1281c8e518d5SBorislav Petkov /* 1282c8e518d5SBorislav Petkov * if 1283c8e518d5SBorislav Petkov * we have a valid hole && 1284c8e518d5SBorislav Petkov * sys_addr > 4Gb 1285c8e518d5SBorislav Petkov * 1286c8e518d5SBorislav Petkov * remove hole 1287c8e518d5SBorislav Petkov * else 1288c8e518d5SBorislav Petkov * remove dram base to normalize to DCT address 1289c8e518d5SBorislav Petkov */ 1290972ea17aSBorislav Petkov if (dhar_valid(pvt) && (sys_addr >= BIT_64(32))) 1291bc21fa57SBorislav Petkov chan_off = hole_off; 12926163b5d4SDoug Thompson else 1293c8e518d5SBorislav Petkov chan_off = dram_base; 12946163b5d4SDoug Thompson } 12956163b5d4SDoug Thompson 1296c8e518d5SBorislav Petkov return (sys_addr & GENMASK(6,47)) - (chan_off & GENMASK(23,47)); 12976163b5d4SDoug Thompson } 12986163b5d4SDoug Thompson 12996163b5d4SDoug Thompson /* 13006163b5d4SDoug Thompson * checks if the csrow passed in is marked as SPARED, if so returns the new 13016163b5d4SDoug Thompson * spare row 13026163b5d4SDoug Thompson */ 130311c75eadSBorislav Petkov static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow) 13046163b5d4SDoug Thompson { 1305614ec9d8SBorislav Petkov int tmp_cs; 13066163b5d4SDoug Thompson 1307614ec9d8SBorislav Petkov if (online_spare_swap_done(pvt, dct) && 1308614ec9d8SBorislav Petkov csrow == online_spare_bad_dramcs(pvt, dct)) { 1309614ec9d8SBorislav Petkov 1310614ec9d8SBorislav Petkov for_each_chip_select(tmp_cs, dct, pvt) { 1311614ec9d8SBorislav Petkov if (chip_select_base(tmp_cs, dct, pvt) & 0x2) { 1312614ec9d8SBorislav Petkov csrow = tmp_cs; 1313614ec9d8SBorislav Petkov break; 1314614ec9d8SBorislav Petkov } 1315614ec9d8SBorislav Petkov } 13166163b5d4SDoug Thompson } 13176163b5d4SDoug Thompson return csrow; 13186163b5d4SDoug Thompson } 13196163b5d4SDoug Thompson 13206163b5d4SDoug Thompson /* 13216163b5d4SDoug Thompson * Iterate over the DRAM DCT "base" and "mask" registers looking for a 13226163b5d4SDoug Thompson * SystemAddr match on the specified 'ChannelSelect' and 'NodeID' 13236163b5d4SDoug Thompson * 13246163b5d4SDoug Thompson * Return: 13256163b5d4SDoug Thompson * -EINVAL: NOT FOUND 13266163b5d4SDoug Thompson * 0..csrow = Chip-Select Row 13276163b5d4SDoug Thompson */ 1328b15f0fcaSBorislav Petkov static int f1x_lookup_addr_in_dct(u64 in_addr, u32 nid, u8 dct) 13296163b5d4SDoug Thompson { 13306163b5d4SDoug Thompson struct mem_ctl_info *mci; 13316163b5d4SDoug Thompson struct amd64_pvt *pvt; 133211c75eadSBorislav Petkov u64 cs_base, cs_mask; 13336163b5d4SDoug Thompson int cs_found = -EINVAL; 13346163b5d4SDoug Thompson int csrow; 13356163b5d4SDoug Thompson 1336cc4d8860SBorislav Petkov mci = mcis[nid]; 13376163b5d4SDoug Thompson if (!mci) 13386163b5d4SDoug Thompson return cs_found; 13396163b5d4SDoug Thompson 13406163b5d4SDoug Thompson pvt = mci->pvt_info; 13416163b5d4SDoug Thompson 134211c75eadSBorislav Petkov debugf1("input addr: 0x%llx, DCT: %d\n", in_addr, dct); 13436163b5d4SDoug Thompson 134411c75eadSBorislav Petkov for_each_chip_select(csrow, dct, pvt) { 134511c75eadSBorislav Petkov if (!csrow_enabled(csrow, dct, pvt)) 13466163b5d4SDoug Thompson continue; 13476163b5d4SDoug Thompson 134811c75eadSBorislav Petkov get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask); 13496163b5d4SDoug Thompson 135011c75eadSBorislav Petkov debugf1(" CSROW=%d CSBase=0x%llx CSMask=0x%llx\n", 13516163b5d4SDoug Thompson csrow, cs_base, cs_mask); 13526163b5d4SDoug Thompson 135311c75eadSBorislav Petkov cs_mask = ~cs_mask; 13546163b5d4SDoug Thompson 135511c75eadSBorislav Petkov debugf1(" (InputAddr & ~CSMask)=0x%llx " 135611c75eadSBorislav Petkov "(CSBase & ~CSMask)=0x%llx\n", 135711c75eadSBorislav Petkov (in_addr & cs_mask), (cs_base & cs_mask)); 13586163b5d4SDoug Thompson 135911c75eadSBorislav Petkov if ((in_addr & cs_mask) == (cs_base & cs_mask)) { 136011c75eadSBorislav Petkov cs_found = f10_process_possible_spare(pvt, dct, csrow); 13616163b5d4SDoug Thompson 13626163b5d4SDoug Thompson debugf1(" MATCH csrow=%d\n", cs_found); 13636163b5d4SDoug Thompson break; 13646163b5d4SDoug Thompson } 13656163b5d4SDoug Thompson } 13666163b5d4SDoug Thompson return cs_found; 13676163b5d4SDoug Thompson } 13686163b5d4SDoug Thompson 136995b0ef55SBorislav Petkov /* 137095b0ef55SBorislav Petkov * See F2x10C. Non-interleaved graphics framebuffer memory under the 16G is 137195b0ef55SBorislav Petkov * swapped with a region located at the bottom of memory so that the GPU can use 137295b0ef55SBorislav Petkov * the interleaved region and thus two channels. 137395b0ef55SBorislav Petkov */ 1374b15f0fcaSBorislav Petkov static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr) 137595b0ef55SBorislav Petkov { 137695b0ef55SBorislav Petkov u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr; 137795b0ef55SBorislav Petkov 137895b0ef55SBorislav Petkov if (boot_cpu_data.x86 == 0x10) { 137995b0ef55SBorislav Petkov /* only revC3 and revE have that feature */ 138095b0ef55SBorislav Petkov if (boot_cpu_data.x86_model < 4 || 138195b0ef55SBorislav Petkov (boot_cpu_data.x86_model < 0xa && 138295b0ef55SBorislav Petkov boot_cpu_data.x86_mask < 3)) 138395b0ef55SBorislav Petkov return sys_addr; 138495b0ef55SBorislav Petkov } 138595b0ef55SBorislav Petkov 138695b0ef55SBorislav Petkov amd64_read_dct_pci_cfg(pvt, SWAP_INTLV_REG, &swap_reg); 138795b0ef55SBorislav Petkov 138895b0ef55SBorislav Petkov if (!(swap_reg & 0x1)) 138995b0ef55SBorislav Petkov return sys_addr; 139095b0ef55SBorislav Petkov 139195b0ef55SBorislav Petkov swap_base = (swap_reg >> 3) & 0x7f; 139295b0ef55SBorislav Petkov swap_limit = (swap_reg >> 11) & 0x7f; 139395b0ef55SBorislav Petkov rgn_size = (swap_reg >> 20) & 0x7f; 139495b0ef55SBorislav Petkov tmp_addr = sys_addr >> 27; 139595b0ef55SBorislav Petkov 139695b0ef55SBorislav Petkov if (!(sys_addr >> 34) && 139795b0ef55SBorislav Petkov (((tmp_addr >= swap_base) && 139895b0ef55SBorislav Petkov (tmp_addr <= swap_limit)) || 139995b0ef55SBorislav Petkov (tmp_addr < rgn_size))) 140095b0ef55SBorislav Petkov return sys_addr ^ (u64)swap_base << 27; 140195b0ef55SBorislav Petkov 140295b0ef55SBorislav Petkov return sys_addr; 140395b0ef55SBorislav Petkov } 140495b0ef55SBorislav Petkov 1405f71d0a05SDoug Thompson /* For a given @dram_range, check if @sys_addr falls within it. */ 1406e761359aSBorislav Petkov static int f1x_match_to_this_node(struct amd64_pvt *pvt, unsigned range, 1407f71d0a05SDoug Thompson u64 sys_addr, int *nid, int *chan_sel) 1408f71d0a05SDoug Thompson { 1409229a7a11SBorislav Petkov int cs_found = -EINVAL; 1410c8e518d5SBorislav Petkov u64 chan_addr; 14115d4b58e8SBorislav Petkov u32 dct_sel_base; 141211c75eadSBorislav Petkov u8 channel; 1413229a7a11SBorislav Petkov bool high_range = false; 1414f71d0a05SDoug Thompson 14157f19bf75SBorislav Petkov u8 node_id = dram_dst_node(pvt, range); 1416229a7a11SBorislav Petkov u8 intlv_en = dram_intlv_en(pvt, range); 14177f19bf75SBorislav Petkov u32 intlv_sel = dram_intlv_sel(pvt, range); 1418f71d0a05SDoug Thompson 1419c8e518d5SBorislav Petkov debugf1("(range %d) SystemAddr= 0x%llx Limit=0x%llx\n", 1420c8e518d5SBorislav Petkov range, sys_addr, get_dram_limit(pvt, range)); 1421f71d0a05SDoug Thompson 1422355fba60SBorislav Petkov if (dhar_valid(pvt) && 1423355fba60SBorislav Petkov dhar_base(pvt) <= sys_addr && 1424355fba60SBorislav Petkov sys_addr < BIT_64(32)) { 1425355fba60SBorislav Petkov amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n", 1426355fba60SBorislav Petkov sys_addr); 1427f71d0a05SDoug Thompson return -EINVAL; 1428355fba60SBorislav Petkov } 1429355fba60SBorislav Petkov 1430f030ddfbSBorislav Petkov if (intlv_en && (intlv_sel != ((sys_addr >> 12) & intlv_en))) 1431355fba60SBorislav Petkov return -EINVAL; 1432f71d0a05SDoug Thompson 1433b15f0fcaSBorislav Petkov sys_addr = f1x_swap_interleaved_region(pvt, sys_addr); 143495b0ef55SBorislav Petkov 1435f71d0a05SDoug Thompson dct_sel_base = dct_sel_baseaddr(pvt); 1436f71d0a05SDoug Thompson 1437f71d0a05SDoug Thompson /* 1438f71d0a05SDoug Thompson * check whether addresses >= DctSelBaseAddr[47:27] are to be used to 1439f71d0a05SDoug Thompson * select between DCT0 and DCT1. 1440f71d0a05SDoug Thompson */ 1441f71d0a05SDoug Thompson if (dct_high_range_enabled(pvt) && 1442f71d0a05SDoug Thompson !dct_ganging_enabled(pvt) && 1443f71d0a05SDoug Thompson ((sys_addr >> 27) >= (dct_sel_base >> 11))) 1444229a7a11SBorislav Petkov high_range = true; 1445f71d0a05SDoug Thompson 1446b15f0fcaSBorislav Petkov channel = f1x_determine_channel(pvt, sys_addr, high_range, intlv_en); 1447f71d0a05SDoug Thompson 1448b15f0fcaSBorislav Petkov chan_addr = f1x_get_norm_dct_addr(pvt, range, sys_addr, 1449c8e518d5SBorislav Petkov high_range, dct_sel_base); 1450f71d0a05SDoug Thompson 1451e2f79dbdSBorislav Petkov /* Remove node interleaving, see F1x120 */ 1452e2f79dbdSBorislav Petkov if (intlv_en) 1453e2f79dbdSBorislav Petkov chan_addr = ((chan_addr >> (12 + hweight8(intlv_en))) << 12) | 1454e2f79dbdSBorislav Petkov (chan_addr & 0xfff); 1455f71d0a05SDoug Thompson 14565d4b58e8SBorislav Petkov /* remove channel interleave */ 1457f71d0a05SDoug Thompson if (dct_interleave_enabled(pvt) && 1458f71d0a05SDoug Thompson !dct_high_range_enabled(pvt) && 1459f71d0a05SDoug Thompson !dct_ganging_enabled(pvt)) { 14605d4b58e8SBorislav Petkov 14615d4b58e8SBorislav Petkov if (dct_sel_interleave_addr(pvt) != 1) { 14625d4b58e8SBorislav Petkov if (dct_sel_interleave_addr(pvt) == 0x3) 14635d4b58e8SBorislav Petkov /* hash 9 */ 14645d4b58e8SBorislav Petkov chan_addr = ((chan_addr >> 10) << 9) | 14655d4b58e8SBorislav Petkov (chan_addr & 0x1ff); 14665d4b58e8SBorislav Petkov else 14675d4b58e8SBorislav Petkov /* A[6] or hash 6 */ 14685d4b58e8SBorislav Petkov chan_addr = ((chan_addr >> 7) << 6) | 14695d4b58e8SBorislav Petkov (chan_addr & 0x3f); 14705d4b58e8SBorislav Petkov } else 14715d4b58e8SBorislav Petkov /* A[12] */ 14725d4b58e8SBorislav Petkov chan_addr = ((chan_addr >> 13) << 12) | 14735d4b58e8SBorislav Petkov (chan_addr & 0xfff); 1474f71d0a05SDoug Thompson } 1475f71d0a05SDoug Thompson 14765d4b58e8SBorislav Petkov debugf1(" Normalized DCT addr: 0x%llx\n", chan_addr); 1477f71d0a05SDoug Thompson 1478b15f0fcaSBorislav Petkov cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, channel); 1479f71d0a05SDoug Thompson 1480f71d0a05SDoug Thompson if (cs_found >= 0) { 1481f71d0a05SDoug Thompson *nid = node_id; 1482f71d0a05SDoug Thompson *chan_sel = channel; 1483f71d0a05SDoug Thompson } 1484f71d0a05SDoug Thompson return cs_found; 1485f71d0a05SDoug Thompson } 1486f71d0a05SDoug Thompson 1487b15f0fcaSBorislav Petkov static int f1x_translate_sysaddr_to_cs(struct amd64_pvt *pvt, u64 sys_addr, 1488f71d0a05SDoug Thompson int *node, int *chan_sel) 1489f71d0a05SDoug Thompson { 1490e761359aSBorislav Petkov int cs_found = -EINVAL; 1491e761359aSBorislav Petkov unsigned range; 1492f71d0a05SDoug Thompson 14937f19bf75SBorislav Petkov for (range = 0; range < DRAM_RANGES; range++) { 1494f71d0a05SDoug Thompson 14957f19bf75SBorislav Petkov if (!dram_rw(pvt, range)) 1496f71d0a05SDoug Thompson continue; 1497f71d0a05SDoug Thompson 14987f19bf75SBorislav Petkov if ((get_dram_base(pvt, range) <= sys_addr) && 14997f19bf75SBorislav Petkov (get_dram_limit(pvt, range) >= sys_addr)) { 1500f71d0a05SDoug Thompson 1501b15f0fcaSBorislav Petkov cs_found = f1x_match_to_this_node(pvt, range, 1502f71d0a05SDoug Thompson sys_addr, node, 1503f71d0a05SDoug Thompson chan_sel); 1504f71d0a05SDoug Thompson if (cs_found >= 0) 1505f71d0a05SDoug Thompson break; 1506f71d0a05SDoug Thompson } 1507f71d0a05SDoug Thompson } 1508f71d0a05SDoug Thompson return cs_found; 1509f71d0a05SDoug Thompson } 1510f71d0a05SDoug Thompson 1511f71d0a05SDoug Thompson /* 1512bdc30a0cSBorislav Petkov * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps 1513bdc30a0cSBorislav Petkov * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW). 1514f71d0a05SDoug Thompson * 1515bdc30a0cSBorislav Petkov * The @sys_addr is usually an error address received from the hardware 1516bdc30a0cSBorislav Petkov * (MCX_ADDR). 1517f71d0a05SDoug Thompson */ 1518b15f0fcaSBorislav Petkov static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr, 1519f192c7b1SBorislav Petkov u16 syndrome) 1520f71d0a05SDoug Thompson { 1521f71d0a05SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 1522f71d0a05SDoug Thompson u32 page, offset; 1523f71d0a05SDoug Thompson int nid, csrow, chan = 0; 1524f71d0a05SDoug Thompson 1525b15f0fcaSBorislav Petkov csrow = f1x_translate_sysaddr_to_cs(pvt, sys_addr, &nid, &chan); 1526f71d0a05SDoug Thompson 1527bdc30a0cSBorislav Petkov if (csrow < 0) { 1528bdc30a0cSBorislav Petkov edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR); 1529bdc30a0cSBorislav Petkov return; 1530bdc30a0cSBorislav Petkov } 1531bdc30a0cSBorislav Petkov 1532f71d0a05SDoug Thompson error_address_to_page_and_offset(sys_addr, &page, &offset); 1533f71d0a05SDoug Thompson 1534f71d0a05SDoug Thompson /* 1535bdc30a0cSBorislav Petkov * We need the syndromes for channel detection only when we're 1536bdc30a0cSBorislav Petkov * ganged. Otherwise @chan should already contain the channel at 1537bdc30a0cSBorislav Petkov * this point. 1538f71d0a05SDoug Thompson */ 1539a97fa68eSBorislav Petkov if (dct_ganging_enabled(pvt)) 1540bfc04aecSBorislav Petkov chan = get_channel_from_ecc_syndrome(mci, syndrome); 1541f71d0a05SDoug Thompson 1542bdc30a0cSBorislav Petkov if (chan >= 0) 1543bdc30a0cSBorislav Petkov edac_mc_handle_ce(mci, page, offset, syndrome, csrow, chan, 1544bdc30a0cSBorislav Petkov EDAC_MOD_STR); 1545bdc30a0cSBorislav Petkov else 1546bdc30a0cSBorislav Petkov /* 1547bdc30a0cSBorislav Petkov * Channel unknown, report all channels on this CSROW as failed. 1548bdc30a0cSBorislav Petkov */ 1549bdc30a0cSBorislav Petkov for (chan = 0; chan < mci->csrows[csrow].nr_channels; chan++) 1550f71d0a05SDoug Thompson edac_mc_handle_ce(mci, page, offset, syndrome, 1551f71d0a05SDoug Thompson csrow, chan, EDAC_MOD_STR); 1552f71d0a05SDoug Thompson } 1553f71d0a05SDoug Thompson 1554f71d0a05SDoug Thompson /* 15558566c4dfSBorislav Petkov * debug routine to display the memory sizes of all logical DIMMs and its 1556cb328507SBorislav Petkov * CSROWs 1557f71d0a05SDoug Thompson */ 15588c671751SBorislav Petkov static void amd64_debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl) 1559f71d0a05SDoug Thompson { 1560603adaf6SBorislav Petkov int dimm, size0, size1, factor = 0; 1561525a1b20SBorislav Petkov u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases; 1562525a1b20SBorislav Petkov u32 dbam = ctrl ? pvt->dbam1 : pvt->dbam0; 1563f71d0a05SDoug Thompson 15648566c4dfSBorislav Petkov if (boot_cpu_data.x86 == 0xf) { 156541d8bfabSBorislav Petkov if (pvt->dclr0 & WIDTH_128) 1566603adaf6SBorislav Petkov factor = 1; 1567603adaf6SBorislav Petkov 15688566c4dfSBorislav Petkov /* K8 families < revF not supported yet */ 15691433eb99SBorislav Petkov if (pvt->ext_model < K8_REV_F) 15708566c4dfSBorislav Petkov return; 15718566c4dfSBorislav Petkov else 15728566c4dfSBorislav Petkov WARN_ON(ctrl != 0); 15738566c4dfSBorislav Petkov } 15748566c4dfSBorislav Petkov 15754d796364SBorislav Petkov dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1 : pvt->dbam0; 157611c75eadSBorislav Petkov dcsb = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->csels[1].csbases 157711c75eadSBorislav Petkov : pvt->csels[0].csbases; 1578f71d0a05SDoug Thompson 15794d796364SBorislav Petkov debugf1("F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n", ctrl, dbam); 1580f71d0a05SDoug Thompson 15818566c4dfSBorislav Petkov edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl); 15828566c4dfSBorislav Petkov 1583f71d0a05SDoug Thompson /* Dump memory sizes for DIMM and its CSROWs */ 1584f71d0a05SDoug Thompson for (dimm = 0; dimm < 4; dimm++) { 1585f71d0a05SDoug Thompson 1586f71d0a05SDoug Thompson size0 = 0; 158711c75eadSBorislav Petkov if (dcsb[dimm*2] & DCSB_CS_ENABLE) 158841d8bfabSBorislav Petkov size0 = pvt->ops->dbam_to_cs(pvt, ctrl, 158941d8bfabSBorislav Petkov DBAM_DIMM(dimm, dbam)); 1590f71d0a05SDoug Thompson 1591f71d0a05SDoug Thompson size1 = 0; 159211c75eadSBorislav Petkov if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE) 159341d8bfabSBorislav Petkov size1 = pvt->ops->dbam_to_cs(pvt, ctrl, 159441d8bfabSBorislav Petkov DBAM_DIMM(dimm, dbam)); 1595f71d0a05SDoug Thompson 159624f9a7feSBorislav Petkov amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n", 1597603adaf6SBorislav Petkov dimm * 2, size0 << factor, 1598603adaf6SBorislav Petkov dimm * 2 + 1, size1 << factor); 1599f71d0a05SDoug Thompson } 1600f71d0a05SDoug Thompson } 1601f71d0a05SDoug Thompson 16024d37607aSDoug Thompson static struct amd64_family_type amd64_family_types[] = { 16034d37607aSDoug Thompson [K8_CPUS] = { 16040092b20dSBorislav Petkov .ctl_name = "K8", 16058d5b5d9cSBorislav Petkov .f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP, 16068d5b5d9cSBorislav Petkov .f3_id = PCI_DEVICE_ID_AMD_K8_NB_MISC, 16074d37607aSDoug Thompson .ops = { 16084d37607aSDoug Thompson .early_channel_count = k8_early_channel_count, 16094d37607aSDoug Thompson .map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow, 16101433eb99SBorislav Petkov .dbam_to_cs = k8_dbam_to_chip_select, 1611b2b0c605SBorislav Petkov .read_dct_pci_cfg = k8_read_dct_pci_cfg, 16124d37607aSDoug Thompson } 16134d37607aSDoug Thompson }, 16144d37607aSDoug Thompson [F10_CPUS] = { 16150092b20dSBorislav Petkov .ctl_name = "F10h", 16168d5b5d9cSBorislav Petkov .f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP, 16178d5b5d9cSBorislav Petkov .f3_id = PCI_DEVICE_ID_AMD_10H_NB_MISC, 16184d37607aSDoug Thompson .ops = { 16197d20d14dSBorislav Petkov .early_channel_count = f1x_early_channel_count, 1620b15f0fcaSBorislav Petkov .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, 16211433eb99SBorislav Petkov .dbam_to_cs = f10_dbam_to_chip_select, 1622b2b0c605SBorislav Petkov .read_dct_pci_cfg = f10_read_dct_pci_cfg, 1623b2b0c605SBorislav Petkov } 1624b2b0c605SBorislav Petkov }, 1625b2b0c605SBorislav Petkov [F15_CPUS] = { 1626b2b0c605SBorislav Petkov .ctl_name = "F15h", 1627df71a053SBorislav Petkov .f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1, 1628df71a053SBorislav Petkov .f3_id = PCI_DEVICE_ID_AMD_15H_NB_F3, 1629b2b0c605SBorislav Petkov .ops = { 16307d20d14dSBorislav Petkov .early_channel_count = f1x_early_channel_count, 1631b15f0fcaSBorislav Petkov .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, 163241d8bfabSBorislav Petkov .dbam_to_cs = f15_dbam_to_chip_select, 1633b2b0c605SBorislav Petkov .read_dct_pci_cfg = f15_read_dct_pci_cfg, 16344d37607aSDoug Thompson } 16354d37607aSDoug Thompson }, 16364d37607aSDoug Thompson }; 16374d37607aSDoug Thompson 16384d37607aSDoug Thompson static struct pci_dev *pci_get_related_function(unsigned int vendor, 16394d37607aSDoug Thompson unsigned int device, 16404d37607aSDoug Thompson struct pci_dev *related) 16414d37607aSDoug Thompson { 16424d37607aSDoug Thompson struct pci_dev *dev = NULL; 16434d37607aSDoug Thompson 16444d37607aSDoug Thompson dev = pci_get_device(vendor, device, dev); 16454d37607aSDoug Thompson while (dev) { 16464d37607aSDoug Thompson if ((dev->bus->number == related->bus->number) && 16474d37607aSDoug Thompson (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn))) 16484d37607aSDoug Thompson break; 16494d37607aSDoug Thompson dev = pci_get_device(vendor, device, dev); 16504d37607aSDoug Thompson } 16514d37607aSDoug Thompson 16524d37607aSDoug Thompson return dev; 16534d37607aSDoug Thompson } 16544d37607aSDoug Thompson 1655b1289d6fSDoug Thompson /* 1656bfc04aecSBorislav Petkov * These are tables of eigenvectors (one per line) which can be used for the 1657bfc04aecSBorislav Petkov * construction of the syndrome tables. The modified syndrome search algorithm 1658bfc04aecSBorislav Petkov * uses those to find the symbol in error and thus the DIMM. 1659b1289d6fSDoug Thompson * 1660bfc04aecSBorislav Petkov * Algorithm courtesy of Ross LaFetra from AMD. 1661b1289d6fSDoug Thompson */ 1662bfc04aecSBorislav Petkov static u16 x4_vectors[] = { 1663bfc04aecSBorislav Petkov 0x2f57, 0x1afe, 0x66cc, 0xdd88, 1664bfc04aecSBorislav Petkov 0x11eb, 0x3396, 0x7f4c, 0xeac8, 1665bfc04aecSBorislav Petkov 0x0001, 0x0002, 0x0004, 0x0008, 1666bfc04aecSBorislav Petkov 0x1013, 0x3032, 0x4044, 0x8088, 1667bfc04aecSBorislav Petkov 0x106b, 0x30d6, 0x70fc, 0xe0a8, 1668bfc04aecSBorislav Petkov 0x4857, 0xc4fe, 0x13cc, 0x3288, 1669bfc04aecSBorislav Petkov 0x1ac5, 0x2f4a, 0x5394, 0xa1e8, 1670bfc04aecSBorislav Petkov 0x1f39, 0x251e, 0xbd6c, 0x6bd8, 1671bfc04aecSBorislav Petkov 0x15c1, 0x2a42, 0x89ac, 0x4758, 1672bfc04aecSBorislav Petkov 0x2b03, 0x1602, 0x4f0c, 0xca08, 1673bfc04aecSBorislav Petkov 0x1f07, 0x3a0e, 0x6b04, 0xbd08, 1674bfc04aecSBorislav Petkov 0x8ba7, 0x465e, 0x244c, 0x1cc8, 1675bfc04aecSBorislav Petkov 0x2b87, 0x164e, 0x642c, 0xdc18, 1676bfc04aecSBorislav Petkov 0x40b9, 0x80de, 0x1094, 0x20e8, 1677bfc04aecSBorislav Petkov 0x27db, 0x1eb6, 0x9dac, 0x7b58, 1678bfc04aecSBorislav Petkov 0x11c1, 0x2242, 0x84ac, 0x4c58, 1679bfc04aecSBorislav Petkov 0x1be5, 0x2d7a, 0x5e34, 0xa718, 1680bfc04aecSBorislav Petkov 0x4b39, 0x8d1e, 0x14b4, 0x28d8, 1681bfc04aecSBorislav Petkov 0x4c97, 0xc87e, 0x11fc, 0x33a8, 1682bfc04aecSBorislav Petkov 0x8e97, 0x497e, 0x2ffc, 0x1aa8, 1683bfc04aecSBorislav Petkov 0x16b3, 0x3d62, 0x4f34, 0x8518, 1684bfc04aecSBorislav Petkov 0x1e2f, 0x391a, 0x5cac, 0xf858, 1685bfc04aecSBorislav Petkov 0x1d9f, 0x3b7a, 0x572c, 0xfe18, 1686bfc04aecSBorislav Petkov 0x15f5, 0x2a5a, 0x5264, 0xa3b8, 1687bfc04aecSBorislav Petkov 0x1dbb, 0x3b66, 0x715c, 0xe3f8, 1688bfc04aecSBorislav Petkov 0x4397, 0xc27e, 0x17fc, 0x3ea8, 1689bfc04aecSBorislav Petkov 0x1617, 0x3d3e, 0x6464, 0xb8b8, 1690bfc04aecSBorislav Petkov 0x23ff, 0x12aa, 0xab6c, 0x56d8, 1691bfc04aecSBorislav Petkov 0x2dfb, 0x1ba6, 0x913c, 0x7328, 1692bfc04aecSBorislav Petkov 0x185d, 0x2ca6, 0x7914, 0x9e28, 1693bfc04aecSBorislav Petkov 0x171b, 0x3e36, 0x7d7c, 0xebe8, 1694bfc04aecSBorislav Petkov 0x4199, 0x82ee, 0x19f4, 0x2e58, 1695bfc04aecSBorislav Petkov 0x4807, 0xc40e, 0x130c, 0x3208, 1696bfc04aecSBorislav Petkov 0x1905, 0x2e0a, 0x5804, 0xac08, 1697bfc04aecSBorislav Petkov 0x213f, 0x132a, 0xadfc, 0x5ba8, 1698bfc04aecSBorislav Petkov 0x19a9, 0x2efe, 0xb5cc, 0x6f88, 1699b1289d6fSDoug Thompson }; 1700b1289d6fSDoug Thompson 1701bfc04aecSBorislav Petkov static u16 x8_vectors[] = { 1702bfc04aecSBorislav Petkov 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480, 1703bfc04aecSBorislav Petkov 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80, 1704bfc04aecSBorislav Petkov 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80, 1705bfc04aecSBorislav Petkov 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80, 1706bfc04aecSBorislav Petkov 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780, 1707bfc04aecSBorislav Petkov 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080, 1708bfc04aecSBorislav Petkov 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080, 1709bfc04aecSBorislav Petkov 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080, 1710bfc04aecSBorislav Petkov 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80, 1711bfc04aecSBorislav Petkov 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580, 1712bfc04aecSBorislav Petkov 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880, 1713bfc04aecSBorislav Petkov 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280, 1714bfc04aecSBorislav Petkov 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180, 1715bfc04aecSBorislav Petkov 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580, 1716bfc04aecSBorislav Petkov 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280, 1717bfc04aecSBorislav Petkov 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180, 1718bfc04aecSBorislav Petkov 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080, 1719bfc04aecSBorislav Petkov 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080, 1720bfc04aecSBorislav Petkov 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000, 1721bfc04aecSBorislav Petkov }; 1722bfc04aecSBorislav Petkov 1723d34a6ecdSBorislav Petkov static int decode_syndrome(u16 syndrome, u16 *vectors, unsigned num_vecs, 1724d34a6ecdSBorislav Petkov unsigned v_dim) 1725b1289d6fSDoug Thompson { 1726bfc04aecSBorislav Petkov unsigned int i, err_sym; 1727b1289d6fSDoug Thompson 1728bfc04aecSBorislav Petkov for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) { 1729bfc04aecSBorislav Petkov u16 s = syndrome; 1730d34a6ecdSBorislav Petkov unsigned v_idx = err_sym * v_dim; 1731d34a6ecdSBorislav Petkov unsigned v_end = (err_sym + 1) * v_dim; 1732b1289d6fSDoug Thompson 1733bfc04aecSBorislav Petkov /* walk over all 16 bits of the syndrome */ 1734bfc04aecSBorislav Petkov for (i = 1; i < (1U << 16); i <<= 1) { 1735bfc04aecSBorislav Petkov 1736bfc04aecSBorislav Petkov /* if bit is set in that eigenvector... */ 1737bfc04aecSBorislav Petkov if (v_idx < v_end && vectors[v_idx] & i) { 1738bfc04aecSBorislav Petkov u16 ev_comp = vectors[v_idx++]; 1739bfc04aecSBorislav Petkov 1740bfc04aecSBorislav Petkov /* ... and bit set in the modified syndrome, */ 1741bfc04aecSBorislav Petkov if (s & i) { 1742bfc04aecSBorislav Petkov /* remove it. */ 1743bfc04aecSBorislav Petkov s ^= ev_comp; 1744bfc04aecSBorislav Petkov 1745bfc04aecSBorislav Petkov if (!s) 1746bfc04aecSBorislav Petkov return err_sym; 1747bfc04aecSBorislav Petkov } 1748bfc04aecSBorislav Petkov 1749bfc04aecSBorislav Petkov } else if (s & i) 1750bfc04aecSBorislav Petkov /* can't get to zero, move to next symbol */ 1751bfc04aecSBorislav Petkov break; 1752bfc04aecSBorislav Petkov } 1753b1289d6fSDoug Thompson } 1754b1289d6fSDoug Thompson 1755b1289d6fSDoug Thompson debugf0("syndrome(%x) not found\n", syndrome); 1756b1289d6fSDoug Thompson return -1; 1757b1289d6fSDoug Thompson } 1758d27bf6faSDoug Thompson 1759bfc04aecSBorislav Petkov static int map_err_sym_to_channel(int err_sym, int sym_size) 1760bfc04aecSBorislav Petkov { 1761bfc04aecSBorislav Petkov if (sym_size == 4) 1762bfc04aecSBorislav Petkov switch (err_sym) { 1763bfc04aecSBorislav Petkov case 0x20: 1764bfc04aecSBorislav Petkov case 0x21: 1765bfc04aecSBorislav Petkov return 0; 1766bfc04aecSBorislav Petkov break; 1767bfc04aecSBorislav Petkov case 0x22: 1768bfc04aecSBorislav Petkov case 0x23: 1769bfc04aecSBorislav Petkov return 1; 1770bfc04aecSBorislav Petkov break; 1771bfc04aecSBorislav Petkov default: 1772bfc04aecSBorislav Petkov return err_sym >> 4; 1773bfc04aecSBorislav Petkov break; 1774bfc04aecSBorislav Petkov } 1775bfc04aecSBorislav Petkov /* x8 symbols */ 1776bfc04aecSBorislav Petkov else 1777bfc04aecSBorislav Petkov switch (err_sym) { 1778bfc04aecSBorislav Petkov /* imaginary bits not in a DIMM */ 1779bfc04aecSBorislav Petkov case 0x10: 1780bfc04aecSBorislav Petkov WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n", 1781bfc04aecSBorislav Petkov err_sym); 1782bfc04aecSBorislav Petkov return -1; 1783bfc04aecSBorislav Petkov break; 1784bfc04aecSBorislav Petkov 1785bfc04aecSBorislav Petkov case 0x11: 1786bfc04aecSBorislav Petkov return 0; 1787bfc04aecSBorislav Petkov break; 1788bfc04aecSBorislav Petkov case 0x12: 1789bfc04aecSBorislav Petkov return 1; 1790bfc04aecSBorislav Petkov break; 1791bfc04aecSBorislav Petkov default: 1792bfc04aecSBorislav Petkov return err_sym >> 3; 1793bfc04aecSBorislav Petkov break; 1794bfc04aecSBorislav Petkov } 1795bfc04aecSBorislav Petkov return -1; 1796bfc04aecSBorislav Petkov } 1797bfc04aecSBorislav Petkov 1798bfc04aecSBorislav Petkov static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome) 1799bfc04aecSBorislav Petkov { 1800bfc04aecSBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 1801ad6a32e9SBorislav Petkov int err_sym = -1; 1802bfc04aecSBorislav Petkov 1803a3b7db09SBorislav Petkov if (pvt->ecc_sym_sz == 8) 1804bfc04aecSBorislav Petkov err_sym = decode_syndrome(syndrome, x8_vectors, 1805ad6a32e9SBorislav Petkov ARRAY_SIZE(x8_vectors), 1806a3b7db09SBorislav Petkov pvt->ecc_sym_sz); 1807a3b7db09SBorislav Petkov else if (pvt->ecc_sym_sz == 4) 1808ad6a32e9SBorislav Petkov err_sym = decode_syndrome(syndrome, x4_vectors, 1809ad6a32e9SBorislav Petkov ARRAY_SIZE(x4_vectors), 1810a3b7db09SBorislav Petkov pvt->ecc_sym_sz); 1811ad6a32e9SBorislav Petkov else { 1812a3b7db09SBorislav Petkov amd64_warn("Illegal syndrome type: %u\n", pvt->ecc_sym_sz); 1813ad6a32e9SBorislav Petkov return err_sym; 1814bfc04aecSBorislav Petkov } 1815ad6a32e9SBorislav Petkov 1816a3b7db09SBorislav Petkov return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz); 181741c31044SBorislav Petkov } 1818bfc04aecSBorislav Petkov 1819d27bf6faSDoug Thompson /* 1820d27bf6faSDoug Thompson * Handle any Correctable Errors (CEs) that have occurred. Check for valid ERROR 1821d27bf6faSDoug Thompson * ADDRESS and process. 1822d27bf6faSDoug Thompson */ 1823f192c7b1SBorislav Petkov static void amd64_handle_ce(struct mem_ctl_info *mci, struct mce *m) 1824d27bf6faSDoug Thompson { 1825d27bf6faSDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 182644e9e2eeSBorislav Petkov u64 sys_addr; 1827f192c7b1SBorislav Petkov u16 syndrome; 1828d27bf6faSDoug Thompson 1829d27bf6faSDoug Thompson /* Ensure that the Error Address is VALID */ 1830f192c7b1SBorislav Petkov if (!(m->status & MCI_STATUS_ADDRV)) { 183124f9a7feSBorislav Petkov amd64_mc_err(mci, "HW has no ERROR_ADDRESS available\n"); 1832d27bf6faSDoug Thompson edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR); 1833d27bf6faSDoug Thompson return; 1834d27bf6faSDoug Thompson } 1835d27bf6faSDoug Thompson 183670046624SBorislav Petkov sys_addr = get_error_address(m); 1837f192c7b1SBorislav Petkov syndrome = extract_syndrome(m->status); 1838d27bf6faSDoug Thompson 183924f9a7feSBorislav Petkov amd64_mc_err(mci, "CE ERROR_ADDRESS= 0x%llx\n", sys_addr); 1840d27bf6faSDoug Thompson 1841f192c7b1SBorislav Petkov pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, syndrome); 1842d27bf6faSDoug Thompson } 1843d27bf6faSDoug Thompson 1844d27bf6faSDoug Thompson /* Handle any Un-correctable Errors (UEs) */ 1845f192c7b1SBorislav Petkov static void amd64_handle_ue(struct mem_ctl_info *mci, struct mce *m) 1846d27bf6faSDoug Thompson { 18471f6bcee7SBorislav Petkov struct mem_ctl_info *log_mci, *src_mci = NULL; 1848d27bf6faSDoug Thompson int csrow; 184944e9e2eeSBorislav Petkov u64 sys_addr; 1850d27bf6faSDoug Thompson u32 page, offset; 1851d27bf6faSDoug Thompson 1852d27bf6faSDoug Thompson log_mci = mci; 1853d27bf6faSDoug Thompson 1854f192c7b1SBorislav Petkov if (!(m->status & MCI_STATUS_ADDRV)) { 185524f9a7feSBorislav Petkov amd64_mc_err(mci, "HW has no ERROR_ADDRESS available\n"); 1856d27bf6faSDoug Thompson edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR); 1857d27bf6faSDoug Thompson return; 1858d27bf6faSDoug Thompson } 1859d27bf6faSDoug Thompson 186070046624SBorislav Petkov sys_addr = get_error_address(m); 1861d27bf6faSDoug Thompson 1862d27bf6faSDoug Thompson /* 1863d27bf6faSDoug Thompson * Find out which node the error address belongs to. This may be 1864d27bf6faSDoug Thompson * different from the node that detected the error. 1865d27bf6faSDoug Thompson */ 186644e9e2eeSBorislav Petkov src_mci = find_mc_by_sys_addr(mci, sys_addr); 1867d27bf6faSDoug Thompson if (!src_mci) { 186824f9a7feSBorislav Petkov amd64_mc_err(mci, "ERROR ADDRESS (0x%lx) NOT mapped to a MC\n", 186944e9e2eeSBorislav Petkov (unsigned long)sys_addr); 1870d27bf6faSDoug Thompson edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR); 1871d27bf6faSDoug Thompson return; 1872d27bf6faSDoug Thompson } 1873d27bf6faSDoug Thompson 1874d27bf6faSDoug Thompson log_mci = src_mci; 1875d27bf6faSDoug Thompson 187644e9e2eeSBorislav Petkov csrow = sys_addr_to_csrow(log_mci, sys_addr); 1877d27bf6faSDoug Thompson if (csrow < 0) { 187824f9a7feSBorislav Petkov amd64_mc_err(mci, "ERROR_ADDRESS (0x%lx) NOT mapped to CS\n", 187944e9e2eeSBorislav Petkov (unsigned long)sys_addr); 1880d27bf6faSDoug Thompson edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR); 1881d27bf6faSDoug Thompson } else { 188244e9e2eeSBorislav Petkov error_address_to_page_and_offset(sys_addr, &page, &offset); 1883d27bf6faSDoug Thompson edac_mc_handle_ue(log_mci, page, offset, csrow, EDAC_MOD_STR); 1884d27bf6faSDoug Thompson } 1885d27bf6faSDoug Thompson } 1886d27bf6faSDoug Thompson 1887549d042dSBorislav Petkov static inline void __amd64_decode_bus_error(struct mem_ctl_info *mci, 1888f192c7b1SBorislav Petkov struct mce *m) 1889d27bf6faSDoug Thompson { 1890f192c7b1SBorislav Petkov u16 ec = EC(m->status); 1891f192c7b1SBorislav Petkov u8 xec = XEC(m->status, 0x1f); 1892f192c7b1SBorislav Petkov u8 ecc_type = (m->status >> 45) & 0x3; 1893d27bf6faSDoug Thompson 1894b70ef010SBorislav Petkov /* Bail early out if this was an 'observed' error */ 18955980bb9cSBorislav Petkov if (PP(ec) == NBSL_PP_OBS) 1896b70ef010SBorislav Petkov return; 1897d27bf6faSDoug Thompson 1898ecaf5606SBorislav Petkov /* Do only ECC errors */ 1899ecaf5606SBorislav Petkov if (xec && xec != F10_NBSL_EXT_ERR_ECC) 1900d27bf6faSDoug Thompson return; 1901d27bf6faSDoug Thompson 1902ecaf5606SBorislav Petkov if (ecc_type == 2) 1903f192c7b1SBorislav Petkov amd64_handle_ce(mci, m); 1904ecaf5606SBorislav Petkov else if (ecc_type == 1) 1905f192c7b1SBorislav Petkov amd64_handle_ue(mci, m); 1906d27bf6faSDoug Thompson } 1907d27bf6faSDoug Thompson 19087cfd4a87SBorislav Petkov void amd64_decode_bus_error(int node_id, struct mce *m, u32 nbcfg) 1909d27bf6faSDoug Thompson { 1910cc4d8860SBorislav Petkov struct mem_ctl_info *mci = mcis[node_id]; 1911d27bf6faSDoug Thompson 1912f192c7b1SBorislav Petkov __amd64_decode_bus_error(mci, m); 1913d27bf6faSDoug Thompson } 1914d27bf6faSDoug Thompson 19150ec449eeSDoug Thompson /* 19168d5b5d9cSBorislav Petkov * Use pvt->F2 which contains the F2 CPU PCI device to get the related 1917bbd0c1f6SBorislav Petkov * F1 (AddrMap) and F3 (Misc) devices. Return negative value on error. 19180ec449eeSDoug Thompson */ 1919360b7f3cSBorislav Petkov static int reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 f1_id, u16 f3_id) 19200ec449eeSDoug Thompson { 19210ec449eeSDoug Thompson /* Reserve the ADDRESS MAP Device */ 19228d5b5d9cSBorislav Petkov pvt->F1 = pci_get_related_function(pvt->F2->vendor, f1_id, pvt->F2); 19238d5b5d9cSBorislav Petkov if (!pvt->F1) { 192424f9a7feSBorislav Petkov amd64_err("error address map device not found: " 19250ec449eeSDoug Thompson "vendor %x device 0x%x (broken BIOS?)\n", 1926bbd0c1f6SBorislav Petkov PCI_VENDOR_ID_AMD, f1_id); 1927bbd0c1f6SBorislav Petkov return -ENODEV; 19280ec449eeSDoug Thompson } 19290ec449eeSDoug Thompson 19300ec449eeSDoug Thompson /* Reserve the MISC Device */ 19318d5b5d9cSBorislav Petkov pvt->F3 = pci_get_related_function(pvt->F2->vendor, f3_id, pvt->F2); 19328d5b5d9cSBorislav Petkov if (!pvt->F3) { 19338d5b5d9cSBorislav Petkov pci_dev_put(pvt->F1); 19348d5b5d9cSBorislav Petkov pvt->F1 = NULL; 19350ec449eeSDoug Thompson 193624f9a7feSBorislav Petkov amd64_err("error F3 device not found: " 19370ec449eeSDoug Thompson "vendor %x device 0x%x (broken BIOS?)\n", 1938bbd0c1f6SBorislav Petkov PCI_VENDOR_ID_AMD, f3_id); 19398d5b5d9cSBorislav Petkov 1940bbd0c1f6SBorislav Petkov return -ENODEV; 19410ec449eeSDoug Thompson } 19428d5b5d9cSBorislav Petkov debugf1("F1: %s\n", pci_name(pvt->F1)); 19438d5b5d9cSBorislav Petkov debugf1("F2: %s\n", pci_name(pvt->F2)); 19448d5b5d9cSBorislav Petkov debugf1("F3: %s\n", pci_name(pvt->F3)); 19450ec449eeSDoug Thompson 19460ec449eeSDoug Thompson return 0; 19470ec449eeSDoug Thompson } 19480ec449eeSDoug Thompson 1949360b7f3cSBorislav Petkov static void free_mc_sibling_devs(struct amd64_pvt *pvt) 19500ec449eeSDoug Thompson { 19518d5b5d9cSBorislav Petkov pci_dev_put(pvt->F1); 19528d5b5d9cSBorislav Petkov pci_dev_put(pvt->F3); 19530ec449eeSDoug Thompson } 19540ec449eeSDoug Thompson 19550ec449eeSDoug Thompson /* 19560ec449eeSDoug Thompson * Retrieve the hardware registers of the memory controller (this includes the 19570ec449eeSDoug Thompson * 'Address Map' and 'Misc' device regs) 19580ec449eeSDoug Thompson */ 1959360b7f3cSBorislav Petkov static void read_mc_regs(struct amd64_pvt *pvt) 19600ec449eeSDoug Thompson { 1961a3b7db09SBorislav Petkov struct cpuinfo_x86 *c = &boot_cpu_data; 19620ec449eeSDoug Thompson u64 msr_val; 1963ad6a32e9SBorislav Petkov u32 tmp; 1964e761359aSBorislav Petkov unsigned range; 19650ec449eeSDoug Thompson 19660ec449eeSDoug Thompson /* 19670ec449eeSDoug Thompson * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since 19680ec449eeSDoug Thompson * those are Read-As-Zero 19690ec449eeSDoug Thompson */ 1970e97f8bb8SBorislav Petkov rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem); 1971e97f8bb8SBorislav Petkov debugf0(" TOP_MEM: 0x%016llx\n", pvt->top_mem); 19720ec449eeSDoug Thompson 19730ec449eeSDoug Thompson /* check first whether TOP_MEM2 is enabled */ 19740ec449eeSDoug Thompson rdmsrl(MSR_K8_SYSCFG, msr_val); 19750ec449eeSDoug Thompson if (msr_val & (1U << 21)) { 1976e97f8bb8SBorislav Petkov rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2); 1977e97f8bb8SBorislav Petkov debugf0(" TOP_MEM2: 0x%016llx\n", pvt->top_mem2); 19780ec449eeSDoug Thompson } else 19790ec449eeSDoug Thompson debugf0(" TOP_MEM2 disabled.\n"); 19800ec449eeSDoug Thompson 19815980bb9cSBorislav Petkov amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap); 19820ec449eeSDoug Thompson 19835a5d2371SBorislav Petkov read_dram_ctl_register(pvt); 19840ec449eeSDoug Thompson 19857f19bf75SBorislav Petkov for (range = 0; range < DRAM_RANGES; range++) { 19867f19bf75SBorislav Petkov u8 rw; 19870ec449eeSDoug Thompson 19887f19bf75SBorislav Petkov /* read settings for this DRAM range */ 19897f19bf75SBorislav Petkov read_dram_base_limit_regs(pvt, range); 1990e97f8bb8SBorislav Petkov 19917f19bf75SBorislav Petkov rw = dram_rw(pvt, range); 19927f19bf75SBorislav Petkov if (!rw) 19937f19bf75SBorislav Petkov continue; 19947f19bf75SBorislav Petkov 19957f19bf75SBorislav Petkov debugf1(" DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n", 19967f19bf75SBorislav Petkov range, 19977f19bf75SBorislav Petkov get_dram_base(pvt, range), 19987f19bf75SBorislav Petkov get_dram_limit(pvt, range)); 19997f19bf75SBorislav Petkov 20007f19bf75SBorislav Petkov debugf1(" IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n", 20017f19bf75SBorislav Petkov dram_intlv_en(pvt, range) ? "Enabled" : "Disabled", 20027f19bf75SBorislav Petkov (rw & 0x1) ? "R" : "-", 20037f19bf75SBorislav Petkov (rw & 0x2) ? "W" : "-", 20047f19bf75SBorislav Petkov dram_intlv_sel(pvt, range), 20057f19bf75SBorislav Petkov dram_dst_node(pvt, range)); 20060ec449eeSDoug Thompson } 20070ec449eeSDoug Thompson 2008b2b0c605SBorislav Petkov read_dct_base_mask(pvt); 20090ec449eeSDoug Thompson 2010bc21fa57SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar); 2011525a1b20SBorislav Petkov amd64_read_dct_pci_cfg(pvt, DBAM0, &pvt->dbam0); 20120ec449eeSDoug Thompson 20138d5b5d9cSBorislav Petkov amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare); 20140ec449eeSDoug Thompson 2015cb328507SBorislav Petkov amd64_read_dct_pci_cfg(pvt, DCLR0, &pvt->dclr0); 2016cb328507SBorislav Petkov amd64_read_dct_pci_cfg(pvt, DCHR0, &pvt->dchr0); 20170ec449eeSDoug Thompson 201878da121eSBorislav Petkov if (!dct_ganging_enabled(pvt)) { 2019cb328507SBorislav Petkov amd64_read_dct_pci_cfg(pvt, DCLR1, &pvt->dclr1); 2020cb328507SBorislav Petkov amd64_read_dct_pci_cfg(pvt, DCHR1, &pvt->dchr1); 20210ec449eeSDoug Thompson } 2022b2b0c605SBorislav Petkov 2023a3b7db09SBorislav Petkov pvt->ecc_sym_sz = 4; 2024a3b7db09SBorislav Petkov 2025a3b7db09SBorislav Petkov if (c->x86 >= 0x10) { 20268d5b5d9cSBorislav Petkov amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp); 2027525a1b20SBorislav Petkov amd64_read_dct_pci_cfg(pvt, DBAM1, &pvt->dbam1); 2028a3b7db09SBorislav Petkov 2029a3b7db09SBorislav Petkov /* F10h, revD and later can do x8 ECC too */ 2030a3b7db09SBorislav Petkov if ((c->x86 > 0x10 || c->x86_model > 7) && tmp & BIT(25)) 2031a3b7db09SBorislav Petkov pvt->ecc_sym_sz = 8; 2032525a1b20SBorislav Petkov } 2033b2b0c605SBorislav Petkov dump_misc_regs(pvt); 20340ec449eeSDoug Thompson } 20350ec449eeSDoug Thompson 20360ec449eeSDoug Thompson /* 20370ec449eeSDoug Thompson * NOTE: CPU Revision Dependent code 20380ec449eeSDoug Thompson * 20390ec449eeSDoug Thompson * Input: 204011c75eadSBorislav Petkov * @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1) 20410ec449eeSDoug Thompson * k8 private pointer to --> 20420ec449eeSDoug Thompson * DRAM Bank Address mapping register 20430ec449eeSDoug Thompson * node_id 20440ec449eeSDoug Thompson * DCL register where dual_channel_active is 20450ec449eeSDoug Thompson * 20460ec449eeSDoug Thompson * The DBAM register consists of 4 sets of 4 bits each definitions: 20470ec449eeSDoug Thompson * 20480ec449eeSDoug Thompson * Bits: CSROWs 20490ec449eeSDoug Thompson * 0-3 CSROWs 0 and 1 20500ec449eeSDoug Thompson * 4-7 CSROWs 2 and 3 20510ec449eeSDoug Thompson * 8-11 CSROWs 4 and 5 20520ec449eeSDoug Thompson * 12-15 CSROWs 6 and 7 20530ec449eeSDoug Thompson * 20540ec449eeSDoug Thompson * Values range from: 0 to 15 20550ec449eeSDoug Thompson * The meaning of the values depends on CPU revision and dual-channel state, 20560ec449eeSDoug Thompson * see relevant BKDG more info. 20570ec449eeSDoug Thompson * 20580ec449eeSDoug Thompson * The memory controller provides for total of only 8 CSROWs in its current 20590ec449eeSDoug Thompson * architecture. Each "pair" of CSROWs normally represents just one DIMM in 20600ec449eeSDoug Thompson * single channel or two (2) DIMMs in dual channel mode. 20610ec449eeSDoug Thompson * 20620ec449eeSDoug Thompson * The following code logic collapses the various tables for CSROW based on CPU 20630ec449eeSDoug Thompson * revision. 20640ec449eeSDoug Thompson * 20650ec449eeSDoug Thompson * Returns: 20660ec449eeSDoug Thompson * The number of PAGE_SIZE pages on the specified CSROW number it 20670ec449eeSDoug Thompson * encompasses 20680ec449eeSDoug Thompson * 20690ec449eeSDoug Thompson */ 207041d8bfabSBorislav Petkov static u32 amd64_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr) 20710ec449eeSDoug Thompson { 20721433eb99SBorislav Petkov u32 cs_mode, nr_pages; 20730ec449eeSDoug Thompson 20740ec449eeSDoug Thompson /* 20750ec449eeSDoug Thompson * The math on this doesn't look right on the surface because x/2*4 can 20760ec449eeSDoug Thompson * be simplified to x*2 but this expression makes use of the fact that 20770ec449eeSDoug Thompson * it is integral math where 1/2=0. This intermediate value becomes the 20780ec449eeSDoug Thompson * number of bits to shift the DBAM register to extract the proper CSROW 20790ec449eeSDoug Thompson * field. 20800ec449eeSDoug Thompson */ 20811433eb99SBorislav Petkov cs_mode = (pvt->dbam0 >> ((csrow_nr / 2) * 4)) & 0xF; 20820ec449eeSDoug Thompson 208341d8bfabSBorislav Petkov nr_pages = pvt->ops->dbam_to_cs(pvt, dct, cs_mode) << (20 - PAGE_SHIFT); 20840ec449eeSDoug Thompson 20850ec449eeSDoug Thompson /* 20860ec449eeSDoug Thompson * If dual channel then double the memory size of single channel. 20870ec449eeSDoug Thompson * Channel count is 1 or 2 20880ec449eeSDoug Thompson */ 20890ec449eeSDoug Thompson nr_pages <<= (pvt->channel_count - 1); 20900ec449eeSDoug Thompson 20911433eb99SBorislav Petkov debugf0(" (csrow=%d) DBAM map index= %d\n", csrow_nr, cs_mode); 20920ec449eeSDoug Thompson debugf0(" nr_pages= %u channel-count = %d\n", 20930ec449eeSDoug Thompson nr_pages, pvt->channel_count); 20940ec449eeSDoug Thompson 20950ec449eeSDoug Thompson return nr_pages; 20960ec449eeSDoug Thompson } 20970ec449eeSDoug Thompson 20980ec449eeSDoug Thompson /* 20990ec449eeSDoug Thompson * Initialize the array of csrow attribute instances, based on the values 21000ec449eeSDoug Thompson * from pci config hardware registers. 21010ec449eeSDoug Thompson */ 2102360b7f3cSBorislav Petkov static int init_csrows(struct mem_ctl_info *mci) 21030ec449eeSDoug Thompson { 21040ec449eeSDoug Thompson struct csrow_info *csrow; 21052299ef71SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 210611c75eadSBorislav Petkov u64 input_addr_min, input_addr_max, sys_addr, base, mask; 21072299ef71SBorislav Petkov u32 val; 21086ba5dcdcSBorislav Petkov int i, empty = 1; 21090ec449eeSDoug Thompson 2110a97fa68eSBorislav Petkov amd64_read_pci_cfg(pvt->F3, NBCFG, &val); 21110ec449eeSDoug Thompson 21122299ef71SBorislav Petkov pvt->nbcfg = val; 21130ec449eeSDoug Thompson 21142299ef71SBorislav Petkov debugf0("node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n", 21152299ef71SBorislav Petkov pvt->mc_node_id, val, 2116a97fa68eSBorislav Petkov !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE)); 21170ec449eeSDoug Thompson 211811c75eadSBorislav Petkov for_each_chip_select(i, 0, pvt) { 21190ec449eeSDoug Thompson csrow = &mci->csrows[i]; 21200ec449eeSDoug Thompson 212111c75eadSBorislav Petkov if (!csrow_enabled(i, 0, pvt)) { 21220ec449eeSDoug Thompson debugf1("----CSROW %d EMPTY for node %d\n", i, 21230ec449eeSDoug Thompson pvt->mc_node_id); 21240ec449eeSDoug Thompson continue; 21250ec449eeSDoug Thompson } 21260ec449eeSDoug Thompson 21270ec449eeSDoug Thompson debugf1("----CSROW %d VALID for MC node %d\n", 21280ec449eeSDoug Thompson i, pvt->mc_node_id); 21290ec449eeSDoug Thompson 21300ec449eeSDoug Thompson empty = 0; 213141d8bfabSBorislav Petkov csrow->nr_pages = amd64_csrow_nr_pages(pvt, 0, i); 21320ec449eeSDoug Thompson find_csrow_limits(mci, i, &input_addr_min, &input_addr_max); 21330ec449eeSDoug Thompson sys_addr = input_addr_to_sys_addr(mci, input_addr_min); 21340ec449eeSDoug Thompson csrow->first_page = (u32) (sys_addr >> PAGE_SHIFT); 21350ec449eeSDoug Thompson sys_addr = input_addr_to_sys_addr(mci, input_addr_max); 21360ec449eeSDoug Thompson csrow->last_page = (u32) (sys_addr >> PAGE_SHIFT); 213711c75eadSBorislav Petkov 213811c75eadSBorislav Petkov get_cs_base_and_mask(pvt, i, 0, &base, &mask); 213911c75eadSBorislav Petkov csrow->page_mask = ~mask; 21400ec449eeSDoug Thompson /* 8 bytes of resolution */ 21410ec449eeSDoug Thompson 214224f9a7feSBorislav Petkov csrow->mtype = amd64_determine_memory_type(pvt, i); 21430ec449eeSDoug Thompson 21440ec449eeSDoug Thompson debugf1(" for MC node %d csrow %d:\n", pvt->mc_node_id, i); 21450ec449eeSDoug Thompson debugf1(" input_addr_min: 0x%lx input_addr_max: 0x%lx\n", 21460ec449eeSDoug Thompson (unsigned long)input_addr_min, 21470ec449eeSDoug Thompson (unsigned long)input_addr_max); 21480ec449eeSDoug Thompson debugf1(" sys_addr: 0x%lx page_mask: 0x%lx\n", 21490ec449eeSDoug Thompson (unsigned long)sys_addr, csrow->page_mask); 21500ec449eeSDoug Thompson debugf1(" nr_pages: %u first_page: 0x%lx " 21510ec449eeSDoug Thompson "last_page: 0x%lx\n", 21520ec449eeSDoug Thompson (unsigned)csrow->nr_pages, 21530ec449eeSDoug Thompson csrow->first_page, csrow->last_page); 21540ec449eeSDoug Thompson 21550ec449eeSDoug Thompson /* 21560ec449eeSDoug Thompson * determine whether CHIPKILL or JUST ECC or NO ECC is operating 21570ec449eeSDoug Thompson */ 2158a97fa68eSBorislav Petkov if (pvt->nbcfg & NBCFG_ECC_ENABLE) 21590ec449eeSDoug Thompson csrow->edac_mode = 2160a97fa68eSBorislav Petkov (pvt->nbcfg & NBCFG_CHIPKILL) ? 21610ec449eeSDoug Thompson EDAC_S4ECD4ED : EDAC_SECDED; 21620ec449eeSDoug Thompson else 21630ec449eeSDoug Thompson csrow->edac_mode = EDAC_NONE; 21640ec449eeSDoug Thompson } 21650ec449eeSDoug Thompson 21660ec449eeSDoug Thompson return empty; 21670ec449eeSDoug Thompson } 2168d27bf6faSDoug Thompson 216906724535SBorislav Petkov /* get all cores on this DCT */ 2170b487c33eSBorislav Petkov static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, unsigned nid) 2171f9431992SDoug Thompson { 217206724535SBorislav Petkov int cpu; 2173f9431992SDoug Thompson 217406724535SBorislav Petkov for_each_online_cpu(cpu) 217506724535SBorislav Petkov if (amd_get_nb_id(cpu) == nid) 217606724535SBorislav Petkov cpumask_set_cpu(cpu, mask); 2177f9431992SDoug Thompson } 2178f9431992SDoug Thompson 2179f9431992SDoug Thompson /* check MCG_CTL on all the cpus on this node */ 2180b487c33eSBorislav Petkov static bool amd64_nb_mce_bank_enabled_on_node(unsigned nid) 2181f9431992SDoug Thompson { 2182ba578cb3SRusty Russell cpumask_var_t mask; 218350542251SBorislav Petkov int cpu, nbe; 218406724535SBorislav Petkov bool ret = false; 2185f9431992SDoug Thompson 2186ba578cb3SRusty Russell if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) { 218724f9a7feSBorislav Petkov amd64_warn("%s: Error allocating mask\n", __func__); 218806724535SBorislav Petkov return false; 218906724535SBorislav Petkov } 219006724535SBorislav Petkov 2191ba578cb3SRusty Russell get_cpus_on_this_dct_cpumask(mask, nid); 219206724535SBorislav Petkov 2193ba578cb3SRusty Russell rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs); 2194ba578cb3SRusty Russell 2195ba578cb3SRusty Russell for_each_cpu(cpu, mask) { 219650542251SBorislav Petkov struct msr *reg = per_cpu_ptr(msrs, cpu); 21975980bb9cSBorislav Petkov nbe = reg->l & MSR_MCGCTL_NBE; 219806724535SBorislav Petkov 219906724535SBorislav Petkov debugf0("core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n", 220050542251SBorislav Petkov cpu, reg->q, 220106724535SBorislav Petkov (nbe ? "enabled" : "disabled")); 220206724535SBorislav Petkov 220306724535SBorislav Petkov if (!nbe) 220406724535SBorislav Petkov goto out; 220506724535SBorislav Petkov } 220606724535SBorislav Petkov ret = true; 220706724535SBorislav Petkov 220806724535SBorislav Petkov out: 2209ba578cb3SRusty Russell free_cpumask_var(mask); 2210f9431992SDoug Thompson return ret; 2211f9431992SDoug Thompson } 2212f9431992SDoug Thompson 22132299ef71SBorislav Petkov static int toggle_ecc_err_reporting(struct ecc_settings *s, u8 nid, bool on) 2214f6d6ae96SBorislav Petkov { 2215f6d6ae96SBorislav Petkov cpumask_var_t cmask; 221650542251SBorislav Petkov int cpu; 2217f6d6ae96SBorislav Petkov 2218f6d6ae96SBorislav Petkov if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) { 221924f9a7feSBorislav Petkov amd64_warn("%s: error allocating mask\n", __func__); 2220f6d6ae96SBorislav Petkov return false; 2221f6d6ae96SBorislav Petkov } 2222f6d6ae96SBorislav Petkov 2223ae7bb7c6SBorislav Petkov get_cpus_on_this_dct_cpumask(cmask, nid); 2224f6d6ae96SBorislav Petkov 2225f6d6ae96SBorislav Petkov rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs); 2226f6d6ae96SBorislav Petkov 2227f6d6ae96SBorislav Petkov for_each_cpu(cpu, cmask) { 2228f6d6ae96SBorislav Petkov 222950542251SBorislav Petkov struct msr *reg = per_cpu_ptr(msrs, cpu); 223050542251SBorislav Petkov 2231f6d6ae96SBorislav Petkov if (on) { 22325980bb9cSBorislav Petkov if (reg->l & MSR_MCGCTL_NBE) 2233ae7bb7c6SBorislav Petkov s->flags.nb_mce_enable = 1; 2234f6d6ae96SBorislav Petkov 22355980bb9cSBorislav Petkov reg->l |= MSR_MCGCTL_NBE; 2236f6d6ae96SBorislav Petkov } else { 2237f6d6ae96SBorislav Petkov /* 2238d95cf4deSBorislav Petkov * Turn off NB MCE reporting only when it was off before 2239f6d6ae96SBorislav Petkov */ 2240ae7bb7c6SBorislav Petkov if (!s->flags.nb_mce_enable) 22415980bb9cSBorislav Petkov reg->l &= ~MSR_MCGCTL_NBE; 2242f6d6ae96SBorislav Petkov } 2243f6d6ae96SBorislav Petkov } 2244f6d6ae96SBorislav Petkov wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs); 2245f6d6ae96SBorislav Petkov 2246f6d6ae96SBorislav Petkov free_cpumask_var(cmask); 2247f6d6ae96SBorislav Petkov 2248f6d6ae96SBorislav Petkov return 0; 2249f6d6ae96SBorislav Petkov } 2250f6d6ae96SBorislav Petkov 22512299ef71SBorislav Petkov static bool enable_ecc_error_reporting(struct ecc_settings *s, u8 nid, 22522299ef71SBorislav Petkov struct pci_dev *F3) 2253f6d6ae96SBorislav Petkov { 22542299ef71SBorislav Petkov bool ret = true; 2255c9f4f26eSBorislav Petkov u32 value, mask = 0x3; /* UECC/CECC enable */ 2256f6d6ae96SBorislav Petkov 22572299ef71SBorislav Petkov if (toggle_ecc_err_reporting(s, nid, ON)) { 22582299ef71SBorislav Petkov amd64_warn("Error enabling ECC reporting over MCGCTL!\n"); 22592299ef71SBorislav Petkov return false; 22602299ef71SBorislav Petkov } 22612299ef71SBorislav Petkov 2262c9f4f26eSBorislav Petkov amd64_read_pci_cfg(F3, NBCTL, &value); 2263f6d6ae96SBorislav Petkov 2264ae7bb7c6SBorislav Petkov s->old_nbctl = value & mask; 2265ae7bb7c6SBorislav Petkov s->nbctl_valid = true; 2266f6d6ae96SBorislav Petkov 2267f6d6ae96SBorislav Petkov value |= mask; 2268c9f4f26eSBorislav Petkov amd64_write_pci_cfg(F3, NBCTL, value); 2269f6d6ae96SBorislav Petkov 2270a97fa68eSBorislav Petkov amd64_read_pci_cfg(F3, NBCFG, &value); 2271f6d6ae96SBorislav Petkov 2272a97fa68eSBorislav Petkov debugf0("1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n", 2273a97fa68eSBorislav Petkov nid, value, !!(value & NBCFG_ECC_ENABLE)); 2274f6d6ae96SBorislav Petkov 2275a97fa68eSBorislav Petkov if (!(value & NBCFG_ECC_ENABLE)) { 227624f9a7feSBorislav Petkov amd64_warn("DRAM ECC disabled on this node, enabling...\n"); 2277f6d6ae96SBorislav Petkov 2278ae7bb7c6SBorislav Petkov s->flags.nb_ecc_prev = 0; 2279d95cf4deSBorislav Petkov 2280f6d6ae96SBorislav Petkov /* Attempt to turn on DRAM ECC Enable */ 2281a97fa68eSBorislav Petkov value |= NBCFG_ECC_ENABLE; 2282a97fa68eSBorislav Petkov amd64_write_pci_cfg(F3, NBCFG, value); 2283f6d6ae96SBorislav Petkov 2284a97fa68eSBorislav Petkov amd64_read_pci_cfg(F3, NBCFG, &value); 2285f6d6ae96SBorislav Petkov 2286a97fa68eSBorislav Petkov if (!(value & NBCFG_ECC_ENABLE)) { 228724f9a7feSBorislav Petkov amd64_warn("Hardware rejected DRAM ECC enable," 228824f9a7feSBorislav Petkov "check memory DIMM configuration.\n"); 22892299ef71SBorislav Petkov ret = false; 2290f6d6ae96SBorislav Petkov } else { 229124f9a7feSBorislav Petkov amd64_info("Hardware accepted DRAM ECC Enable\n"); 2292f6d6ae96SBorislav Petkov } 2293d95cf4deSBorislav Petkov } else { 2294ae7bb7c6SBorislav Petkov s->flags.nb_ecc_prev = 1; 2295f6d6ae96SBorislav Petkov } 2296d95cf4deSBorislav Petkov 2297a97fa68eSBorislav Petkov debugf0("2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n", 2298a97fa68eSBorislav Petkov nid, value, !!(value & NBCFG_ECC_ENABLE)); 2299f6d6ae96SBorislav Petkov 23002299ef71SBorislav Petkov return ret; 2301f6d6ae96SBorislav Petkov } 2302f6d6ae96SBorislav Petkov 2303360b7f3cSBorislav Petkov static void restore_ecc_error_reporting(struct ecc_settings *s, u8 nid, 2304360b7f3cSBorislav Petkov struct pci_dev *F3) 2305f6d6ae96SBorislav Petkov { 2306c9f4f26eSBorislav Petkov u32 value, mask = 0x3; /* UECC/CECC enable */ 2307c9f4f26eSBorislav Petkov 2308f6d6ae96SBorislav Petkov 2309ae7bb7c6SBorislav Petkov if (!s->nbctl_valid) 2310f6d6ae96SBorislav Petkov return; 2311f6d6ae96SBorislav Petkov 2312c9f4f26eSBorislav Petkov amd64_read_pci_cfg(F3, NBCTL, &value); 2313f6d6ae96SBorislav Petkov value &= ~mask; 2314ae7bb7c6SBorislav Petkov value |= s->old_nbctl; 2315f6d6ae96SBorislav Petkov 2316c9f4f26eSBorislav Petkov amd64_write_pci_cfg(F3, NBCTL, value); 2317f6d6ae96SBorislav Petkov 2318ae7bb7c6SBorislav Petkov /* restore previous BIOS DRAM ECC "off" setting we force-enabled */ 2319ae7bb7c6SBorislav Petkov if (!s->flags.nb_ecc_prev) { 2320a97fa68eSBorislav Petkov amd64_read_pci_cfg(F3, NBCFG, &value); 2321a97fa68eSBorislav Petkov value &= ~NBCFG_ECC_ENABLE; 2322a97fa68eSBorislav Petkov amd64_write_pci_cfg(F3, NBCFG, value); 2323d95cf4deSBorislav Petkov } 2324d95cf4deSBorislav Petkov 2325d95cf4deSBorislav Petkov /* restore the NB Enable MCGCTL bit */ 23262299ef71SBorislav Petkov if (toggle_ecc_err_reporting(s, nid, OFF)) 232724f9a7feSBorislav Petkov amd64_warn("Error restoring NB MCGCTL settings!\n"); 2328f6d6ae96SBorislav Petkov } 2329f6d6ae96SBorislav Petkov 2330f9431992SDoug Thompson /* 23312299ef71SBorislav Petkov * EDAC requires that the BIOS have ECC enabled before 23322299ef71SBorislav Petkov * taking over the processing of ECC errors. A command line 23332299ef71SBorislav Petkov * option allows to force-enable hardware ECC later in 23342299ef71SBorislav Petkov * enable_ecc_error_reporting(). 2335f9431992SDoug Thompson */ 2336cab4d277SBorislav Petkov static const char *ecc_msg = 2337cab4d277SBorislav Petkov "ECC disabled in the BIOS or no ECC capability, module will not load.\n" 2338cab4d277SBorislav Petkov " Either enable ECC checking or force module loading by setting " 2339cab4d277SBorislav Petkov "'ecc_enable_override'.\n" 2340cab4d277SBorislav Petkov " (Note that use of the override may cause unknown side effects.)\n"; 2341be3468e8SBorislav Petkov 23422299ef71SBorislav Petkov static bool ecc_enabled(struct pci_dev *F3, u8 nid) 2343f9431992SDoug Thompson { 2344f9431992SDoug Thompson u32 value; 23452299ef71SBorislav Petkov u8 ecc_en = 0; 234606724535SBorislav Petkov bool nb_mce_en = false; 2347f9431992SDoug Thompson 2348a97fa68eSBorislav Petkov amd64_read_pci_cfg(F3, NBCFG, &value); 2349f9431992SDoug Thompson 2350a97fa68eSBorislav Petkov ecc_en = !!(value & NBCFG_ECC_ENABLE); 23512299ef71SBorislav Petkov amd64_info("DRAM ECC %s.\n", (ecc_en ? "enabled" : "disabled")); 2352be3468e8SBorislav Petkov 23532299ef71SBorislav Petkov nb_mce_en = amd64_nb_mce_bank_enabled_on_node(nid); 235406724535SBorislav Petkov if (!nb_mce_en) 23552299ef71SBorislav Petkov amd64_notice("NB MCE bank disabled, set MSR " 23562299ef71SBorislav Petkov "0x%08x[4] on node %d to enable.\n", 23572299ef71SBorislav Petkov MSR_IA32_MCG_CTL, nid); 2358be3468e8SBorislav Petkov 23592299ef71SBorislav Petkov if (!ecc_en || !nb_mce_en) { 236024f9a7feSBorislav Petkov amd64_notice("%s", ecc_msg); 23612299ef71SBorislav Petkov return false; 2362be3468e8SBorislav Petkov } 23632299ef71SBorislav Petkov return true; 2364f9431992SDoug Thompson } 2365f9431992SDoug Thompson 23667d6034d3SDoug Thompson struct mcidev_sysfs_attribute sysfs_attrs[ARRAY_SIZE(amd64_dbg_attrs) + 23677d6034d3SDoug Thompson ARRAY_SIZE(amd64_inj_attrs) + 23687d6034d3SDoug Thompson 1]; 23697d6034d3SDoug Thompson 23707d6034d3SDoug Thompson struct mcidev_sysfs_attribute terminator = { .attr = { .name = NULL } }; 23717d6034d3SDoug Thompson 2372360b7f3cSBorislav Petkov static void set_mc_sysfs_attrs(struct mem_ctl_info *mci) 23737d6034d3SDoug Thompson { 23747d6034d3SDoug Thompson unsigned int i = 0, j = 0; 23757d6034d3SDoug Thompson 23767d6034d3SDoug Thompson for (; i < ARRAY_SIZE(amd64_dbg_attrs); i++) 23777d6034d3SDoug Thompson sysfs_attrs[i] = amd64_dbg_attrs[i]; 23787d6034d3SDoug Thompson 2379a135cef7SBorislav Petkov if (boot_cpu_data.x86 >= 0x10) 23807d6034d3SDoug Thompson for (j = 0; j < ARRAY_SIZE(amd64_inj_attrs); j++, i++) 23817d6034d3SDoug Thompson sysfs_attrs[i] = amd64_inj_attrs[j]; 23827d6034d3SDoug Thompson 23837d6034d3SDoug Thompson sysfs_attrs[i] = terminator; 23847d6034d3SDoug Thompson 23857d6034d3SDoug Thompson mci->mc_driver_sysfs_attributes = sysfs_attrs; 23867d6034d3SDoug Thompson } 23877d6034d3SDoug Thompson 2388df71a053SBorislav Petkov static void setup_mci_misc_attrs(struct mem_ctl_info *mci, 2389df71a053SBorislav Petkov struct amd64_family_type *fam) 23907d6034d3SDoug Thompson { 23917d6034d3SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 23927d6034d3SDoug Thompson 23937d6034d3SDoug Thompson mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2; 23947d6034d3SDoug Thompson mci->edac_ctl_cap = EDAC_FLAG_NONE; 23957d6034d3SDoug Thompson 23965980bb9cSBorislav Petkov if (pvt->nbcap & NBCAP_SECDED) 23977d6034d3SDoug Thompson mci->edac_ctl_cap |= EDAC_FLAG_SECDED; 23987d6034d3SDoug Thompson 23995980bb9cSBorislav Petkov if (pvt->nbcap & NBCAP_CHIPKILL) 24007d6034d3SDoug Thompson mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED; 24017d6034d3SDoug Thompson 24027d6034d3SDoug Thompson mci->edac_cap = amd64_determine_edac_cap(pvt); 24037d6034d3SDoug Thompson mci->mod_name = EDAC_MOD_STR; 24047d6034d3SDoug Thompson mci->mod_ver = EDAC_AMD64_VERSION; 2405df71a053SBorislav Petkov mci->ctl_name = fam->ctl_name; 24068d5b5d9cSBorislav Petkov mci->dev_name = pci_name(pvt->F2); 24077d6034d3SDoug Thompson mci->ctl_page_to_phys = NULL; 24087d6034d3SDoug Thompson 24097d6034d3SDoug Thompson /* memory scrubber interface */ 24107d6034d3SDoug Thompson mci->set_sdram_scrub_rate = amd64_set_scrub_rate; 24117d6034d3SDoug Thompson mci->get_sdram_scrub_rate = amd64_get_scrub_rate; 24127d6034d3SDoug Thompson } 24137d6034d3SDoug Thompson 24140092b20dSBorislav Petkov /* 24150092b20dSBorislav Petkov * returns a pointer to the family descriptor on success, NULL otherwise. 24160092b20dSBorislav Petkov */ 24170092b20dSBorislav Petkov static struct amd64_family_type *amd64_per_family_init(struct amd64_pvt *pvt) 2418395ae783SBorislav Petkov { 24190092b20dSBorislav Petkov u8 fam = boot_cpu_data.x86; 24200092b20dSBorislav Petkov struct amd64_family_type *fam_type = NULL; 24210092b20dSBorislav Petkov 24220092b20dSBorislav Petkov switch (fam) { 2423395ae783SBorislav Petkov case 0xf: 24240092b20dSBorislav Petkov fam_type = &amd64_family_types[K8_CPUS]; 2425b8cfa02fSBorislav Petkov pvt->ops = &amd64_family_types[K8_CPUS].ops; 2426395ae783SBorislav Petkov break; 2427df71a053SBorislav Petkov 2428395ae783SBorislav Petkov case 0x10: 24290092b20dSBorislav Petkov fam_type = &amd64_family_types[F10_CPUS]; 2430b8cfa02fSBorislav Petkov pvt->ops = &amd64_family_types[F10_CPUS].ops; 2431df71a053SBorislav Petkov break; 2432df71a053SBorislav Petkov 2433df71a053SBorislav Petkov case 0x15: 2434df71a053SBorislav Petkov fam_type = &amd64_family_types[F15_CPUS]; 2435df71a053SBorislav Petkov pvt->ops = &amd64_family_types[F15_CPUS].ops; 2436395ae783SBorislav Petkov break; 2437395ae783SBorislav Petkov 2438395ae783SBorislav Petkov default: 243924f9a7feSBorislav Petkov amd64_err("Unsupported family!\n"); 24400092b20dSBorislav Petkov return NULL; 2441395ae783SBorislav Petkov } 24420092b20dSBorislav Petkov 2443b8cfa02fSBorislav Petkov pvt->ext_model = boot_cpu_data.x86_model >> 4; 2444b8cfa02fSBorislav Petkov 2445df71a053SBorislav Petkov amd64_info("%s %sdetected (node %d).\n", fam_type->ctl_name, 24460092b20dSBorislav Petkov (fam == 0xf ? 24470092b20dSBorislav Petkov (pvt->ext_model >= K8_REV_F ? "revF or later " 24480092b20dSBorislav Petkov : "revE or earlier ") 244924f9a7feSBorislav Petkov : ""), pvt->mc_node_id); 24500092b20dSBorislav Petkov return fam_type; 2451395ae783SBorislav Petkov } 2452395ae783SBorislav Petkov 24532299ef71SBorislav Petkov static int amd64_init_one_instance(struct pci_dev *F2) 24547d6034d3SDoug Thompson { 24557d6034d3SDoug Thompson struct amd64_pvt *pvt = NULL; 24560092b20dSBorislav Petkov struct amd64_family_type *fam_type = NULL; 2457360b7f3cSBorislav Petkov struct mem_ctl_info *mci = NULL; 24587d6034d3SDoug Thompson int err = 0, ret; 2459360b7f3cSBorislav Petkov u8 nid = get_node_id(F2); 24607d6034d3SDoug Thompson 24617d6034d3SDoug Thompson ret = -ENOMEM; 24627d6034d3SDoug Thompson pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL); 24637d6034d3SDoug Thompson if (!pvt) 2464360b7f3cSBorislav Petkov goto err_ret; 24657d6034d3SDoug Thompson 2466360b7f3cSBorislav Petkov pvt->mc_node_id = nid; 24678d5b5d9cSBorislav Petkov pvt->F2 = F2; 24687d6034d3SDoug Thompson 2469395ae783SBorislav Petkov ret = -EINVAL; 24700092b20dSBorislav Petkov fam_type = amd64_per_family_init(pvt); 24710092b20dSBorislav Petkov if (!fam_type) 2472395ae783SBorislav Petkov goto err_free; 2473395ae783SBorislav Petkov 24747d6034d3SDoug Thompson ret = -ENODEV; 2475360b7f3cSBorislav Petkov err = reserve_mc_sibling_devs(pvt, fam_type->f1_id, fam_type->f3_id); 24767d6034d3SDoug Thompson if (err) 24777d6034d3SDoug Thompson goto err_free; 24787d6034d3SDoug Thompson 2479360b7f3cSBorislav Petkov read_mc_regs(pvt); 24807d6034d3SDoug Thompson 24817d6034d3SDoug Thompson /* 24827d6034d3SDoug Thompson * We need to determine how many memory channels there are. Then use 24837d6034d3SDoug Thompson * that information for calculating the size of the dynamic instance 2484360b7f3cSBorislav Petkov * tables in the 'mci' structure. 24857d6034d3SDoug Thompson */ 2486360b7f3cSBorislav Petkov ret = -EINVAL; 24877d6034d3SDoug Thompson pvt->channel_count = pvt->ops->early_channel_count(pvt); 24887d6034d3SDoug Thompson if (pvt->channel_count < 0) 2489360b7f3cSBorislav Petkov goto err_siblings; 24907d6034d3SDoug Thompson 24917d6034d3SDoug Thompson ret = -ENOMEM; 249211c75eadSBorislav Petkov mci = edac_mc_alloc(0, pvt->csels[0].b_cnt, pvt->channel_count, nid); 24937d6034d3SDoug Thompson if (!mci) 2494360b7f3cSBorislav Petkov goto err_siblings; 24957d6034d3SDoug Thompson 24967d6034d3SDoug Thompson mci->pvt_info = pvt; 24978d5b5d9cSBorislav Petkov mci->dev = &pvt->F2->dev; 24987d6034d3SDoug Thompson 2499df71a053SBorislav Petkov setup_mci_misc_attrs(mci, fam_type); 2500360b7f3cSBorislav Petkov 2501360b7f3cSBorislav Petkov if (init_csrows(mci)) 25027d6034d3SDoug Thompson mci->edac_cap = EDAC_FLAG_NONE; 25037d6034d3SDoug Thompson 2504360b7f3cSBorislav Petkov set_mc_sysfs_attrs(mci); 25057d6034d3SDoug Thompson 25067d6034d3SDoug Thompson ret = -ENODEV; 25077d6034d3SDoug Thompson if (edac_mc_add_mc(mci)) { 25087d6034d3SDoug Thompson debugf1("failed edac_mc_add_mc()\n"); 25097d6034d3SDoug Thompson goto err_add_mc; 25107d6034d3SDoug Thompson } 25117d6034d3SDoug Thompson 2512549d042dSBorislav Petkov /* register stuff with EDAC MCE */ 2513549d042dSBorislav Petkov if (report_gart_errors) 2514549d042dSBorislav Petkov amd_report_gart_errors(true); 2515549d042dSBorislav Petkov 2516549d042dSBorislav Petkov amd_register_ecc_decoder(amd64_decode_bus_error); 2517549d042dSBorislav Petkov 2518360b7f3cSBorislav Petkov mcis[nid] = mci; 2519360b7f3cSBorislav Petkov 2520360b7f3cSBorislav Petkov atomic_inc(&drv_instances); 2521360b7f3cSBorislav Petkov 25227d6034d3SDoug Thompson return 0; 25237d6034d3SDoug Thompson 25247d6034d3SDoug Thompson err_add_mc: 25257d6034d3SDoug Thompson edac_mc_free(mci); 25267d6034d3SDoug Thompson 2527360b7f3cSBorislav Petkov err_siblings: 2528360b7f3cSBorislav Petkov free_mc_sibling_devs(pvt); 25297d6034d3SDoug Thompson 2530360b7f3cSBorislav Petkov err_free: 2531360b7f3cSBorislav Petkov kfree(pvt); 25327d6034d3SDoug Thompson 2533360b7f3cSBorislav Petkov err_ret: 25347d6034d3SDoug Thompson return ret; 25357d6034d3SDoug Thompson } 25367d6034d3SDoug Thompson 25372299ef71SBorislav Petkov static int __devinit amd64_probe_one_instance(struct pci_dev *pdev, 25387d6034d3SDoug Thompson const struct pci_device_id *mc_type) 25397d6034d3SDoug Thompson { 2540ae7bb7c6SBorislav Petkov u8 nid = get_node_id(pdev); 25412299ef71SBorislav Petkov struct pci_dev *F3 = node_to_amd_nb(nid)->misc; 2542ae7bb7c6SBorislav Petkov struct ecc_settings *s; 25432299ef71SBorislav Petkov int ret = 0; 25447d6034d3SDoug Thompson 25457d6034d3SDoug Thompson ret = pci_enable_device(pdev); 2546b8cfa02fSBorislav Petkov if (ret < 0) { 25477d6034d3SDoug Thompson debugf0("ret=%d\n", ret); 2548b8cfa02fSBorislav Petkov return -EIO; 2549b8cfa02fSBorislav Petkov } 2550b8cfa02fSBorislav Petkov 2551ae7bb7c6SBorislav Petkov ret = -ENOMEM; 2552ae7bb7c6SBorislav Petkov s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL); 2553ae7bb7c6SBorislav Petkov if (!s) 25542299ef71SBorislav Petkov goto err_out; 2555ae7bb7c6SBorislav Petkov 2556ae7bb7c6SBorislav Petkov ecc_stngs[nid] = s; 2557ae7bb7c6SBorislav Petkov 25582299ef71SBorislav Petkov if (!ecc_enabled(F3, nid)) { 25592299ef71SBorislav Petkov ret = -ENODEV; 25602299ef71SBorislav Petkov 25612299ef71SBorislav Petkov if (!ecc_enable_override) 25622299ef71SBorislav Petkov goto err_enable; 25632299ef71SBorislav Petkov 25642299ef71SBorislav Petkov amd64_warn("Forcing ECC on!\n"); 25652299ef71SBorislav Petkov 25662299ef71SBorislav Petkov if (!enable_ecc_error_reporting(s, nid, F3)) 25672299ef71SBorislav Petkov goto err_enable; 25682299ef71SBorislav Petkov } 25692299ef71SBorislav Petkov 25702299ef71SBorislav Petkov ret = amd64_init_one_instance(pdev); 2571360b7f3cSBorislav Petkov if (ret < 0) { 2572ae7bb7c6SBorislav Petkov amd64_err("Error probing instance: %d\n", nid); 2573360b7f3cSBorislav Petkov restore_ecc_error_reporting(s, nid, F3); 2574360b7f3cSBorislav Petkov } 25757d6034d3SDoug Thompson 25767d6034d3SDoug Thompson return ret; 25772299ef71SBorislav Petkov 25782299ef71SBorislav Petkov err_enable: 25792299ef71SBorislav Petkov kfree(s); 25802299ef71SBorislav Petkov ecc_stngs[nid] = NULL; 25812299ef71SBorislav Petkov 25822299ef71SBorislav Petkov err_out: 25832299ef71SBorislav Petkov return ret; 25847d6034d3SDoug Thompson } 25857d6034d3SDoug Thompson 25867d6034d3SDoug Thompson static void __devexit amd64_remove_one_instance(struct pci_dev *pdev) 25877d6034d3SDoug Thompson { 25887d6034d3SDoug Thompson struct mem_ctl_info *mci; 25897d6034d3SDoug Thompson struct amd64_pvt *pvt; 2590360b7f3cSBorislav Petkov u8 nid = get_node_id(pdev); 2591360b7f3cSBorislav Petkov struct pci_dev *F3 = node_to_amd_nb(nid)->misc; 2592360b7f3cSBorislav Petkov struct ecc_settings *s = ecc_stngs[nid]; 25937d6034d3SDoug Thompson 25947d6034d3SDoug Thompson /* Remove from EDAC CORE tracking list */ 25957d6034d3SDoug Thompson mci = edac_mc_del_mc(&pdev->dev); 25967d6034d3SDoug Thompson if (!mci) 25977d6034d3SDoug Thompson return; 25987d6034d3SDoug Thompson 25997d6034d3SDoug Thompson pvt = mci->pvt_info; 26007d6034d3SDoug Thompson 2601360b7f3cSBorislav Petkov restore_ecc_error_reporting(s, nid, F3); 26027d6034d3SDoug Thompson 2603360b7f3cSBorislav Petkov free_mc_sibling_devs(pvt); 26047d6034d3SDoug Thompson 2605549d042dSBorislav Petkov /* unregister from EDAC MCE */ 2606549d042dSBorislav Petkov amd_report_gart_errors(false); 2607549d042dSBorislav Petkov amd_unregister_ecc_decoder(amd64_decode_bus_error); 2608549d042dSBorislav Petkov 2609360b7f3cSBorislav Petkov kfree(ecc_stngs[nid]); 2610360b7f3cSBorislav Petkov ecc_stngs[nid] = NULL; 2611ae7bb7c6SBorislav Petkov 26127d6034d3SDoug Thompson /* Free the EDAC CORE resources */ 26138f68ed97SBorislav Petkov mci->pvt_info = NULL; 2614360b7f3cSBorislav Petkov mcis[nid] = NULL; 26158f68ed97SBorislav Petkov 26168f68ed97SBorislav Petkov kfree(pvt); 26177d6034d3SDoug Thompson edac_mc_free(mci); 26187d6034d3SDoug Thompson } 26197d6034d3SDoug Thompson 26207d6034d3SDoug Thompson /* 26217d6034d3SDoug Thompson * This table is part of the interface for loading drivers for PCI devices. The 26227d6034d3SDoug Thompson * PCI core identifies what devices are on a system during boot, and then 26237d6034d3SDoug Thompson * inquiry this table to see if this driver is for a given device found. 26247d6034d3SDoug Thompson */ 26257d6034d3SDoug Thompson static const struct pci_device_id amd64_pci_table[] __devinitdata = { 26267d6034d3SDoug Thompson { 26277d6034d3SDoug Thompson .vendor = PCI_VENDOR_ID_AMD, 26287d6034d3SDoug Thompson .device = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL, 26297d6034d3SDoug Thompson .subvendor = PCI_ANY_ID, 26307d6034d3SDoug Thompson .subdevice = PCI_ANY_ID, 26317d6034d3SDoug Thompson .class = 0, 26327d6034d3SDoug Thompson .class_mask = 0, 26337d6034d3SDoug Thompson }, 26347d6034d3SDoug Thompson { 26357d6034d3SDoug Thompson .vendor = PCI_VENDOR_ID_AMD, 26367d6034d3SDoug Thompson .device = PCI_DEVICE_ID_AMD_10H_NB_DRAM, 26377d6034d3SDoug Thompson .subvendor = PCI_ANY_ID, 26387d6034d3SDoug Thompson .subdevice = PCI_ANY_ID, 26397d6034d3SDoug Thompson .class = 0, 26407d6034d3SDoug Thompson .class_mask = 0, 26417d6034d3SDoug Thompson }, 2642df71a053SBorislav Petkov { 2643df71a053SBorislav Petkov .vendor = PCI_VENDOR_ID_AMD, 2644df71a053SBorislav Petkov .device = PCI_DEVICE_ID_AMD_15H_NB_F2, 2645df71a053SBorislav Petkov .subvendor = PCI_ANY_ID, 2646df71a053SBorislav Petkov .subdevice = PCI_ANY_ID, 2647df71a053SBorislav Petkov .class = 0, 2648df71a053SBorislav Petkov .class_mask = 0, 2649df71a053SBorislav Petkov }, 2650df71a053SBorislav Petkov 26517d6034d3SDoug Thompson {0, } 26527d6034d3SDoug Thompson }; 26537d6034d3SDoug Thompson MODULE_DEVICE_TABLE(pci, amd64_pci_table); 26547d6034d3SDoug Thompson 26557d6034d3SDoug Thompson static struct pci_driver amd64_pci_driver = { 26567d6034d3SDoug Thompson .name = EDAC_MOD_STR, 26572299ef71SBorislav Petkov .probe = amd64_probe_one_instance, 26587d6034d3SDoug Thompson .remove = __devexit_p(amd64_remove_one_instance), 26597d6034d3SDoug Thompson .id_table = amd64_pci_table, 26607d6034d3SDoug Thompson }; 26617d6034d3SDoug Thompson 2662360b7f3cSBorislav Petkov static void setup_pci_device(void) 26637d6034d3SDoug Thompson { 26647d6034d3SDoug Thompson struct mem_ctl_info *mci; 26657d6034d3SDoug Thompson struct amd64_pvt *pvt; 26667d6034d3SDoug Thompson 26677d6034d3SDoug Thompson if (amd64_ctl_pci) 26687d6034d3SDoug Thompson return; 26697d6034d3SDoug Thompson 2670cc4d8860SBorislav Petkov mci = mcis[0]; 26717d6034d3SDoug Thompson if (mci) { 26727d6034d3SDoug Thompson 26737d6034d3SDoug Thompson pvt = mci->pvt_info; 26747d6034d3SDoug Thompson amd64_ctl_pci = 26758d5b5d9cSBorislav Petkov edac_pci_create_generic_ctl(&pvt->F2->dev, EDAC_MOD_STR); 26767d6034d3SDoug Thompson 26777d6034d3SDoug Thompson if (!amd64_ctl_pci) { 26787d6034d3SDoug Thompson pr_warning("%s(): Unable to create PCI control\n", 26797d6034d3SDoug Thompson __func__); 26807d6034d3SDoug Thompson 26817d6034d3SDoug Thompson pr_warning("%s(): PCI error report via EDAC not set\n", 26827d6034d3SDoug Thompson __func__); 26837d6034d3SDoug Thompson } 26847d6034d3SDoug Thompson } 26857d6034d3SDoug Thompson } 26867d6034d3SDoug Thompson 26877d6034d3SDoug Thompson static int __init amd64_edac_init(void) 26887d6034d3SDoug Thompson { 2689360b7f3cSBorislav Petkov int err = -ENODEV; 26907d6034d3SDoug Thompson 2691df71a053SBorislav Petkov printk(KERN_INFO "AMD64 EDAC driver v%s\n", EDAC_AMD64_VERSION); 26927d6034d3SDoug Thompson 26937d6034d3SDoug Thompson opstate_init(); 26947d6034d3SDoug Thompson 26959653a5c7SHans Rosenfeld if (amd_cache_northbridges() < 0) 269656b34b91SBorislav Petkov goto err_ret; 26977d6034d3SDoug Thompson 2698cc4d8860SBorislav Petkov err = -ENOMEM; 2699cc4d8860SBorislav Petkov mcis = kzalloc(amd_nb_num() * sizeof(mcis[0]), GFP_KERNEL); 2700ae7bb7c6SBorislav Petkov ecc_stngs = kzalloc(amd_nb_num() * sizeof(ecc_stngs[0]), GFP_KERNEL); 2701360b7f3cSBorislav Petkov if (!(mcis && ecc_stngs)) 2702a9f0fbe2SBorislav Petkov goto err_free; 2703cc4d8860SBorislav Petkov 270450542251SBorislav Petkov msrs = msrs_alloc(); 270556b34b91SBorislav Petkov if (!msrs) 2706360b7f3cSBorislav Petkov goto err_free; 270750542251SBorislav Petkov 27087d6034d3SDoug Thompson err = pci_register_driver(&amd64_pci_driver); 27097d6034d3SDoug Thompson if (err) 271056b34b91SBorislav Petkov goto err_pci; 27117d6034d3SDoug Thompson 271256b34b91SBorislav Petkov err = -ENODEV; 2713360b7f3cSBorislav Petkov if (!atomic_read(&drv_instances)) 2714360b7f3cSBorislav Petkov goto err_no_instances; 27157d6034d3SDoug Thompson 2716360b7f3cSBorislav Petkov setup_pci_device(); 27177d6034d3SDoug Thompson return 0; 27187d6034d3SDoug Thompson 2719360b7f3cSBorislav Petkov err_no_instances: 27207d6034d3SDoug Thompson pci_unregister_driver(&amd64_pci_driver); 2721cc4d8860SBorislav Petkov 272256b34b91SBorislav Petkov err_pci: 272356b34b91SBorislav Petkov msrs_free(msrs); 272456b34b91SBorislav Petkov msrs = NULL; 2725cc4d8860SBorislav Petkov 2726360b7f3cSBorislav Petkov err_free: 2727360b7f3cSBorislav Petkov kfree(mcis); 2728360b7f3cSBorislav Petkov mcis = NULL; 2729360b7f3cSBorislav Petkov 2730360b7f3cSBorislav Petkov kfree(ecc_stngs); 2731360b7f3cSBorislav Petkov ecc_stngs = NULL; 2732360b7f3cSBorislav Petkov 273356b34b91SBorislav Petkov err_ret: 27347d6034d3SDoug Thompson return err; 27357d6034d3SDoug Thompson } 27367d6034d3SDoug Thompson 27377d6034d3SDoug Thompson static void __exit amd64_edac_exit(void) 27387d6034d3SDoug Thompson { 27397d6034d3SDoug Thompson if (amd64_ctl_pci) 27407d6034d3SDoug Thompson edac_pci_release_generic_ctl(amd64_ctl_pci); 27417d6034d3SDoug Thompson 27427d6034d3SDoug Thompson pci_unregister_driver(&amd64_pci_driver); 274350542251SBorislav Petkov 2744ae7bb7c6SBorislav Petkov kfree(ecc_stngs); 2745ae7bb7c6SBorislav Petkov ecc_stngs = NULL; 2746ae7bb7c6SBorislav Petkov 2747cc4d8860SBorislav Petkov kfree(mcis); 2748cc4d8860SBorislav Petkov mcis = NULL; 2749cc4d8860SBorislav Petkov 275050542251SBorislav Petkov msrs_free(msrs); 275150542251SBorislav Petkov msrs = NULL; 27527d6034d3SDoug Thompson } 27537d6034d3SDoug Thompson 27547d6034d3SDoug Thompson module_init(amd64_edac_init); 27557d6034d3SDoug Thompson module_exit(amd64_edac_exit); 27567d6034d3SDoug Thompson 27577d6034d3SDoug Thompson MODULE_LICENSE("GPL"); 27587d6034d3SDoug Thompson MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, " 27597d6034d3SDoug Thompson "Dave Peterson, Thayne Harbaugh"); 27607d6034d3SDoug Thompson MODULE_DESCRIPTION("MC support for AMD64 memory controllers - " 27617d6034d3SDoug Thompson EDAC_AMD64_VERSION); 27627d6034d3SDoug Thompson 27637d6034d3SDoug Thompson module_param(edac_op_state, int, 0444); 27647d6034d3SDoug Thompson MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI"); 2765