12bc65418SDoug Thompson #include "amd64_edac.h" 223ac4ae8SAndreas Herrmann #include <asm/amd_nb.h> 32bc65418SDoug Thompson 42bc65418SDoug Thompson static struct edac_pci_ctl_info *amd64_ctl_pci; 52bc65418SDoug Thompson 62bc65418SDoug Thompson static int report_gart_errors; 72bc65418SDoug Thompson module_param(report_gart_errors, int, 0644); 82bc65418SDoug Thompson 92bc65418SDoug Thompson /* 102bc65418SDoug Thompson * Set by command line parameter. If BIOS has enabled the ECC, this override is 112bc65418SDoug Thompson * cleared to prevent re-enabling the hardware by this driver. 122bc65418SDoug Thompson */ 132bc65418SDoug Thompson static int ecc_enable_override; 142bc65418SDoug Thompson module_param(ecc_enable_override, int, 0644); 152bc65418SDoug Thompson 16a29d8b8eSTejun Heo static struct msr __percpu *msrs; 1750542251SBorislav Petkov 18360b7f3cSBorislav Petkov /* 19360b7f3cSBorislav Petkov * count successfully initialized driver instances for setup_pci_device() 20360b7f3cSBorislav Petkov */ 21360b7f3cSBorislav Petkov static atomic_t drv_instances = ATOMIC_INIT(0); 22360b7f3cSBorislav Petkov 23cc4d8860SBorislav Petkov /* Per-node driver instances */ 24cc4d8860SBorislav Petkov static struct mem_ctl_info **mcis; 25ae7bb7c6SBorislav Petkov static struct ecc_settings **ecc_stngs; 262bc65418SDoug Thompson 272bc65418SDoug Thompson /* 28b70ef010SBorislav Petkov * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing 29b70ef010SBorislav Petkov * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching- 30b70ef010SBorislav Petkov * or higher value'. 31b70ef010SBorislav Petkov * 32b70ef010SBorislav Petkov *FIXME: Produce a better mapping/linearisation. 33b70ef010SBorislav Petkov */ 3439094443SBorislav Petkov struct scrubrate { 3539094443SBorislav Petkov u32 scrubval; /* bit pattern for scrub rate */ 3639094443SBorislav Petkov u32 bandwidth; /* bandwidth consumed (bytes/sec) */ 3739094443SBorislav Petkov } scrubrates[] = { 38b70ef010SBorislav Petkov { 0x01, 1600000000UL}, 39b70ef010SBorislav Petkov { 0x02, 800000000UL}, 40b70ef010SBorislav Petkov { 0x03, 400000000UL}, 41b70ef010SBorislav Petkov { 0x04, 200000000UL}, 42b70ef010SBorislav Petkov { 0x05, 100000000UL}, 43b70ef010SBorislav Petkov { 0x06, 50000000UL}, 44b70ef010SBorislav Petkov { 0x07, 25000000UL}, 45b70ef010SBorislav Petkov { 0x08, 12284069UL}, 46b70ef010SBorislav Petkov { 0x09, 6274509UL}, 47b70ef010SBorislav Petkov { 0x0A, 3121951UL}, 48b70ef010SBorislav Petkov { 0x0B, 1560975UL}, 49b70ef010SBorislav Petkov { 0x0C, 781440UL}, 50b70ef010SBorislav Petkov { 0x0D, 390720UL}, 51b70ef010SBorislav Petkov { 0x0E, 195300UL}, 52b70ef010SBorislav Petkov { 0x0F, 97650UL}, 53b70ef010SBorislav Petkov { 0x10, 48854UL}, 54b70ef010SBorislav Petkov { 0x11, 24427UL}, 55b70ef010SBorislav Petkov { 0x12, 12213UL}, 56b70ef010SBorislav Petkov { 0x13, 6101UL}, 57b70ef010SBorislav Petkov { 0x14, 3051UL}, 58b70ef010SBorislav Petkov { 0x15, 1523UL}, 59b70ef010SBorislav Petkov { 0x16, 761UL}, 60b70ef010SBorislav Petkov { 0x00, 0UL}, /* scrubbing off */ 61b70ef010SBorislav Petkov }; 62b70ef010SBorislav Petkov 63b2b0c605SBorislav Petkov static int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset, 64b2b0c605SBorislav Petkov u32 *val, const char *func) 65b2b0c605SBorislav Petkov { 66b2b0c605SBorislav Petkov int err = 0; 67b2b0c605SBorislav Petkov 68b2b0c605SBorislav Petkov err = pci_read_config_dword(pdev, offset, val); 69b2b0c605SBorislav Petkov if (err) 70b2b0c605SBorislav Petkov amd64_warn("%s: error reading F%dx%03x.\n", 71b2b0c605SBorislav Petkov func, PCI_FUNC(pdev->devfn), offset); 72b2b0c605SBorislav Petkov 73b2b0c605SBorislav Petkov return err; 74b2b0c605SBorislav Petkov } 75b2b0c605SBorislav Petkov 76b2b0c605SBorislav Petkov int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset, 77b2b0c605SBorislav Petkov u32 val, const char *func) 78b2b0c605SBorislav Petkov { 79b2b0c605SBorislav Petkov int err = 0; 80b2b0c605SBorislav Petkov 81b2b0c605SBorislav Petkov err = pci_write_config_dword(pdev, offset, val); 82b2b0c605SBorislav Petkov if (err) 83b2b0c605SBorislav Petkov amd64_warn("%s: error writing to F%dx%03x.\n", 84b2b0c605SBorislav Petkov func, PCI_FUNC(pdev->devfn), offset); 85b2b0c605SBorislav Petkov 86b2b0c605SBorislav Petkov return err; 87b2b0c605SBorislav Petkov } 88b2b0c605SBorislav Petkov 89b2b0c605SBorislav Petkov /* 90b2b0c605SBorislav Petkov * 91b2b0c605SBorislav Petkov * Depending on the family, F2 DCT reads need special handling: 92b2b0c605SBorislav Petkov * 93b2b0c605SBorislav Petkov * K8: has a single DCT only 94b2b0c605SBorislav Petkov * 95b2b0c605SBorislav Petkov * F10h: each DCT has its own set of regs 96b2b0c605SBorislav Petkov * DCT0 -> F2x040.. 97b2b0c605SBorislav Petkov * DCT1 -> F2x140.. 98b2b0c605SBorislav Petkov * 99b2b0c605SBorislav Petkov * F15h: we select which DCT we access using F1x10C[DctCfgSel] 100b2b0c605SBorislav Petkov * 101b2b0c605SBorislav Petkov */ 102b2b0c605SBorislav Petkov static int k8_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val, 103b2b0c605SBorislav Petkov const char *func) 104b2b0c605SBorislav Petkov { 105b2b0c605SBorislav Petkov if (addr >= 0x100) 106b2b0c605SBorislav Petkov return -EINVAL; 107b2b0c605SBorislav Petkov 108b2b0c605SBorislav Petkov return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func); 109b2b0c605SBorislav Petkov } 110b2b0c605SBorislav Petkov 111b2b0c605SBorislav Petkov static int f10_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val, 112b2b0c605SBorislav Petkov const char *func) 113b2b0c605SBorislav Petkov { 114b2b0c605SBorislav Petkov return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func); 115b2b0c605SBorislav Petkov } 116b2b0c605SBorislav Petkov 117b2b0c605SBorislav Petkov static int f15_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val, 118b2b0c605SBorislav Petkov const char *func) 119b2b0c605SBorislav Petkov { 120b2b0c605SBorislav Petkov u32 reg = 0; 121b2b0c605SBorislav Petkov u8 dct = 0; 122b2b0c605SBorislav Petkov 123b2b0c605SBorislav Petkov if (addr >= 0x140 && addr <= 0x1a0) { 124b2b0c605SBorislav Petkov dct = 1; 125b2b0c605SBorislav Petkov addr -= 0x100; 126b2b0c605SBorislav Petkov } 127b2b0c605SBorislav Petkov 128b2b0c605SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, ®); 129b2b0c605SBorislav Petkov reg &= 0xfffffffe; 130b2b0c605SBorislav Petkov reg |= dct; 131b2b0c605SBorislav Petkov amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg); 132b2b0c605SBorislav Petkov 133b2b0c605SBorislav Petkov return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func); 134b2b0c605SBorislav Petkov } 135b2b0c605SBorislav Petkov 136b70ef010SBorislav Petkov /* 1372bc65418SDoug Thompson * Memory scrubber control interface. For K8, memory scrubbing is handled by 1382bc65418SDoug Thompson * hardware and can involve L2 cache, dcache as well as the main memory. With 1392bc65418SDoug Thompson * F10, this is extended to L3 cache scrubbing on CPU models sporting that 1402bc65418SDoug Thompson * functionality. 1412bc65418SDoug Thompson * 1422bc65418SDoug Thompson * This causes the "units" for the scrubbing speed to vary from 64 byte blocks 1432bc65418SDoug Thompson * (dram) over to cache lines. This is nasty, so we will use bandwidth in 1442bc65418SDoug Thompson * bytes/sec for the setting. 1452bc65418SDoug Thompson * 1462bc65418SDoug Thompson * Currently, we only do dram scrubbing. If the scrubbing is done in software on 1472bc65418SDoug Thompson * other archs, we might not have access to the caches directly. 1482bc65418SDoug Thompson */ 1492bc65418SDoug Thompson 1502bc65418SDoug Thompson /* 1512bc65418SDoug Thompson * scan the scrub rate mapping table for a close or matching bandwidth value to 1522bc65418SDoug Thompson * issue. If requested is too big, then use last maximum value found. 1532bc65418SDoug Thompson */ 154395ae783SBorislav Petkov static int __amd64_set_scrub_rate(struct pci_dev *ctl, u32 new_bw, u32 min_rate) 1552bc65418SDoug Thompson { 1562bc65418SDoug Thompson u32 scrubval; 1572bc65418SDoug Thompson int i; 1582bc65418SDoug Thompson 1592bc65418SDoug Thompson /* 1602bc65418SDoug Thompson * map the configured rate (new_bw) to a value specific to the AMD64 1612bc65418SDoug Thompson * memory controller and apply to register. Search for the first 1622bc65418SDoug Thompson * bandwidth entry that is greater or equal than the setting requested 1632bc65418SDoug Thompson * and program that. If at last entry, turn off DRAM scrubbing. 1642bc65418SDoug Thompson */ 1652bc65418SDoug Thompson for (i = 0; i < ARRAY_SIZE(scrubrates); i++) { 1662bc65418SDoug Thompson /* 1672bc65418SDoug Thompson * skip scrub rates which aren't recommended 1682bc65418SDoug Thompson * (see F10 BKDG, F3x58) 1692bc65418SDoug Thompson */ 170395ae783SBorislav Petkov if (scrubrates[i].scrubval < min_rate) 1712bc65418SDoug Thompson continue; 1722bc65418SDoug Thompson 1732bc65418SDoug Thompson if (scrubrates[i].bandwidth <= new_bw) 1742bc65418SDoug Thompson break; 1752bc65418SDoug Thompson 1762bc65418SDoug Thompson /* 1772bc65418SDoug Thompson * if no suitable bandwidth found, turn off DRAM scrubbing 1782bc65418SDoug Thompson * entirely by falling back to the last element in the 1792bc65418SDoug Thompson * scrubrates array. 1802bc65418SDoug Thompson */ 1812bc65418SDoug Thompson } 1822bc65418SDoug Thompson 1832bc65418SDoug Thompson scrubval = scrubrates[i].scrubval; 1842bc65418SDoug Thompson 1855980bb9cSBorislav Petkov pci_write_bits32(ctl, SCRCTRL, scrubval, 0x001F); 1862bc65418SDoug Thompson 18739094443SBorislav Petkov if (scrubval) 18839094443SBorislav Petkov return scrubrates[i].bandwidth; 18939094443SBorislav Petkov 1902bc65418SDoug Thompson return 0; 1912bc65418SDoug Thompson } 1922bc65418SDoug Thompson 193395ae783SBorislav Petkov static int amd64_set_scrub_rate(struct mem_ctl_info *mci, u32 bw) 1942bc65418SDoug Thompson { 1952bc65418SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 19687b3e0e6SBorislav Petkov u32 min_scrubrate = 0x5; 1972bc65418SDoug Thompson 19887b3e0e6SBorislav Petkov if (boot_cpu_data.x86 == 0xf) 19987b3e0e6SBorislav Petkov min_scrubrate = 0x0; 20087b3e0e6SBorislav Petkov 20187b3e0e6SBorislav Petkov return __amd64_set_scrub_rate(pvt->F3, bw, min_scrubrate); 2022bc65418SDoug Thompson } 2032bc65418SDoug Thompson 20439094443SBorislav Petkov static int amd64_get_scrub_rate(struct mem_ctl_info *mci) 2052bc65418SDoug Thompson { 2062bc65418SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 2072bc65418SDoug Thompson u32 scrubval = 0; 20839094443SBorislav Petkov int i, retval = -EINVAL; 2092bc65418SDoug Thompson 2105980bb9cSBorislav Petkov amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval); 2112bc65418SDoug Thompson 2122bc65418SDoug Thompson scrubval = scrubval & 0x001F; 2132bc65418SDoug Thompson 21424f9a7feSBorislav Petkov amd64_debug("pci-read, sdram scrub control value: %d\n", scrubval); 2152bc65418SDoug Thompson 216926311fdSRoel Kluin for (i = 0; i < ARRAY_SIZE(scrubrates); i++) { 2172bc65418SDoug Thompson if (scrubrates[i].scrubval == scrubval) { 21839094443SBorislav Petkov retval = scrubrates[i].bandwidth; 2192bc65418SDoug Thompson break; 2202bc65418SDoug Thompson } 2212bc65418SDoug Thompson } 22239094443SBorislav Petkov return retval; 2232bc65418SDoug Thompson } 2242bc65418SDoug Thompson 2256775763aSDoug Thompson /* 2267f19bf75SBorislav Petkov * returns true if the SysAddr given by sys_addr matches the 2277f19bf75SBorislav Petkov * DRAM base/limit associated with node_id 2286775763aSDoug Thompson */ 229b487c33eSBorislav Petkov static bool amd64_base_limit_match(struct amd64_pvt *pvt, u64 sys_addr, 230b487c33eSBorislav Petkov unsigned nid) 2316775763aSDoug Thompson { 2327f19bf75SBorislav Petkov u64 addr; 2336775763aSDoug Thompson 2346775763aSDoug Thompson /* The K8 treats this as a 40-bit value. However, bits 63-40 will be 2356775763aSDoug Thompson * all ones if the most significant implemented address bit is 1. 2366775763aSDoug Thompson * Here we discard bits 63-40. See section 3.4.2 of AMD publication 2376775763aSDoug Thompson * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1 2386775763aSDoug Thompson * Application Programming. 2396775763aSDoug Thompson */ 2406775763aSDoug Thompson addr = sys_addr & 0x000000ffffffffffull; 2416775763aSDoug Thompson 2427f19bf75SBorislav Petkov return ((addr >= get_dram_base(pvt, nid)) && 2437f19bf75SBorislav Petkov (addr <= get_dram_limit(pvt, nid))); 2446775763aSDoug Thompson } 2456775763aSDoug Thompson 2466775763aSDoug Thompson /* 2476775763aSDoug Thompson * Attempt to map a SysAddr to a node. On success, return a pointer to the 2486775763aSDoug Thompson * mem_ctl_info structure for the node that the SysAddr maps to. 2496775763aSDoug Thompson * 2506775763aSDoug Thompson * On failure, return NULL. 2516775763aSDoug Thompson */ 2526775763aSDoug Thompson static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci, 2536775763aSDoug Thompson u64 sys_addr) 2546775763aSDoug Thompson { 2556775763aSDoug Thompson struct amd64_pvt *pvt; 256b487c33eSBorislav Petkov unsigned node_id; 2576775763aSDoug Thompson u32 intlv_en, bits; 2586775763aSDoug Thompson 2596775763aSDoug Thompson /* 2606775763aSDoug Thompson * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section 2616775763aSDoug Thompson * 3.4.4.2) registers to map the SysAddr to a node ID. 2626775763aSDoug Thompson */ 2636775763aSDoug Thompson pvt = mci->pvt_info; 2646775763aSDoug Thompson 2656775763aSDoug Thompson /* 2666775763aSDoug Thompson * The value of this field should be the same for all DRAM Base 2676775763aSDoug Thompson * registers. Therefore we arbitrarily choose to read it from the 2686775763aSDoug Thompson * register for node 0. 2696775763aSDoug Thompson */ 2707f19bf75SBorislav Petkov intlv_en = dram_intlv_en(pvt, 0); 2716775763aSDoug Thompson 2726775763aSDoug Thompson if (intlv_en == 0) { 2737f19bf75SBorislav Petkov for (node_id = 0; node_id < DRAM_RANGES; node_id++) { 2746775763aSDoug Thompson if (amd64_base_limit_match(pvt, sys_addr, node_id)) 2756775763aSDoug Thompson goto found; 2766775763aSDoug Thompson } 2778edc5445SBorislav Petkov goto err_no_match; 2788edc5445SBorislav Petkov } 2796775763aSDoug Thompson 28072f158feSBorislav Petkov if (unlikely((intlv_en != 0x01) && 28172f158feSBorislav Petkov (intlv_en != 0x03) && 28272f158feSBorislav Petkov (intlv_en != 0x07))) { 28324f9a7feSBorislav Petkov amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en); 2846775763aSDoug Thompson return NULL; 2856775763aSDoug Thompson } 2866775763aSDoug Thompson 2876775763aSDoug Thompson bits = (((u32) sys_addr) >> 12) & intlv_en; 2886775763aSDoug Thompson 2896775763aSDoug Thompson for (node_id = 0; ; ) { 2907f19bf75SBorislav Petkov if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits) 2916775763aSDoug Thompson break; /* intlv_sel field matches */ 2926775763aSDoug Thompson 2937f19bf75SBorislav Petkov if (++node_id >= DRAM_RANGES) 2946775763aSDoug Thompson goto err_no_match; 2956775763aSDoug Thompson } 2966775763aSDoug Thompson 2976775763aSDoug Thompson /* sanity test for sys_addr */ 2986775763aSDoug Thompson if (unlikely(!amd64_base_limit_match(pvt, sys_addr, node_id))) { 29924f9a7feSBorislav Petkov amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address" 30024f9a7feSBorislav Petkov "range for node %d with node interleaving enabled.\n", 3018edc5445SBorislav Petkov __func__, sys_addr, node_id); 3026775763aSDoug Thompson return NULL; 3036775763aSDoug Thompson } 3046775763aSDoug Thompson 3056775763aSDoug Thompson found: 306b487c33eSBorislav Petkov return edac_mc_find((int)node_id); 3076775763aSDoug Thompson 3086775763aSDoug Thompson err_no_match: 3096775763aSDoug Thompson debugf2("sys_addr 0x%lx doesn't match any node\n", 3106775763aSDoug Thompson (unsigned long)sys_addr); 3116775763aSDoug Thompson 3126775763aSDoug Thompson return NULL; 3136775763aSDoug Thompson } 314e2ce7255SDoug Thompson 315e2ce7255SDoug Thompson /* 31611c75eadSBorislav Petkov * compute the CS base address of the @csrow on the DRAM controller @dct. 31711c75eadSBorislav Petkov * For details see F2x[5C:40] in the processor's BKDG 318e2ce7255SDoug Thompson */ 31911c75eadSBorislav Petkov static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct, 32011c75eadSBorislav Petkov u64 *base, u64 *mask) 321e2ce7255SDoug Thompson { 32211c75eadSBorislav Petkov u64 csbase, csmask, base_bits, mask_bits; 32311c75eadSBorislav Petkov u8 addr_shift; 32411c75eadSBorislav Petkov 32511c75eadSBorislav Petkov if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) { 32611c75eadSBorislav Petkov csbase = pvt->csels[dct].csbases[csrow]; 32711c75eadSBorislav Petkov csmask = pvt->csels[dct].csmasks[csrow]; 32811c75eadSBorislav Petkov base_bits = GENMASK(21, 31) | GENMASK(9, 15); 32911c75eadSBorislav Petkov mask_bits = GENMASK(21, 29) | GENMASK(9, 15); 33011c75eadSBorislav Petkov addr_shift = 4; 33111c75eadSBorislav Petkov } else { 33211c75eadSBorislav Petkov csbase = pvt->csels[dct].csbases[csrow]; 33311c75eadSBorislav Petkov csmask = pvt->csels[dct].csmasks[csrow >> 1]; 33411c75eadSBorislav Petkov addr_shift = 8; 33511c75eadSBorislav Petkov 33611c75eadSBorislav Petkov if (boot_cpu_data.x86 == 0x15) 33711c75eadSBorislav Petkov base_bits = mask_bits = GENMASK(19,30) | GENMASK(5,13); 33811c75eadSBorislav Petkov else 33911c75eadSBorislav Petkov base_bits = mask_bits = GENMASK(19,28) | GENMASK(5,13); 340e2ce7255SDoug Thompson } 341e2ce7255SDoug Thompson 34211c75eadSBorislav Petkov *base = (csbase & base_bits) << addr_shift; 343e2ce7255SDoug Thompson 34411c75eadSBorislav Petkov *mask = ~0ULL; 34511c75eadSBorislav Petkov /* poke holes for the csmask */ 34611c75eadSBorislav Petkov *mask &= ~(mask_bits << addr_shift); 34711c75eadSBorislav Petkov /* OR them in */ 34811c75eadSBorislav Petkov *mask |= (csmask & mask_bits) << addr_shift; 349e2ce7255SDoug Thompson } 350e2ce7255SDoug Thompson 35111c75eadSBorislav Petkov #define for_each_chip_select(i, dct, pvt) \ 35211c75eadSBorislav Petkov for (i = 0; i < pvt->csels[dct].b_cnt; i++) 35311c75eadSBorislav Petkov 354614ec9d8SBorislav Petkov #define chip_select_base(i, dct, pvt) \ 355614ec9d8SBorislav Petkov pvt->csels[dct].csbases[i] 356614ec9d8SBorislav Petkov 35711c75eadSBorislav Petkov #define for_each_chip_select_mask(i, dct, pvt) \ 35811c75eadSBorislav Petkov for (i = 0; i < pvt->csels[dct].m_cnt; i++) 35911c75eadSBorislav Petkov 360e2ce7255SDoug Thompson /* 361e2ce7255SDoug Thompson * @input_addr is an InputAddr associated with the node given by mci. Return the 362e2ce7255SDoug Thompson * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr). 363e2ce7255SDoug Thompson */ 364e2ce7255SDoug Thompson static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr) 365e2ce7255SDoug Thompson { 366e2ce7255SDoug Thompson struct amd64_pvt *pvt; 367e2ce7255SDoug Thompson int csrow; 368e2ce7255SDoug Thompson u64 base, mask; 369e2ce7255SDoug Thompson 370e2ce7255SDoug Thompson pvt = mci->pvt_info; 371e2ce7255SDoug Thompson 37211c75eadSBorislav Petkov for_each_chip_select(csrow, 0, pvt) { 37311c75eadSBorislav Petkov if (!csrow_enabled(csrow, 0, pvt)) 374e2ce7255SDoug Thompson continue; 375e2ce7255SDoug Thompson 37611c75eadSBorislav Petkov get_cs_base_and_mask(pvt, csrow, 0, &base, &mask); 37711c75eadSBorislav Petkov 37811c75eadSBorislav Petkov mask = ~mask; 379e2ce7255SDoug Thompson 380e2ce7255SDoug Thompson if ((input_addr & mask) == (base & mask)) { 381e2ce7255SDoug Thompson debugf2("InputAddr 0x%lx matches csrow %d (node %d)\n", 382e2ce7255SDoug Thompson (unsigned long)input_addr, csrow, 383e2ce7255SDoug Thompson pvt->mc_node_id); 384e2ce7255SDoug Thompson 385e2ce7255SDoug Thompson return csrow; 386e2ce7255SDoug Thompson } 387e2ce7255SDoug Thompson } 388e2ce7255SDoug Thompson debugf2("no matching csrow for InputAddr 0x%lx (MC node %d)\n", 389e2ce7255SDoug Thompson (unsigned long)input_addr, pvt->mc_node_id); 390e2ce7255SDoug Thompson 391e2ce7255SDoug Thompson return -1; 392e2ce7255SDoug Thompson } 393e2ce7255SDoug Thompson 394e2ce7255SDoug Thompson /* 395e2ce7255SDoug Thompson * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094) 396e2ce7255SDoug Thompson * for the node represented by mci. Info is passed back in *hole_base, 397e2ce7255SDoug Thompson * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if 398e2ce7255SDoug Thompson * info is invalid. Info may be invalid for either of the following reasons: 399e2ce7255SDoug Thompson * 400e2ce7255SDoug Thompson * - The revision of the node is not E or greater. In this case, the DRAM Hole 401e2ce7255SDoug Thompson * Address Register does not exist. 402e2ce7255SDoug Thompson * 403e2ce7255SDoug Thompson * - The DramHoleValid bit is cleared in the DRAM Hole Address Register, 404e2ce7255SDoug Thompson * indicating that its contents are not valid. 405e2ce7255SDoug Thompson * 406e2ce7255SDoug Thompson * The values passed back in *hole_base, *hole_offset, and *hole_size are 407e2ce7255SDoug Thompson * complete 32-bit values despite the fact that the bitfields in the DHAR 408e2ce7255SDoug Thompson * only represent bits 31-24 of the base and offset values. 409e2ce7255SDoug Thompson */ 410e2ce7255SDoug Thompson int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base, 411e2ce7255SDoug Thompson u64 *hole_offset, u64 *hole_size) 412e2ce7255SDoug Thompson { 413e2ce7255SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 414e2ce7255SDoug Thompson u64 base; 415e2ce7255SDoug Thompson 416e2ce7255SDoug Thompson /* only revE and later have the DRAM Hole Address Register */ 4171433eb99SBorislav Petkov if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_E) { 418e2ce7255SDoug Thompson debugf1(" revision %d for node %d does not support DHAR\n", 419e2ce7255SDoug Thompson pvt->ext_model, pvt->mc_node_id); 420e2ce7255SDoug Thompson return 1; 421e2ce7255SDoug Thompson } 422e2ce7255SDoug Thompson 423bc21fa57SBorislav Petkov /* valid for Fam10h and above */ 424c8e518d5SBorislav Petkov if (boot_cpu_data.x86 >= 0x10 && !dhar_mem_hoist_valid(pvt)) { 425e2ce7255SDoug Thompson debugf1(" Dram Memory Hoisting is DISABLED on this system\n"); 426e2ce7255SDoug Thompson return 1; 427e2ce7255SDoug Thompson } 428e2ce7255SDoug Thompson 429c8e518d5SBorislav Petkov if (!dhar_valid(pvt)) { 430e2ce7255SDoug Thompson debugf1(" Dram Memory Hoisting is DISABLED on this node %d\n", 431e2ce7255SDoug Thompson pvt->mc_node_id); 432e2ce7255SDoug Thompson return 1; 433e2ce7255SDoug Thompson } 434e2ce7255SDoug Thompson 435e2ce7255SDoug Thompson /* This node has Memory Hoisting */ 436e2ce7255SDoug Thompson 437e2ce7255SDoug Thompson /* +------------------+--------------------+--------------------+----- 438e2ce7255SDoug Thompson * | memory | DRAM hole | relocated | 439e2ce7255SDoug Thompson * | [0, (x - 1)] | [x, 0xffffffff] | addresses from | 440e2ce7255SDoug Thompson * | | | DRAM hole | 441e2ce7255SDoug Thompson * | | | [0x100000000, | 442e2ce7255SDoug Thompson * | | | (0x100000000+ | 443e2ce7255SDoug Thompson * | | | (0xffffffff-x))] | 444e2ce7255SDoug Thompson * +------------------+--------------------+--------------------+----- 445e2ce7255SDoug Thompson * 446e2ce7255SDoug Thompson * Above is a diagram of physical memory showing the DRAM hole and the 447e2ce7255SDoug Thompson * relocated addresses from the DRAM hole. As shown, the DRAM hole 448e2ce7255SDoug Thompson * starts at address x (the base address) and extends through address 449e2ce7255SDoug Thompson * 0xffffffff. The DRAM Hole Address Register (DHAR) relocates the 450e2ce7255SDoug Thompson * addresses in the hole so that they start at 0x100000000. 451e2ce7255SDoug Thompson */ 452e2ce7255SDoug Thompson 453bc21fa57SBorislav Petkov base = dhar_base(pvt); 454e2ce7255SDoug Thompson 455e2ce7255SDoug Thompson *hole_base = base; 456e2ce7255SDoug Thompson *hole_size = (0x1ull << 32) - base; 457e2ce7255SDoug Thompson 458e2ce7255SDoug Thompson if (boot_cpu_data.x86 > 0xf) 459bc21fa57SBorislav Petkov *hole_offset = f10_dhar_offset(pvt); 460e2ce7255SDoug Thompson else 461bc21fa57SBorislav Petkov *hole_offset = k8_dhar_offset(pvt); 462e2ce7255SDoug Thompson 463e2ce7255SDoug Thompson debugf1(" DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n", 464e2ce7255SDoug Thompson pvt->mc_node_id, (unsigned long)*hole_base, 465e2ce7255SDoug Thompson (unsigned long)*hole_offset, (unsigned long)*hole_size); 466e2ce7255SDoug Thompson 467e2ce7255SDoug Thompson return 0; 468e2ce7255SDoug Thompson } 469e2ce7255SDoug Thompson EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info); 470e2ce7255SDoug Thompson 47193c2df58SDoug Thompson /* 47293c2df58SDoug Thompson * Return the DramAddr that the SysAddr given by @sys_addr maps to. It is 47393c2df58SDoug Thompson * assumed that sys_addr maps to the node given by mci. 47493c2df58SDoug Thompson * 47593c2df58SDoug Thompson * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section 47693c2df58SDoug Thompson * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a 47793c2df58SDoug Thompson * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled, 47893c2df58SDoug Thompson * then it is also involved in translating a SysAddr to a DramAddr. Sections 47993c2df58SDoug Thompson * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting. 48093c2df58SDoug Thompson * These parts of the documentation are unclear. I interpret them as follows: 48193c2df58SDoug Thompson * 48293c2df58SDoug Thompson * When node n receives a SysAddr, it processes the SysAddr as follows: 48393c2df58SDoug Thompson * 48493c2df58SDoug Thompson * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM 48593c2df58SDoug Thompson * Limit registers for node n. If the SysAddr is not within the range 48693c2df58SDoug Thompson * specified by the base and limit values, then node n ignores the Sysaddr 48793c2df58SDoug Thompson * (since it does not map to node n). Otherwise continue to step 2 below. 48893c2df58SDoug Thompson * 48993c2df58SDoug Thompson * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is 49093c2df58SDoug Thompson * disabled so skip to step 3 below. Otherwise see if the SysAddr is within 49193c2df58SDoug Thompson * the range of relocated addresses (starting at 0x100000000) from the DRAM 49293c2df58SDoug Thompson * hole. If not, skip to step 3 below. Else get the value of the 49393c2df58SDoug Thompson * DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the 49493c2df58SDoug Thompson * offset defined by this value from the SysAddr. 49593c2df58SDoug Thompson * 49693c2df58SDoug Thompson * 3. Obtain the base address for node n from the DRAMBase field of the DRAM 49793c2df58SDoug Thompson * Base register for node n. To obtain the DramAddr, subtract the base 49893c2df58SDoug Thompson * address from the SysAddr, as shown near the start of section 3.4.4 (p.70). 49993c2df58SDoug Thompson */ 50093c2df58SDoug Thompson static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr) 50193c2df58SDoug Thompson { 5027f19bf75SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 50393c2df58SDoug Thompson u64 dram_base, hole_base, hole_offset, hole_size, dram_addr; 50493c2df58SDoug Thompson int ret = 0; 50593c2df58SDoug Thompson 5067f19bf75SBorislav Petkov dram_base = get_dram_base(pvt, pvt->mc_node_id); 50793c2df58SDoug Thompson 50893c2df58SDoug Thompson ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset, 50993c2df58SDoug Thompson &hole_size); 51093c2df58SDoug Thompson if (!ret) { 51193c2df58SDoug Thompson if ((sys_addr >= (1ull << 32)) && 51293c2df58SDoug Thompson (sys_addr < ((1ull << 32) + hole_size))) { 51393c2df58SDoug Thompson /* use DHAR to translate SysAddr to DramAddr */ 51493c2df58SDoug Thompson dram_addr = sys_addr - hole_offset; 51593c2df58SDoug Thompson 51693c2df58SDoug Thompson debugf2("using DHAR to translate SysAddr 0x%lx to " 51793c2df58SDoug Thompson "DramAddr 0x%lx\n", 51893c2df58SDoug Thompson (unsigned long)sys_addr, 51993c2df58SDoug Thompson (unsigned long)dram_addr); 52093c2df58SDoug Thompson 52193c2df58SDoug Thompson return dram_addr; 52293c2df58SDoug Thompson } 52393c2df58SDoug Thompson } 52493c2df58SDoug Thompson 52593c2df58SDoug Thompson /* 52693c2df58SDoug Thompson * Translate the SysAddr to a DramAddr as shown near the start of 52793c2df58SDoug Thompson * section 3.4.4 (p. 70). Although sys_addr is a 64-bit value, the k8 52893c2df58SDoug Thompson * only deals with 40-bit values. Therefore we discard bits 63-40 of 52993c2df58SDoug Thompson * sys_addr below. If bit 39 of sys_addr is 1 then the bits we 53093c2df58SDoug Thompson * discard are all 1s. Otherwise the bits we discard are all 0s. See 53193c2df58SDoug Thompson * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture 53293c2df58SDoug Thompson * Programmer's Manual Volume 1 Application Programming. 53393c2df58SDoug Thompson */ 534f678b8ccSBorislav Petkov dram_addr = (sys_addr & GENMASK(0, 39)) - dram_base; 53593c2df58SDoug Thompson 53693c2df58SDoug Thompson debugf2("using DRAM Base register to translate SysAddr 0x%lx to " 53793c2df58SDoug Thompson "DramAddr 0x%lx\n", (unsigned long)sys_addr, 53893c2df58SDoug Thompson (unsigned long)dram_addr); 53993c2df58SDoug Thompson return dram_addr; 54093c2df58SDoug Thompson } 54193c2df58SDoug Thompson 54293c2df58SDoug Thompson /* 54393c2df58SDoug Thompson * @intlv_en is the value of the IntlvEn field from a DRAM Base register 54493c2df58SDoug Thompson * (section 3.4.4.1). Return the number of bits from a SysAddr that are used 54593c2df58SDoug Thompson * for node interleaving. 54693c2df58SDoug Thompson */ 54793c2df58SDoug Thompson static int num_node_interleave_bits(unsigned intlv_en) 54893c2df58SDoug Thompson { 54993c2df58SDoug Thompson static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 }; 55093c2df58SDoug Thompson int n; 55193c2df58SDoug Thompson 55293c2df58SDoug Thompson BUG_ON(intlv_en > 7); 55393c2df58SDoug Thompson n = intlv_shift_table[intlv_en]; 55493c2df58SDoug Thompson return n; 55593c2df58SDoug Thompson } 55693c2df58SDoug Thompson 55793c2df58SDoug Thompson /* Translate the DramAddr given by @dram_addr to an InputAddr. */ 55893c2df58SDoug Thompson static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr) 55993c2df58SDoug Thompson { 56093c2df58SDoug Thompson struct amd64_pvt *pvt; 56193c2df58SDoug Thompson int intlv_shift; 56293c2df58SDoug Thompson u64 input_addr; 56393c2df58SDoug Thompson 56493c2df58SDoug Thompson pvt = mci->pvt_info; 56593c2df58SDoug Thompson 56693c2df58SDoug Thompson /* 56793c2df58SDoug Thompson * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E) 56893c2df58SDoug Thompson * concerning translating a DramAddr to an InputAddr. 56993c2df58SDoug Thompson */ 5707f19bf75SBorislav Petkov intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0)); 571f678b8ccSBorislav Petkov input_addr = ((dram_addr >> intlv_shift) & GENMASK(12, 35)) + 57293c2df58SDoug Thompson (dram_addr & 0xfff); 57393c2df58SDoug Thompson 57493c2df58SDoug Thompson debugf2(" Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n", 57593c2df58SDoug Thompson intlv_shift, (unsigned long)dram_addr, 57693c2df58SDoug Thompson (unsigned long)input_addr); 57793c2df58SDoug Thompson 57893c2df58SDoug Thompson return input_addr; 57993c2df58SDoug Thompson } 58093c2df58SDoug Thompson 58193c2df58SDoug Thompson /* 58293c2df58SDoug Thompson * Translate the SysAddr represented by @sys_addr to an InputAddr. It is 58393c2df58SDoug Thompson * assumed that @sys_addr maps to the node given by mci. 58493c2df58SDoug Thompson */ 58593c2df58SDoug Thompson static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr) 58693c2df58SDoug Thompson { 58793c2df58SDoug Thompson u64 input_addr; 58893c2df58SDoug Thompson 58993c2df58SDoug Thompson input_addr = 59093c2df58SDoug Thompson dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr)); 59193c2df58SDoug Thompson 59293c2df58SDoug Thompson debugf2("SysAdddr 0x%lx translates to InputAddr 0x%lx\n", 59393c2df58SDoug Thompson (unsigned long)sys_addr, (unsigned long)input_addr); 59493c2df58SDoug Thompson 59593c2df58SDoug Thompson return input_addr; 59693c2df58SDoug Thompson } 59793c2df58SDoug Thompson 59893c2df58SDoug Thompson 59993c2df58SDoug Thompson /* 60093c2df58SDoug Thompson * @input_addr is an InputAddr associated with the node represented by mci. 60193c2df58SDoug Thompson * Translate @input_addr to a DramAddr and return the result. 60293c2df58SDoug Thompson */ 60393c2df58SDoug Thompson static u64 input_addr_to_dram_addr(struct mem_ctl_info *mci, u64 input_addr) 60493c2df58SDoug Thompson { 60593c2df58SDoug Thompson struct amd64_pvt *pvt; 606b487c33eSBorislav Petkov unsigned node_id, intlv_shift; 60793c2df58SDoug Thompson u64 bits, dram_addr; 60893c2df58SDoug Thompson u32 intlv_sel; 60993c2df58SDoug Thompson 61093c2df58SDoug Thompson /* 61193c2df58SDoug Thompson * Near the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E) 61293c2df58SDoug Thompson * shows how to translate a DramAddr to an InputAddr. Here we reverse 61393c2df58SDoug Thompson * this procedure. When translating from a DramAddr to an InputAddr, the 61493c2df58SDoug Thompson * bits used for node interleaving are discarded. Here we recover these 61593c2df58SDoug Thompson * bits from the IntlvSel field of the DRAM Limit register (section 61693c2df58SDoug Thompson * 3.4.4.2) for the node that input_addr is associated with. 61793c2df58SDoug Thompson */ 61893c2df58SDoug Thompson pvt = mci->pvt_info; 61993c2df58SDoug Thompson node_id = pvt->mc_node_id; 620b487c33eSBorislav Petkov 621b487c33eSBorislav Petkov BUG_ON(node_id > 7); 62293c2df58SDoug Thompson 6237f19bf75SBorislav Petkov intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0)); 62493c2df58SDoug Thompson if (intlv_shift == 0) { 62593c2df58SDoug Thompson debugf1(" InputAddr 0x%lx translates to DramAddr of " 62693c2df58SDoug Thompson "same value\n", (unsigned long)input_addr); 62793c2df58SDoug Thompson 62893c2df58SDoug Thompson return input_addr; 62993c2df58SDoug Thompson } 63093c2df58SDoug Thompson 631f678b8ccSBorislav Petkov bits = ((input_addr & GENMASK(12, 35)) << intlv_shift) + 63293c2df58SDoug Thompson (input_addr & 0xfff); 63393c2df58SDoug Thompson 6347f19bf75SBorislav Petkov intlv_sel = dram_intlv_sel(pvt, node_id) & ((1 << intlv_shift) - 1); 63593c2df58SDoug Thompson dram_addr = bits + (intlv_sel << 12); 63693c2df58SDoug Thompson 63793c2df58SDoug Thompson debugf1("InputAddr 0x%lx translates to DramAddr 0x%lx " 63893c2df58SDoug Thompson "(%d node interleave bits)\n", (unsigned long)input_addr, 63993c2df58SDoug Thompson (unsigned long)dram_addr, intlv_shift); 64093c2df58SDoug Thompson 64193c2df58SDoug Thompson return dram_addr; 64293c2df58SDoug Thompson } 64393c2df58SDoug Thompson 64493c2df58SDoug Thompson /* 64593c2df58SDoug Thompson * @dram_addr is a DramAddr that maps to the node represented by mci. Convert 64693c2df58SDoug Thompson * @dram_addr to a SysAddr. 64793c2df58SDoug Thompson */ 64893c2df58SDoug Thompson static u64 dram_addr_to_sys_addr(struct mem_ctl_info *mci, u64 dram_addr) 64993c2df58SDoug Thompson { 65093c2df58SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 6517f19bf75SBorislav Petkov u64 hole_base, hole_offset, hole_size, base, sys_addr; 65293c2df58SDoug Thompson int ret = 0; 65393c2df58SDoug Thompson 65493c2df58SDoug Thompson ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset, 65593c2df58SDoug Thompson &hole_size); 65693c2df58SDoug Thompson if (!ret) { 65793c2df58SDoug Thompson if ((dram_addr >= hole_base) && 65893c2df58SDoug Thompson (dram_addr < (hole_base + hole_size))) { 65993c2df58SDoug Thompson sys_addr = dram_addr + hole_offset; 66093c2df58SDoug Thompson 66193c2df58SDoug Thompson debugf1("using DHAR to translate DramAddr 0x%lx to " 66293c2df58SDoug Thompson "SysAddr 0x%lx\n", (unsigned long)dram_addr, 66393c2df58SDoug Thompson (unsigned long)sys_addr); 66493c2df58SDoug Thompson 66593c2df58SDoug Thompson return sys_addr; 66693c2df58SDoug Thompson } 66793c2df58SDoug Thompson } 66893c2df58SDoug Thompson 6697f19bf75SBorislav Petkov base = get_dram_base(pvt, pvt->mc_node_id); 67093c2df58SDoug Thompson sys_addr = dram_addr + base; 67193c2df58SDoug Thompson 67293c2df58SDoug Thompson /* 67393c2df58SDoug Thompson * The sys_addr we have computed up to this point is a 40-bit value 67493c2df58SDoug Thompson * because the k8 deals with 40-bit values. However, the value we are 67593c2df58SDoug Thompson * supposed to return is a full 64-bit physical address. The AMD 67693c2df58SDoug Thompson * x86-64 architecture specifies that the most significant implemented 67793c2df58SDoug Thompson * address bit through bit 63 of a physical address must be either all 67893c2df58SDoug Thompson * 0s or all 1s. Therefore we sign-extend the 40-bit sys_addr to a 67993c2df58SDoug Thompson * 64-bit value below. See section 3.4.2 of AMD publication 24592: 68093c2df58SDoug Thompson * AMD x86-64 Architecture Programmer's Manual Volume 1 Application 68193c2df58SDoug Thompson * Programming. 68293c2df58SDoug Thompson */ 68393c2df58SDoug Thompson sys_addr |= ~((sys_addr & (1ull << 39)) - 1); 68493c2df58SDoug Thompson 68593c2df58SDoug Thompson debugf1(" Node %d, DramAddr 0x%lx to SysAddr 0x%lx\n", 68693c2df58SDoug Thompson pvt->mc_node_id, (unsigned long)dram_addr, 68793c2df58SDoug Thompson (unsigned long)sys_addr); 68893c2df58SDoug Thompson 68993c2df58SDoug Thompson return sys_addr; 69093c2df58SDoug Thompson } 69193c2df58SDoug Thompson 69293c2df58SDoug Thompson /* 69393c2df58SDoug Thompson * @input_addr is an InputAddr associated with the node given by mci. Translate 69493c2df58SDoug Thompson * @input_addr to a SysAddr. 69593c2df58SDoug Thompson */ 69693c2df58SDoug Thompson static inline u64 input_addr_to_sys_addr(struct mem_ctl_info *mci, 69793c2df58SDoug Thompson u64 input_addr) 69893c2df58SDoug Thompson { 69993c2df58SDoug Thompson return dram_addr_to_sys_addr(mci, 70093c2df58SDoug Thompson input_addr_to_dram_addr(mci, input_addr)); 70193c2df58SDoug Thompson } 70293c2df58SDoug Thompson 70393c2df58SDoug Thompson /* 70493c2df58SDoug Thompson * Find the minimum and maximum InputAddr values that map to the given @csrow. 70593c2df58SDoug Thompson * Pass back these values in *input_addr_min and *input_addr_max. 70693c2df58SDoug Thompson */ 70793c2df58SDoug Thompson static void find_csrow_limits(struct mem_ctl_info *mci, int csrow, 70893c2df58SDoug Thompson u64 *input_addr_min, u64 *input_addr_max) 70993c2df58SDoug Thompson { 71093c2df58SDoug Thompson struct amd64_pvt *pvt; 71193c2df58SDoug Thompson u64 base, mask; 71293c2df58SDoug Thompson 71393c2df58SDoug Thompson pvt = mci->pvt_info; 71411c75eadSBorislav Petkov BUG_ON((csrow < 0) || (csrow >= pvt->csels[0].b_cnt)); 71593c2df58SDoug Thompson 71611c75eadSBorislav Petkov get_cs_base_and_mask(pvt, csrow, 0, &base, &mask); 71793c2df58SDoug Thompson 71893c2df58SDoug Thompson *input_addr_min = base & ~mask; 71911c75eadSBorislav Petkov *input_addr_max = base | mask; 72093c2df58SDoug Thompson } 72193c2df58SDoug Thompson 72293c2df58SDoug Thompson /* Map the Error address to a PAGE and PAGE OFFSET. */ 72393c2df58SDoug Thompson static inline void error_address_to_page_and_offset(u64 error_address, 72493c2df58SDoug Thompson u32 *page, u32 *offset) 72593c2df58SDoug Thompson { 72693c2df58SDoug Thompson *page = (u32) (error_address >> PAGE_SHIFT); 72793c2df58SDoug Thompson *offset = ((u32) error_address) & ~PAGE_MASK; 72893c2df58SDoug Thompson } 72993c2df58SDoug Thompson 73093c2df58SDoug Thompson /* 73193c2df58SDoug Thompson * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address 73293c2df58SDoug Thompson * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers 73393c2df58SDoug Thompson * of a node that detected an ECC memory error. mci represents the node that 73493c2df58SDoug Thompson * the error address maps to (possibly different from the node that detected 73593c2df58SDoug Thompson * the error). Return the number of the csrow that sys_addr maps to, or -1 on 73693c2df58SDoug Thompson * error. 73793c2df58SDoug Thompson */ 73893c2df58SDoug Thompson static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr) 73993c2df58SDoug Thompson { 74093c2df58SDoug Thompson int csrow; 74193c2df58SDoug Thompson 74293c2df58SDoug Thompson csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr)); 74393c2df58SDoug Thompson 74493c2df58SDoug Thompson if (csrow == -1) 74524f9a7feSBorislav Petkov amd64_mc_err(mci, "Failed to translate InputAddr to csrow for " 74693c2df58SDoug Thompson "address 0x%lx\n", (unsigned long)sys_addr); 74793c2df58SDoug Thompson return csrow; 74893c2df58SDoug Thompson } 749e2ce7255SDoug Thompson 750bfc04aecSBorislav Petkov static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16); 7512da11654SDoug Thompson 7522da11654SDoug Thompson /* 7532da11654SDoug Thompson * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs 7542da11654SDoug Thompson * are ECC capable. 7552da11654SDoug Thompson */ 7562da11654SDoug Thompson static enum edac_type amd64_determine_edac_cap(struct amd64_pvt *pvt) 7572da11654SDoug Thompson { 758cb328507SBorislav Petkov u8 bit; 759584fcff4SBorislav Petkov enum dev_type edac_cap = EDAC_FLAG_NONE; 7602da11654SDoug Thompson 7611433eb99SBorislav Petkov bit = (boot_cpu_data.x86 > 0xf || pvt->ext_model >= K8_REV_F) 7622da11654SDoug Thompson ? 19 7632da11654SDoug Thompson : 17; 7642da11654SDoug Thompson 765584fcff4SBorislav Petkov if (pvt->dclr0 & BIT(bit)) 7662da11654SDoug Thompson edac_cap = EDAC_FLAG_SECDED; 7672da11654SDoug Thompson 7682da11654SDoug Thompson return edac_cap; 7692da11654SDoug Thompson } 7702da11654SDoug Thompson 7712da11654SDoug Thompson 7728566c4dfSBorislav Petkov static void amd64_debug_display_dimm_sizes(int ctrl, struct amd64_pvt *pvt); 7732da11654SDoug Thompson 77468798e17SBorislav Petkov static void amd64_dump_dramcfg_low(u32 dclr, int chan) 77568798e17SBorislav Petkov { 77668798e17SBorislav Petkov debugf1("F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr); 77768798e17SBorislav Petkov 77868798e17SBorislav Petkov debugf1(" DIMM type: %sbuffered; all DIMMs support ECC: %s\n", 77968798e17SBorislav Petkov (dclr & BIT(16)) ? "un" : "", 78068798e17SBorislav Petkov (dclr & BIT(19)) ? "yes" : "no"); 78168798e17SBorislav Petkov 78268798e17SBorislav Petkov debugf1(" PAR/ERR parity: %s\n", 78368798e17SBorislav Petkov (dclr & BIT(8)) ? "enabled" : "disabled"); 78468798e17SBorislav Petkov 785cb328507SBorislav Petkov if (boot_cpu_data.x86 == 0x10) 78668798e17SBorislav Petkov debugf1(" DCT 128bit mode width: %s\n", 78768798e17SBorislav Petkov (dclr & BIT(11)) ? "128b" : "64b"); 78868798e17SBorislav Petkov 78968798e17SBorislav Petkov debugf1(" x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n", 79068798e17SBorislav Petkov (dclr & BIT(12)) ? "yes" : "no", 79168798e17SBorislav Petkov (dclr & BIT(13)) ? "yes" : "no", 79268798e17SBorislav Petkov (dclr & BIT(14)) ? "yes" : "no", 79368798e17SBorislav Petkov (dclr & BIT(15)) ? "yes" : "no"); 79468798e17SBorislav Petkov } 79568798e17SBorislav Petkov 7962da11654SDoug Thompson /* Display and decode various NB registers for debug purposes. */ 797b2b0c605SBorislav Petkov static void dump_misc_regs(struct amd64_pvt *pvt) 7982da11654SDoug Thompson { 79968798e17SBorislav Petkov debugf1("F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap); 8002da11654SDoug Thompson 80168798e17SBorislav Petkov debugf1(" NB two channel DRAM capable: %s\n", 8025980bb9cSBorislav Petkov (pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no"); 80368798e17SBorislav Petkov 80468798e17SBorislav Petkov debugf1(" ECC capable: %s, ChipKill ECC capable: %s\n", 8055980bb9cSBorislav Petkov (pvt->nbcap & NBCAP_SECDED) ? "yes" : "no", 8065980bb9cSBorislav Petkov (pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no"); 80768798e17SBorislav Petkov 80868798e17SBorislav Petkov amd64_dump_dramcfg_low(pvt->dclr0, 0); 8092da11654SDoug Thompson 8108de1d91eSBorislav Petkov debugf1("F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare); 8112da11654SDoug Thompson 8128de1d91eSBorislav Petkov debugf1("F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, " 8138de1d91eSBorislav Petkov "offset: 0x%08x\n", 814bc21fa57SBorislav Petkov pvt->dhar, dhar_base(pvt), 815bc21fa57SBorislav Petkov (boot_cpu_data.x86 == 0xf) ? k8_dhar_offset(pvt) 816bc21fa57SBorislav Petkov : f10_dhar_offset(pvt)); 8172da11654SDoug Thompson 818c8e518d5SBorislav Petkov debugf1(" DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no"); 8192da11654SDoug Thompson 8208566c4dfSBorislav Petkov amd64_debug_display_dimm_sizes(0, pvt); 8214d796364SBorislav Petkov 8224d796364SBorislav Petkov /* everything below this point is Fam10h and above */ 8234d796364SBorislav Petkov if (boot_cpu_data.x86 == 0xf) 8242da11654SDoug Thompson return; 8254d796364SBorislav Petkov 8264d796364SBorislav Petkov amd64_debug_display_dimm_sizes(1, pvt); 8272da11654SDoug Thompson 828a3b7db09SBorislav Petkov amd64_info("using %s syndromes.\n", ((pvt->ecc_sym_sz == 8) ? "x8" : "x4")); 829ad6a32e9SBorislav Petkov 8308de1d91eSBorislav Petkov /* Only if NOT ganged does dclr1 have valid info */ 83168798e17SBorislav Petkov if (!dct_ganging_enabled(pvt)) 83268798e17SBorislav Petkov amd64_dump_dramcfg_low(pvt->dclr1, 1); 8332da11654SDoug Thompson } 8342da11654SDoug Thompson 83594be4bffSDoug Thompson /* 83611c75eadSBorislav Petkov * see BKDG, F2x[1,0][5C:40], F2[1,0][6C:60] 83794be4bffSDoug Thompson */ 83811c75eadSBorislav Petkov static void prep_chip_selects(struct amd64_pvt *pvt) 83994be4bffSDoug Thompson { 8401433eb99SBorislav Petkov if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) { 84111c75eadSBorislav Petkov pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8; 84211c75eadSBorislav Petkov pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8; 8439d858bb1SBorislav Petkov } else { 84411c75eadSBorislav Petkov pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8; 84511c75eadSBorislav Petkov pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4; 8469d858bb1SBorislav Petkov } 84794be4bffSDoug Thompson } 84894be4bffSDoug Thompson 84994be4bffSDoug Thompson /* 85011c75eadSBorislav Petkov * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers 85194be4bffSDoug Thompson */ 852b2b0c605SBorislav Petkov static void read_dct_base_mask(struct amd64_pvt *pvt) 85394be4bffSDoug Thompson { 85411c75eadSBorislav Petkov int cs; 85594be4bffSDoug Thompson 85611c75eadSBorislav Petkov prep_chip_selects(pvt); 85794be4bffSDoug Thompson 85811c75eadSBorislav Petkov for_each_chip_select(cs, 0, pvt) { 85971d2a32eSBorislav Petkov int reg0 = DCSB0 + (cs * 4); 86071d2a32eSBorislav Petkov int reg1 = DCSB1 + (cs * 4); 86111c75eadSBorislav Petkov u32 *base0 = &pvt->csels[0].csbases[cs]; 86211c75eadSBorislav Petkov u32 *base1 = &pvt->csels[1].csbases[cs]; 863b2b0c605SBorislav Petkov 86411c75eadSBorislav Petkov if (!amd64_read_dct_pci_cfg(pvt, reg0, base0)) 86594be4bffSDoug Thompson debugf0(" DCSB0[%d]=0x%08x reg: F2x%x\n", 86611c75eadSBorislav Petkov cs, *base0, reg0); 86794be4bffSDoug Thompson 86811c75eadSBorislav Petkov if (boot_cpu_data.x86 == 0xf || dct_ganging_enabled(pvt)) 86911c75eadSBorislav Petkov continue; 870b2b0c605SBorislav Petkov 87111c75eadSBorislav Petkov if (!amd64_read_dct_pci_cfg(pvt, reg1, base1)) 87294be4bffSDoug Thompson debugf0(" DCSB1[%d]=0x%08x reg: F2x%x\n", 87311c75eadSBorislav Petkov cs, *base1, reg1); 87494be4bffSDoug Thompson } 87594be4bffSDoug Thompson 87611c75eadSBorislav Petkov for_each_chip_select_mask(cs, 0, pvt) { 87771d2a32eSBorislav Petkov int reg0 = DCSM0 + (cs * 4); 87871d2a32eSBorislav Petkov int reg1 = DCSM1 + (cs * 4); 87911c75eadSBorislav Petkov u32 *mask0 = &pvt->csels[0].csmasks[cs]; 88011c75eadSBorislav Petkov u32 *mask1 = &pvt->csels[1].csmasks[cs]; 881b2b0c605SBorislav Petkov 88211c75eadSBorislav Petkov if (!amd64_read_dct_pci_cfg(pvt, reg0, mask0)) 88394be4bffSDoug Thompson debugf0(" DCSM0[%d]=0x%08x reg: F2x%x\n", 88411c75eadSBorislav Petkov cs, *mask0, reg0); 88594be4bffSDoug Thompson 88611c75eadSBorislav Petkov if (boot_cpu_data.x86 == 0xf || dct_ganging_enabled(pvt)) 88711c75eadSBorislav Petkov continue; 888b2b0c605SBorislav Petkov 88911c75eadSBorislav Petkov if (!amd64_read_dct_pci_cfg(pvt, reg1, mask1)) 89094be4bffSDoug Thompson debugf0(" DCSM1[%d]=0x%08x reg: F2x%x\n", 89111c75eadSBorislav Petkov cs, *mask1, reg1); 89294be4bffSDoug Thompson } 8936ba5dcdcSBorislav Petkov } 89494be4bffSDoug Thompson 89524f9a7feSBorislav Petkov static enum mem_type amd64_determine_memory_type(struct amd64_pvt *pvt, int cs) 89694be4bffSDoug Thompson { 89794be4bffSDoug Thompson enum mem_type type; 89894be4bffSDoug Thompson 899cb328507SBorislav Petkov /* F15h supports only DDR3 */ 900cb328507SBorislav Petkov if (boot_cpu_data.x86 >= 0x15) 901cb328507SBorislav Petkov type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3; 902cb328507SBorislav Petkov else if (boot_cpu_data.x86 == 0x10 || pvt->ext_model >= K8_REV_F) { 9036b4c0bdeSBorislav Petkov if (pvt->dchr0 & DDR3_MODE) 9046b4c0bdeSBorislav Petkov type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3; 9056b4c0bdeSBorislav Petkov else 90694be4bffSDoug Thompson type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2; 90794be4bffSDoug Thompson } else { 90894be4bffSDoug Thompson type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR; 90994be4bffSDoug Thompson } 91094be4bffSDoug Thompson 91124f9a7feSBorislav Petkov amd64_info("CS%d: %s\n", cs, edac_mem_types[type]); 91294be4bffSDoug Thompson 91394be4bffSDoug Thompson return type; 91494be4bffSDoug Thompson } 91594be4bffSDoug Thompson 916cb328507SBorislav Petkov /* Get the number of DCT channels the memory controller is using. */ 917ddff876dSDoug Thompson static int k8_early_channel_count(struct amd64_pvt *pvt) 918ddff876dSDoug Thompson { 919cb328507SBorislav Petkov int flag; 920ddff876dSDoug Thompson 9219f56da0eSBorislav Petkov if (pvt->ext_model >= K8_REV_F) 922ddff876dSDoug Thompson /* RevF (NPT) and later */ 92341d8bfabSBorislav Petkov flag = pvt->dclr0 & WIDTH_128; 9249f56da0eSBorislav Petkov else 925ddff876dSDoug Thompson /* RevE and earlier */ 926ddff876dSDoug Thompson flag = pvt->dclr0 & REVE_WIDTH_128; 927ddff876dSDoug Thompson 928ddff876dSDoug Thompson /* not used */ 929ddff876dSDoug Thompson pvt->dclr1 = 0; 930ddff876dSDoug Thompson 931ddff876dSDoug Thompson return (flag) ? 2 : 1; 932ddff876dSDoug Thompson } 933ddff876dSDoug Thompson 93470046624SBorislav Petkov /* On F10h and later ErrAddr is MC4_ADDR[47:1] */ 93570046624SBorislav Petkov static u64 get_error_address(struct mce *m) 936ddff876dSDoug Thompson { 93770046624SBorislav Petkov u8 start_bit = 1; 93870046624SBorislav Petkov u8 end_bit = 47; 93970046624SBorislav Petkov 94070046624SBorislav Petkov if (boot_cpu_data.x86 == 0xf) { 94170046624SBorislav Petkov start_bit = 3; 94270046624SBorislav Petkov end_bit = 39; 94370046624SBorislav Petkov } 94470046624SBorislav Petkov 94570046624SBorislav Petkov return m->addr & GENMASK(start_bit, end_bit); 946ddff876dSDoug Thompson } 947ddff876dSDoug Thompson 9487f19bf75SBorislav Petkov static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range) 949ddff876dSDoug Thompson { 95071d2a32eSBorislav Petkov int off = range << 3; 951ddff876dSDoug Thompson 9527f19bf75SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off, &pvt->ranges[range].base.lo); 9537f19bf75SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo); 954ddff876dSDoug Thompson 9557f19bf75SBorislav Petkov if (boot_cpu_data.x86 == 0xf) 9567f19bf75SBorislav Petkov return; 957ddff876dSDoug Thompson 9587f19bf75SBorislav Petkov if (!dram_rw(pvt, range)) 9597f19bf75SBorislav Petkov return; 960ddff876dSDoug Thompson 9617f19bf75SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off, &pvt->ranges[range].base.hi); 9627f19bf75SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi); 963ddff876dSDoug Thompson } 964ddff876dSDoug Thompson 965f192c7b1SBorislav Petkov static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr, 966f192c7b1SBorislav Petkov u16 syndrome) 967ddff876dSDoug Thompson { 968ddff876dSDoug Thompson struct mem_ctl_info *src_mci; 969f192c7b1SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 970ddff876dSDoug Thompson int channel, csrow; 971ddff876dSDoug Thompson u32 page, offset; 972ddff876dSDoug Thompson 973ddff876dSDoug Thompson /* CHIPKILL enabled */ 974f192c7b1SBorislav Petkov if (pvt->nbcfg & NBCFG_CHIPKILL) { 975bfc04aecSBorislav Petkov channel = get_channel_from_ecc_syndrome(mci, syndrome); 976ddff876dSDoug Thompson if (channel < 0) { 977ddff876dSDoug Thompson /* 978ddff876dSDoug Thompson * Syndrome didn't map, so we don't know which of the 979ddff876dSDoug Thompson * 2 DIMMs is in error. So we need to ID 'both' of them 980ddff876dSDoug Thompson * as suspect. 981ddff876dSDoug Thompson */ 98224f9a7feSBorislav Petkov amd64_mc_warn(mci, "unknown syndrome 0x%04x - possible " 983ad6a32e9SBorislav Petkov "error reporting race\n", syndrome); 984ddff876dSDoug Thompson edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR); 985ddff876dSDoug Thompson return; 986ddff876dSDoug Thompson } 987ddff876dSDoug Thompson } else { 988ddff876dSDoug Thompson /* 989ddff876dSDoug Thompson * non-chipkill ecc mode 990ddff876dSDoug Thompson * 991ddff876dSDoug Thompson * The k8 documentation is unclear about how to determine the 992ddff876dSDoug Thompson * channel number when using non-chipkill memory. This method 993ddff876dSDoug Thompson * was obtained from email communication with someone at AMD. 994ddff876dSDoug Thompson * (Wish the email was placed in this comment - norsk) 995ddff876dSDoug Thompson */ 99644e9e2eeSBorislav Petkov channel = ((sys_addr & BIT(3)) != 0); 997ddff876dSDoug Thompson } 998ddff876dSDoug Thompson 999ddff876dSDoug Thompson /* 1000ddff876dSDoug Thompson * Find out which node the error address belongs to. This may be 1001ddff876dSDoug Thompson * different from the node that detected the error. 1002ddff876dSDoug Thompson */ 100344e9e2eeSBorislav Petkov src_mci = find_mc_by_sys_addr(mci, sys_addr); 10042cff18c2SKeith Mannthey if (!src_mci) { 100524f9a7feSBorislav Petkov amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n", 100644e9e2eeSBorislav Petkov (unsigned long)sys_addr); 1007ddff876dSDoug Thompson edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR); 1008ddff876dSDoug Thompson return; 1009ddff876dSDoug Thompson } 1010ddff876dSDoug Thompson 101144e9e2eeSBorislav Petkov /* Now map the sys_addr to a CSROW */ 101244e9e2eeSBorislav Petkov csrow = sys_addr_to_csrow(src_mci, sys_addr); 1013ddff876dSDoug Thompson if (csrow < 0) { 1014ddff876dSDoug Thompson edac_mc_handle_ce_no_info(src_mci, EDAC_MOD_STR); 1015ddff876dSDoug Thompson } else { 101644e9e2eeSBorislav Petkov error_address_to_page_and_offset(sys_addr, &page, &offset); 1017ddff876dSDoug Thompson 1018ddff876dSDoug Thompson edac_mc_handle_ce(src_mci, page, offset, syndrome, csrow, 1019ddff876dSDoug Thompson channel, EDAC_MOD_STR); 1020ddff876dSDoug Thompson } 1021ddff876dSDoug Thompson } 1022ddff876dSDoug Thompson 102341d8bfabSBorislav Petkov static int ddr2_cs_size(unsigned i, bool dct_width) 1024ddff876dSDoug Thompson { 102541d8bfabSBorislav Petkov unsigned shift = 0; 1026ddff876dSDoug Thompson 102741d8bfabSBorislav Petkov if (i <= 2) 102841d8bfabSBorislav Petkov shift = i; 102941d8bfabSBorislav Petkov else if (!(i & 0x1)) 103041d8bfabSBorislav Petkov shift = i >> 1; 10311433eb99SBorislav Petkov else 103241d8bfabSBorislav Petkov shift = (i + 1) >> 1; 1033ddff876dSDoug Thompson 103441d8bfabSBorislav Petkov return 128 << (shift + !!dct_width); 103541d8bfabSBorislav Petkov } 103641d8bfabSBorislav Petkov 103741d8bfabSBorislav Petkov static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, 103841d8bfabSBorislav Petkov unsigned cs_mode) 103941d8bfabSBorislav Petkov { 104041d8bfabSBorislav Petkov u32 dclr = dct ? pvt->dclr1 : pvt->dclr0; 104141d8bfabSBorislav Petkov 104241d8bfabSBorislav Petkov if (pvt->ext_model >= K8_REV_F) { 104341d8bfabSBorislav Petkov WARN_ON(cs_mode > 11); 104441d8bfabSBorislav Petkov return ddr2_cs_size(cs_mode, dclr & WIDTH_128); 104541d8bfabSBorislav Petkov } 104641d8bfabSBorislav Petkov else if (pvt->ext_model >= K8_REV_D) { 104741d8bfabSBorislav Petkov WARN_ON(cs_mode > 10); 104841d8bfabSBorislav Petkov 104941d8bfabSBorislav Petkov if (cs_mode == 3 || cs_mode == 8) 105041d8bfabSBorislav Petkov return 32 << (cs_mode - 1); 105141d8bfabSBorislav Petkov else 105241d8bfabSBorislav Petkov return 32 << cs_mode; 105341d8bfabSBorislav Petkov } 105441d8bfabSBorislav Petkov else { 105541d8bfabSBorislav Petkov WARN_ON(cs_mode > 6); 105641d8bfabSBorislav Petkov return 32 << cs_mode; 105741d8bfabSBorislav Petkov } 1058ddff876dSDoug Thompson } 1059ddff876dSDoug Thompson 10601afd3c98SDoug Thompson /* 10611afd3c98SDoug Thompson * Get the number of DCT channels in use. 10621afd3c98SDoug Thompson * 10631afd3c98SDoug Thompson * Return: 10641afd3c98SDoug Thompson * number of Memory Channels in operation 10651afd3c98SDoug Thompson * Pass back: 10661afd3c98SDoug Thompson * contents of the DCL0_LOW register 10671afd3c98SDoug Thompson */ 10687d20d14dSBorislav Petkov static int f1x_early_channel_count(struct amd64_pvt *pvt) 10691afd3c98SDoug Thompson { 10706ba5dcdcSBorislav Petkov int i, j, channels = 0; 1071ddff876dSDoug Thompson 10727d20d14dSBorislav Petkov /* On F10h, if we are in 128 bit mode, then we are using 2 channels */ 107341d8bfabSBorislav Petkov if (boot_cpu_data.x86 == 0x10 && (pvt->dclr0 & WIDTH_128)) 10747d20d14dSBorislav Petkov return 2; 10751afd3c98SDoug Thompson 10761afd3c98SDoug Thompson /* 1077d16149e8SBorislav Petkov * Need to check if in unganged mode: In such, there are 2 channels, 1078d16149e8SBorislav Petkov * but they are not in 128 bit mode and thus the above 'dclr0' status 1079d16149e8SBorislav Petkov * bit will be OFF. 10801afd3c98SDoug Thompson * 10811afd3c98SDoug Thompson * Need to check DCT0[0] and DCT1[0] to see if only one of them has 10821afd3c98SDoug Thompson * their CSEnable bit on. If so, then SINGLE DIMM case. 10831afd3c98SDoug Thompson */ 1084d16149e8SBorislav Petkov debugf0("Data width is not 128 bits - need more decoding\n"); 10851afd3c98SDoug Thompson 10861afd3c98SDoug Thompson /* 10871afd3c98SDoug Thompson * Check DRAM Bank Address Mapping values for each DIMM to see if there 10881afd3c98SDoug Thompson * is more than just one DIMM present in unganged mode. Need to check 10891afd3c98SDoug Thompson * both controllers since DIMMs can be placed in either one. 10901afd3c98SDoug Thompson */ 1091525a1b20SBorislav Petkov for (i = 0; i < 2; i++) { 1092525a1b20SBorislav Petkov u32 dbam = (i ? pvt->dbam1 : pvt->dbam0); 10931afd3c98SDoug Thompson 109457a30854SWan Wei for (j = 0; j < 4; j++) { 109557a30854SWan Wei if (DBAM_DIMM(j, dbam) > 0) { 10961afd3c98SDoug Thompson channels++; 109757a30854SWan Wei break; 10981afd3c98SDoug Thompson } 109957a30854SWan Wei } 110057a30854SWan Wei } 11011afd3c98SDoug Thompson 1102d16149e8SBorislav Petkov if (channels > 2) 1103d16149e8SBorislav Petkov channels = 2; 1104d16149e8SBorislav Petkov 110524f9a7feSBorislav Petkov amd64_info("MCT channel count: %d\n", channels); 11061afd3c98SDoug Thompson 11071afd3c98SDoug Thompson return channels; 11081afd3c98SDoug Thompson } 11091afd3c98SDoug Thompson 111041d8bfabSBorislav Petkov static int ddr3_cs_size(unsigned i, bool dct_width) 11111afd3c98SDoug Thompson { 111241d8bfabSBorislav Petkov unsigned shift = 0; 111341d8bfabSBorislav Petkov int cs_size = 0; 111441d8bfabSBorislav Petkov 111541d8bfabSBorislav Petkov if (i == 0 || i == 3 || i == 4) 111641d8bfabSBorislav Petkov cs_size = -1; 111741d8bfabSBorislav Petkov else if (i <= 2) 111841d8bfabSBorislav Petkov shift = i; 111941d8bfabSBorislav Petkov else if (i == 12) 112041d8bfabSBorislav Petkov shift = 7; 112141d8bfabSBorislav Petkov else if (!(i & 0x1)) 112241d8bfabSBorislav Petkov shift = i >> 1; 112341d8bfabSBorislav Petkov else 112441d8bfabSBorislav Petkov shift = (i + 1) >> 1; 112541d8bfabSBorislav Petkov 112641d8bfabSBorislav Petkov if (cs_size != -1) 112741d8bfabSBorislav Petkov cs_size = (128 * (1 << !!dct_width)) << shift; 112841d8bfabSBorislav Petkov 112941d8bfabSBorislav Petkov return cs_size; 113041d8bfabSBorislav Petkov } 113141d8bfabSBorislav Petkov 113241d8bfabSBorislav Petkov static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, 113341d8bfabSBorislav Petkov unsigned cs_mode) 113441d8bfabSBorislav Petkov { 113541d8bfabSBorislav Petkov u32 dclr = dct ? pvt->dclr1 : pvt->dclr0; 113641d8bfabSBorislav Petkov 113741d8bfabSBorislav Petkov WARN_ON(cs_mode > 11); 11381433eb99SBorislav Petkov 11391433eb99SBorislav Petkov if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE) 114041d8bfabSBorislav Petkov return ddr3_cs_size(cs_mode, dclr & WIDTH_128); 11411433eb99SBorislav Petkov else 114241d8bfabSBorislav Petkov return ddr2_cs_size(cs_mode, dclr & WIDTH_128); 114341d8bfabSBorislav Petkov } 11441433eb99SBorislav Petkov 114541d8bfabSBorislav Petkov /* 114641d8bfabSBorislav Petkov * F15h supports only 64bit DCT interfaces 114741d8bfabSBorislav Petkov */ 114841d8bfabSBorislav Petkov static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, 114941d8bfabSBorislav Petkov unsigned cs_mode) 115041d8bfabSBorislav Petkov { 115141d8bfabSBorislav Petkov WARN_ON(cs_mode > 12); 115241d8bfabSBorislav Petkov 115341d8bfabSBorislav Petkov return ddr3_cs_size(cs_mode, false); 11541afd3c98SDoug Thompson } 11551afd3c98SDoug Thompson 11565a5d2371SBorislav Petkov static void read_dram_ctl_register(struct amd64_pvt *pvt) 11576163b5d4SDoug Thompson { 11586163b5d4SDoug Thompson 11595a5d2371SBorislav Petkov if (boot_cpu_data.x86 == 0xf) 11605a5d2371SBorislav Petkov return; 11615a5d2371SBorislav Petkov 116278da121eSBorislav Petkov if (!amd64_read_dct_pci_cfg(pvt, DCT_SEL_LO, &pvt->dct_sel_lo)) { 116378da121eSBorislav Petkov debugf0("F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n", 116478da121eSBorislav Petkov pvt->dct_sel_lo, dct_sel_baseaddr(pvt)); 11656163b5d4SDoug Thompson 11665a5d2371SBorislav Petkov debugf0(" DCTs operate in %s mode.\n", 11675a5d2371SBorislav Petkov (dct_ganging_enabled(pvt) ? "ganged" : "unganged")); 11686163b5d4SDoug Thompson 116972381bd5SBorislav Petkov if (!dct_ganging_enabled(pvt)) 117072381bd5SBorislav Petkov debugf0(" Address range split per DCT: %s\n", 117172381bd5SBorislav Petkov (dct_high_range_enabled(pvt) ? "yes" : "no")); 117272381bd5SBorislav Petkov 117378da121eSBorislav Petkov debugf0(" data interleave for ECC: %s, " 117472381bd5SBorislav Petkov "DRAM cleared since last warm reset: %s\n", 117572381bd5SBorislav Petkov (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"), 117672381bd5SBorislav Petkov (dct_memory_cleared(pvt) ? "yes" : "no")); 117772381bd5SBorislav Petkov 117878da121eSBorislav Petkov debugf0(" channel interleave: %s, " 117978da121eSBorislav Petkov "interleave bits selector: 0x%x\n", 118072381bd5SBorislav Petkov (dct_interleave_enabled(pvt) ? "enabled" : "disabled"), 11816163b5d4SDoug Thompson dct_sel_interleave_addr(pvt)); 11826163b5d4SDoug Thompson } 11836163b5d4SDoug Thompson 118478da121eSBorislav Petkov amd64_read_dct_pci_cfg(pvt, DCT_SEL_HI, &pvt->dct_sel_hi); 11856163b5d4SDoug Thompson } 11866163b5d4SDoug Thompson 1187f71d0a05SDoug Thompson /* 1188229a7a11SBorislav Petkov * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory 1189f71d0a05SDoug Thompson * Interleaving Modes. 1190f71d0a05SDoug Thompson */ 1191b15f0fcaSBorislav Petkov static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr, 1192229a7a11SBorislav Petkov bool hi_range_sel, u8 intlv_en) 11936163b5d4SDoug Thompson { 1194151fa71cSBorislav Petkov u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1; 11956163b5d4SDoug Thompson 11966163b5d4SDoug Thompson if (dct_ganging_enabled(pvt)) 1197229a7a11SBorislav Petkov return 0; 1198229a7a11SBorislav Petkov 1199229a7a11SBorislav Petkov if (hi_range_sel) 1200229a7a11SBorislav Petkov return dct_sel_high; 1201229a7a11SBorislav Petkov 1202f71d0a05SDoug Thompson /* 1203f71d0a05SDoug Thompson * see F2x110[DctSelIntLvAddr] - channel interleave mode 1204f71d0a05SDoug Thompson */ 1205229a7a11SBorislav Petkov if (dct_interleave_enabled(pvt)) { 1206229a7a11SBorislav Petkov u8 intlv_addr = dct_sel_interleave_addr(pvt); 12076163b5d4SDoug Thompson 1208229a7a11SBorislav Petkov /* return DCT select function: 0=DCT0, 1=DCT1 */ 1209229a7a11SBorislav Petkov if (!intlv_addr) 1210229a7a11SBorislav Petkov return sys_addr >> 6 & 1; 12116163b5d4SDoug Thompson 1212229a7a11SBorislav Petkov if (intlv_addr & 0x2) { 1213229a7a11SBorislav Petkov u8 shift = intlv_addr & 0x1 ? 9 : 6; 1214229a7a11SBorislav Petkov u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) % 2; 1215229a7a11SBorislav Petkov 1216229a7a11SBorislav Petkov return ((sys_addr >> shift) & 1) ^ temp; 12176163b5d4SDoug Thompson } 12186163b5d4SDoug Thompson 1219229a7a11SBorislav Petkov return (sys_addr >> (12 + hweight8(intlv_en))) & 1; 1220229a7a11SBorislav Petkov } 1221229a7a11SBorislav Petkov 1222229a7a11SBorislav Petkov if (dct_high_range_enabled(pvt)) 1223229a7a11SBorislav Petkov return ~dct_sel_high & 1; 12246163b5d4SDoug Thompson 12256163b5d4SDoug Thompson return 0; 12266163b5d4SDoug Thompson } 12276163b5d4SDoug Thompson 1228c8e518d5SBorislav Petkov /* Convert the sys_addr to the normalized DCT address */ 1229b15f0fcaSBorislav Petkov static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, int range, 1230c8e518d5SBorislav Petkov u64 sys_addr, bool hi_rng, 1231c8e518d5SBorislav Petkov u32 dct_sel_base_addr) 12326163b5d4SDoug Thompson { 12336163b5d4SDoug Thompson u64 chan_off; 1234c8e518d5SBorislav Petkov u64 dram_base = get_dram_base(pvt, range); 1235c8e518d5SBorislav Petkov u64 hole_off = f10_dhar_offset(pvt); 1236c8e518d5SBorislav Petkov u64 dct_sel_base_off = (pvt->dct_sel_hi & 0xFFFFFC00) << 16; 12376163b5d4SDoug Thompson 1238c8e518d5SBorislav Petkov if (hi_rng) { 1239c8e518d5SBorislav Petkov /* 1240c8e518d5SBorislav Petkov * if 1241c8e518d5SBorislav Petkov * base address of high range is below 4Gb 1242c8e518d5SBorislav Petkov * (bits [47:27] at [31:11]) 1243c8e518d5SBorislav Petkov * DRAM address space on this DCT is hoisted above 4Gb && 1244c8e518d5SBorislav Petkov * sys_addr > 4Gb 1245c8e518d5SBorislav Petkov * 1246c8e518d5SBorislav Petkov * remove hole offset from sys_addr 1247c8e518d5SBorislav Petkov * else 1248c8e518d5SBorislav Petkov * remove high range offset from sys_addr 1249c8e518d5SBorislav Petkov */ 1250c8e518d5SBorislav Petkov if ((!(dct_sel_base_addr >> 16) || 1251c8e518d5SBorislav Petkov dct_sel_base_addr < dhar_base(pvt)) && 1252972ea17aSBorislav Petkov dhar_valid(pvt) && 1253c8e518d5SBorislav Petkov (sys_addr >= BIT_64(32))) 1254bc21fa57SBorislav Petkov chan_off = hole_off; 12556163b5d4SDoug Thompson else 12566163b5d4SDoug Thompson chan_off = dct_sel_base_off; 12576163b5d4SDoug Thompson } else { 1258c8e518d5SBorislav Petkov /* 1259c8e518d5SBorislav Petkov * if 1260c8e518d5SBorislav Petkov * we have a valid hole && 1261c8e518d5SBorislav Petkov * sys_addr > 4Gb 1262c8e518d5SBorislav Petkov * 1263c8e518d5SBorislav Petkov * remove hole 1264c8e518d5SBorislav Petkov * else 1265c8e518d5SBorislav Petkov * remove dram base to normalize to DCT address 1266c8e518d5SBorislav Petkov */ 1267972ea17aSBorislav Petkov if (dhar_valid(pvt) && (sys_addr >= BIT_64(32))) 1268bc21fa57SBorislav Petkov chan_off = hole_off; 12696163b5d4SDoug Thompson else 1270c8e518d5SBorislav Petkov chan_off = dram_base; 12716163b5d4SDoug Thompson } 12726163b5d4SDoug Thompson 1273c8e518d5SBorislav Petkov return (sys_addr & GENMASK(6,47)) - (chan_off & GENMASK(23,47)); 12746163b5d4SDoug Thompson } 12756163b5d4SDoug Thompson 12766163b5d4SDoug Thompson /* 12776163b5d4SDoug Thompson * checks if the csrow passed in is marked as SPARED, if so returns the new 12786163b5d4SDoug Thompson * spare row 12796163b5d4SDoug Thompson */ 128011c75eadSBorislav Petkov static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow) 12816163b5d4SDoug Thompson { 1282614ec9d8SBorislav Petkov int tmp_cs; 12836163b5d4SDoug Thompson 1284614ec9d8SBorislav Petkov if (online_spare_swap_done(pvt, dct) && 1285614ec9d8SBorislav Petkov csrow == online_spare_bad_dramcs(pvt, dct)) { 1286614ec9d8SBorislav Petkov 1287614ec9d8SBorislav Petkov for_each_chip_select(tmp_cs, dct, pvt) { 1288614ec9d8SBorislav Petkov if (chip_select_base(tmp_cs, dct, pvt) & 0x2) { 1289614ec9d8SBorislav Petkov csrow = tmp_cs; 1290614ec9d8SBorislav Petkov break; 1291614ec9d8SBorislav Petkov } 1292614ec9d8SBorislav Petkov } 12936163b5d4SDoug Thompson } 12946163b5d4SDoug Thompson return csrow; 12956163b5d4SDoug Thompson } 12966163b5d4SDoug Thompson 12976163b5d4SDoug Thompson /* 12986163b5d4SDoug Thompson * Iterate over the DRAM DCT "base" and "mask" registers looking for a 12996163b5d4SDoug Thompson * SystemAddr match on the specified 'ChannelSelect' and 'NodeID' 13006163b5d4SDoug Thompson * 13016163b5d4SDoug Thompson * Return: 13026163b5d4SDoug Thompson * -EINVAL: NOT FOUND 13036163b5d4SDoug Thompson * 0..csrow = Chip-Select Row 13046163b5d4SDoug Thompson */ 1305b15f0fcaSBorislav Petkov static int f1x_lookup_addr_in_dct(u64 in_addr, u32 nid, u8 dct) 13066163b5d4SDoug Thompson { 13076163b5d4SDoug Thompson struct mem_ctl_info *mci; 13086163b5d4SDoug Thompson struct amd64_pvt *pvt; 130911c75eadSBorislav Petkov u64 cs_base, cs_mask; 13106163b5d4SDoug Thompson int cs_found = -EINVAL; 13116163b5d4SDoug Thompson int csrow; 13126163b5d4SDoug Thompson 1313cc4d8860SBorislav Petkov mci = mcis[nid]; 13146163b5d4SDoug Thompson if (!mci) 13156163b5d4SDoug Thompson return cs_found; 13166163b5d4SDoug Thompson 13176163b5d4SDoug Thompson pvt = mci->pvt_info; 13186163b5d4SDoug Thompson 131911c75eadSBorislav Petkov debugf1("input addr: 0x%llx, DCT: %d\n", in_addr, dct); 13206163b5d4SDoug Thompson 132111c75eadSBorislav Petkov for_each_chip_select(csrow, dct, pvt) { 132211c75eadSBorislav Petkov if (!csrow_enabled(csrow, dct, pvt)) 13236163b5d4SDoug Thompson continue; 13246163b5d4SDoug Thompson 132511c75eadSBorislav Petkov get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask); 13266163b5d4SDoug Thompson 132711c75eadSBorislav Petkov debugf1(" CSROW=%d CSBase=0x%llx CSMask=0x%llx\n", 13286163b5d4SDoug Thompson csrow, cs_base, cs_mask); 13296163b5d4SDoug Thompson 133011c75eadSBorislav Petkov cs_mask = ~cs_mask; 13316163b5d4SDoug Thompson 133211c75eadSBorislav Petkov debugf1(" (InputAddr & ~CSMask)=0x%llx " 133311c75eadSBorislav Petkov "(CSBase & ~CSMask)=0x%llx\n", 133411c75eadSBorislav Petkov (in_addr & cs_mask), (cs_base & cs_mask)); 13356163b5d4SDoug Thompson 133611c75eadSBorislav Petkov if ((in_addr & cs_mask) == (cs_base & cs_mask)) { 133711c75eadSBorislav Petkov cs_found = f10_process_possible_spare(pvt, dct, csrow); 13386163b5d4SDoug Thompson 13396163b5d4SDoug Thompson debugf1(" MATCH csrow=%d\n", cs_found); 13406163b5d4SDoug Thompson break; 13416163b5d4SDoug Thompson } 13426163b5d4SDoug Thompson } 13436163b5d4SDoug Thompson return cs_found; 13446163b5d4SDoug Thompson } 13456163b5d4SDoug Thompson 134695b0ef55SBorislav Petkov /* 134795b0ef55SBorislav Petkov * See F2x10C. Non-interleaved graphics framebuffer memory under the 16G is 134895b0ef55SBorislav Petkov * swapped with a region located at the bottom of memory so that the GPU can use 134995b0ef55SBorislav Petkov * the interleaved region and thus two channels. 135095b0ef55SBorislav Petkov */ 1351b15f0fcaSBorislav Petkov static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr) 135295b0ef55SBorislav Petkov { 135395b0ef55SBorislav Petkov u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr; 135495b0ef55SBorislav Petkov 135595b0ef55SBorislav Petkov if (boot_cpu_data.x86 == 0x10) { 135695b0ef55SBorislav Petkov /* only revC3 and revE have that feature */ 135795b0ef55SBorislav Petkov if (boot_cpu_data.x86_model < 4 || 135895b0ef55SBorislav Petkov (boot_cpu_data.x86_model < 0xa && 135995b0ef55SBorislav Petkov boot_cpu_data.x86_mask < 3)) 136095b0ef55SBorislav Petkov return sys_addr; 136195b0ef55SBorislav Petkov } 136295b0ef55SBorislav Petkov 136395b0ef55SBorislav Petkov amd64_read_dct_pci_cfg(pvt, SWAP_INTLV_REG, &swap_reg); 136495b0ef55SBorislav Petkov 136595b0ef55SBorislav Petkov if (!(swap_reg & 0x1)) 136695b0ef55SBorislav Petkov return sys_addr; 136795b0ef55SBorislav Petkov 136895b0ef55SBorislav Petkov swap_base = (swap_reg >> 3) & 0x7f; 136995b0ef55SBorislav Petkov swap_limit = (swap_reg >> 11) & 0x7f; 137095b0ef55SBorislav Petkov rgn_size = (swap_reg >> 20) & 0x7f; 137195b0ef55SBorislav Petkov tmp_addr = sys_addr >> 27; 137295b0ef55SBorislav Petkov 137395b0ef55SBorislav Petkov if (!(sys_addr >> 34) && 137495b0ef55SBorislav Petkov (((tmp_addr >= swap_base) && 137595b0ef55SBorislav Petkov (tmp_addr <= swap_limit)) || 137695b0ef55SBorislav Petkov (tmp_addr < rgn_size))) 137795b0ef55SBorislav Petkov return sys_addr ^ (u64)swap_base << 27; 137895b0ef55SBorislav Petkov 137995b0ef55SBorislav Petkov return sys_addr; 138095b0ef55SBorislav Petkov } 138195b0ef55SBorislav Petkov 1382f71d0a05SDoug Thompson /* For a given @dram_range, check if @sys_addr falls within it. */ 1383b15f0fcaSBorislav Petkov static int f1x_match_to_this_node(struct amd64_pvt *pvt, int range, 1384f71d0a05SDoug Thompson u64 sys_addr, int *nid, int *chan_sel) 1385f71d0a05SDoug Thompson { 1386229a7a11SBorislav Petkov int cs_found = -EINVAL; 1387c8e518d5SBorislav Petkov u64 chan_addr; 13885d4b58e8SBorislav Petkov u32 dct_sel_base; 138911c75eadSBorislav Petkov u8 channel; 1390229a7a11SBorislav Petkov bool high_range = false; 1391f71d0a05SDoug Thompson 13927f19bf75SBorislav Petkov u8 node_id = dram_dst_node(pvt, range); 1393229a7a11SBorislav Petkov u8 intlv_en = dram_intlv_en(pvt, range); 13947f19bf75SBorislav Petkov u32 intlv_sel = dram_intlv_sel(pvt, range); 1395f71d0a05SDoug Thompson 1396c8e518d5SBorislav Petkov debugf1("(range %d) SystemAddr= 0x%llx Limit=0x%llx\n", 1397c8e518d5SBorislav Petkov range, sys_addr, get_dram_limit(pvt, range)); 1398f71d0a05SDoug Thompson 1399355fba60SBorislav Petkov if (dhar_valid(pvt) && 1400355fba60SBorislav Petkov dhar_base(pvt) <= sys_addr && 1401355fba60SBorislav Petkov sys_addr < BIT_64(32)) { 1402355fba60SBorislav Petkov amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n", 1403355fba60SBorislav Petkov sys_addr); 1404f71d0a05SDoug Thompson return -EINVAL; 1405355fba60SBorislav Petkov } 1406355fba60SBorislav Petkov 1407355fba60SBorislav Petkov if (intlv_en && 1408355fba60SBorislav Petkov (intlv_sel != ((sys_addr >> 12) & intlv_en))) { 1409355fba60SBorislav Petkov amd64_warn("Botched intlv bits, en: 0x%x, sel: 0x%x\n", 1410355fba60SBorislav Petkov intlv_en, intlv_sel); 1411355fba60SBorislav Petkov return -EINVAL; 1412355fba60SBorislav Petkov } 1413f71d0a05SDoug Thompson 1414b15f0fcaSBorislav Petkov sys_addr = f1x_swap_interleaved_region(pvt, sys_addr); 141595b0ef55SBorislav Petkov 1416f71d0a05SDoug Thompson dct_sel_base = dct_sel_baseaddr(pvt); 1417f71d0a05SDoug Thompson 1418f71d0a05SDoug Thompson /* 1419f71d0a05SDoug Thompson * check whether addresses >= DctSelBaseAddr[47:27] are to be used to 1420f71d0a05SDoug Thompson * select between DCT0 and DCT1. 1421f71d0a05SDoug Thompson */ 1422f71d0a05SDoug Thompson if (dct_high_range_enabled(pvt) && 1423f71d0a05SDoug Thompson !dct_ganging_enabled(pvt) && 1424f71d0a05SDoug Thompson ((sys_addr >> 27) >= (dct_sel_base >> 11))) 1425229a7a11SBorislav Petkov high_range = true; 1426f71d0a05SDoug Thompson 1427b15f0fcaSBorislav Petkov channel = f1x_determine_channel(pvt, sys_addr, high_range, intlv_en); 1428f71d0a05SDoug Thompson 1429b15f0fcaSBorislav Petkov chan_addr = f1x_get_norm_dct_addr(pvt, range, sys_addr, 1430c8e518d5SBorislav Petkov high_range, dct_sel_base); 1431f71d0a05SDoug Thompson 1432e2f79dbdSBorislav Petkov /* Remove node interleaving, see F1x120 */ 1433e2f79dbdSBorislav Petkov if (intlv_en) 1434e2f79dbdSBorislav Petkov chan_addr = ((chan_addr >> (12 + hweight8(intlv_en))) << 12) | 1435e2f79dbdSBorislav Petkov (chan_addr & 0xfff); 1436f71d0a05SDoug Thompson 14375d4b58e8SBorislav Petkov /* remove channel interleave */ 1438f71d0a05SDoug Thompson if (dct_interleave_enabled(pvt) && 1439f71d0a05SDoug Thompson !dct_high_range_enabled(pvt) && 1440f71d0a05SDoug Thompson !dct_ganging_enabled(pvt)) { 14415d4b58e8SBorislav Petkov 14425d4b58e8SBorislav Petkov if (dct_sel_interleave_addr(pvt) != 1) { 14435d4b58e8SBorislav Petkov if (dct_sel_interleave_addr(pvt) == 0x3) 14445d4b58e8SBorislav Petkov /* hash 9 */ 14455d4b58e8SBorislav Petkov chan_addr = ((chan_addr >> 10) << 9) | 14465d4b58e8SBorislav Petkov (chan_addr & 0x1ff); 14475d4b58e8SBorislav Petkov else 14485d4b58e8SBorislav Petkov /* A[6] or hash 6 */ 14495d4b58e8SBorislav Petkov chan_addr = ((chan_addr >> 7) << 6) | 14505d4b58e8SBorislav Petkov (chan_addr & 0x3f); 14515d4b58e8SBorislav Petkov } else 14525d4b58e8SBorislav Petkov /* A[12] */ 14535d4b58e8SBorislav Petkov chan_addr = ((chan_addr >> 13) << 12) | 14545d4b58e8SBorislav Petkov (chan_addr & 0xfff); 1455f71d0a05SDoug Thompson } 1456f71d0a05SDoug Thompson 14575d4b58e8SBorislav Petkov debugf1(" Normalized DCT addr: 0x%llx\n", chan_addr); 1458f71d0a05SDoug Thompson 1459b15f0fcaSBorislav Petkov cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, channel); 1460f71d0a05SDoug Thompson 1461f71d0a05SDoug Thompson if (cs_found >= 0) { 1462f71d0a05SDoug Thompson *nid = node_id; 1463f71d0a05SDoug Thompson *chan_sel = channel; 1464f71d0a05SDoug Thompson } 1465f71d0a05SDoug Thompson return cs_found; 1466f71d0a05SDoug Thompson } 1467f71d0a05SDoug Thompson 1468b15f0fcaSBorislav Petkov static int f1x_translate_sysaddr_to_cs(struct amd64_pvt *pvt, u64 sys_addr, 1469f71d0a05SDoug Thompson int *node, int *chan_sel) 1470f71d0a05SDoug Thompson { 14717f19bf75SBorislav Petkov int range, cs_found = -EINVAL; 1472f71d0a05SDoug Thompson 14737f19bf75SBorislav Petkov for (range = 0; range < DRAM_RANGES; range++) { 1474f71d0a05SDoug Thompson 14757f19bf75SBorislav Petkov if (!dram_rw(pvt, range)) 1476f71d0a05SDoug Thompson continue; 1477f71d0a05SDoug Thompson 14787f19bf75SBorislav Petkov if ((get_dram_base(pvt, range) <= sys_addr) && 14797f19bf75SBorislav Petkov (get_dram_limit(pvt, range) >= sys_addr)) { 1480f71d0a05SDoug Thompson 1481b15f0fcaSBorislav Petkov cs_found = f1x_match_to_this_node(pvt, range, 1482f71d0a05SDoug Thompson sys_addr, node, 1483f71d0a05SDoug Thompson chan_sel); 1484f71d0a05SDoug Thompson if (cs_found >= 0) 1485f71d0a05SDoug Thompson break; 1486f71d0a05SDoug Thompson } 1487f71d0a05SDoug Thompson } 1488f71d0a05SDoug Thompson return cs_found; 1489f71d0a05SDoug Thompson } 1490f71d0a05SDoug Thompson 1491f71d0a05SDoug Thompson /* 1492bdc30a0cSBorislav Petkov * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps 1493bdc30a0cSBorislav Petkov * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW). 1494f71d0a05SDoug Thompson * 1495bdc30a0cSBorislav Petkov * The @sys_addr is usually an error address received from the hardware 1496bdc30a0cSBorislav Petkov * (MCX_ADDR). 1497f71d0a05SDoug Thompson */ 1498b15f0fcaSBorislav Petkov static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr, 1499f192c7b1SBorislav Petkov u16 syndrome) 1500f71d0a05SDoug Thompson { 1501f71d0a05SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 1502f71d0a05SDoug Thompson u32 page, offset; 1503f71d0a05SDoug Thompson int nid, csrow, chan = 0; 1504f71d0a05SDoug Thompson 1505b15f0fcaSBorislav Petkov csrow = f1x_translate_sysaddr_to_cs(pvt, sys_addr, &nid, &chan); 1506f71d0a05SDoug Thompson 1507bdc30a0cSBorislav Petkov if (csrow < 0) { 1508bdc30a0cSBorislav Petkov edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR); 1509bdc30a0cSBorislav Petkov return; 1510bdc30a0cSBorislav Petkov } 1511bdc30a0cSBorislav Petkov 1512f71d0a05SDoug Thompson error_address_to_page_and_offset(sys_addr, &page, &offset); 1513f71d0a05SDoug Thompson 1514f71d0a05SDoug Thompson /* 1515bdc30a0cSBorislav Petkov * We need the syndromes for channel detection only when we're 1516bdc30a0cSBorislav Petkov * ganged. Otherwise @chan should already contain the channel at 1517bdc30a0cSBorislav Petkov * this point. 1518f71d0a05SDoug Thompson */ 1519a97fa68eSBorislav Petkov if (dct_ganging_enabled(pvt)) 1520bfc04aecSBorislav Petkov chan = get_channel_from_ecc_syndrome(mci, syndrome); 1521f71d0a05SDoug Thompson 1522bdc30a0cSBorislav Petkov if (chan >= 0) 1523bdc30a0cSBorislav Petkov edac_mc_handle_ce(mci, page, offset, syndrome, csrow, chan, 1524bdc30a0cSBorislav Petkov EDAC_MOD_STR); 1525bdc30a0cSBorislav Petkov else 1526bdc30a0cSBorislav Petkov /* 1527bdc30a0cSBorislav Petkov * Channel unknown, report all channels on this CSROW as failed. 1528bdc30a0cSBorislav Petkov */ 1529bdc30a0cSBorislav Petkov for (chan = 0; chan < mci->csrows[csrow].nr_channels; chan++) 1530f71d0a05SDoug Thompson edac_mc_handle_ce(mci, page, offset, syndrome, 1531f71d0a05SDoug Thompson csrow, chan, EDAC_MOD_STR); 1532f71d0a05SDoug Thompson } 1533f71d0a05SDoug Thompson 1534f71d0a05SDoug Thompson /* 15358566c4dfSBorislav Petkov * debug routine to display the memory sizes of all logical DIMMs and its 1536cb328507SBorislav Petkov * CSROWs 1537f71d0a05SDoug Thompson */ 15388566c4dfSBorislav Petkov static void amd64_debug_display_dimm_sizes(int ctrl, struct amd64_pvt *pvt) 1539f71d0a05SDoug Thompson { 1540603adaf6SBorislav Petkov int dimm, size0, size1, factor = 0; 1541525a1b20SBorislav Petkov u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases; 1542525a1b20SBorislav Petkov u32 dbam = ctrl ? pvt->dbam1 : pvt->dbam0; 1543f71d0a05SDoug Thompson 15448566c4dfSBorislav Petkov if (boot_cpu_data.x86 == 0xf) { 154541d8bfabSBorislav Petkov if (pvt->dclr0 & WIDTH_128) 1546603adaf6SBorislav Petkov factor = 1; 1547603adaf6SBorislav Petkov 15488566c4dfSBorislav Petkov /* K8 families < revF not supported yet */ 15491433eb99SBorislav Petkov if (pvt->ext_model < K8_REV_F) 15508566c4dfSBorislav Petkov return; 15518566c4dfSBorislav Petkov else 15528566c4dfSBorislav Petkov WARN_ON(ctrl != 0); 15538566c4dfSBorislav Petkov } 15548566c4dfSBorislav Petkov 15554d796364SBorislav Petkov dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1 : pvt->dbam0; 155611c75eadSBorislav Petkov dcsb = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->csels[1].csbases 155711c75eadSBorislav Petkov : pvt->csels[0].csbases; 1558f71d0a05SDoug Thompson 15594d796364SBorislav Petkov debugf1("F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n", ctrl, dbam); 1560f71d0a05SDoug Thompson 15618566c4dfSBorislav Petkov edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl); 15628566c4dfSBorislav Petkov 1563f71d0a05SDoug Thompson /* Dump memory sizes for DIMM and its CSROWs */ 1564f71d0a05SDoug Thompson for (dimm = 0; dimm < 4; dimm++) { 1565f71d0a05SDoug Thompson 1566f71d0a05SDoug Thompson size0 = 0; 156711c75eadSBorislav Petkov if (dcsb[dimm*2] & DCSB_CS_ENABLE) 156841d8bfabSBorislav Petkov size0 = pvt->ops->dbam_to_cs(pvt, ctrl, 156941d8bfabSBorislav Petkov DBAM_DIMM(dimm, dbam)); 1570f71d0a05SDoug Thompson 1571f71d0a05SDoug Thompson size1 = 0; 157211c75eadSBorislav Petkov if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE) 157341d8bfabSBorislav Petkov size1 = pvt->ops->dbam_to_cs(pvt, ctrl, 157441d8bfabSBorislav Petkov DBAM_DIMM(dimm, dbam)); 1575f71d0a05SDoug Thompson 157624f9a7feSBorislav Petkov amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n", 1577603adaf6SBorislav Petkov dimm * 2, size0 << factor, 1578603adaf6SBorislav Petkov dimm * 2 + 1, size1 << factor); 1579f71d0a05SDoug Thompson } 1580f71d0a05SDoug Thompson } 1581f71d0a05SDoug Thompson 15824d37607aSDoug Thompson static struct amd64_family_type amd64_family_types[] = { 15834d37607aSDoug Thompson [K8_CPUS] = { 15840092b20dSBorislav Petkov .ctl_name = "K8", 15858d5b5d9cSBorislav Petkov .f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP, 15868d5b5d9cSBorislav Petkov .f3_id = PCI_DEVICE_ID_AMD_K8_NB_MISC, 15874d37607aSDoug Thompson .ops = { 15884d37607aSDoug Thompson .early_channel_count = k8_early_channel_count, 15894d37607aSDoug Thompson .map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow, 15901433eb99SBorislav Petkov .dbam_to_cs = k8_dbam_to_chip_select, 1591b2b0c605SBorislav Petkov .read_dct_pci_cfg = k8_read_dct_pci_cfg, 15924d37607aSDoug Thompson } 15934d37607aSDoug Thompson }, 15944d37607aSDoug Thompson [F10_CPUS] = { 15950092b20dSBorislav Petkov .ctl_name = "F10h", 15968d5b5d9cSBorislav Petkov .f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP, 15978d5b5d9cSBorislav Petkov .f3_id = PCI_DEVICE_ID_AMD_10H_NB_MISC, 15984d37607aSDoug Thompson .ops = { 15997d20d14dSBorislav Petkov .early_channel_count = f1x_early_channel_count, 1600b15f0fcaSBorislav Petkov .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, 16011433eb99SBorislav Petkov .dbam_to_cs = f10_dbam_to_chip_select, 1602b2b0c605SBorislav Petkov .read_dct_pci_cfg = f10_read_dct_pci_cfg, 1603b2b0c605SBorislav Petkov } 1604b2b0c605SBorislav Petkov }, 1605b2b0c605SBorislav Petkov [F15_CPUS] = { 1606b2b0c605SBorislav Petkov .ctl_name = "F15h", 1607df71a053SBorislav Petkov .f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1, 1608df71a053SBorislav Petkov .f3_id = PCI_DEVICE_ID_AMD_15H_NB_F3, 1609b2b0c605SBorislav Petkov .ops = { 16107d20d14dSBorislav Petkov .early_channel_count = f1x_early_channel_count, 1611b15f0fcaSBorislav Petkov .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, 161241d8bfabSBorislav Petkov .dbam_to_cs = f15_dbam_to_chip_select, 1613b2b0c605SBorislav Petkov .read_dct_pci_cfg = f15_read_dct_pci_cfg, 16144d37607aSDoug Thompson } 16154d37607aSDoug Thompson }, 16164d37607aSDoug Thompson }; 16174d37607aSDoug Thompson 16184d37607aSDoug Thompson static struct pci_dev *pci_get_related_function(unsigned int vendor, 16194d37607aSDoug Thompson unsigned int device, 16204d37607aSDoug Thompson struct pci_dev *related) 16214d37607aSDoug Thompson { 16224d37607aSDoug Thompson struct pci_dev *dev = NULL; 16234d37607aSDoug Thompson 16244d37607aSDoug Thompson dev = pci_get_device(vendor, device, dev); 16254d37607aSDoug Thompson while (dev) { 16264d37607aSDoug Thompson if ((dev->bus->number == related->bus->number) && 16274d37607aSDoug Thompson (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn))) 16284d37607aSDoug Thompson break; 16294d37607aSDoug Thompson dev = pci_get_device(vendor, device, dev); 16304d37607aSDoug Thompson } 16314d37607aSDoug Thompson 16324d37607aSDoug Thompson return dev; 16334d37607aSDoug Thompson } 16344d37607aSDoug Thompson 1635b1289d6fSDoug Thompson /* 1636bfc04aecSBorislav Petkov * These are tables of eigenvectors (one per line) which can be used for the 1637bfc04aecSBorislav Petkov * construction of the syndrome tables. The modified syndrome search algorithm 1638bfc04aecSBorislav Petkov * uses those to find the symbol in error and thus the DIMM. 1639b1289d6fSDoug Thompson * 1640bfc04aecSBorislav Petkov * Algorithm courtesy of Ross LaFetra from AMD. 1641b1289d6fSDoug Thompson */ 1642bfc04aecSBorislav Petkov static u16 x4_vectors[] = { 1643bfc04aecSBorislav Petkov 0x2f57, 0x1afe, 0x66cc, 0xdd88, 1644bfc04aecSBorislav Petkov 0x11eb, 0x3396, 0x7f4c, 0xeac8, 1645bfc04aecSBorislav Petkov 0x0001, 0x0002, 0x0004, 0x0008, 1646bfc04aecSBorislav Petkov 0x1013, 0x3032, 0x4044, 0x8088, 1647bfc04aecSBorislav Petkov 0x106b, 0x30d6, 0x70fc, 0xe0a8, 1648bfc04aecSBorislav Petkov 0x4857, 0xc4fe, 0x13cc, 0x3288, 1649bfc04aecSBorislav Petkov 0x1ac5, 0x2f4a, 0x5394, 0xa1e8, 1650bfc04aecSBorislav Petkov 0x1f39, 0x251e, 0xbd6c, 0x6bd8, 1651bfc04aecSBorislav Petkov 0x15c1, 0x2a42, 0x89ac, 0x4758, 1652bfc04aecSBorislav Petkov 0x2b03, 0x1602, 0x4f0c, 0xca08, 1653bfc04aecSBorislav Petkov 0x1f07, 0x3a0e, 0x6b04, 0xbd08, 1654bfc04aecSBorislav Petkov 0x8ba7, 0x465e, 0x244c, 0x1cc8, 1655bfc04aecSBorislav Petkov 0x2b87, 0x164e, 0x642c, 0xdc18, 1656bfc04aecSBorislav Petkov 0x40b9, 0x80de, 0x1094, 0x20e8, 1657bfc04aecSBorislav Petkov 0x27db, 0x1eb6, 0x9dac, 0x7b58, 1658bfc04aecSBorislav Petkov 0x11c1, 0x2242, 0x84ac, 0x4c58, 1659bfc04aecSBorislav Petkov 0x1be5, 0x2d7a, 0x5e34, 0xa718, 1660bfc04aecSBorislav Petkov 0x4b39, 0x8d1e, 0x14b4, 0x28d8, 1661bfc04aecSBorislav Petkov 0x4c97, 0xc87e, 0x11fc, 0x33a8, 1662bfc04aecSBorislav Petkov 0x8e97, 0x497e, 0x2ffc, 0x1aa8, 1663bfc04aecSBorislav Petkov 0x16b3, 0x3d62, 0x4f34, 0x8518, 1664bfc04aecSBorislav Petkov 0x1e2f, 0x391a, 0x5cac, 0xf858, 1665bfc04aecSBorislav Petkov 0x1d9f, 0x3b7a, 0x572c, 0xfe18, 1666bfc04aecSBorislav Petkov 0x15f5, 0x2a5a, 0x5264, 0xa3b8, 1667bfc04aecSBorislav Petkov 0x1dbb, 0x3b66, 0x715c, 0xe3f8, 1668bfc04aecSBorislav Petkov 0x4397, 0xc27e, 0x17fc, 0x3ea8, 1669bfc04aecSBorislav Petkov 0x1617, 0x3d3e, 0x6464, 0xb8b8, 1670bfc04aecSBorislav Petkov 0x23ff, 0x12aa, 0xab6c, 0x56d8, 1671bfc04aecSBorislav Petkov 0x2dfb, 0x1ba6, 0x913c, 0x7328, 1672bfc04aecSBorislav Petkov 0x185d, 0x2ca6, 0x7914, 0x9e28, 1673bfc04aecSBorislav Petkov 0x171b, 0x3e36, 0x7d7c, 0xebe8, 1674bfc04aecSBorislav Petkov 0x4199, 0x82ee, 0x19f4, 0x2e58, 1675bfc04aecSBorislav Petkov 0x4807, 0xc40e, 0x130c, 0x3208, 1676bfc04aecSBorislav Petkov 0x1905, 0x2e0a, 0x5804, 0xac08, 1677bfc04aecSBorislav Petkov 0x213f, 0x132a, 0xadfc, 0x5ba8, 1678bfc04aecSBorislav Petkov 0x19a9, 0x2efe, 0xb5cc, 0x6f88, 1679b1289d6fSDoug Thompson }; 1680b1289d6fSDoug Thompson 1681bfc04aecSBorislav Petkov static u16 x8_vectors[] = { 1682bfc04aecSBorislav Petkov 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480, 1683bfc04aecSBorislav Petkov 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80, 1684bfc04aecSBorislav Petkov 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80, 1685bfc04aecSBorislav Petkov 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80, 1686bfc04aecSBorislav Petkov 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780, 1687bfc04aecSBorislav Petkov 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080, 1688bfc04aecSBorislav Petkov 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080, 1689bfc04aecSBorislav Petkov 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080, 1690bfc04aecSBorislav Petkov 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80, 1691bfc04aecSBorislav Petkov 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580, 1692bfc04aecSBorislav Petkov 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880, 1693bfc04aecSBorislav Petkov 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280, 1694bfc04aecSBorislav Petkov 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180, 1695bfc04aecSBorislav Petkov 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580, 1696bfc04aecSBorislav Petkov 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280, 1697bfc04aecSBorislav Petkov 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180, 1698bfc04aecSBorislav Petkov 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080, 1699bfc04aecSBorislav Petkov 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080, 1700bfc04aecSBorislav Petkov 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000, 1701bfc04aecSBorislav Petkov }; 1702bfc04aecSBorislav Petkov 1703bfc04aecSBorislav Petkov static int decode_syndrome(u16 syndrome, u16 *vectors, int num_vecs, 1704bfc04aecSBorislav Petkov int v_dim) 1705b1289d6fSDoug Thompson { 1706bfc04aecSBorislav Petkov unsigned int i, err_sym; 1707b1289d6fSDoug Thompson 1708bfc04aecSBorislav Petkov for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) { 1709bfc04aecSBorislav Petkov u16 s = syndrome; 1710bfc04aecSBorislav Petkov int v_idx = err_sym * v_dim; 1711bfc04aecSBorislav Petkov int v_end = (err_sym + 1) * v_dim; 1712b1289d6fSDoug Thompson 1713bfc04aecSBorislav Petkov /* walk over all 16 bits of the syndrome */ 1714bfc04aecSBorislav Petkov for (i = 1; i < (1U << 16); i <<= 1) { 1715bfc04aecSBorislav Petkov 1716bfc04aecSBorislav Petkov /* if bit is set in that eigenvector... */ 1717bfc04aecSBorislav Petkov if (v_idx < v_end && vectors[v_idx] & i) { 1718bfc04aecSBorislav Petkov u16 ev_comp = vectors[v_idx++]; 1719bfc04aecSBorislav Petkov 1720bfc04aecSBorislav Petkov /* ... and bit set in the modified syndrome, */ 1721bfc04aecSBorislav Petkov if (s & i) { 1722bfc04aecSBorislav Petkov /* remove it. */ 1723bfc04aecSBorislav Petkov s ^= ev_comp; 1724bfc04aecSBorislav Petkov 1725bfc04aecSBorislav Petkov if (!s) 1726bfc04aecSBorislav Petkov return err_sym; 1727bfc04aecSBorislav Petkov } 1728bfc04aecSBorislav Petkov 1729bfc04aecSBorislav Petkov } else if (s & i) 1730bfc04aecSBorislav Petkov /* can't get to zero, move to next symbol */ 1731bfc04aecSBorislav Petkov break; 1732bfc04aecSBorislav Petkov } 1733b1289d6fSDoug Thompson } 1734b1289d6fSDoug Thompson 1735b1289d6fSDoug Thompson debugf0("syndrome(%x) not found\n", syndrome); 1736b1289d6fSDoug Thompson return -1; 1737b1289d6fSDoug Thompson } 1738d27bf6faSDoug Thompson 1739bfc04aecSBorislav Petkov static int map_err_sym_to_channel(int err_sym, int sym_size) 1740bfc04aecSBorislav Petkov { 1741bfc04aecSBorislav Petkov if (sym_size == 4) 1742bfc04aecSBorislav Petkov switch (err_sym) { 1743bfc04aecSBorislav Petkov case 0x20: 1744bfc04aecSBorislav Petkov case 0x21: 1745bfc04aecSBorislav Petkov return 0; 1746bfc04aecSBorislav Petkov break; 1747bfc04aecSBorislav Petkov case 0x22: 1748bfc04aecSBorislav Petkov case 0x23: 1749bfc04aecSBorislav Petkov return 1; 1750bfc04aecSBorislav Petkov break; 1751bfc04aecSBorislav Petkov default: 1752bfc04aecSBorislav Petkov return err_sym >> 4; 1753bfc04aecSBorislav Petkov break; 1754bfc04aecSBorislav Petkov } 1755bfc04aecSBorislav Petkov /* x8 symbols */ 1756bfc04aecSBorislav Petkov else 1757bfc04aecSBorislav Petkov switch (err_sym) { 1758bfc04aecSBorislav Petkov /* imaginary bits not in a DIMM */ 1759bfc04aecSBorislav Petkov case 0x10: 1760bfc04aecSBorislav Petkov WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n", 1761bfc04aecSBorislav Petkov err_sym); 1762bfc04aecSBorislav Petkov return -1; 1763bfc04aecSBorislav Petkov break; 1764bfc04aecSBorislav Petkov 1765bfc04aecSBorislav Petkov case 0x11: 1766bfc04aecSBorislav Petkov return 0; 1767bfc04aecSBorislav Petkov break; 1768bfc04aecSBorislav Petkov case 0x12: 1769bfc04aecSBorislav Petkov return 1; 1770bfc04aecSBorislav Petkov break; 1771bfc04aecSBorislav Petkov default: 1772bfc04aecSBorislav Petkov return err_sym >> 3; 1773bfc04aecSBorislav Petkov break; 1774bfc04aecSBorislav Petkov } 1775bfc04aecSBorislav Petkov return -1; 1776bfc04aecSBorislav Petkov } 1777bfc04aecSBorislav Petkov 1778bfc04aecSBorislav Petkov static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome) 1779bfc04aecSBorislav Petkov { 1780bfc04aecSBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 1781ad6a32e9SBorislav Petkov int err_sym = -1; 1782bfc04aecSBorislav Petkov 1783a3b7db09SBorislav Petkov if (pvt->ecc_sym_sz == 8) 1784bfc04aecSBorislav Petkov err_sym = decode_syndrome(syndrome, x8_vectors, 1785ad6a32e9SBorislav Petkov ARRAY_SIZE(x8_vectors), 1786a3b7db09SBorislav Petkov pvt->ecc_sym_sz); 1787a3b7db09SBorislav Petkov else if (pvt->ecc_sym_sz == 4) 1788ad6a32e9SBorislav Petkov err_sym = decode_syndrome(syndrome, x4_vectors, 1789ad6a32e9SBorislav Petkov ARRAY_SIZE(x4_vectors), 1790a3b7db09SBorislav Petkov pvt->ecc_sym_sz); 1791ad6a32e9SBorislav Petkov else { 1792a3b7db09SBorislav Petkov amd64_warn("Illegal syndrome type: %u\n", pvt->ecc_sym_sz); 1793ad6a32e9SBorislav Petkov return err_sym; 1794bfc04aecSBorislav Petkov } 1795ad6a32e9SBorislav Petkov 1796a3b7db09SBorislav Petkov return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz); 179741c31044SBorislav Petkov } 1798bfc04aecSBorislav Petkov 1799d27bf6faSDoug Thompson /* 1800d27bf6faSDoug Thompson * Handle any Correctable Errors (CEs) that have occurred. Check for valid ERROR 1801d27bf6faSDoug Thompson * ADDRESS and process. 1802d27bf6faSDoug Thompson */ 1803f192c7b1SBorislav Petkov static void amd64_handle_ce(struct mem_ctl_info *mci, struct mce *m) 1804d27bf6faSDoug Thompson { 1805d27bf6faSDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 180644e9e2eeSBorislav Petkov u64 sys_addr; 1807f192c7b1SBorislav Petkov u16 syndrome; 1808d27bf6faSDoug Thompson 1809d27bf6faSDoug Thompson /* Ensure that the Error Address is VALID */ 1810f192c7b1SBorislav Petkov if (!(m->status & MCI_STATUS_ADDRV)) { 181124f9a7feSBorislav Petkov amd64_mc_err(mci, "HW has no ERROR_ADDRESS available\n"); 1812d27bf6faSDoug Thompson edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR); 1813d27bf6faSDoug Thompson return; 1814d27bf6faSDoug Thompson } 1815d27bf6faSDoug Thompson 181670046624SBorislav Petkov sys_addr = get_error_address(m); 1817f192c7b1SBorislav Petkov syndrome = extract_syndrome(m->status); 1818d27bf6faSDoug Thompson 181924f9a7feSBorislav Petkov amd64_mc_err(mci, "CE ERROR_ADDRESS= 0x%llx\n", sys_addr); 1820d27bf6faSDoug Thompson 1821f192c7b1SBorislav Petkov pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, syndrome); 1822d27bf6faSDoug Thompson } 1823d27bf6faSDoug Thompson 1824d27bf6faSDoug Thompson /* Handle any Un-correctable Errors (UEs) */ 1825f192c7b1SBorislav Petkov static void amd64_handle_ue(struct mem_ctl_info *mci, struct mce *m) 1826d27bf6faSDoug Thompson { 18271f6bcee7SBorislav Petkov struct mem_ctl_info *log_mci, *src_mci = NULL; 1828d27bf6faSDoug Thompson int csrow; 182944e9e2eeSBorislav Petkov u64 sys_addr; 1830d27bf6faSDoug Thompson u32 page, offset; 1831d27bf6faSDoug Thompson 1832d27bf6faSDoug Thompson log_mci = mci; 1833d27bf6faSDoug Thompson 1834f192c7b1SBorislav Petkov if (!(m->status & MCI_STATUS_ADDRV)) { 183524f9a7feSBorislav Petkov amd64_mc_err(mci, "HW has no ERROR_ADDRESS available\n"); 1836d27bf6faSDoug Thompson edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR); 1837d27bf6faSDoug Thompson return; 1838d27bf6faSDoug Thompson } 1839d27bf6faSDoug Thompson 184070046624SBorislav Petkov sys_addr = get_error_address(m); 1841d27bf6faSDoug Thompson 1842d27bf6faSDoug Thompson /* 1843d27bf6faSDoug Thompson * Find out which node the error address belongs to. This may be 1844d27bf6faSDoug Thompson * different from the node that detected the error. 1845d27bf6faSDoug Thompson */ 184644e9e2eeSBorislav Petkov src_mci = find_mc_by_sys_addr(mci, sys_addr); 1847d27bf6faSDoug Thompson if (!src_mci) { 184824f9a7feSBorislav Petkov amd64_mc_err(mci, "ERROR ADDRESS (0x%lx) NOT mapped to a MC\n", 184944e9e2eeSBorislav Petkov (unsigned long)sys_addr); 1850d27bf6faSDoug Thompson edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR); 1851d27bf6faSDoug Thompson return; 1852d27bf6faSDoug Thompson } 1853d27bf6faSDoug Thompson 1854d27bf6faSDoug Thompson log_mci = src_mci; 1855d27bf6faSDoug Thompson 185644e9e2eeSBorislav Petkov csrow = sys_addr_to_csrow(log_mci, sys_addr); 1857d27bf6faSDoug Thompson if (csrow < 0) { 185824f9a7feSBorislav Petkov amd64_mc_err(mci, "ERROR_ADDRESS (0x%lx) NOT mapped to CS\n", 185944e9e2eeSBorislav Petkov (unsigned long)sys_addr); 1860d27bf6faSDoug Thompson edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR); 1861d27bf6faSDoug Thompson } else { 186244e9e2eeSBorislav Petkov error_address_to_page_and_offset(sys_addr, &page, &offset); 1863d27bf6faSDoug Thompson edac_mc_handle_ue(log_mci, page, offset, csrow, EDAC_MOD_STR); 1864d27bf6faSDoug Thompson } 1865d27bf6faSDoug Thompson } 1866d27bf6faSDoug Thompson 1867549d042dSBorislav Petkov static inline void __amd64_decode_bus_error(struct mem_ctl_info *mci, 1868f192c7b1SBorislav Petkov struct mce *m) 1869d27bf6faSDoug Thompson { 1870f192c7b1SBorislav Petkov u16 ec = EC(m->status); 1871f192c7b1SBorislav Petkov u8 xec = XEC(m->status, 0x1f); 1872f192c7b1SBorislav Petkov u8 ecc_type = (m->status >> 45) & 0x3; 1873d27bf6faSDoug Thompson 1874b70ef010SBorislav Petkov /* Bail early out if this was an 'observed' error */ 18755980bb9cSBorislav Petkov if (PP(ec) == NBSL_PP_OBS) 1876b70ef010SBorislav Petkov return; 1877d27bf6faSDoug Thompson 1878ecaf5606SBorislav Petkov /* Do only ECC errors */ 1879ecaf5606SBorislav Petkov if (xec && xec != F10_NBSL_EXT_ERR_ECC) 1880d27bf6faSDoug Thompson return; 1881d27bf6faSDoug Thompson 1882ecaf5606SBorislav Petkov if (ecc_type == 2) 1883f192c7b1SBorislav Petkov amd64_handle_ce(mci, m); 1884ecaf5606SBorislav Petkov else if (ecc_type == 1) 1885f192c7b1SBorislav Petkov amd64_handle_ue(mci, m); 1886d27bf6faSDoug Thompson } 1887d27bf6faSDoug Thompson 18887cfd4a87SBorislav Petkov void amd64_decode_bus_error(int node_id, struct mce *m, u32 nbcfg) 1889d27bf6faSDoug Thompson { 1890cc4d8860SBorislav Petkov struct mem_ctl_info *mci = mcis[node_id]; 1891d27bf6faSDoug Thompson 1892f192c7b1SBorislav Petkov __amd64_decode_bus_error(mci, m); 1893d27bf6faSDoug Thompson } 1894d27bf6faSDoug Thompson 18950ec449eeSDoug Thompson /* 18968d5b5d9cSBorislav Petkov * Use pvt->F2 which contains the F2 CPU PCI device to get the related 1897bbd0c1f6SBorislav Petkov * F1 (AddrMap) and F3 (Misc) devices. Return negative value on error. 18980ec449eeSDoug Thompson */ 1899360b7f3cSBorislav Petkov static int reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 f1_id, u16 f3_id) 19000ec449eeSDoug Thompson { 19010ec449eeSDoug Thompson /* Reserve the ADDRESS MAP Device */ 19028d5b5d9cSBorislav Petkov pvt->F1 = pci_get_related_function(pvt->F2->vendor, f1_id, pvt->F2); 19038d5b5d9cSBorislav Petkov if (!pvt->F1) { 190424f9a7feSBorislav Petkov amd64_err("error address map device not found: " 19050ec449eeSDoug Thompson "vendor %x device 0x%x (broken BIOS?)\n", 1906bbd0c1f6SBorislav Petkov PCI_VENDOR_ID_AMD, f1_id); 1907bbd0c1f6SBorislav Petkov return -ENODEV; 19080ec449eeSDoug Thompson } 19090ec449eeSDoug Thompson 19100ec449eeSDoug Thompson /* Reserve the MISC Device */ 19118d5b5d9cSBorislav Petkov pvt->F3 = pci_get_related_function(pvt->F2->vendor, f3_id, pvt->F2); 19128d5b5d9cSBorislav Petkov if (!pvt->F3) { 19138d5b5d9cSBorislav Petkov pci_dev_put(pvt->F1); 19148d5b5d9cSBorislav Petkov pvt->F1 = NULL; 19150ec449eeSDoug Thompson 191624f9a7feSBorislav Petkov amd64_err("error F3 device not found: " 19170ec449eeSDoug Thompson "vendor %x device 0x%x (broken BIOS?)\n", 1918bbd0c1f6SBorislav Petkov PCI_VENDOR_ID_AMD, f3_id); 19198d5b5d9cSBorislav Petkov 1920bbd0c1f6SBorislav Petkov return -ENODEV; 19210ec449eeSDoug Thompson } 19228d5b5d9cSBorislav Petkov debugf1("F1: %s\n", pci_name(pvt->F1)); 19238d5b5d9cSBorislav Petkov debugf1("F2: %s\n", pci_name(pvt->F2)); 19248d5b5d9cSBorislav Petkov debugf1("F3: %s\n", pci_name(pvt->F3)); 19250ec449eeSDoug Thompson 19260ec449eeSDoug Thompson return 0; 19270ec449eeSDoug Thompson } 19280ec449eeSDoug Thompson 1929360b7f3cSBorislav Petkov static void free_mc_sibling_devs(struct amd64_pvt *pvt) 19300ec449eeSDoug Thompson { 19318d5b5d9cSBorislav Petkov pci_dev_put(pvt->F1); 19328d5b5d9cSBorislav Petkov pci_dev_put(pvt->F3); 19330ec449eeSDoug Thompson } 19340ec449eeSDoug Thompson 19350ec449eeSDoug Thompson /* 19360ec449eeSDoug Thompson * Retrieve the hardware registers of the memory controller (this includes the 19370ec449eeSDoug Thompson * 'Address Map' and 'Misc' device regs) 19380ec449eeSDoug Thompson */ 1939360b7f3cSBorislav Petkov static void read_mc_regs(struct amd64_pvt *pvt) 19400ec449eeSDoug Thompson { 1941a3b7db09SBorislav Petkov struct cpuinfo_x86 *c = &boot_cpu_data; 19420ec449eeSDoug Thompson u64 msr_val; 1943ad6a32e9SBorislav Petkov u32 tmp; 19447f19bf75SBorislav Petkov int range; 19450ec449eeSDoug Thompson 19460ec449eeSDoug Thompson /* 19470ec449eeSDoug Thompson * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since 19480ec449eeSDoug Thompson * those are Read-As-Zero 19490ec449eeSDoug Thompson */ 1950e97f8bb8SBorislav Petkov rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem); 1951e97f8bb8SBorislav Petkov debugf0(" TOP_MEM: 0x%016llx\n", pvt->top_mem); 19520ec449eeSDoug Thompson 19530ec449eeSDoug Thompson /* check first whether TOP_MEM2 is enabled */ 19540ec449eeSDoug Thompson rdmsrl(MSR_K8_SYSCFG, msr_val); 19550ec449eeSDoug Thompson if (msr_val & (1U << 21)) { 1956e97f8bb8SBorislav Petkov rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2); 1957e97f8bb8SBorislav Petkov debugf0(" TOP_MEM2: 0x%016llx\n", pvt->top_mem2); 19580ec449eeSDoug Thompson } else 19590ec449eeSDoug Thompson debugf0(" TOP_MEM2 disabled.\n"); 19600ec449eeSDoug Thompson 19615980bb9cSBorislav Petkov amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap); 19620ec449eeSDoug Thompson 19635a5d2371SBorislav Petkov read_dram_ctl_register(pvt); 19640ec449eeSDoug Thompson 19657f19bf75SBorislav Petkov for (range = 0; range < DRAM_RANGES; range++) { 19667f19bf75SBorislav Petkov u8 rw; 19670ec449eeSDoug Thompson 19687f19bf75SBorislav Petkov /* read settings for this DRAM range */ 19697f19bf75SBorislav Petkov read_dram_base_limit_regs(pvt, range); 1970e97f8bb8SBorislav Petkov 19717f19bf75SBorislav Petkov rw = dram_rw(pvt, range); 19727f19bf75SBorislav Petkov if (!rw) 19737f19bf75SBorislav Petkov continue; 19747f19bf75SBorislav Petkov 19757f19bf75SBorislav Petkov debugf1(" DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n", 19767f19bf75SBorislav Petkov range, 19777f19bf75SBorislav Petkov get_dram_base(pvt, range), 19787f19bf75SBorislav Petkov get_dram_limit(pvt, range)); 19797f19bf75SBorislav Petkov 19807f19bf75SBorislav Petkov debugf1(" IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n", 19817f19bf75SBorislav Petkov dram_intlv_en(pvt, range) ? "Enabled" : "Disabled", 19827f19bf75SBorislav Petkov (rw & 0x1) ? "R" : "-", 19837f19bf75SBorislav Petkov (rw & 0x2) ? "W" : "-", 19847f19bf75SBorislav Petkov dram_intlv_sel(pvt, range), 19857f19bf75SBorislav Petkov dram_dst_node(pvt, range)); 19860ec449eeSDoug Thompson } 19870ec449eeSDoug Thompson 1988b2b0c605SBorislav Petkov read_dct_base_mask(pvt); 19890ec449eeSDoug Thompson 1990bc21fa57SBorislav Petkov amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar); 1991525a1b20SBorislav Petkov amd64_read_dct_pci_cfg(pvt, DBAM0, &pvt->dbam0); 19920ec449eeSDoug Thompson 19938d5b5d9cSBorislav Petkov amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare); 19940ec449eeSDoug Thompson 1995cb328507SBorislav Petkov amd64_read_dct_pci_cfg(pvt, DCLR0, &pvt->dclr0); 1996cb328507SBorislav Petkov amd64_read_dct_pci_cfg(pvt, DCHR0, &pvt->dchr0); 19970ec449eeSDoug Thompson 199878da121eSBorislav Petkov if (!dct_ganging_enabled(pvt)) { 1999cb328507SBorislav Petkov amd64_read_dct_pci_cfg(pvt, DCLR1, &pvt->dclr1); 2000cb328507SBorislav Petkov amd64_read_dct_pci_cfg(pvt, DCHR1, &pvt->dchr1); 20010ec449eeSDoug Thompson } 2002b2b0c605SBorislav Petkov 2003a3b7db09SBorislav Petkov pvt->ecc_sym_sz = 4; 2004a3b7db09SBorislav Petkov 2005a3b7db09SBorislav Petkov if (c->x86 >= 0x10) { 20068d5b5d9cSBorislav Petkov amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp); 2007525a1b20SBorislav Petkov amd64_read_dct_pci_cfg(pvt, DBAM1, &pvt->dbam1); 2008a3b7db09SBorislav Petkov 2009a3b7db09SBorislav Petkov /* F10h, revD and later can do x8 ECC too */ 2010a3b7db09SBorislav Petkov if ((c->x86 > 0x10 || c->x86_model > 7) && tmp & BIT(25)) 2011a3b7db09SBorislav Petkov pvt->ecc_sym_sz = 8; 2012525a1b20SBorislav Petkov } 2013b2b0c605SBorislav Petkov dump_misc_regs(pvt); 20140ec449eeSDoug Thompson } 20150ec449eeSDoug Thompson 20160ec449eeSDoug Thompson /* 20170ec449eeSDoug Thompson * NOTE: CPU Revision Dependent code 20180ec449eeSDoug Thompson * 20190ec449eeSDoug Thompson * Input: 202011c75eadSBorislav Petkov * @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1) 20210ec449eeSDoug Thompson * k8 private pointer to --> 20220ec449eeSDoug Thompson * DRAM Bank Address mapping register 20230ec449eeSDoug Thompson * node_id 20240ec449eeSDoug Thompson * DCL register where dual_channel_active is 20250ec449eeSDoug Thompson * 20260ec449eeSDoug Thompson * The DBAM register consists of 4 sets of 4 bits each definitions: 20270ec449eeSDoug Thompson * 20280ec449eeSDoug Thompson * Bits: CSROWs 20290ec449eeSDoug Thompson * 0-3 CSROWs 0 and 1 20300ec449eeSDoug Thompson * 4-7 CSROWs 2 and 3 20310ec449eeSDoug Thompson * 8-11 CSROWs 4 and 5 20320ec449eeSDoug Thompson * 12-15 CSROWs 6 and 7 20330ec449eeSDoug Thompson * 20340ec449eeSDoug Thompson * Values range from: 0 to 15 20350ec449eeSDoug Thompson * The meaning of the values depends on CPU revision and dual-channel state, 20360ec449eeSDoug Thompson * see relevant BKDG more info. 20370ec449eeSDoug Thompson * 20380ec449eeSDoug Thompson * The memory controller provides for total of only 8 CSROWs in its current 20390ec449eeSDoug Thompson * architecture. Each "pair" of CSROWs normally represents just one DIMM in 20400ec449eeSDoug Thompson * single channel or two (2) DIMMs in dual channel mode. 20410ec449eeSDoug Thompson * 20420ec449eeSDoug Thompson * The following code logic collapses the various tables for CSROW based on CPU 20430ec449eeSDoug Thompson * revision. 20440ec449eeSDoug Thompson * 20450ec449eeSDoug Thompson * Returns: 20460ec449eeSDoug Thompson * The number of PAGE_SIZE pages on the specified CSROW number it 20470ec449eeSDoug Thompson * encompasses 20480ec449eeSDoug Thompson * 20490ec449eeSDoug Thompson */ 205041d8bfabSBorislav Petkov static u32 amd64_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr) 20510ec449eeSDoug Thompson { 20521433eb99SBorislav Petkov u32 cs_mode, nr_pages; 20530ec449eeSDoug Thompson 20540ec449eeSDoug Thompson /* 20550ec449eeSDoug Thompson * The math on this doesn't look right on the surface because x/2*4 can 20560ec449eeSDoug Thompson * be simplified to x*2 but this expression makes use of the fact that 20570ec449eeSDoug Thompson * it is integral math where 1/2=0. This intermediate value becomes the 20580ec449eeSDoug Thompson * number of bits to shift the DBAM register to extract the proper CSROW 20590ec449eeSDoug Thompson * field. 20600ec449eeSDoug Thompson */ 20611433eb99SBorislav Petkov cs_mode = (pvt->dbam0 >> ((csrow_nr / 2) * 4)) & 0xF; 20620ec449eeSDoug Thompson 206341d8bfabSBorislav Petkov nr_pages = pvt->ops->dbam_to_cs(pvt, dct, cs_mode) << (20 - PAGE_SHIFT); 20640ec449eeSDoug Thompson 20650ec449eeSDoug Thompson /* 20660ec449eeSDoug Thompson * If dual channel then double the memory size of single channel. 20670ec449eeSDoug Thompson * Channel count is 1 or 2 20680ec449eeSDoug Thompson */ 20690ec449eeSDoug Thompson nr_pages <<= (pvt->channel_count - 1); 20700ec449eeSDoug Thompson 20711433eb99SBorislav Petkov debugf0(" (csrow=%d) DBAM map index= %d\n", csrow_nr, cs_mode); 20720ec449eeSDoug Thompson debugf0(" nr_pages= %u channel-count = %d\n", 20730ec449eeSDoug Thompson nr_pages, pvt->channel_count); 20740ec449eeSDoug Thompson 20750ec449eeSDoug Thompson return nr_pages; 20760ec449eeSDoug Thompson } 20770ec449eeSDoug Thompson 20780ec449eeSDoug Thompson /* 20790ec449eeSDoug Thompson * Initialize the array of csrow attribute instances, based on the values 20800ec449eeSDoug Thompson * from pci config hardware registers. 20810ec449eeSDoug Thompson */ 2082360b7f3cSBorislav Petkov static int init_csrows(struct mem_ctl_info *mci) 20830ec449eeSDoug Thompson { 20840ec449eeSDoug Thompson struct csrow_info *csrow; 20852299ef71SBorislav Petkov struct amd64_pvt *pvt = mci->pvt_info; 208611c75eadSBorislav Petkov u64 input_addr_min, input_addr_max, sys_addr, base, mask; 20872299ef71SBorislav Petkov u32 val; 20886ba5dcdcSBorislav Petkov int i, empty = 1; 20890ec449eeSDoug Thompson 2090a97fa68eSBorislav Petkov amd64_read_pci_cfg(pvt->F3, NBCFG, &val); 20910ec449eeSDoug Thompson 20922299ef71SBorislav Petkov pvt->nbcfg = val; 20930ec449eeSDoug Thompson 20942299ef71SBorislav Petkov debugf0("node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n", 20952299ef71SBorislav Petkov pvt->mc_node_id, val, 2096a97fa68eSBorislav Petkov !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE)); 20970ec449eeSDoug Thompson 209811c75eadSBorislav Petkov for_each_chip_select(i, 0, pvt) { 20990ec449eeSDoug Thompson csrow = &mci->csrows[i]; 21000ec449eeSDoug Thompson 210111c75eadSBorislav Petkov if (!csrow_enabled(i, 0, pvt)) { 21020ec449eeSDoug Thompson debugf1("----CSROW %d EMPTY for node %d\n", i, 21030ec449eeSDoug Thompson pvt->mc_node_id); 21040ec449eeSDoug Thompson continue; 21050ec449eeSDoug Thompson } 21060ec449eeSDoug Thompson 21070ec449eeSDoug Thompson debugf1("----CSROW %d VALID for MC node %d\n", 21080ec449eeSDoug Thompson i, pvt->mc_node_id); 21090ec449eeSDoug Thompson 21100ec449eeSDoug Thompson empty = 0; 211141d8bfabSBorislav Petkov csrow->nr_pages = amd64_csrow_nr_pages(pvt, 0, i); 21120ec449eeSDoug Thompson find_csrow_limits(mci, i, &input_addr_min, &input_addr_max); 21130ec449eeSDoug Thompson sys_addr = input_addr_to_sys_addr(mci, input_addr_min); 21140ec449eeSDoug Thompson csrow->first_page = (u32) (sys_addr >> PAGE_SHIFT); 21150ec449eeSDoug Thompson sys_addr = input_addr_to_sys_addr(mci, input_addr_max); 21160ec449eeSDoug Thompson csrow->last_page = (u32) (sys_addr >> PAGE_SHIFT); 211711c75eadSBorislav Petkov 211811c75eadSBorislav Petkov get_cs_base_and_mask(pvt, i, 0, &base, &mask); 211911c75eadSBorislav Petkov csrow->page_mask = ~mask; 21200ec449eeSDoug Thompson /* 8 bytes of resolution */ 21210ec449eeSDoug Thompson 212224f9a7feSBorislav Petkov csrow->mtype = amd64_determine_memory_type(pvt, i); 21230ec449eeSDoug Thompson 21240ec449eeSDoug Thompson debugf1(" for MC node %d csrow %d:\n", pvt->mc_node_id, i); 21250ec449eeSDoug Thompson debugf1(" input_addr_min: 0x%lx input_addr_max: 0x%lx\n", 21260ec449eeSDoug Thompson (unsigned long)input_addr_min, 21270ec449eeSDoug Thompson (unsigned long)input_addr_max); 21280ec449eeSDoug Thompson debugf1(" sys_addr: 0x%lx page_mask: 0x%lx\n", 21290ec449eeSDoug Thompson (unsigned long)sys_addr, csrow->page_mask); 21300ec449eeSDoug Thompson debugf1(" nr_pages: %u first_page: 0x%lx " 21310ec449eeSDoug Thompson "last_page: 0x%lx\n", 21320ec449eeSDoug Thompson (unsigned)csrow->nr_pages, 21330ec449eeSDoug Thompson csrow->first_page, csrow->last_page); 21340ec449eeSDoug Thompson 21350ec449eeSDoug Thompson /* 21360ec449eeSDoug Thompson * determine whether CHIPKILL or JUST ECC or NO ECC is operating 21370ec449eeSDoug Thompson */ 2138a97fa68eSBorislav Petkov if (pvt->nbcfg & NBCFG_ECC_ENABLE) 21390ec449eeSDoug Thompson csrow->edac_mode = 2140a97fa68eSBorislav Petkov (pvt->nbcfg & NBCFG_CHIPKILL) ? 21410ec449eeSDoug Thompson EDAC_S4ECD4ED : EDAC_SECDED; 21420ec449eeSDoug Thompson else 21430ec449eeSDoug Thompson csrow->edac_mode = EDAC_NONE; 21440ec449eeSDoug Thompson } 21450ec449eeSDoug Thompson 21460ec449eeSDoug Thompson return empty; 21470ec449eeSDoug Thompson } 2148d27bf6faSDoug Thompson 214906724535SBorislav Petkov /* get all cores on this DCT */ 2150b487c33eSBorislav Petkov static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, unsigned nid) 2151f9431992SDoug Thompson { 215206724535SBorislav Petkov int cpu; 2153f9431992SDoug Thompson 215406724535SBorislav Petkov for_each_online_cpu(cpu) 215506724535SBorislav Petkov if (amd_get_nb_id(cpu) == nid) 215606724535SBorislav Petkov cpumask_set_cpu(cpu, mask); 2157f9431992SDoug Thompson } 2158f9431992SDoug Thompson 2159f9431992SDoug Thompson /* check MCG_CTL on all the cpus on this node */ 2160b487c33eSBorislav Petkov static bool amd64_nb_mce_bank_enabled_on_node(unsigned nid) 2161f9431992SDoug Thompson { 2162ba578cb3SRusty Russell cpumask_var_t mask; 216350542251SBorislav Petkov int cpu, nbe; 216406724535SBorislav Petkov bool ret = false; 2165f9431992SDoug Thompson 2166ba578cb3SRusty Russell if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) { 216724f9a7feSBorislav Petkov amd64_warn("%s: Error allocating mask\n", __func__); 216806724535SBorislav Petkov return false; 216906724535SBorislav Petkov } 217006724535SBorislav Petkov 2171ba578cb3SRusty Russell get_cpus_on_this_dct_cpumask(mask, nid); 217206724535SBorislav Petkov 2173ba578cb3SRusty Russell rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs); 2174ba578cb3SRusty Russell 2175ba578cb3SRusty Russell for_each_cpu(cpu, mask) { 217650542251SBorislav Petkov struct msr *reg = per_cpu_ptr(msrs, cpu); 21775980bb9cSBorislav Petkov nbe = reg->l & MSR_MCGCTL_NBE; 217806724535SBorislav Petkov 217906724535SBorislav Petkov debugf0("core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n", 218050542251SBorislav Petkov cpu, reg->q, 218106724535SBorislav Petkov (nbe ? "enabled" : "disabled")); 218206724535SBorislav Petkov 218306724535SBorislav Petkov if (!nbe) 218406724535SBorislav Petkov goto out; 218506724535SBorislav Petkov } 218606724535SBorislav Petkov ret = true; 218706724535SBorislav Petkov 218806724535SBorislav Petkov out: 2189ba578cb3SRusty Russell free_cpumask_var(mask); 2190f9431992SDoug Thompson return ret; 2191f9431992SDoug Thompson } 2192f9431992SDoug Thompson 21932299ef71SBorislav Petkov static int toggle_ecc_err_reporting(struct ecc_settings *s, u8 nid, bool on) 2194f6d6ae96SBorislav Petkov { 2195f6d6ae96SBorislav Petkov cpumask_var_t cmask; 219650542251SBorislav Petkov int cpu; 2197f6d6ae96SBorislav Petkov 2198f6d6ae96SBorislav Petkov if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) { 219924f9a7feSBorislav Petkov amd64_warn("%s: error allocating mask\n", __func__); 2200f6d6ae96SBorislav Petkov return false; 2201f6d6ae96SBorislav Petkov } 2202f6d6ae96SBorislav Petkov 2203ae7bb7c6SBorislav Petkov get_cpus_on_this_dct_cpumask(cmask, nid); 2204f6d6ae96SBorislav Petkov 2205f6d6ae96SBorislav Petkov rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs); 2206f6d6ae96SBorislav Petkov 2207f6d6ae96SBorislav Petkov for_each_cpu(cpu, cmask) { 2208f6d6ae96SBorislav Petkov 220950542251SBorislav Petkov struct msr *reg = per_cpu_ptr(msrs, cpu); 221050542251SBorislav Petkov 2211f6d6ae96SBorislav Petkov if (on) { 22125980bb9cSBorislav Petkov if (reg->l & MSR_MCGCTL_NBE) 2213ae7bb7c6SBorislav Petkov s->flags.nb_mce_enable = 1; 2214f6d6ae96SBorislav Petkov 22155980bb9cSBorislav Petkov reg->l |= MSR_MCGCTL_NBE; 2216f6d6ae96SBorislav Petkov } else { 2217f6d6ae96SBorislav Petkov /* 2218d95cf4deSBorislav Petkov * Turn off NB MCE reporting only when it was off before 2219f6d6ae96SBorislav Petkov */ 2220ae7bb7c6SBorislav Petkov if (!s->flags.nb_mce_enable) 22215980bb9cSBorislav Petkov reg->l &= ~MSR_MCGCTL_NBE; 2222f6d6ae96SBorislav Petkov } 2223f6d6ae96SBorislav Petkov } 2224f6d6ae96SBorislav Petkov wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs); 2225f6d6ae96SBorislav Petkov 2226f6d6ae96SBorislav Petkov free_cpumask_var(cmask); 2227f6d6ae96SBorislav Petkov 2228f6d6ae96SBorislav Petkov return 0; 2229f6d6ae96SBorislav Petkov } 2230f6d6ae96SBorislav Petkov 22312299ef71SBorislav Petkov static bool enable_ecc_error_reporting(struct ecc_settings *s, u8 nid, 22322299ef71SBorislav Petkov struct pci_dev *F3) 2233f6d6ae96SBorislav Petkov { 22342299ef71SBorislav Petkov bool ret = true; 2235c9f4f26eSBorislav Petkov u32 value, mask = 0x3; /* UECC/CECC enable */ 2236f6d6ae96SBorislav Petkov 22372299ef71SBorislav Petkov if (toggle_ecc_err_reporting(s, nid, ON)) { 22382299ef71SBorislav Petkov amd64_warn("Error enabling ECC reporting over MCGCTL!\n"); 22392299ef71SBorislav Petkov return false; 22402299ef71SBorislav Petkov } 22412299ef71SBorislav Petkov 2242c9f4f26eSBorislav Petkov amd64_read_pci_cfg(F3, NBCTL, &value); 2243f6d6ae96SBorislav Petkov 2244ae7bb7c6SBorislav Petkov s->old_nbctl = value & mask; 2245ae7bb7c6SBorislav Petkov s->nbctl_valid = true; 2246f6d6ae96SBorislav Petkov 2247f6d6ae96SBorislav Petkov value |= mask; 2248c9f4f26eSBorislav Petkov amd64_write_pci_cfg(F3, NBCTL, value); 2249f6d6ae96SBorislav Petkov 2250a97fa68eSBorislav Petkov amd64_read_pci_cfg(F3, NBCFG, &value); 2251f6d6ae96SBorislav Petkov 2252a97fa68eSBorislav Petkov debugf0("1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n", 2253a97fa68eSBorislav Petkov nid, value, !!(value & NBCFG_ECC_ENABLE)); 2254f6d6ae96SBorislav Petkov 2255a97fa68eSBorislav Petkov if (!(value & NBCFG_ECC_ENABLE)) { 225624f9a7feSBorislav Petkov amd64_warn("DRAM ECC disabled on this node, enabling...\n"); 2257f6d6ae96SBorislav Petkov 2258ae7bb7c6SBorislav Petkov s->flags.nb_ecc_prev = 0; 2259d95cf4deSBorislav Petkov 2260f6d6ae96SBorislav Petkov /* Attempt to turn on DRAM ECC Enable */ 2261a97fa68eSBorislav Petkov value |= NBCFG_ECC_ENABLE; 2262a97fa68eSBorislav Petkov amd64_write_pci_cfg(F3, NBCFG, value); 2263f6d6ae96SBorislav Petkov 2264a97fa68eSBorislav Petkov amd64_read_pci_cfg(F3, NBCFG, &value); 2265f6d6ae96SBorislav Petkov 2266a97fa68eSBorislav Petkov if (!(value & NBCFG_ECC_ENABLE)) { 226724f9a7feSBorislav Petkov amd64_warn("Hardware rejected DRAM ECC enable," 226824f9a7feSBorislav Petkov "check memory DIMM configuration.\n"); 22692299ef71SBorislav Petkov ret = false; 2270f6d6ae96SBorislav Petkov } else { 227124f9a7feSBorislav Petkov amd64_info("Hardware accepted DRAM ECC Enable\n"); 2272f6d6ae96SBorislav Petkov } 2273d95cf4deSBorislav Petkov } else { 2274ae7bb7c6SBorislav Petkov s->flags.nb_ecc_prev = 1; 2275f6d6ae96SBorislav Petkov } 2276d95cf4deSBorislav Petkov 2277a97fa68eSBorislav Petkov debugf0("2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n", 2278a97fa68eSBorislav Petkov nid, value, !!(value & NBCFG_ECC_ENABLE)); 2279f6d6ae96SBorislav Petkov 22802299ef71SBorislav Petkov return ret; 2281f6d6ae96SBorislav Petkov } 2282f6d6ae96SBorislav Petkov 2283360b7f3cSBorislav Petkov static void restore_ecc_error_reporting(struct ecc_settings *s, u8 nid, 2284360b7f3cSBorislav Petkov struct pci_dev *F3) 2285f6d6ae96SBorislav Petkov { 2286c9f4f26eSBorislav Petkov u32 value, mask = 0x3; /* UECC/CECC enable */ 2287c9f4f26eSBorislav Petkov 2288f6d6ae96SBorislav Petkov 2289ae7bb7c6SBorislav Petkov if (!s->nbctl_valid) 2290f6d6ae96SBorislav Petkov return; 2291f6d6ae96SBorislav Petkov 2292c9f4f26eSBorislav Petkov amd64_read_pci_cfg(F3, NBCTL, &value); 2293f6d6ae96SBorislav Petkov value &= ~mask; 2294ae7bb7c6SBorislav Petkov value |= s->old_nbctl; 2295f6d6ae96SBorislav Petkov 2296c9f4f26eSBorislav Petkov amd64_write_pci_cfg(F3, NBCTL, value); 2297f6d6ae96SBorislav Petkov 2298ae7bb7c6SBorislav Petkov /* restore previous BIOS DRAM ECC "off" setting we force-enabled */ 2299ae7bb7c6SBorislav Petkov if (!s->flags.nb_ecc_prev) { 2300a97fa68eSBorislav Petkov amd64_read_pci_cfg(F3, NBCFG, &value); 2301a97fa68eSBorislav Petkov value &= ~NBCFG_ECC_ENABLE; 2302a97fa68eSBorislav Petkov amd64_write_pci_cfg(F3, NBCFG, value); 2303d95cf4deSBorislav Petkov } 2304d95cf4deSBorislav Petkov 2305d95cf4deSBorislav Petkov /* restore the NB Enable MCGCTL bit */ 23062299ef71SBorislav Petkov if (toggle_ecc_err_reporting(s, nid, OFF)) 230724f9a7feSBorislav Petkov amd64_warn("Error restoring NB MCGCTL settings!\n"); 2308f6d6ae96SBorislav Petkov } 2309f6d6ae96SBorislav Petkov 2310f9431992SDoug Thompson /* 23112299ef71SBorislav Petkov * EDAC requires that the BIOS have ECC enabled before 23122299ef71SBorislav Petkov * taking over the processing of ECC errors. A command line 23132299ef71SBorislav Petkov * option allows to force-enable hardware ECC later in 23142299ef71SBorislav Petkov * enable_ecc_error_reporting(). 2315f9431992SDoug Thompson */ 2316cab4d277SBorislav Petkov static const char *ecc_msg = 2317cab4d277SBorislav Petkov "ECC disabled in the BIOS or no ECC capability, module will not load.\n" 2318cab4d277SBorislav Petkov " Either enable ECC checking or force module loading by setting " 2319cab4d277SBorislav Petkov "'ecc_enable_override'.\n" 2320cab4d277SBorislav Petkov " (Note that use of the override may cause unknown side effects.)\n"; 2321be3468e8SBorislav Petkov 23222299ef71SBorislav Petkov static bool ecc_enabled(struct pci_dev *F3, u8 nid) 2323f9431992SDoug Thompson { 2324f9431992SDoug Thompson u32 value; 23252299ef71SBorislav Petkov u8 ecc_en = 0; 232606724535SBorislav Petkov bool nb_mce_en = false; 2327f9431992SDoug Thompson 2328a97fa68eSBorislav Petkov amd64_read_pci_cfg(F3, NBCFG, &value); 2329f9431992SDoug Thompson 2330a97fa68eSBorislav Petkov ecc_en = !!(value & NBCFG_ECC_ENABLE); 23312299ef71SBorislav Petkov amd64_info("DRAM ECC %s.\n", (ecc_en ? "enabled" : "disabled")); 2332be3468e8SBorislav Petkov 23332299ef71SBorislav Petkov nb_mce_en = amd64_nb_mce_bank_enabled_on_node(nid); 233406724535SBorislav Petkov if (!nb_mce_en) 23352299ef71SBorislav Petkov amd64_notice("NB MCE bank disabled, set MSR " 23362299ef71SBorislav Petkov "0x%08x[4] on node %d to enable.\n", 23372299ef71SBorislav Petkov MSR_IA32_MCG_CTL, nid); 2338be3468e8SBorislav Petkov 23392299ef71SBorislav Petkov if (!ecc_en || !nb_mce_en) { 234024f9a7feSBorislav Petkov amd64_notice("%s", ecc_msg); 23412299ef71SBorislav Petkov return false; 2342be3468e8SBorislav Petkov } 23432299ef71SBorislav Petkov return true; 2344f9431992SDoug Thompson } 2345f9431992SDoug Thompson 23467d6034d3SDoug Thompson struct mcidev_sysfs_attribute sysfs_attrs[ARRAY_SIZE(amd64_dbg_attrs) + 23477d6034d3SDoug Thompson ARRAY_SIZE(amd64_inj_attrs) + 23487d6034d3SDoug Thompson 1]; 23497d6034d3SDoug Thompson 23507d6034d3SDoug Thompson struct mcidev_sysfs_attribute terminator = { .attr = { .name = NULL } }; 23517d6034d3SDoug Thompson 2352360b7f3cSBorislav Petkov static void set_mc_sysfs_attrs(struct mem_ctl_info *mci) 23537d6034d3SDoug Thompson { 23547d6034d3SDoug Thompson unsigned int i = 0, j = 0; 23557d6034d3SDoug Thompson 23567d6034d3SDoug Thompson for (; i < ARRAY_SIZE(amd64_dbg_attrs); i++) 23577d6034d3SDoug Thompson sysfs_attrs[i] = amd64_dbg_attrs[i]; 23587d6034d3SDoug Thompson 2359a135cef7SBorislav Petkov if (boot_cpu_data.x86 >= 0x10) 23607d6034d3SDoug Thompson for (j = 0; j < ARRAY_SIZE(amd64_inj_attrs); j++, i++) 23617d6034d3SDoug Thompson sysfs_attrs[i] = amd64_inj_attrs[j]; 23627d6034d3SDoug Thompson 23637d6034d3SDoug Thompson sysfs_attrs[i] = terminator; 23647d6034d3SDoug Thompson 23657d6034d3SDoug Thompson mci->mc_driver_sysfs_attributes = sysfs_attrs; 23667d6034d3SDoug Thompson } 23677d6034d3SDoug Thompson 2368df71a053SBorislav Petkov static void setup_mci_misc_attrs(struct mem_ctl_info *mci, 2369df71a053SBorislav Petkov struct amd64_family_type *fam) 23707d6034d3SDoug Thompson { 23717d6034d3SDoug Thompson struct amd64_pvt *pvt = mci->pvt_info; 23727d6034d3SDoug Thompson 23737d6034d3SDoug Thompson mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2; 23747d6034d3SDoug Thompson mci->edac_ctl_cap = EDAC_FLAG_NONE; 23757d6034d3SDoug Thompson 23765980bb9cSBorislav Petkov if (pvt->nbcap & NBCAP_SECDED) 23777d6034d3SDoug Thompson mci->edac_ctl_cap |= EDAC_FLAG_SECDED; 23787d6034d3SDoug Thompson 23795980bb9cSBorislav Petkov if (pvt->nbcap & NBCAP_CHIPKILL) 23807d6034d3SDoug Thompson mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED; 23817d6034d3SDoug Thompson 23827d6034d3SDoug Thompson mci->edac_cap = amd64_determine_edac_cap(pvt); 23837d6034d3SDoug Thompson mci->mod_name = EDAC_MOD_STR; 23847d6034d3SDoug Thompson mci->mod_ver = EDAC_AMD64_VERSION; 2385df71a053SBorislav Petkov mci->ctl_name = fam->ctl_name; 23868d5b5d9cSBorislav Petkov mci->dev_name = pci_name(pvt->F2); 23877d6034d3SDoug Thompson mci->ctl_page_to_phys = NULL; 23887d6034d3SDoug Thompson 23897d6034d3SDoug Thompson /* memory scrubber interface */ 23907d6034d3SDoug Thompson mci->set_sdram_scrub_rate = amd64_set_scrub_rate; 23917d6034d3SDoug Thompson mci->get_sdram_scrub_rate = amd64_get_scrub_rate; 23927d6034d3SDoug Thompson } 23937d6034d3SDoug Thompson 23940092b20dSBorislav Petkov /* 23950092b20dSBorislav Petkov * returns a pointer to the family descriptor on success, NULL otherwise. 23960092b20dSBorislav Petkov */ 23970092b20dSBorislav Petkov static struct amd64_family_type *amd64_per_family_init(struct amd64_pvt *pvt) 2398395ae783SBorislav Petkov { 23990092b20dSBorislav Petkov u8 fam = boot_cpu_data.x86; 24000092b20dSBorislav Petkov struct amd64_family_type *fam_type = NULL; 24010092b20dSBorislav Petkov 24020092b20dSBorislav Petkov switch (fam) { 2403395ae783SBorislav Petkov case 0xf: 24040092b20dSBorislav Petkov fam_type = &amd64_family_types[K8_CPUS]; 2405b8cfa02fSBorislav Petkov pvt->ops = &amd64_family_types[K8_CPUS].ops; 2406395ae783SBorislav Petkov break; 2407df71a053SBorislav Petkov 2408395ae783SBorislav Petkov case 0x10: 24090092b20dSBorislav Petkov fam_type = &amd64_family_types[F10_CPUS]; 2410b8cfa02fSBorislav Petkov pvt->ops = &amd64_family_types[F10_CPUS].ops; 2411df71a053SBorislav Petkov break; 2412df71a053SBorislav Petkov 2413df71a053SBorislav Petkov case 0x15: 2414df71a053SBorislav Petkov fam_type = &amd64_family_types[F15_CPUS]; 2415df71a053SBorislav Petkov pvt->ops = &amd64_family_types[F15_CPUS].ops; 2416395ae783SBorislav Petkov break; 2417395ae783SBorislav Petkov 2418395ae783SBorislav Petkov default: 241924f9a7feSBorislav Petkov amd64_err("Unsupported family!\n"); 24200092b20dSBorislav Petkov return NULL; 2421395ae783SBorislav Petkov } 24220092b20dSBorislav Petkov 2423b8cfa02fSBorislav Petkov pvt->ext_model = boot_cpu_data.x86_model >> 4; 2424b8cfa02fSBorislav Petkov 2425df71a053SBorislav Petkov amd64_info("%s %sdetected (node %d).\n", fam_type->ctl_name, 24260092b20dSBorislav Petkov (fam == 0xf ? 24270092b20dSBorislav Petkov (pvt->ext_model >= K8_REV_F ? "revF or later " 24280092b20dSBorislav Petkov : "revE or earlier ") 242924f9a7feSBorislav Petkov : ""), pvt->mc_node_id); 24300092b20dSBorislav Petkov return fam_type; 2431395ae783SBorislav Petkov } 2432395ae783SBorislav Petkov 24332299ef71SBorislav Petkov static int amd64_init_one_instance(struct pci_dev *F2) 24347d6034d3SDoug Thompson { 24357d6034d3SDoug Thompson struct amd64_pvt *pvt = NULL; 24360092b20dSBorislav Petkov struct amd64_family_type *fam_type = NULL; 2437360b7f3cSBorislav Petkov struct mem_ctl_info *mci = NULL; 24387d6034d3SDoug Thompson int err = 0, ret; 2439360b7f3cSBorislav Petkov u8 nid = get_node_id(F2); 24407d6034d3SDoug Thompson 24417d6034d3SDoug Thompson ret = -ENOMEM; 24427d6034d3SDoug Thompson pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL); 24437d6034d3SDoug Thompson if (!pvt) 2444360b7f3cSBorislav Petkov goto err_ret; 24457d6034d3SDoug Thompson 2446360b7f3cSBorislav Petkov pvt->mc_node_id = nid; 24478d5b5d9cSBorislav Petkov pvt->F2 = F2; 24487d6034d3SDoug Thompson 2449395ae783SBorislav Petkov ret = -EINVAL; 24500092b20dSBorislav Petkov fam_type = amd64_per_family_init(pvt); 24510092b20dSBorislav Petkov if (!fam_type) 2452395ae783SBorislav Petkov goto err_free; 2453395ae783SBorislav Petkov 24547d6034d3SDoug Thompson ret = -ENODEV; 2455360b7f3cSBorislav Petkov err = reserve_mc_sibling_devs(pvt, fam_type->f1_id, fam_type->f3_id); 24567d6034d3SDoug Thompson if (err) 24577d6034d3SDoug Thompson goto err_free; 24587d6034d3SDoug Thompson 2459360b7f3cSBorislav Petkov read_mc_regs(pvt); 24607d6034d3SDoug Thompson 24617d6034d3SDoug Thompson /* 24627d6034d3SDoug Thompson * We need to determine how many memory channels there are. Then use 24637d6034d3SDoug Thompson * that information for calculating the size of the dynamic instance 2464360b7f3cSBorislav Petkov * tables in the 'mci' structure. 24657d6034d3SDoug Thompson */ 2466360b7f3cSBorislav Petkov ret = -EINVAL; 24677d6034d3SDoug Thompson pvt->channel_count = pvt->ops->early_channel_count(pvt); 24687d6034d3SDoug Thompson if (pvt->channel_count < 0) 2469360b7f3cSBorislav Petkov goto err_siblings; 24707d6034d3SDoug Thompson 24717d6034d3SDoug Thompson ret = -ENOMEM; 247211c75eadSBorislav Petkov mci = edac_mc_alloc(0, pvt->csels[0].b_cnt, pvt->channel_count, nid); 24737d6034d3SDoug Thompson if (!mci) 2474360b7f3cSBorislav Petkov goto err_siblings; 24757d6034d3SDoug Thompson 24767d6034d3SDoug Thompson mci->pvt_info = pvt; 24778d5b5d9cSBorislav Petkov mci->dev = &pvt->F2->dev; 24787d6034d3SDoug Thompson 2479df71a053SBorislav Petkov setup_mci_misc_attrs(mci, fam_type); 2480360b7f3cSBorislav Petkov 2481360b7f3cSBorislav Petkov if (init_csrows(mci)) 24827d6034d3SDoug Thompson mci->edac_cap = EDAC_FLAG_NONE; 24837d6034d3SDoug Thompson 2484360b7f3cSBorislav Petkov set_mc_sysfs_attrs(mci); 24857d6034d3SDoug Thompson 24867d6034d3SDoug Thompson ret = -ENODEV; 24877d6034d3SDoug Thompson if (edac_mc_add_mc(mci)) { 24887d6034d3SDoug Thompson debugf1("failed edac_mc_add_mc()\n"); 24897d6034d3SDoug Thompson goto err_add_mc; 24907d6034d3SDoug Thompson } 24917d6034d3SDoug Thompson 2492549d042dSBorislav Petkov /* register stuff with EDAC MCE */ 2493549d042dSBorislav Petkov if (report_gart_errors) 2494549d042dSBorislav Petkov amd_report_gart_errors(true); 2495549d042dSBorislav Petkov 2496549d042dSBorislav Petkov amd_register_ecc_decoder(amd64_decode_bus_error); 2497549d042dSBorislav Petkov 2498360b7f3cSBorislav Petkov mcis[nid] = mci; 2499360b7f3cSBorislav Petkov 2500360b7f3cSBorislav Petkov atomic_inc(&drv_instances); 2501360b7f3cSBorislav Petkov 25027d6034d3SDoug Thompson return 0; 25037d6034d3SDoug Thompson 25047d6034d3SDoug Thompson err_add_mc: 25057d6034d3SDoug Thompson edac_mc_free(mci); 25067d6034d3SDoug Thompson 2507360b7f3cSBorislav Petkov err_siblings: 2508360b7f3cSBorislav Petkov free_mc_sibling_devs(pvt); 25097d6034d3SDoug Thompson 2510360b7f3cSBorislav Petkov err_free: 2511360b7f3cSBorislav Petkov kfree(pvt); 25127d6034d3SDoug Thompson 2513360b7f3cSBorislav Petkov err_ret: 25147d6034d3SDoug Thompson return ret; 25157d6034d3SDoug Thompson } 25167d6034d3SDoug Thompson 25172299ef71SBorislav Petkov static int __devinit amd64_probe_one_instance(struct pci_dev *pdev, 25187d6034d3SDoug Thompson const struct pci_device_id *mc_type) 25197d6034d3SDoug Thompson { 2520ae7bb7c6SBorislav Petkov u8 nid = get_node_id(pdev); 25212299ef71SBorislav Petkov struct pci_dev *F3 = node_to_amd_nb(nid)->misc; 2522ae7bb7c6SBorislav Petkov struct ecc_settings *s; 25232299ef71SBorislav Petkov int ret = 0; 25247d6034d3SDoug Thompson 25257d6034d3SDoug Thompson ret = pci_enable_device(pdev); 2526b8cfa02fSBorislav Petkov if (ret < 0) { 25277d6034d3SDoug Thompson debugf0("ret=%d\n", ret); 2528b8cfa02fSBorislav Petkov return -EIO; 2529b8cfa02fSBorislav Petkov } 2530b8cfa02fSBorislav Petkov 2531ae7bb7c6SBorislav Petkov ret = -ENOMEM; 2532ae7bb7c6SBorislav Petkov s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL); 2533ae7bb7c6SBorislav Petkov if (!s) 25342299ef71SBorislav Petkov goto err_out; 2535ae7bb7c6SBorislav Petkov 2536ae7bb7c6SBorislav Petkov ecc_stngs[nid] = s; 2537ae7bb7c6SBorislav Petkov 25382299ef71SBorislav Petkov if (!ecc_enabled(F3, nid)) { 25392299ef71SBorislav Petkov ret = -ENODEV; 25402299ef71SBorislav Petkov 25412299ef71SBorislav Petkov if (!ecc_enable_override) 25422299ef71SBorislav Petkov goto err_enable; 25432299ef71SBorislav Petkov 25442299ef71SBorislav Petkov amd64_warn("Forcing ECC on!\n"); 25452299ef71SBorislav Petkov 25462299ef71SBorislav Petkov if (!enable_ecc_error_reporting(s, nid, F3)) 25472299ef71SBorislav Petkov goto err_enable; 25482299ef71SBorislav Petkov } 25492299ef71SBorislav Petkov 25502299ef71SBorislav Petkov ret = amd64_init_one_instance(pdev); 2551360b7f3cSBorislav Petkov if (ret < 0) { 2552ae7bb7c6SBorislav Petkov amd64_err("Error probing instance: %d\n", nid); 2553360b7f3cSBorislav Petkov restore_ecc_error_reporting(s, nid, F3); 2554360b7f3cSBorislav Petkov } 25557d6034d3SDoug Thompson 25567d6034d3SDoug Thompson return ret; 25572299ef71SBorislav Petkov 25582299ef71SBorislav Petkov err_enable: 25592299ef71SBorislav Petkov kfree(s); 25602299ef71SBorislav Petkov ecc_stngs[nid] = NULL; 25612299ef71SBorislav Petkov 25622299ef71SBorislav Petkov err_out: 25632299ef71SBorislav Petkov return ret; 25647d6034d3SDoug Thompson } 25657d6034d3SDoug Thompson 25667d6034d3SDoug Thompson static void __devexit amd64_remove_one_instance(struct pci_dev *pdev) 25677d6034d3SDoug Thompson { 25687d6034d3SDoug Thompson struct mem_ctl_info *mci; 25697d6034d3SDoug Thompson struct amd64_pvt *pvt; 2570360b7f3cSBorislav Petkov u8 nid = get_node_id(pdev); 2571360b7f3cSBorislav Petkov struct pci_dev *F3 = node_to_amd_nb(nid)->misc; 2572360b7f3cSBorislav Petkov struct ecc_settings *s = ecc_stngs[nid]; 25737d6034d3SDoug Thompson 25747d6034d3SDoug Thompson /* Remove from EDAC CORE tracking list */ 25757d6034d3SDoug Thompson mci = edac_mc_del_mc(&pdev->dev); 25767d6034d3SDoug Thompson if (!mci) 25777d6034d3SDoug Thompson return; 25787d6034d3SDoug Thompson 25797d6034d3SDoug Thompson pvt = mci->pvt_info; 25807d6034d3SDoug Thompson 2581360b7f3cSBorislav Petkov restore_ecc_error_reporting(s, nid, F3); 25827d6034d3SDoug Thompson 2583360b7f3cSBorislav Petkov free_mc_sibling_devs(pvt); 25847d6034d3SDoug Thompson 2585549d042dSBorislav Petkov /* unregister from EDAC MCE */ 2586549d042dSBorislav Petkov amd_report_gart_errors(false); 2587549d042dSBorislav Petkov amd_unregister_ecc_decoder(amd64_decode_bus_error); 2588549d042dSBorislav Petkov 2589360b7f3cSBorislav Petkov kfree(ecc_stngs[nid]); 2590360b7f3cSBorislav Petkov ecc_stngs[nid] = NULL; 2591ae7bb7c6SBorislav Petkov 25927d6034d3SDoug Thompson /* Free the EDAC CORE resources */ 25938f68ed97SBorislav Petkov mci->pvt_info = NULL; 2594360b7f3cSBorislav Petkov mcis[nid] = NULL; 25958f68ed97SBorislav Petkov 25968f68ed97SBorislav Petkov kfree(pvt); 25977d6034d3SDoug Thompson edac_mc_free(mci); 25987d6034d3SDoug Thompson } 25997d6034d3SDoug Thompson 26007d6034d3SDoug Thompson /* 26017d6034d3SDoug Thompson * This table is part of the interface for loading drivers for PCI devices. The 26027d6034d3SDoug Thompson * PCI core identifies what devices are on a system during boot, and then 26037d6034d3SDoug Thompson * inquiry this table to see if this driver is for a given device found. 26047d6034d3SDoug Thompson */ 26057d6034d3SDoug Thompson static const struct pci_device_id amd64_pci_table[] __devinitdata = { 26067d6034d3SDoug Thompson { 26077d6034d3SDoug Thompson .vendor = PCI_VENDOR_ID_AMD, 26087d6034d3SDoug Thompson .device = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL, 26097d6034d3SDoug Thompson .subvendor = PCI_ANY_ID, 26107d6034d3SDoug Thompson .subdevice = PCI_ANY_ID, 26117d6034d3SDoug Thompson .class = 0, 26127d6034d3SDoug Thompson .class_mask = 0, 26137d6034d3SDoug Thompson }, 26147d6034d3SDoug Thompson { 26157d6034d3SDoug Thompson .vendor = PCI_VENDOR_ID_AMD, 26167d6034d3SDoug Thompson .device = PCI_DEVICE_ID_AMD_10H_NB_DRAM, 26177d6034d3SDoug Thompson .subvendor = PCI_ANY_ID, 26187d6034d3SDoug Thompson .subdevice = PCI_ANY_ID, 26197d6034d3SDoug Thompson .class = 0, 26207d6034d3SDoug Thompson .class_mask = 0, 26217d6034d3SDoug Thompson }, 2622df71a053SBorislav Petkov { 2623df71a053SBorislav Petkov .vendor = PCI_VENDOR_ID_AMD, 2624df71a053SBorislav Petkov .device = PCI_DEVICE_ID_AMD_15H_NB_F2, 2625df71a053SBorislav Petkov .subvendor = PCI_ANY_ID, 2626df71a053SBorislav Petkov .subdevice = PCI_ANY_ID, 2627df71a053SBorislav Petkov .class = 0, 2628df71a053SBorislav Petkov .class_mask = 0, 2629df71a053SBorislav Petkov }, 2630df71a053SBorislav Petkov 26317d6034d3SDoug Thompson {0, } 26327d6034d3SDoug Thompson }; 26337d6034d3SDoug Thompson MODULE_DEVICE_TABLE(pci, amd64_pci_table); 26347d6034d3SDoug Thompson 26357d6034d3SDoug Thompson static struct pci_driver amd64_pci_driver = { 26367d6034d3SDoug Thompson .name = EDAC_MOD_STR, 26372299ef71SBorislav Petkov .probe = amd64_probe_one_instance, 26387d6034d3SDoug Thompson .remove = __devexit_p(amd64_remove_one_instance), 26397d6034d3SDoug Thompson .id_table = amd64_pci_table, 26407d6034d3SDoug Thompson }; 26417d6034d3SDoug Thompson 2642360b7f3cSBorislav Petkov static void setup_pci_device(void) 26437d6034d3SDoug Thompson { 26447d6034d3SDoug Thompson struct mem_ctl_info *mci; 26457d6034d3SDoug Thompson struct amd64_pvt *pvt; 26467d6034d3SDoug Thompson 26477d6034d3SDoug Thompson if (amd64_ctl_pci) 26487d6034d3SDoug Thompson return; 26497d6034d3SDoug Thompson 2650cc4d8860SBorislav Petkov mci = mcis[0]; 26517d6034d3SDoug Thompson if (mci) { 26527d6034d3SDoug Thompson 26537d6034d3SDoug Thompson pvt = mci->pvt_info; 26547d6034d3SDoug Thompson amd64_ctl_pci = 26558d5b5d9cSBorislav Petkov edac_pci_create_generic_ctl(&pvt->F2->dev, EDAC_MOD_STR); 26567d6034d3SDoug Thompson 26577d6034d3SDoug Thompson if (!amd64_ctl_pci) { 26587d6034d3SDoug Thompson pr_warning("%s(): Unable to create PCI control\n", 26597d6034d3SDoug Thompson __func__); 26607d6034d3SDoug Thompson 26617d6034d3SDoug Thompson pr_warning("%s(): PCI error report via EDAC not set\n", 26627d6034d3SDoug Thompson __func__); 26637d6034d3SDoug Thompson } 26647d6034d3SDoug Thompson } 26657d6034d3SDoug Thompson } 26667d6034d3SDoug Thompson 26677d6034d3SDoug Thompson static int __init amd64_edac_init(void) 26687d6034d3SDoug Thompson { 2669360b7f3cSBorislav Petkov int err = -ENODEV; 26707d6034d3SDoug Thompson 2671df71a053SBorislav Petkov printk(KERN_INFO "AMD64 EDAC driver v%s\n", EDAC_AMD64_VERSION); 26727d6034d3SDoug Thompson 26737d6034d3SDoug Thompson opstate_init(); 26747d6034d3SDoug Thompson 26759653a5c7SHans Rosenfeld if (amd_cache_northbridges() < 0) 267656b34b91SBorislav Petkov goto err_ret; 26777d6034d3SDoug Thompson 2678cc4d8860SBorislav Petkov err = -ENOMEM; 2679cc4d8860SBorislav Petkov mcis = kzalloc(amd_nb_num() * sizeof(mcis[0]), GFP_KERNEL); 2680ae7bb7c6SBorislav Petkov ecc_stngs = kzalloc(amd_nb_num() * sizeof(ecc_stngs[0]), GFP_KERNEL); 2681360b7f3cSBorislav Petkov if (!(mcis && ecc_stngs)) 2682cc4d8860SBorislav Petkov goto err_ret; 2683cc4d8860SBorislav Petkov 268450542251SBorislav Petkov msrs = msrs_alloc(); 268556b34b91SBorislav Petkov if (!msrs) 2686360b7f3cSBorislav Petkov goto err_free; 268750542251SBorislav Petkov 26887d6034d3SDoug Thompson err = pci_register_driver(&amd64_pci_driver); 26897d6034d3SDoug Thompson if (err) 269056b34b91SBorislav Petkov goto err_pci; 26917d6034d3SDoug Thompson 269256b34b91SBorislav Petkov err = -ENODEV; 2693360b7f3cSBorislav Petkov if (!atomic_read(&drv_instances)) 2694360b7f3cSBorislav Petkov goto err_no_instances; 26957d6034d3SDoug Thompson 2696360b7f3cSBorislav Petkov setup_pci_device(); 26977d6034d3SDoug Thompson return 0; 26987d6034d3SDoug Thompson 2699360b7f3cSBorislav Petkov err_no_instances: 27007d6034d3SDoug Thompson pci_unregister_driver(&amd64_pci_driver); 2701cc4d8860SBorislav Petkov 270256b34b91SBorislav Petkov err_pci: 270356b34b91SBorislav Petkov msrs_free(msrs); 270456b34b91SBorislav Petkov msrs = NULL; 2705cc4d8860SBorislav Petkov 2706360b7f3cSBorislav Petkov err_free: 2707360b7f3cSBorislav Petkov kfree(mcis); 2708360b7f3cSBorislav Petkov mcis = NULL; 2709360b7f3cSBorislav Petkov 2710360b7f3cSBorislav Petkov kfree(ecc_stngs); 2711360b7f3cSBorislav Petkov ecc_stngs = NULL; 2712360b7f3cSBorislav Petkov 271356b34b91SBorislav Petkov err_ret: 27147d6034d3SDoug Thompson return err; 27157d6034d3SDoug Thompson } 27167d6034d3SDoug Thompson 27177d6034d3SDoug Thompson static void __exit amd64_edac_exit(void) 27187d6034d3SDoug Thompson { 27197d6034d3SDoug Thompson if (amd64_ctl_pci) 27207d6034d3SDoug Thompson edac_pci_release_generic_ctl(amd64_ctl_pci); 27217d6034d3SDoug Thompson 27227d6034d3SDoug Thompson pci_unregister_driver(&amd64_pci_driver); 272350542251SBorislav Petkov 2724ae7bb7c6SBorislav Petkov kfree(ecc_stngs); 2725ae7bb7c6SBorislav Petkov ecc_stngs = NULL; 2726ae7bb7c6SBorislav Petkov 2727cc4d8860SBorislav Petkov kfree(mcis); 2728cc4d8860SBorislav Petkov mcis = NULL; 2729cc4d8860SBorislav Petkov 273050542251SBorislav Petkov msrs_free(msrs); 273150542251SBorislav Petkov msrs = NULL; 27327d6034d3SDoug Thompson } 27337d6034d3SDoug Thompson 27347d6034d3SDoug Thompson module_init(amd64_edac_init); 27357d6034d3SDoug Thompson module_exit(amd64_edac_exit); 27367d6034d3SDoug Thompson 27377d6034d3SDoug Thompson MODULE_LICENSE("GPL"); 27387d6034d3SDoug Thompson MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, " 27397d6034d3SDoug Thompson "Dave Peterson, Thayne Harbaugh"); 27407d6034d3SDoug Thompson MODULE_DESCRIPTION("MC support for AMD64 memory controllers - " 27417d6034d3SDoug Thompson EDAC_AMD64_VERSION); 27427d6034d3SDoug Thompson 27437d6034d3SDoug Thompson module_param(edac_op_state, int, 0444); 27447d6034d3SDoug Thompson MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI"); 2745